1 | # To set up a new LDAP server: |
---|
2 | |
---|
3 | # Temporarily move away the existing slapd-scripts folder |
---|
4 | mv /etc/dirsrv/slapd-scripts{,.bak} |
---|
5 | |
---|
6 | # Setup directory server |
---|
7 | /usr/sbin/setup-ds.pl |
---|
8 | # - Choose a typical install |
---|
9 | # - Tell it to use the fedora-ds user and group |
---|
10 | # - Directory server identifier: scripts |
---|
11 | # - Suffix: dc=scripts,dc=mit,dc=edu |
---|
12 | # - Input directory manager password |
---|
13 | # (this can be found in ~/.ldapvirc) |
---|
14 | |
---|
15 | # Move the schema back |
---|
16 | cp -R /etc/dirsrv/slapd-scripts.bak/{.svn,*} /etc/dirsrv/slapd-scripts |
---|
17 | rm -Rf /etc/dirsrv/slapd-scripts.bak |
---|
18 | |
---|
19 | # Check and make sure the sysconfig references the correct keytab |
---|
20 | /etc/sysconfig/dirsrv-scripts |
---|
21 | |
---|
22 | # Turn dirsrv off: |
---|
23 | systemctl stop dirsrv@scripts.service |
---|
24 | |
---|
25 | # Apply the following configuration changes. If you're editing |
---|
26 | # dse.ldif, you don't want dirsrv to be on, otherwise it will |
---|
27 | # overwrite your changes. [XXX: show how to do these changes with |
---|
28 | # dsconf, which is the "blessed" method, although it seems |
---|
29 | # dsconf only exists for Red Hat] |
---|
30 | |
---|
31 | vim /etc/dirsrv/slapd-scripts/dse.ldif |
---|
32 | <<<EOF |
---|
33 | |
---|
34 | # Inside cn=config. These changes definitely require a restart. |
---|
35 | nsslapd-ldapilisten: on |
---|
36 | nsslapd-syntaxcheck: off |
---|
37 | |
---|
38 | # We need to turn off syntax check because our schema is wrong and too |
---|
39 | # restrictive on some value. This should get fixed. |
---|
40 | |
---|
41 | # Add these blocks |
---|
42 | |
---|
43 | # mapname, mapping, sasl, config |
---|
44 | # This is the most liberal mapping you can have for SASL: you can |
---|
45 | # basically add authentication for any given GSSAPI mechanism by |
---|
46 | # explicitly creating the UID for that SASL string. |
---|
47 | dn: cn=mapname,cn=mapping,cn=sasl,cn=config |
---|
48 | objectClass: top |
---|
49 | objectClass: nsSaslMapping |
---|
50 | cn: mapname |
---|
51 | nsSaslMapRegexString: \(.*\) |
---|
52 | nsSaslMapBaseDNTemplate: uid=\1,ou=People,dc=scripts,dc=mit,dc=edu |
---|
53 | nsSaslMapFilterTemplate: (objectClass=posixAccount) |
---|
54 | |
---|
55 | EOF; |
---|
56 | |
---|
57 | systemctl start dirsrv@scripts.service |
---|
58 | |
---|
59 | ldapvi -b cn=config |
---|
60 | # Add these indexes (8 of them): |
---|
61 | |
---|
62 | <<<EOF |
---|
63 | |
---|
64 | add cn=apacheServerName, cn=index, cn=userRoot, cn=ldbm database, cn=plugins, cn=config |
---|
65 | objectClass: top |
---|
66 | objectClass: nsIndex |
---|
67 | cn: apacheServerName |
---|
68 | nsSystemIndex: false |
---|
69 | nsIndexType: eq |
---|
70 | nsIndexType: pres |
---|
71 | |
---|
72 | add cn=apacheServerAlias, cn=index, cn=userRoot, cn=ldbm database, cn=plugins, cn=config |
---|
73 | objectClass: top |
---|
74 | objectClass: nsIndex |
---|
75 | cn: apacheServerAlias |
---|
76 | nsSystemIndex: false |
---|
77 | nsIndexType: eq |
---|
78 | nsIndexType: pres |
---|
79 | |
---|
80 | add cn=scriptsVhostName, cn=index, cn=userRoot, cn=ldbm database, cn=plugins, cn=config |
---|
81 | objectClass: top |
---|
82 | objectClass: nsIndex |
---|
83 | cn: scriptsVhostName |
---|
84 | nsSystemIndex: false |
---|
85 | nsIndexType: eq |
---|
86 | nsIndexType: pres |
---|
87 | |
---|
88 | add cn=scriptsVhostAlias, cn=index, cn=userRoot, cn=ldbm database, cn=plugins, cn=config |
---|
89 | objectClass: top |
---|
90 | objectClass: nsIndex |
---|
91 | cn: scriptsVhostAlias |
---|
92 | nsSystemIndex: false |
---|
93 | nsIndexType: eq |
---|
94 | nsIndexType: pres |
---|
95 | |
---|
96 | add cn=scriptsVhostAccount, cn=index, cn=userRoot, cn=ldbm database, cn=plugins, cn=config |
---|
97 | objectClass: top |
---|
98 | objectClass: nsIndex |
---|
99 | cn: scriptsVhostAccount |
---|
100 | nsSystemIndex: false |
---|
101 | nsIndexType: eq |
---|
102 | nsIndexType: pres |
---|
103 | |
---|
104 | add cn=memberuid, cn=index, cn=userRoot, cn=ldbm database, cn=plugins, cn=config |
---|
105 | objectClass: top |
---|
106 | objectClass: nsIndex |
---|
107 | cn: memberuid |
---|
108 | nsSystemIndex: false |
---|
109 | nsIndexType: eq |
---|
110 | nsIndexType: pres |
---|
111 | |
---|
112 | add cn=uidnumber, cn=index, cn=userRoot, cn=ldbm database, cn=plugins, cn=config |
---|
113 | objectClass: top |
---|
114 | objectClass: nsIndex |
---|
115 | cn: uidnumber |
---|
116 | nsSystemIndex: false |
---|
117 | nsIndexType: eq |
---|
118 | nsIndexType: pres |
---|
119 | |
---|
120 | add cn=gidnumber, cn=index, cn=userRoot, cn=ldbm database, cn=plugins, cn=config |
---|
121 | objectClass: top |
---|
122 | objectClass: nsIndex |
---|
123 | cn: gidnumber |
---|
124 | nsSystemIndex: false |
---|
125 | nsIndexType: eq |
---|
126 | nsIndexType: pres |
---|
127 | |
---|
128 | EOF; |
---|
129 | |
---|
130 | - Build the indexes for all the fields: |
---|
131 | |
---|
132 | /usr/lib64/dirsrv/slapd-scripts/db2index.pl -D "cn=Directory Manager" -j /etc/signup-ldap-pw -n userRoot |
---|
133 | |
---|
134 | (/etc/signup-ldap-pw is the LDAP root password, make sure it's |
---|
135 | chmodded correctly and chowned to signup. Also, make sure it doesn't |
---|
136 | have a trailing newline!) |
---|
137 | |
---|
138 | - Watch for the indexing operations to finish with this command: |
---|
139 | |
---|
140 | ldapsearch -x -y /etc/signup-ldap-pw -D 'cn=Directory Manager' -b cn=tasks,cn=config |
---|
141 | |
---|
142 | (look for nktaskstatus) |
---|
143 | |
---|
144 | - Set up replication. |
---|
145 | |
---|
146 | We used to tell people to go execute |
---|
147 | http://directory.fedoraproject.org/sources/contrib/mmr.pl manually |
---|
148 | (manually because that script assumes only two masters and we have |
---|
149 | every one of our servers set up as a master.) However, those |
---|
150 | instructions are inaccurate, because we use GSSAPI, not SSL and |
---|
151 | because the initializing procedure is actually prone to a race |
---|
152 | condition. Here are some better instructions. |
---|
153 | |
---|
154 | LDAP replication is based around producers and consumers. Producers |
---|
155 | push changes in LDAP to consumers: these arrangements are called |
---|
156 | "replication agreements" and the producer will hold a |
---|
157 | nsDS5ReplicationAgreement object that represents this commitment, |
---|
158 | as well as some extra configuration to say who consumers will accept |
---|
159 | replication data from (a nsDS5Replica). |
---|
160 | |
---|
161 | The procedure, at a high level, is this: |
---|
162 | |
---|
163 | 1. Pick an arbitrary existing master. The current server will |
---|
164 | be configured as a slave to that master. Initialize a changelog, |
---|
165 | then request a replication to populate our server with |
---|
166 | information. |
---|
167 | |
---|
168 | M1 <---> M2 ---> S |
---|
169 | |
---|
170 | 2. Configure the new server to be replicated back. |
---|
171 | |
---|
172 | M1 <---> M2 <---> S |
---|
173 | |
---|
174 | 3. Set up the rest of the replication agreements. |
---|
175 | |
---|
176 | M1 <---> M2 |
---|
177 | ^ ^ |
---|
178 | | | |
---|
179 | +--> S <--+ |
---|
180 | |
---|
181 | 4. Push a change from every existing server (to the new server), and |
---|
182 | then a change from the new server to (all) the existing servers. |
---|
183 | In addition to merely testing that replication works, this will |
---|
184 | set up the servers' changelogs properly. |
---|
185 | |
---|
186 | If this step is not completed before any server's LDAP server |
---|
187 | shuts down, then the replication agreements will fall apart the |
---|
188 | next time a change is made. You may wish to intentionally reboot |
---|
189 | any servers that look like they want to crash _before_ beginning |
---|
190 | this process. |
---|
191 | |
---|
192 | Here's how you do it. |
---|
193 | |
---|
194 | NOTE: There's this spiffy new tool MMR hammer which automates some of |
---|
195 | this process. Check the "MMR Hammer" sections to see how. Install it |
---|
196 | here: https://github.com/ezyang/mmr-hammer |
---|
197 | |
---|
198 | 0. Tell -c scripts not to go off and reboot servers until you're |
---|
199 | done (or to get any rebooting done with first). |
---|
200 | |
---|
201 | 1. Pull open the replication part of the database. It's fairly empty |
---|
202 | right now. |
---|
203 | |
---|
204 | ldapvi -b cn=\"dc=scripts,dc=mit,dc=edu\",cn=mapping\ tree,cn=config |
---|
205 | |
---|
206 | 2. Configure the server $SLAVE (this server) to accept $MASTER |
---|
207 | replications by adding the following LDAP entries: |
---|
208 | |
---|
209 | add cn=replica, cn="dc=scripts,dc=mit,dc=edu", cn=mapping tree, cn=config |
---|
210 | objectClass: top |
---|
211 | objectClass: nsDS5Replica |
---|
212 | cn: replica |
---|
213 | nsDS5ReplicaId: $REPLICA_ID |
---|
214 | nsDS5ReplicaRoot: dc=scripts,dc=mit,dc=edu |
---|
215 | nsDS5Flags: 1 |
---|
216 | nsDS5ReplicaBindDN: uid=ldap/bees-knees.mit.edu,ou=People,dc=scripts,dc=mit,dc=edu |
---|
217 | nsDS5ReplicaBindDN: uid=ldap/busy-beaver.mit.edu,ou=People,dc=scripts,dc=mit,dc=edu |
---|
218 | nsDS5ReplicaBindDN: uid=ldap/cats-whiskers.mit.edu,ou=People,dc=scripts,dc=mit,dc=edu |
---|
219 | nsDS5ReplicaBindDN: uid=ldap/pancake-bunny.mit.edu,ou=People,dc=scripts,dc=mit,dc=edu |
---|
220 | nsDS5ReplicaBindDN: uid=ldap/whole-enchilada.mit.edu,ou=People,dc=scripts,dc=mit,dc=edu |
---|
221 | nsDS5ReplicaBindDN: uid=ldap/real-mccoy.mit.edu,ou=People,dc=scripts,dc=mit,dc=edu |
---|
222 | nsDS5ReplicaBindDN: uid=ldap/better-mousetrap.mit.edu,ou=People,dc=scripts,dc=mit,dc=edu |
---|
223 | nsDS5ReplicaBindDN: uid=ldap/old-faithful.mit.edu,ou=People,dc=scripts,dc=mit,dc=edu |
---|
224 | nsDS5ReplicaBindDN: uid=ldap/shining-armor.mit.edu,ou=People,dc=scripts,dc=mit,dc=edu |
---|
225 | nsDS5ReplicaBindDN: uid=ldap/golden-egg.mit.edu,ou=People,dc=scripts,dc=mit,dc=edu |
---|
226 | nsds5ReplicaPurgeDelay: 604800 |
---|
227 | nsds5ReplicaLegacyConsumer: off |
---|
228 | nsDS5ReplicaType: 3 |
---|
229 | |
---|
230 | $REPLICA_ID is the scripts$N number (stella $HOSTNAME to find |
---|
231 | out.) You might wonder why we are binding to all servers; |
---|
232 | weren't we going to replicate from only one server? That is |
---|
233 | correct, however, simply binding won't mean we will receive |
---|
234 | updates; we have to setup the $MASTER to send data $SLAVE. |
---|
235 | |
---|
236 | 3. Although we allowed those uids to bind, that user information |
---|
237 | doesn't exist on $SLAVE yet. So you'll need to create the entry |
---|
238 | for just $MASTER. |
---|
239 | |
---|
240 | REMEMBER: You need to use FOO.mit.edu for the names! Otherwise you will get |
---|
241 | unauthorized errors. |
---|
242 | |
---|
243 | add uid=ldap/$MASTER,ou=People,dc=scripts,dc=mit,dc=edu |
---|
244 | uid: ldap/$MASTER |
---|
245 | objectClass: account |
---|
246 | objectClass: top |
---|
247 | |
---|
248 | 4. Though our $SLAVE will not be making changes to LDAP, we need to |
---|
249 | initialize the changelog because we intend to be able to do this |
---|
250 | later. |
---|
251 | |
---|
252 | add cn=changelog5,cn=config |
---|
253 | objectclass: top |
---|
254 | objectclass: extensibleObject |
---|
255 | cn: changelog5 |
---|
256 | nsslapd-changelogdir: /etc/dirsrv/slapd-scripts/changelogdb |
---|
257 | |
---|
258 | 5. Ok, now go to your $MASTER server that you picked (it should have |
---|
259 | been one of the hosts mentioned in nsDS5ReplicaBindDN) and tell |
---|
260 | it to replicate to $SLAVE. |
---|
261 | |
---|
262 | The last line runs the replication. This is perhaps the most |
---|
263 | risky step of the process; see below for help debugging problems. |
---|
264 | |
---|
265 | MMR Hammer: |
---|
266 | mmr-hammer -h $MASTER init agreements $SLAVE |
---|
267 | mmr-hammer -h $MASTER update $SLAVE # XXX pick a better name |
---|
268 | |
---|
269 | ldapvi -b cn=\"dc=scripts,dc=mit,dc=edu\",cn=mapping\ tree,cn=config |
---|
270 | |
---|
271 | add cn="GSSAPI Replication to $SLAVE", cn=replica, cn="dc=scripts,dc=mit,dc=edu", cn=mapping tree, cn=config |
---|
272 | objectClass: top |
---|
273 | objectClass: nsDS5ReplicationAgreement |
---|
274 | cn: "GSSAPI Replication to $SLAVE" |
---|
275 | cn: GSSAPI Replication to $SLAVE |
---|
276 | nsDS5ReplicaHost: $SLAVE |
---|
277 | nsDS5ReplicaRoot: dc=scripts,dc=mit,dc=edu |
---|
278 | nsDS5ReplicaPort: 389 |
---|
279 | nsDS5ReplicaTransportInfo: LDAP |
---|
280 | nsDS5ReplicaBindDN: uid=ldap/$MASTER,ou=People,dc=scripts,dc=mit,dc=edu |
---|
281 | nsDS5ReplicaBindMethod: SASL/GSSAPI |
---|
282 | nsDS5ReplicaUpdateSchedule: "0000-2359 0123456" |
---|
283 | nsDS5ReplicaTimeout: 120 |
---|
284 | nsDS5BeginReplicaRefresh: start |
---|
285 | |
---|
286 | 5. Check that the replication is running; the status will be stored |
---|
287 | in the object we've been mucking around with. |
---|
288 | |
---|
289 | If it fails with LDAP Error 49, check /var/log/dirsrv on $MASTER |
---|
290 | for more information. It might be because fedora-ds can't read |
---|
291 | /etc/dirsrv/keytab or because you setup the account on the SLAVE |
---|
292 | incorrectly. |
---|
293 | |
---|
294 | 6. Replicate in the other direction. On $MASTER, add $SLAVE |
---|
295 | as a nsDS5ReplicaBindDN in cn=replica,cn="dc=scripts,dc=mit,dc=edu",cn=mapping tree,cn=config |
---|
296 | Also, add an account for $SLAVE if it doesn't exist already. |
---|
297 | |
---|
298 | add uid=ldap/$SLAVE,ou=People,dc=scripts,dc=mit,dc=edu |
---|
299 | uid: ldap/$SLAVE |
---|
300 | objectClass: account |
---|
301 | objectClass: top |
---|
302 | |
---|
303 | On $SLAVE, |
---|
304 | |
---|
305 | MMR Hammer: mmr-hammer -h $SLAVE init agreements $MASTER |
---|
306 | |
---|
307 | add cn="GSSAPI Replication to $MASTER", cn=replica, cn="dc=scripts,dc=mit,dc=edu", cn=mapping tree, cn=config |
---|
308 | objectClass: top |
---|
309 | objectClass: nsDS5ReplicationAgreement |
---|
310 | cn: "GSSAPI Replication to $MASTER" |
---|
311 | cn: GSSAPI Replication to $MASTER |
---|
312 | nsDS5ReplicaHost: $MASTER |
---|
313 | nsDS5ReplicaRoot: dc=scripts,dc=mit,dc=edu |
---|
314 | nsDS5ReplicaPort: 389 |
---|
315 | nsDS5ReplicaTransportInfo: LDAP |
---|
316 | nsDS5ReplicaBindDN: uid=ldap/$SLAVE,ou=People,dc=scripts,dc=mit,dc=edu |
---|
317 | nsDS5ReplicaBindMethod: SASL/GSSAPI |
---|
318 | nsDS5ReplicaUpdateSchedule: "0000-2359 0123456" |
---|
319 | nsDS5ReplicaTimeout: 120 |
---|
320 | |
---|
321 | If you get a really scary internal server error, that might mean you |
---|
322 | forgot to initialize the changelog. Remove the replication |
---|
323 | agreement (you'll need to turn off dirsrv), add the changelog, and |
---|
324 | then try again. |
---|
325 | |
---|
326 | 7. Repeat step 6 to complete the graph of replications (i.e., from |
---|
327 | every other server to the new server, and from the new server to |
---|
328 | every other server). |
---|
329 | |
---|
330 | Note the only difference between steps 5 and 6 is the lack of |
---|
331 | nsDS5ReplicaRefresh: start. That only needs to be done once, to the |
---|
332 | new server. |
---|
333 | |
---|
334 | With MMR hammer, that's something like: |
---|
335 | |
---|
336 | for i in $SERVER_NAMES; do mmr-hammer -h $i init agreements $SERVER_NAMES; done |
---|
337 | |
---|
338 | 8. If at this point you look at the new server's changelog with |
---|
339 | cl-dump (preferably /mit/scripts/admin/cl-dump.pl, to not prompt you |
---|
340 | for a password), you won't see the servers you added in step 7. So, |
---|
341 | from each of those servers, make a change to some record so it gets |
---|
342 | propagated to the new server, and then one from the new server so it |
---|
343 | gets propagated to all the existing servers' changelogs. This is |
---|
344 | also good for making sure the replication agreements actually work. |
---|
345 | |
---|
346 | With MMR hammer, that's something like: |
---|
347 | |
---|
348 | for i in $SERVER_NAMES; do mmr-hammer -h $i test; sleep 20; done |
---|
349 | |
---|
350 | Troubleshooting |
---|
351 | =============== |
---|
352 | |
---|
353 | LDAP multimaster replication can fail in a number of colorful ways; |
---|
354 | combine that with GSSAPI authentication and it goes exponential. |
---|
355 | |
---|
356 | If authentication is failing with LDAP error 49, check if: |
---|
357 | |
---|
358 | * /etc/dirsrv/keytab |
---|
359 | * fedora-ds is able to read /etc/dirsrv/keytab |
---|
360 | * /etc/hosts has not been modified by Network Manager (you |
---|
361 | /did/ uninstall it, right? Right?) |
---|
362 | |
---|
363 | If the failure is local to a single master, usually you can recover |
---|
364 | by asking another master to refresh that master with: |
---|
365 | |
---|
366 | nsDS5BeginReplicaRefresh: start |
---|
367 | |
---|
368 | In practice, we've also had problems with this technique. Some of them |
---|
369 | include: |
---|
370 | |
---|
371 | * Something like https://bugzilla.redhat.com/show_bug.cgi?id=547503 |
---|
372 | on Fedora 11 ns-slapd, where replication is turned off to do the |
---|
373 | replication, but then it wedges and you need to forcibly kill the |
---|
374 | process. |
---|
375 | |
---|
376 | * Failed LDAP authentication because another master attempted to do |
---|
377 | an incremental update. |
---|
378 | |
---|
379 | * Repropagation of the error because the corrupt master thinks it still |
---|
380 | should push updates. |
---|
381 | |
---|
382 | So the extremely safe method to bring up a crashed master is as follows: |
---|
383 | |
---|
384 | 1. Disable all incoming and outgoing replication agreements by editing |
---|
385 | /etc/dirsrv/slapd-scripts/dse.ldif. You'll need to munge: |
---|
386 | |
---|
387 | nsDS5ReplicaBindDN in cn=replica,cn=dc\3Dscripts\2Cdc\3Dmit\2Cdc\3Dedu,cn=mapping tree,cn=config |
---|
388 | |
---|
389 | and all of the push agreements. Deleting them outright works, but |
---|
390 | means you'll have to reconstruct all of the agreements from scratch. |
---|
391 | |
---|
392 | 2. Bring up the server. |
---|
393 | |
---|
394 | 3. Accept incoming replication data from a single server. |
---|
395 | |
---|
396 | 4. Initiate a full update from that server. |
---|
397 | |
---|
398 | 5. Finish setting up replication as described above. |
---|
399 | |
---|
400 | If your database gets extremely fucked, other servers may not be able |
---|
401 | to authenticate because your authentication information has gone missing. |
---|
402 | In that case, the minimal set of entries you need is: |
---|
403 | |
---|
404 | add dc=scripts,dc=mit,dc=edu |
---|
405 | objectClass: top |
---|
406 | objectClass: domain |
---|
407 | dc: scripts |
---|
408 | |
---|
409 | add ou=People,dc=scripts,dc=mit,dc=edu |
---|
410 | objectClass: top |
---|
411 | objectClass: organizationalunit |
---|
412 | ou: People |
---|
413 | |
---|
414 | add uid=ldap/whole-enchilada.mit.edu,ou=People,dc=scripts,dc=mit,dc=edu |
---|
415 | objectClass: account |
---|
416 | objectClass: top |
---|
417 | uid: ldap/whole-enchilada.mit.edu |
---|