1#! /bin/sh
2# OpenLDAP: pkg/ldap/tests/scripts/test058-syncrepl-asymmetric,v 1.1.2.6 2010/04/19 19:14:36 quanah Exp
3## This work is part of OpenLDAP Software <http://www.openldap.org/>.
4##
5## Copyright 1998-2010 The OpenLDAP Foundation.
6## All rights reserved.
7##
8## Redistribution and use in source and binary forms, with or without
9## modification, are permitted only as authorized by the OpenLDAP
10## Public License.
11##
12## A copy of this license is available in the file LICENSE in the
13## top-level directory of the distribution or, alternatively, at
14## <http://www.OpenLDAP.org/license.html>.
15
16# This script tests a configuration scenario as described in these URLs:
17#
18# http://www.openldap.org/lists/openldap-devel/200806/msg00041.html
19# http://www.openldap.org/lists/openldap-devel/200806/msg00054.html
20#
21# Search for "TEST:" to find each major test this script performs.
22
23# The configuration here consist of 3 "sites", each with a "master" and
24# a "search" server. One of the sites is the "central", the other two
25# are called "site1" and "site2".
26
27# The following notations are used in variable names below to identify
28# these servers, the first number defines the $URL# and $PORT# variable
29# that server uses:
30#
31# 1: SMC_*	Site Master Central
32# 2: SM1_*	Site Master 1
33# 3: SM2_*	Site Master 2
34# 4: SSC_*	Search Site Central
35# 5: SS1_*	Search Site 1
36# 6: SS2_*	Search Site 2
37
38# The master servers all have a set of subordinate databases glued below
39# the same suffix database.  Each of the masters are the master for at
40# least one of these subordinate databases, but there are never more
41# than one master for any single database.  I.e, this is neither a
42# traditional single-master configuration, nor what most people think
43# of as multi-master, but more what can be called multiple masters.
44
45# The central master replicates to the two other masters, and receives
46# updates from them of the backends they are the master for.  There is
47# no direct connection between the other two master servers.  All of the
48# masters have the syncprov overlay configured on the glue database.
49
50# The search servers replicates from the master server at their site.
51# They all have a single database with the glue suffix, but their
52# database configuration doesn't matter much in this test.  (This
53# database layout was originally created before gluing was introduced
54# in OpenLDAP, which is why the search servers doesn't use it).
55
56# The primary objective for gluing the backend databases is not to make
57# them look like one huge database but to create a common search suffix
58# for the clients.  Searching is mostly done on the search servers, only
59# updates are done on the masters.
60
61# It varies which backends that are replicated to which server (hence
62# the name asymmetric in this test).  Access control rules on the
63# masters are used to control what their consumers receives.  The table
64# below gives an overview of which backend (the columns) that are
65# replicated to which server (the rows).  A "M" defines the master for
66# the backend, a "S" is a slave, and "-" means it is not replicated
67# there.  Oh, the table probably looks wrong without the 4-position
68# tab-stops OpenLDAP uses...
69
70#		glue	ou1		ou2		sm1ou1	sm1ou2	sm2ou1	sm2ou2
71# smc	M		M		M		S		S		S		-
72# sm1	S		S		-		M		M		-		-
73# sm2	S		S		S		S		-		M		M
74# ssc	S		S		-		-		S		-		-
75# ss1	S		S		-		S		S		-		-
76# ss2	S		S		S		-		-		S		S
77
78# On the central master syncrepl is configured on the subordinate
79# databases, as it varies which backends that exists on its providers.
80# Had it been used on the glue database then syncrepl would have removed
81# the backends replicated from site1 but not present on site2 when it
82# synchronizes with site2 (and vice versa).
83#
84# All the other servers uses syncrepl on the glue database, since
85# replicating more than one subordinate database from the same master
86# creates (as of the writing of this test script) race conditions that
87# causes the replication to fail, as the race tests at the end shows.
88
89# The databases controlled by syncrepl all have $UPDATEDN as their
90# RootDN, while the master servers has other RootDN values for the
91# backends they are the backend for them self.  This violates the current
92# guidelines for gluing databases, which states that the same rootdn
93# should be used on all of them.  Unfortunately, this cannot be done on
94# site masters 1 and 2.  The backends they manage locally are either not
95# present on the central master, or when so they are not replicated back
96# to their source, which causes syncrepl to try to remove the content of
97# these backends when it synchronizes with the central master.  The
98# differing rootdn values used on the backends controlled by syncrepl
99# and those managed locally prevents it from succeeding in this.  As
100# noted above, moving syncrepl to the subordinate databases is currently
101# not an option since that creates race conditions.
102
103# The binddn values used in the syncrepl configurations are chosen to
104# make the configuration and access control rules easiest to set up.  It
105# occasionally uses a DN that is also used as a RootDN.  This is not a
106# good practice and should not be taken as an example for real
107# configurations!
108
109# This script will print the content of any invalid contextCSN values it
110# detects if the environment variable CSN_VERBOSE is non-empty.  The
111# environment variable RACE_TESTS can be set to the number of race test
112# iterations the script should perform.
113
114if test "$BACKEND" = ldif ; then
115	echo "$backend backend does not support access controls, test skipped"
116	exit 0
117fi
118
119echo "running defines.sh"
120. $SRCDIR/scripts/defines.sh
121
122if test $SYNCPROV = syncprovno; then
123	echo "Syncrepl provider overlay not available, test skipped"
124	exit 0
125fi
126
127SMC_DIR=$TESTDIR/smc
128SM1_DIR=$TESTDIR/sm1
129SM2_DIR=$TESTDIR/sm2
130SS1_DIR=$TESTDIR/ss1
131SS2_DIR=$TESTDIR/ss2
132SSC_DIR=$TESTDIR/ssc
133
134mkdir -p $TESTDIR
135
136for dir in $SMC_DIR $SM1_DIR $SM2_DIR $SS1_DIR $SS2_DIR $SSC_DIR; do
137	mkdir -p $dir $dir/slapd.d $dir/db
138done
139
140mkdir -p $SMC_DIR/ou1 $SMC_DIR/sm1ou1 $SMC_DIR/sm1ou2
141mkdir -p $SMC_DIR/ou2 $SMC_DIR/sm2ou1
142mkdir -p $SM1_DIR/ou1 $SM1_DIR/sm1ou1 $SM1_DIR/sm1ou2
143mkdir -p $SM2_DIR/ou2 $SM2_DIR/sm1ou1 $SM2_DIR/sm2ou1 $SM2_DIR/sm2ou2
144
145cd $TESTDIR
146
147KILLPIDS=
148
149$SLAPPASSWD -g -n >$CONFIGPWF
150
151ID=1
152
153if test $WAIT != 0 ; then
154	RETRY="1 60"
155else
156	RETRY="1 10"
157fi
158
159echo "Initializing master configurations..."
160for dir in $SMC_DIR $SM1_DIR $SM2_DIR; do
161	$SLAPADD -F $dir/slapd.d -n 0 <<EOF
162dn: cn=config
163objectClass: olcGlobal
164cn: config
165olcServerID: $ID
166
167dn: olcDatabase={0}config,cn=config
168objectClass: olcDatabaseConfig
169olcDatabase: {0}config
170olcRootPW:< file://$CONFIGPWF
171
172EOF
173	ID=`expr $ID + 1`
174done
175
176echo "Initializing search configurations..."
177for dir in $SS1_DIR $SS2_DIR $SSC_DIR; do
178	$SLAPADD -F $dir/slapd.d -n 0 <<EOF
179dn: cn=config
180objectClass: olcGlobal
181cn: config
182
183dn: olcDatabase={0}config,cn=config
184objectClass: olcDatabaseConfig
185olcDatabase: {0}config
186olcRootPW:< file://$CONFIGPWF
187
188EOF
189done
190
191echo "Starting central master slapd on TCP/IP port $PORT1..."
192cd $SMC_DIR
193$SLAPD -F slapd.d -h $URI1 -d $LVL $TIMING > $LOG1 2>&1 &
194SMC_PID=$!
195if test $WAIT != 0 ; then
196	echo PID $SMC_PID
197	read foo
198fi
199KILLPIDS="$KILLPIDS $SMC_PID"
200cd $TESTWD
201sleep 1
202echo "Using ldapsearch to check that central master slapd is running..."
203for i in 1 2 3 4 5; do
204	$LDAPSEARCH -s base -b "" -H $URI1 \
205		'objectclass=*' > /dev/null 2>&1
206	RC=$?
207	test $RC = 0 && break
208	echo "Waiting $i seconds for slapd to start..."
209	sleep $i
210done
211if test $RC != 0 ; then
212	echo "ldapsearch failed ($RC)!"
213	test $KILLSERVERS != no && kill -HUP $KILLPIDS
214	exit $RC
215fi
216
217echo "Starting site1 master slapd on TCP/IP port $PORT2..."
218cd $SM1_DIR
219$SLAPD -F slapd.d -h $URI2 -d $LVL $TIMING > $LOG2 2>&1 &
220SM1_PID=$!
221if test $WAIT != 0 ; then
222	echo PID $SM1_PID
223	read foo
224fi
225KILLPIDS="$KILLPIDS $SM1_PID"
226cd $TESTWD
227sleep 1
228echo "Using ldapsearch to check that site1 master is running..."
229for i in 1 2 3 4 5; do
230	$LDAPSEARCH -s base -b "" -H $URI2 \
231		'objectclass=*' > /dev/null 2>&1
232	RC=$?
233	test $RC = 0 && break
234	echo "Waiting $i seconds for slapd to start..."
235	sleep $i
236done
237if test $RC != 0 ; then
238	echo "ldapsearch failed ($RC)!"
239	test $KILLSERVERS != no && kill -HUP $KILLPIDS
240	exit $RC
241fi
242
243echo "Starting site2 master slapd on TCP/IP port $PORT3..."
244cd $SM2_DIR
245$SLAPD -F slapd.d -h $URI3 -d $LVL $TIMING > $LOG3 2>&1 &
246SM2_PID=$!
247if test $WAIT != 0 ; then
248	echo PID $SM2_PID
249	read foo
250fi
251KILLPIDS="$KILLPIDS $SM2_PID"
252cd $TESTWD
253sleep 1
254echo "Using ldapsearch to check that site2 master is running..."
255for i in 1 2 3 4 5; do
256	$LDAPSEARCH -s base -b "" -H $URI3 \
257		'objectclass=*' > /dev/null 2>&1
258	RC=$?
259	test $RC = 0 && break
260	echo "Waiting $i seconds for slapd to start..."
261	sleep $i
262done
263if test $RC != 0 ; then
264	echo "ldapsearch failed ($RC)!"
265	test $KILLSERVERS != no && kill -HUP $KILLPIDS
266	exit $RC
267fi
268
269echo "Starting central search slapd on TCP/IP port $PORT4..."
270cd $SSC_DIR
271$SLAPD -F slapd.d -h $URI4 -d $LVL $TIMING > $LOG4 2>&1 &
272SSC_PID=$!
273if test $WAIT != 0 ; then
274	echo PID $SSC_PID
275	read foo
276fi
277KILLPIDS="$KILLPIDS $SSC_PID"
278cd $TESTWD
279sleep 1
280echo "Using ldapsearch to check that central search slapd is running..."
281for i in 1 2 3 4 5; do
282	$LDAPSEARCH -s base -b "" -H $URI4 \
283		'objectclass=*' > /dev/null 2>&1
284	RC=$?
285	test $RC = 0 && break
286	echo "Waiting $i seconds for slapd to start..."
287	sleep $i
288done
289if test $RC != 0 ; then
290	echo "ldapsearch failed ($RC)!"
291	test $KILLSERVERS != no && kill -HUP $KILLPIDS
292	exit $RC
293fi
294
295
296echo "Starting site1 search slapd on TCP/IP port $PORT5..."
297cd $SS1_DIR
298$SLAPD -F slapd.d -h $URI5 -d $LVL $TIMING > $LOG5 2>&1 &
299SS1_PID=$!
300if test $WAIT != 0 ; then
301	echo PID $SS1_PID
302	read foo
303fi
304KILLPIDS="$KILLPIDS $SS1_PID"
305cd $TESTWD
306sleep 1
307echo "Using ldapsearch to check that site1 search slapd is running..."
308for i in 1 2 3 4 5; do
309	$LDAPSEARCH -s base -b "" -H $URI5 \
310		'objectclass=*' > /dev/null 2>&1
311	RC=$?
312	test $RC = 0 && break
313	echo "Waiting $i seconds for slapd to start..."
314	sleep $i
315done
316if test $RC != 0 ; then
317	echo "ldapsearch failed ($RC)!"
318	test $KILLSERVERS != no && kill -HUP $KILLPIDS
319	exit $RC
320fi
321
322
323echo "Starting site2 search slapd on TCP/IP port $PORT6..."
324cd $SS2_DIR
325$SLAPD -F slapd.d -h $URI6 -d $LVL $TIMING > $LOG6 2>&1 &
326SS2_PID=$!
327if test $WAIT != 0 ; then
328	echo PID $SS2_PID
329	read foo
330fi
331KILLPIDS="$KILLPIDS $SS2_PID"
332cd $TESTWD
333sleep 1
334echo "Using ldapsearch to check that site2 search slapd is running..."
335for i in 1 2 3 4 5; do
336	$LDAPSEARCH -s base -b "" -H $URI6 \
337		'objectclass=*' > /dev/null 2>&1
338	RC=$?
339	test $RC = 0 && break
340	echo "Waiting $i seconds for slapd to start..."
341	sleep $i
342done
343if test $RC != 0 ; then
344	echo "ldapsearch failed ($RC)!"
345	test $KILLSERVERS != no && kill -HUP $KILLPIDS
346	exit $RC
347fi
348
349for uri in $URI1 $URI2 $URI3 $URI4 $URI5 $URI6; do
350	echo "Adding schema on $uri..."
351	$LDAPADD -D cn=config -H $uri -y $CONFIGPWF <<EOF > $TESTOUT 2>&1
352include: file://$ABS_SCHEMADIR/core.ldif
353
354include: file://$ABS_SCHEMADIR/cosine.ldif
355
356include: file://$ABS_SCHEMADIR/inetorgperson.ldif
357
358include: file://$ABS_SCHEMADIR/openldap.ldif
359
360include: file://$ABS_SCHEMADIR/nis.ldif
361EOF
362	RC=$?
363	if test $RC != 0 ; then
364		echo "ldapadd failed for schema config ($RC)!"
365		test $KILLSERVERS != no && kill -HUP $KILLPIDS
366		exit $RC
367	fi
368
369	[ "$BACKENDTYPE" = mod ] || continue
370
371	echo "Adding backend module on $uri..."
372	$LDAPADD -D cn=config -H $uri -y $CONFIGPWF <<EOF >>$TESTOUT 2>&1
373dn: cn=module,cn=config
374objectClass: olcModuleList
375cn: module
376olcModulePath: ../../../servers/slapd/back-$BACKEND
377olcModuleLoad: back_$BACKEND.la
378EOF
379	RC=$?
380	if test $RC != 0 ; then
381		echo "ldapadd failed for backend module ($RC)!"
382		test $KILLSERVERS != no && kill -HUP $KILLPIDS
383		exit $RC
384	fi
385done
386
387echo "Adding database config on central master..."
388if [ "$SYNCPROV" = syncprovmod ]; then
389	$LDAPADD -D cn=config -H $URI1 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
390dn: cn=module,cn=config
391objectClass: olcModuleList
392cn: module
393olcModulePath: ../../../servers/slapd/overlays
394olcModuleLoad: syncprov.la
395
396EOF
397	RC=$?
398	if test $RC != 0 ; then
399		echo "ldapadd failed for moduleLoad ($RC)!"
400		test $KILLSERVERS != no && kill -HUP $KILLPIDS
401		exit $RC
402	fi
403fi
404
405nullExclude="" nullOK="" wantNoObj=32
406test $BACKEND = null && nullExclude="# " nullOK="OK" wantNoObj=0
407
408$LDAPADD -D cn=config -H $URI1 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
409dn: olcDatabase={1}$BACKEND,cn=config
410objectClass: olcDatabaseConfig
411${nullExclude}objectClass: olc${BACKEND}Config
412olcDatabase: {1}$BACKEND
413${nullExclude}olcDbDirectory: $SMC_DIR/db
414olcSuffix: $BASEDN
415olcRootDN: $MANAGERDN
416olcRootPW: $PASSWD
417
418dn: olcOverlay={0}glue,olcDatabase={1}$BACKEND,cn=config
419objectClass: olcOverlayConfig
420olcOverlay: {0}glue
421
422dn: olcOverlay={1}syncprov,olcDatabase={1}$BACKEND,cn=config
423objectClass: olcOverlayConfig
424objectClass: olcSyncProvConfig
425olcOverlay: {1}syncprov
426olcSpCheckpoint: 3 1
427
428dn: olcDatabase={1}$BACKEND,cn=config
429objectClass: olcDatabaseConfig
430${nullExclude}objectClass: olc${BACKEND}Config
431olcDatabase: {1}$BACKEND
432${nullExclude}olcDbDirectory: $SMC_DIR/ou1
433olcSubordinate: TRUE
434olcSuffix: ou=ou1,$BASEDN
435olcRootDN: $MANAGERDN
436
437dn: olcDatabase={2}$BACKEND,cn=config
438objectClass: olcDatabaseConfig
439${nullExclude}objectClass: olc${BACKEND}Config
440olcDatabase: {2}$BACKEND
441${nullExclude}olcDbDirectory: $SMC_DIR/ou2
442olcSubordinate: TRUE
443olcSuffix: ou=ou2,$BASEDN
444olcRootDN: $MANAGERDN
445
446dn: olcDatabase={3}$BACKEND,cn=config
447objectClass: olcDatabaseConfig
448${nullExclude}objectClass: olc${BACKEND}Config
449olcDatabase: {3}$BACKEND
450${nullExclude}olcDbDirectory: $SMC_DIR/sm1ou1
451olcSubordinate: TRUE
452olcSuffix: ou=sm1ou1,$BASEDN
453olcRootDN: $UPDATEDN
454
455dn: olcDatabase={4}$BACKEND,cn=config
456objectClass: olcDatabaseConfig
457${nullExclude}objectClass: olc${BACKEND}Config
458olcDatabase: {4}$BACKEND
459${nullExclude}olcDbDirectory: $SMC_DIR/sm1ou2
460olcSubordinate: TRUE
461olcSuffix: ou=sm1ou2,$BASEDN
462olcRootDN: $UPDATEDN
463
464dn: olcDatabase={5}$BACKEND,cn=config
465objectClass: olcDatabaseConfig
466${nullExclude}objectClass: olc${BACKEND}Config
467olcDatabase: {5}$BACKEND
468${nullExclude}olcDbDirectory: $SMC_DIR/sm2ou1
469olcSubordinate: TRUE
470olcSuffix: ou=sm2ou1,$BASEDN
471olcRootDN: $UPDATEDN
472
473EOF
474RC=$?
475if test $RC != 0 ; then
476	echo "ldapadd failed for central master database config ($RC)!"
477	test $KILLSERVERS != no && kill -HUP $KILLPIDS
478	exit $RC
479fi
480
481echo "Adding database config on site1 master..."
482if [ "$SYNCPROV" = syncprovmod ]; then
483	$LDAPADD -D cn=config -H $URI2 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
484dn: cn=module,cn=config
485objectClass: olcModuleList
486cn: module
487olcModulePath: ../../../servers/slapd/overlays
488olcModuleLoad: syncprov.la
489
490EOF
491	RC=$?
492	if test $RC != 0 ; then
493		echo "ldapadd failed for moduleLoad ($RC)!"
494		test $KILLSERVERS != no && kill -HUP $KILLPIDS
495		exit $RC
496	fi
497fi
498
499$LDAPADD -D cn=config -H $URI2 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
500dn: olcDatabase={1}$BACKEND,cn=config
501objectClass: olcDatabaseConfig
502${nullExclude}objectClass: olc${BACKEND}Config
503olcDatabase: {1}$BACKEND
504${nullExclude}olcDbDirectory: $SM1_DIR/db
505olcSuffix: $BASEDN
506olcRootDN: $UPDATEDN
507
508dn: olcOverlay={0}glue,olcDatabase={1}$BACKEND,cn=config
509objectClass: olcOverlayConfig
510olcOverlay: {0}glue
511
512dn: olcOverlay={1}syncprov,olcDatabase={1}$BACKEND,cn=config
513objectClass: olcOverlayConfig
514objectClass: olcSyncProvConfig
515olcOverlay: {1}syncprov
516
517dn: olcDatabase={1}$BACKEND,cn=config
518objectClass: olcDatabaseConfig
519${nullExclude}objectClass: olc${BACKEND}Config
520olcDatabase: {1}$BACKEND
521${nullExclude}olcDbDirectory: $SM1_DIR/ou1
522olcSubordinate: TRUE
523olcSuffix: ou=ou1,$BASEDN
524olcRootDN: $UPDATEDN
525
526dn: olcDatabase={2}$BACKEND,cn=config
527objectClass: olcDatabaseConfig
528${nullExclude}objectClass: olc${BACKEND}Config
529olcDatabase: {2}$BACKEND
530${nullExclude}olcDbDirectory: $SM1_DIR/sm1ou1
531olcSubordinate: TRUE
532olcSuffix: ou=sm1ou1,$BASEDN
533olcRootDN: ou=sm1ou1,$BASEDN
534olcRootPW: $PASSWD
535
536dn: olcDatabase={3}$BACKEND,cn=config
537objectClass: olcDatabaseConfig
538${nullExclude}objectClass: olc${BACKEND}Config
539olcDatabase: {3}$BACKEND
540${nullExclude}olcDbDirectory: $SM1_DIR/sm1ou2
541olcSubordinate: TRUE
542olcSuffix: ou=sm1ou2,$BASEDN
543olcRootDN: ou=sm1ou1,$BASEDN
544
545EOF
546
547RC=$?
548if test $RC != 0 ; then
549	echo "ldapadd failed for site1 master database config ($RC)!"
550	test $KILLSERVERS != no && kill -HUP $KILLPIDS
551	exit $RC
552fi
553
554echo "Adding database config on site2 master..."
555if [ "$SYNCPROV" = syncprovmod ]; then
556	$LDAPADD -D cn=config -H $URI3 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
557dn: cn=module,cn=config
558objectClass: olcModuleList
559cn: module
560olcModulePath: ../../../servers/slapd/overlays
561olcModuleLoad: syncprov.la
562
563EOF
564	RC=$?
565	if test $RC != 0 ; then
566		echo "ldapadd failed for moduleLoad ($RC)!"
567		test $KILLSERVERS != no && kill -HUP $KILLPIDS
568		exit $RC
569	fi
570fi
571
572$LDAPADD -D cn=config -H $URI3 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
573dn: olcDatabase={1}$BACKEND,cn=config
574objectClass: olcDatabaseConfig
575${nullExclude}objectClass: olc${BACKEND}Config
576olcDatabase: {1}$BACKEND
577${nullExclude}olcDbDirectory: $SM2_DIR/db
578olcSuffix: $BASEDN
579olcRootDN: $UPDATEDN
580
581dn: olcOverlay={0}glue,olcDatabase={1}$BACKEND,cn=config
582objectClass: olcOverlayConfig
583olcOverlay: {0}glue
584
585dn: olcOverlay={1}syncprov,olcDatabase={1}$BACKEND,cn=config
586objectClass: olcOverlayConfig
587objectClass: olcSyncProvConfig
588olcOverlay: {1}syncprov
589olcSpCheckpoint: 1 1
590
591dn: olcDatabase={1}$BACKEND,cn=config
592objectClass: olcDatabaseConfig
593${nullExclude}objectClass: olc${BACKEND}Config
594olcDatabase: {1}$BACKEND
595${nullExclude}olcDbDirectory: $SM2_DIR/ou2
596olcSubordinate: TRUE
597olcSuffix: ou=ou2,$BASEDN
598olcRootDN: $UPDATEDN
599
600dn: olcDatabase={2}$BACKEND,cn=config
601objectClass: olcDatabaseConfig
602${nullExclude}objectClass: olc${BACKEND}Config
603olcDatabase: {2}$BACKEND
604${nullExclude}olcDbDirectory: $SM2_DIR/sm1ou1
605olcSubordinate: TRUE
606olcSuffix: ou=sm1ou1,$BASEDN
607olcRootDN: $UPDATEDN
608
609dn: olcDatabase={3}$BACKEND,cn=config
610objectClass: olcDatabaseConfig
611${nullExclude}objectClass: olc${BACKEND}Config
612olcDatabase: {3}$BACKEND
613${nullExclude}olcDbDirectory: $SM2_DIR/sm2ou1
614olcSubordinate: TRUE
615olcSuffix: ou=sm2ou1,$BASEDN
616olcRootDN: ou=sm2ou1,$BASEDN
617olcRootPW: $PASSWD
618
619dn: olcDatabase={4}$BACKEND,cn=config
620objectClass: olcDatabaseConfig
621${nullExclude}objectClass: olc${BACKEND}Config
622olcDatabase: {4}$BACKEND
623${nullExclude}olcDbDirectory: $SM2_DIR/sm2ou2
624olcSubordinate: TRUE
625olcSuffix: ou=sm2ou2,$BASEDN
626olcRootDN: ou=sm2ou1,$BASEDN
627
628EOF
629
630RC=$?
631if test $RC != 0 ; then
632	echo "ldapadd failed for site2 master database config ($RC)!"
633	test $KILLSERVERS != no && kill -HUP $KILLPIDS
634	exit $RC
635fi
636
637echo "Adding access rules on central master..."
638$LDAPMODIFY -D cn=config -H $URI1 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
639dn: olcDatabase={-1}frontend,cn=config
640changetype: modify
641add: olcAccess
642olcAccess: to dn.exact=dc=example,dc=com
643  by * read
644olcAccess: to dn.subtree=ou=ou1,dc=example,dc=com
645  by * read
646olcAccess: to dn.subtree=ou=ou2,dc=example,dc=com
647  by dn.exact=ou=ou1,dc=example,dc=com none
648  by dn.exact=ou=ou2,dc=example,dc=com read
649  by dn.exact=dc=example,dc=com none
650  by * read
651olcAccess: to dn.subtree=ou=sm1ou1,dc=example,dc=com
652  by dn.exact=ou=ou1,dc=example,dc=com none
653  by dn.exact=ou=ou2,dc=example,dc=com read
654  by dn.exact=dc=example,dc=com none
655  by * read
656olcAccess: to dn.subtree=ou=sm1ou2,dc=example,dc=com
657  by dn.exact=ou=ou1,dc=example,dc=com none
658  by dn.exact=ou=ou2,dc=example,dc=com none
659  by dn.exact=dc=example,dc=com read
660  by * read
661olcAccess: to dn.subtree=ou=sm2ou1,dc=example,dc=com
662  by dn.exact=ou=ou1,dc=example,dc=com none
663  by dn.exact=ou=ou2,dc=example,dc=com none
664  by dn.exact=dc=example,dc=com none
665  by * read
666olcAccess: to * by * read
667
668EOF
669RC=$?
670if test $RC != 0 ; then
671	echo "ldapmodify failed for central master access config ($RC)!"
672	test $KILLSERVERS != no && kill -HUP $KILLPIDS
673	exit $RC
674fi
675
676echo "Adding access rules on site1 master..."
677$LDAPMODIFY -D cn=config -H $URI2 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
678dn: olcDatabase={-1}frontend,cn=config
679changetype: modify
680add: olcAccess
681olcAccess: to dn.subtree=dc=example,dc=com
682  by * read
683olcAccess: to * by * read
684
685EOF
686RC=$?
687if test $RC != 0 ; then
688	echo "ldapmodify failed for site1 master access config ($RC)!"
689	test $KILLSERVERS != no && kill -HUP $KILLPIDS
690	exit $RC
691fi
692
693echo "Adding access rules on site2 master..."
694$LDAPMODIFY -D cn=config -H $URI3 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
695dn: olcDatabase={-1}frontend,cn=config
696changetype: modify
697add: olcAccess
698olcAccess: to dn.exact=dc=example,dc=com
699  by * read
700olcAccess: to dn.subtree=ou=ou2,dc=example,dc=com
701  by * read
702olcAccess: to dn.subtree=ou=sm1ou1,dc=example,dc=com
703  by users none
704  by * read
705olcAccess: to dn.subtree=ou=sm2ou1,dc=example,dc=com
706  by * read
707olcAccess: to dn.subtree=ou=sm2ou2,dc=example,dc=com
708  by dn.exact=dc=example,dc=com read
709  by users none
710  by * read
711olcAccess: to * by * read
712
713EOF
714RC=$?
715if test $RC != 0 ; then
716	echo "ldapmodify failed for site2 master access config ($RC)!"
717	test $KILLSERVERS != no && kill -HUP $KILLPIDS
718	exit $RC
719fi
720
721echo "Adding database config on central search..."
722$LDAPADD -D cn=config -H $URI4 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
723dn: olcDatabase={1}$BACKEND,cn=config
724objectClass: olcDatabaseConfig
725${nullExclude}objectClass: olc${BACKEND}Config
726olcDatabase: {1}$BACKEND
727${nullExclude}olcDbDirectory: $SSC_DIR/db
728olcSuffix: $BASEDN
729olcRootDN: $UPDATEDN
730
731EOF
732RC=$?
733if test $RC != 0 ; then
734	echo "ldapadd failed for central search database config ($RC)!"
735	test $KILLSERVERS != no && kill -HUP $KILLPIDS
736	exit $RC
737fi
738
739echo "Adding database config on site1 search..."
740$LDAPADD -D cn=config -H $URI5 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
741dn: olcDatabase={1}$BACKEND,cn=config
742objectClass: olcDatabaseConfig
743${nullExclude}objectClass: olc${BACKEND}Config
744olcDatabase: {1}$BACKEND
745${nullExclude}olcDbDirectory: $SS1_DIR/db
746olcSuffix: $BASEDN
747olcRootDN: $UPDATEDN
748
749EOF
750RC=$?
751if test $RC != 0 ; then
752	echo "ldapadd failed for site1 search database config ($RC)!"
753	test $KILLSERVERS != no && kill -HUP $KILLPIDS
754	exit $RC
755fi
756
757echo "Adding database config on site2 search..."
758$LDAPADD -D cn=config -H $URI6 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
759dn: olcDatabase={1}$BACKEND,cn=config
760objectClass: olcDatabaseConfig
761${nullExclude}objectClass: olc${BACKEND}Config
762olcDatabase: {1}$BACKEND
763${nullExclude}olcDbDirectory: $SS2_DIR/db
764olcSuffix: $BASEDN
765olcRootDN: $UPDATEDN
766
767EOF
768RC=$?
769if test $RC != 0 ; then
770	echo "ldapadd failed for site2 search database config ($RC)!"
771	test $KILLSERVERS != no && kill -HUP $KILLPIDS
772	exit $RC
773fi
774
775echo "Populating central master..."
776$LDAPADD -D "$MANAGERDN" -H $URI1 -w $PASSWD <<EOF >> $TESTOUT 2>&1
777dn: dc=example,dc=com
778objectClass: top
779objectClass: organization
780objectClass: dcObject
781dc: example
782o: Example, Inc
783userPassword: $PASSWD
784
785dn: ou=ou1,dc=example,dc=com
786objectClass: top
787objectClass: organizationalUnit
788ou: ou1
789userPassword: $PASSWD
790
791dn: ou=ou2,dc=example,dc=com
792objectClass: top
793objectClass: organizationalUnit
794ou: ou2
795userPassword: $PASSWD
796
797EOF
798RC=$?
799if test $RC != 0 ; then
800	echo "ldapadd failed to populate central master entry ($RC)!"
801	test $KILLSERVERS != no && kill -HUP $KILLPIDS
802	exit $RC
803fi
804
805echo "Adding syncrepl on site1 master..."
806$LDAPMODIFY -D cn=config -H $URI2 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
807dn: olcDatabase={4}$BACKEND,cn=config
808changetype: modify
809add: olcSyncRepl
810olcSyncRepl: rid=1 provider=$URI1 searchbase="$BASEDN"
811  binddn="ou=ou1,$BASEDN" bindmethod=simple credentials=$PASSWD
812  type=refreshAndPersist retry="$RETRY" timeout=1
813
814EOF
815RC=$?
816if test $RC != 0 ; then
817	echo "ldapmodify failed to add syncrepl on site1 master ($RC)!"
818	test $KILLSERVERS != no && kill -HUP $KILLPIDS
819	exit $RC
820fi
821
822echo "Adding syncrepl on site2 master..."
823$LDAPMODIFY -D cn=config -H $URI3 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
824dn: olcDatabase={5}$BACKEND,cn=config
825changetype: modify
826add: olcSyncRepl
827olcSyncRepl: rid=1 provider=$URI1 searchbase="$BASEDN"
828  binddn="ou=ou2,$BASEDN" bindmethod=simple credentials=$PASSWD
829  type=refreshAndPersist retry="$RETRY" timeout=1
830
831EOF
832RC=$?
833if test $RC != 0 ; then
834	echo "ldapmodify failed to add syncrepl on site2 master ($RC)!"
835	test $KILLSERVERS != no && kill -HUP $KILLPIDS
836	exit $RC
837fi
838sleep 1
839
840echo "Using ldapsearch to check that site1 master received changes..."
841RC=32
842for i in 1 2 3 4 5; do
843	RESULT=`$LDAPSEARCH -H $URI2 \
844		-s base -b "ou=ou1,$BASEDN" \
845		'(objectClass=*)' 2>&1 | awk '/^dn:/ {print "OK"}'`
846	if test "x$RESULT$nullOK" = "xOK" ; then
847		RC=0
848		break
849	fi
850	echo "Waiting $i seconds for syncrepl to receive changes..."
851	sleep $i
852done
853if test $RC != 0 ; then
854	echo "ldapsearch failed ($RC)!"
855	test $KILLSERVERS != no && kill -HUP $KILLPIDS
856	exit $RC
857fi
858
859echo "Using ldapsearch to check that site2 master received changes..."
860RC=32
861for i in 1 2 3 4 5; do
862	RESULT=`$LDAPSEARCH -H $URI3 \
863		-s base -b "ou=ou1,$BASEDN" \
864		'(objectClass=*)' 2>&1 | awk '/^dn:/ {print "OK"}'`
865	if test "x$RESULT$nullOK" = "xOK" ; then
866		RC=0
867		break
868	fi
869	echo "Waiting $i seconds for syncrepl to receive changes..."
870	sleep $i
871done
872if test $RC != 0 ; then
873	echo "ldapsearch failed ($RC)!"
874	test $KILLSERVERS != no && kill -HUP $KILLPIDS
875	exit $RC
876fi
877
878sleep 1
879
880echo "Populating site1 master..."
881$LDAPADD -D "ou=sm1ou1,$BASEDN" -H $URI2 -w $PASSWD <<EOF >> $TESTOUT 2>&1
882dn: ou=sm1ou1,dc=example,dc=com
883objectClass: top
884objectClass: organizationalUnit
885ou: sm1ou1
886
887dn: ou=sm1ou2,dc=example,dc=com
888objectClass: top
889objectClass: organizationalUnit
890ou: sm1ou2
891
892EOF
893RC=$?
894if test $RC != 0 ; then
895	echo "ldapadd failed to populate site1 master ($RC)!"
896	test $KILLSERVERS != no && kill -HUP $KILLPIDS
897	exit $RC
898fi
899
900sleep 1
901
902echo "Populating site2 master..."
903$LDAPADD -D "ou=sm2ou1,$BASEDN" -H $URI3 -w $PASSWD <<EOF >> $TESTOUT 2>&1
904dn: ou=sm2ou1,dc=example,dc=com
905objectClass: top
906objectClass: organizationalUnit
907ou: sm2ou1
908
909dn: ou=sm2ou2,dc=example,dc=com
910objectClass: top
911objectClass: organizationalUnit
912ou: sm2ou2
913
914EOF
915RC=$?
916if test $RC != 0 ; then
917	echo "ldapadd failed to populate site2 master ($RC)!"
918	test $KILLSERVERS != no && kill -HUP $KILLPIDS
919	exit $RC
920fi
921
922ERRORS=0
923
924# TEST:
925# Stop site1 master when adding syncrepl to the central master.  When
926# site1 master is started again both it and the central master will have
927# the same number of contextCSN values, but the ones on central master
928# will be the newest.  The central master will not update its contextCSN
929# values unless the bug in ITS#5597 have been fixed.
930echo "Stopping site1 master..."
931kill -HUP "$SM1_PID"
932wait "$SM1_PID"
933KILLPIDS=`echo "$KILLPIDS " | sed -e "s/ $SM1_PID / /"`;
934SM1_PID=
935
936echo "Adding syncrepl on central master..."
937$LDAPMODIFY -D cn=config -H $URI1 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
938dn: olcDatabase={3}$BACKEND,cn=config
939changetype: modify
940add: olcSyncRepl
941olcSyncRepl: rid=3 provider=$URI2 searchbase="ou=sm1ou1,$BASEDN"
942  binddn="ou=sm1ou1,$BASEDN" bindmethod=simple credentials=$PASSWD
943  type=refreshAndPersist retry="$RETRY" timeout=1
944
945dn: olcDatabase={5}$BACKEND,cn=config
946changetype: modify
947add: olcSyncRepl
948olcSyncRepl: rid=5 provider=$URI3 searchbase="ou=sm2ou1,$BASEDN"
949  binddn="ou=sm2ou1,$BASEDN" bindmethod=simple credentials=$PASSWD
950  type=refreshAndPersist retry="$RETRY" timeout=1
951
952EOF
953RC=$?
954if test $RC != 0 ; then
955	echo "ldapmodify failed to add syncrepl on central master ($RC)!"
956	test $KILLSERVERS != no && kill -HUP $KILLPIDS
957	exit $RC
958fi
959sleep 1
960echo "Using ldapsearch to check that central master received site2 entries..."
961RC=32
962for i in 1 2 3 4 5; do
963	RESULT=`$LDAPSEARCH -H $URI1 \
964		-s base -b "ou=sm2ou1,$BASEDN" \
965		'(objectClass=*)' 2>&1 | awk '/^dn:/ {print "OK"}'`
966	if test "x$RESULT$nullOK" = "xOK" ; then
967		RC=0
968		break
969	fi
970	echo "Waiting $i seconds for syncrepl to receive changes..."
971	sleep $i
972done
973if test $RC != 0 ; then
974	echo "ldapsearch failed ($RC)!"
975	test $KILLSERVERS != no && kill -HUP $KILLPIDS
976	exit $RC
977fi
978
979if test -z "$SM1_PID" ; then
980	echo "Restarting site1 master slapd on TCP/IP port $PORT2..."
981	cd $SM1_DIR
982	$SLAPD -F slapd.d -h $URI2 -d $LVL $TIMING >> $LOG2 2>&1 &
983	SM1_PID=$!
984	if test $WAIT != 0 ; then
985	    echo PID $SM1_PID
986	    read foo
987	fi
988	KILLPIDS="$KILLPIDS $SM1_PID"
989	cd $TESTWD
990	sleep 1
991fi
992sleep 1
993echo "Using ldapsearch to check that site1 master is running..."
994for i in 1 2 3 4 5; do
995	$LDAPSEARCH -s base -b "" -H $URI2 \
996		'objectclass=*' > /dev/null 2>&1
997	RC=$?
998	test $RC = 0 && break
999	echo "Waiting $i seconds for slapd to start..."
1000	sleep $i
1001done
1002if test $RC != 0 ; then
1003	echo "ldapsearch failed ($RC)!"
1004	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1005	exit $RC
1006fi
1007
1008echo "Using ldapsearch to check that central master received site1 entries..."
1009RC=32
1010for i in 1 2 3 4 5; do
1011	RESULT=`$LDAPSEARCH -H $URI1 \
1012		-s base -b "ou=sm1ou1,$BASEDN" \
1013		'(objectClass=*)' 2>&1 | awk '/^dn:/ {print "OK"}'`
1014	if test "x$RESULT$nullOK" = "xOK" ; then
1015		RC=0
1016		break
1017	fi
1018	echo "Waiting $i seconds for syncrepl to receive changes..."
1019	sleep $i
1020done
1021if test $RC != 0 ; then
1022	echo "ldapsearch failed ($RC)!"
1023	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1024	exit $RC
1025fi
1026
1027# Test done, now some more intialization...
1028
1029echo "Adding syncrepl consumer on central search..."
1030$LDAPMODIFY -D cn=config -H $URI4 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
1031dn: olcDatabase={1}$BACKEND,cn=config
1032changetype: modify
1033add: olcSyncRepl
1034olcSyncRepl: rid=1 provider=$URI1 searchbase="$BASEDN"
1035  binddn="$BASEDN" bindmethod=simple credentials=$PASSWD
1036  type=refreshAndPersist retry="$RETRY" timeout=1
1037
1038EOF
1039RC=$?
1040if test $RC != 0 ; then
1041	echo "ldapmodify failed to add syncrepl on site1 search ($RC)!"
1042	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1043	exit $RC
1044fi
1045
1046echo "Adding syncrepl consumer on site1 search..."
1047$LDAPMODIFY -D cn=config -H $URI5 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
1048dn: olcDatabase={1}$BACKEND,cn=config
1049changetype: modify
1050add: olcSyncRepl
1051olcSyncRepl: rid=1 provider=$URI2 searchbase="$BASEDN"
1052  binddn="$BASEDN" bindmethod=simple credentials=$PASSWD
1053  type=refreshAndPersist retry="$RETRY" timeout=1
1054
1055EOF
1056RC=$?
1057if test $RC != 0 ; then
1058	echo "ldapmodify failed to add syncrepl on site1 search ($RC)!"
1059	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1060	exit $RC
1061fi
1062
1063echo "Adding syncrepl consumer on site2 search..."
1064$LDAPMODIFY -D cn=config -H $URI6 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
1065dn: olcDatabase={1}$BACKEND,cn=config
1066changetype: modify
1067add: olcSyncRepl
1068olcSyncRepl: rid=1 provider=$URI3 searchbase="$BASEDN"
1069  binddn="$BASEDN" bindmethod=simple credentials=$PASSWD
1070  type=refreshAndPersist retry="$RETRY" timeout=1
1071
1072EOF
1073RC=$?
1074if test $RC != 0 ; then
1075	echo "ldapmodify failed to add syncrepl on site2 search ($RC)!"
1076	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1077	exit $RC
1078fi
1079sleep 1
1080
1081echo "Using ldapsearch to check that central search received changes..."
1082RC=32
1083for i in 1 2 3 4 5; do
1084	RESULT=`$LDAPSEARCH -H $URI4 \
1085		-s base -b "$BASEDN" \
1086		'(objectClass=*)' 2>&1 | awk '/^dn:/ {print "OK"}'`
1087	if test "x$RESULT$nullOK" = "xOK" ; then
1088		RC=0
1089		break
1090	fi
1091	echo "Waiting $i seconds for syncrepl to receive changes..."
1092	sleep $i
1093done
1094if test $RC != 0 ; then
1095	echo "ldapsearch failed ($RC)!"
1096	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1097	exit $RC
1098fi
1099
1100echo "Using ldapsearch to check that site1 search received changes..."
1101RC=32
1102for i in 1 2 3 4 5; do
1103	RESULT=`$LDAPSEARCH -H $URI5 \
1104		-s base -b "$BASEDN" \
1105		'(objectClass=*)' 2>&1 | awk '/^dn:/ {print "OK"}'`
1106	if test "x$RESULT$nullOK" = "xOK" ; then
1107		RC=0
1108		break
1109	fi
1110	echo "Waiting $i seconds for syncrepl to receive changes..."
1111	sleep $i
1112done
1113if test $RC != 0 ; then
1114	echo "ldapsearch failed ($RC)!"
1115	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1116	exit $RC
1117fi
1118
1119echo "Using ldapsearch to check that site2 search received changes..."
1120RC=32
1121for i in 1 2 3 4 5; do
1122	RESULT=`$LDAPSEARCH -H $URI6 \
1123		-s base -b "$BASEDN" \
1124		'(objectClass=*)' 2>&1 | awk '/^dn:/ {print "OK"}'`
1125	if test "x$RESULT$nullOK" = "xOK" ; then
1126		RC=0
1127		break
1128	fi
1129	echo "Waiting $i seconds for syncrepl to receive changes..."
1130	sleep $i
1131done
1132if test $RC != 0 ; then
1133	echo "ldapsearch failed ($RC)!"
1134	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1135	exit $RC
1136fi
1137
1138# Create a script that will check the contextCSN values of all servers,
1139# and restart them to re-synchronize if it finds any errors:
1140cat > $TESTDIR/checkcsn.sh <<'EOF'
1141#!/bin/sh
1142
1143CSN_ERRORS=0
1144
1145CSN1=`$LDAPSEARCH -H $URI1 -b $BASEDN -s base contextCSN | grep contextCSN | sort`
1146CSN2=`$LDAPSEARCH -H $URI2 -b $BASEDN -s base contextCSN | grep contextCSN | sort`
1147CSN3=`$LDAPSEARCH -H $URI3 -b $BASEDN -s base contextCSN | grep contextCSN | sort`
1148CSN4=`$LDAPSEARCH -H $URI4 -b $BASEDN -s base contextCSN | grep contextCSN | sort`
1149CSN5=`$LDAPSEARCH -H $URI5 -b $BASEDN -s base contextCSN | grep contextCSN | sort`
1150CSN6=`$LDAPSEARCH -H $URI6 -b $BASEDN -s base contextCSN | grep contextCSN | sort`
1151
1152if test -z "$CSN1" ; then
1153	echo "ERROR: contextCSN empty on central master"
1154	CSN_ERRORS=`expr $CSN_ERRORS + 1`
1155fi
1156nCSN=`echo "$CSN1" | wc -l`
1157if test "$nCSN" -ne 3 ; then
1158	echo "ERROR: Wrong contextCSN count on central master, should be 3"
1159	CSN_ERRORS=`expr $CSN_ERRORS + 1`
1160	if test -n "$CSN_VERBOSE"; then
1161		echo "$CSN1"
1162	fi
1163fi
1164if  test -z "$CSN2" -o "$CSN1" != "$CSN2" ; then
1165	echo "ERROR: contextCSN mismatch between central master and site1 master"
1166	CSN_ERRORS=`expr $CSN_ERRORS + 1`;
1167	if test -n "$CSN_VERBOSE"; then
1168		echo "contextCSN on central master:"
1169		echo "$CSN1"
1170		echo "contextCSN on site1 master:"
1171		echo "$CSN2"
1172	fi
1173fi
1174if  test -z "$CSN3" -o "$CSN1" != "$CSN3" ; then
1175	echo "ERROR: contextCSN mismatch between central master and site2 master"
1176	CSN_ERRORS=`expr $CSN_ERRORS + 1`;
1177	if test -n "$CSN_VERBOSE"; then
1178		echo "contextCSN on central master:"
1179		echo "$CSN1"
1180		echo "contextCSN on site2 master:"
1181		echo "$CSN3"
1182	fi
1183fi
1184if  test -z "$CSN4" -o "$CSN1" != "$CSN4" ; then
1185	echo "ERROR: contextCSN mismatch between central master and central search"
1186	CSN_ERRORS=`expr $CSN_ERRORS + 1`;
1187	if test -n "$CSN_VERBOSE"; then
1188		echo "contextCSN on central master:"
1189		echo "$CSN1"
1190		echo "contextCSN on central search:"
1191		echo "$CSN4"
1192	fi
1193fi
1194if  test -z "$CSN5" -o "$CSN2" != "$CSN5" ; then
1195	echo "ERROR: contextCSN mismatch between site1 master and site1 search"
1196	CSN_ERRORS=`expr $CSN_ERRORS + 1`;
1197	if test -n "$CSN_VERBOSE"; then
1198		echo "contextCSN on site1 master:"
1199		echo "$CSN2"
1200		echo "contextCSN on site1 search:"
1201		echo "$CSN5"
1202	fi
1203fi
1204if  test -z "$CSN6" -o "$CSN3" != "$CSN6" ; then
1205	echo "ERROR: contextCSN mismatch between site2 master and site2 search:"
1206	CSN_ERRORS=`expr $CSN_ERRORS + 1`;
1207	if test -n "$CSN_VERBOSE"; then
1208		echo "contextCSN on site2 master:"
1209		echo "$CSN3"
1210		echo "contextCSN on site2 search:"
1211		echo "$CSN6"
1212	fi
1213fi
1214
1215if test $CSN_ERRORS != 0 ; then
1216	echo "Stopping all servers to synchronize contextCSN..."
1217	kill -HUP  $KILLPIDS
1218	for pid in $KILLPIDS ; do wait $pid ; done
1219	KILLPIDS=
1220
1221	echo "Restarting site1 master slapd on TCP/IP port $PORT2..."
1222	cd $SM1_DIR
1223	$SLAPD -F slapd.d -h $URI2 -d $LVL $TIMING >> $LOG2 2>&1 &
1224	SM1_PID=$!
1225	if test $WAIT != 0 ; then
1226		echo PID $SM1_PID
1227		read foo
1228	fi
1229	KILLPIDS="$KILLPIDS $SM1_PID"
1230	cd $TESTWD
1231	sleep 1
1232	echo "Using ldapsearch to check that site1 master is running..."
1233	for i in 1 2 3 4 5; do
1234		$LDAPSEARCH -s base -b "" -H $URI2 \
1235			'objectclass=*' > /dev/null 2>&1
1236		RC=$?
1237		test $RC = 0 && break
1238		echo "Waiting $i seconds for slapd to start..."
1239		sleep $i
1240	done
1241	if test $RC != 0 ; then
1242		echo "ldapsearch failed ($RC)!"
1243		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1244		exit $RC
1245	fi
1246
1247	echo "Restarting site2 master slapd on TCP/IP port $PORT3..."
1248	cd $SM2_DIR
1249	$SLAPD -F slapd.d -h $URI3 -d $LVL $TIMING >> $LOG3 2>&1 &
1250	SM2_PID=$!
1251	if test $WAIT != 0 ; then
1252		echo PID $SM2_PID
1253		read foo
1254	fi
1255	KILLPIDS="$KILLPIDS $SM2_PID "
1256	cd $TESTWD
1257	sleep 1
1258	echo "Using ldapsearch to check that site2 master is running..."
1259	for i in 1 2 3 4 5; do
1260		$LDAPSEARCH -s base -b "" -H $URI3 \
1261			'objectclass=*' > /dev/null 2>&1
1262		RC=$?
1263		test $RC = 0 && break
1264		echo "Waiting $i seconds for slapd to start..."
1265		sleep $i
1266	done
1267	if test $RC != 0 ; then
1268		echo "ldapsearch failed ($RC)!"
1269		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1270		exit $RC
1271	fi
1272
1273	echo "Restarting central master slapd on TCP/IP port $PORT1..."
1274	cd $SMC_DIR
1275	$SLAPD -F slapd.d -h $URI1 -d $LVL $TIMING >> $LOG1 2>&1 &
1276	SMC_PID=$!
1277	if test $WAIT != 0 ; then
1278		echo PID $SMC_PID
1279		read foo
1280	fi
1281	KILLPIDS="$KILLPIDS $SMC_PID"
1282	cd $TESTWD
1283	sleep 1
1284	echo "Using ldapsearch to check that central master slapd is running..."
1285	for i in 1 2 3 4 5; do
1286		$LDAPSEARCH -s base -b "" -H $URI1 \
1287			'objectclass=*' > /dev/null 2>&1
1288		RC=$?
1289		test $RC = 0 && break
1290		echo "Waiting $i seconds for slapd to start..."
1291		sleep $i
1292	done
1293	if test $RC != 0 ; then
1294		echo "ldapsearch failed ($RC)!"
1295		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1296		exit $RC
1297	fi
1298
1299	echo "Sleeping 5 seconds to allow contextCSN to synchronize..."
1300	sleep 5
1301
1302	echo "Stopping site1 and site2 master..."
1303	kill -HUP  $SM1_PID $SM2_PID
1304	for pid in $SM1_PID $SM2_PID ; do wait $pid ; done
1305	KILLPIDS=" $SMC_PID"
1306
1307	echo "Restarting site1 master slapd on TCP/IP port $PORT2..."
1308	cd $SM1_DIR
1309	$SLAPD -F slapd.d -h $URI2 -d $LVL $TIMING >> $LOG2 2>&1 &
1310	SM1_PID=$!
1311	if test $WAIT != 0 ; then
1312		echo PID $SM1_PID
1313		read foo
1314	fi
1315	KILLPIDS="$KILLPIDS $SM1_PID"
1316	cd $TESTWD
1317	sleep 1
1318	echo "Using ldapsearch to check that site1 master is running..."
1319	for i in 1 2 3 4 5; do
1320		$LDAPSEARCH -s base -b "" -H $URI2 \
1321			'objectclass=*' > /dev/null 2>&1
1322		RC=$?
1323		test $RC = 0 && break
1324		echo "Waiting $i seconds for slapd to start..."
1325		sleep $i
1326	done
1327	if test $RC != 0 ; then
1328		echo "ldapsearch failed ($RC)!"
1329		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1330		exit $RC
1331	fi
1332
1333	echo "Restarting site2 master slapd on TCP/IP port $PORT3..."
1334	cd $SM2_DIR
1335	$SLAPD -F slapd.d -h $URI3 -d $LVL $TIMING >> $LOG3 2>&1 &
1336	SM2_PID=$!
1337	if test $WAIT != 0 ; then
1338		echo PID $SM2_PID
1339		read foo
1340	fi
1341	KILLPIDS="$KILLPIDS $SM2_PID"
1342	cd $TESTWD
1343	sleep 1
1344	echo "Using ldapsearch to check that site2 master is running..."
1345	for i in 1 2 3 4 5; do
1346		$LDAPSEARCH -s base -b "" -H $URI3 \
1347			'objectclass=*' > /dev/null 2>&1
1348		RC=$?
1349		test $RC = 0 && break
1350		echo "Waiting $i seconds for slapd to start..."
1351		sleep $i
1352	done
1353	if test $RC != 0 ; then
1354		echo "ldapsearch failed ($RC)!"
1355		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1356		exit $RC
1357	fi
1358
1359	echo "Sleeping 5 seconds to allow contextCSN to synchronize..."
1360	sleep 5
1361
1362	echo "Restarting central search slapd on TCP/IP port $PORT4..."
1363	cd $SSC_DIR
1364	$SLAPD -F slapd.d -h $URI4 -d $LVL $TIMING >> $LOG4 2>&1 &
1365	SSC_PID=$!
1366	if test $WAIT != 0 ; then
1367		echo PID $SSC_PID
1368		read foo
1369	fi
1370	KILLPIDS="$KILLPIDS $SSC_PID"
1371	cd $TESTWD
1372	sleep 1
1373	echo "Using ldapsearch to check that central search slapd is running..."
1374	for i in 1 2 3 4 5; do
1375		$LDAPSEARCH -s base -b "" -H $URI4 \
1376			'objectclass=*' > /dev/null 2>&1
1377		RC=$?
1378		test $RC = 0 && break
1379		echo "Waiting $i seconds for slapd to start..."
1380		sleep $i
1381	done
1382	if test $RC != 0 ; then
1383		echo "ldapsearch failed ($RC)!"
1384		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1385		exit $RC
1386	fi
1387
1388	echo "Restarting site1 search slapd on TCP/IP port $PORT5..."
1389	cd $SS1_DIR
1390	$SLAPD -F slapd.d -h $URI5 -d $LVL $TIMING >> $LOG5 2>&1 &
1391	SS1_PID=$!
1392	if test $WAIT != 0 ; then
1393		echo PID $SS1_PID
1394		read foo
1395	fi
1396	KILLPIDS="$KILLPIDS $SS1_PID"
1397	cd $TESTWD
1398	sleep 1
1399	echo "Using ldapsearch to check that site1 search slapd is running..."
1400	for i in 1 2 3 4 5; do
1401		$LDAPSEARCH -s base -b "" -H $URI5 \
1402			'objectclass=*' > /dev/null 2>&1
1403		RC=$?
1404		test $RC = 0 && break
1405		echo "Waiting $i seconds for slapd to start..."
1406		sleep $i
1407	done
1408	if test $RC != 0 ; then
1409		echo "ldapsearch failed ($RC)!"
1410		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1411		exit $RC
1412	fi
1413
1414	echo "Restarting site2 search slapd on TCP/IP port $PORT6..."
1415	cd $SS2_DIR
1416	$SLAPD -F slapd.d -h $URI6 -d $LVL $TIMING >> $LOG6 2>&1 &
1417	SS2_PID=$!
1418	if test $WAIT != 0 ; then
1419		echo PID $SS2_PID
1420		read foo
1421	fi
1422	KILLPIDS="$KILLPIDS $SS2_PID"
1423	cd $TESTWD
1424	sleep 1
1425	echo "Using ldapsearch to check that site2 search slapd is running..."
1426	for i in 1 2 3 4 5; do
1427		$LDAPSEARCH -s base -b "" -H $URI6 \
1428			'objectclass=*' > /dev/null 2>&1
1429		RC=$?
1430		test $RC = 0 && break
1431		echo "Waiting $i seconds for slapd to start..."
1432		sleep $i
1433	done
1434	if test $RC != 0 ; then
1435		echo "ldapsearch failed ($RC)!"
1436		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1437		exit $RC
1438	fi
1439
1440	echo "Sleeping 5 seconds to allow contextCSN to synchronize..."
1441	sleep 5
1442
1443	echo "Checking contextCSN after restart..."
1444	CSN1=`$LDAPSEARCH -H $URI1 -b $BASEDN -s base contextCSN | grep contextCSN | sort`
1445	CSN2=`$LDAPSEARCH -H $URI2 -b $BASEDN -s base contextCSN | grep contextCSN | sort`
1446	CSN3=`$LDAPSEARCH -H $URI3 -b $BASEDN -s base contextCSN | grep contextCSN | sort`
1447	CSN4=`$LDAPSEARCH -H $URI4 -b $BASEDN -s base contextCSN | grep contextCSN | sort`
1448	CSN5=`$LDAPSEARCH -H $URI5 -b $BASEDN -s base contextCSN | grep contextCSN | sort`
1449	CSN6=`$LDAPSEARCH -H $URI6 -b $BASEDN -s base contextCSN | grep contextCSN | sort`
1450	if test -z "$CSN1" ; then
1451		echo "ERROR: contextCSN empty on central master"
1452		CSN_ERRORS=`expr $CSN_ERRORS + 1`
1453	fi
1454
1455	if  test -z "$CSN2" -o "$CSN1" != "$CSN2" ; then
1456		echo "ERROR: contextCSN mismatch between central master and site1 master"
1457		CSN_ERRORS=`expr $CSN_ERRORS + 1`;
1458		if test -n "$CSN_VERBOSE"; then
1459			echo "contextCSN on central master:"
1460			echo "$CSN1"
1461			echo "contextCSN on site1 master:"
1462			echo "$CSN2"
1463		fi
1464	fi
1465	if  test -z "$CSN3" -o "$CSN1" != "$CSN3" ; then
1466		echo "ERROR: contextCSN mismatch between central master and site2 master"
1467		CSN_ERRORS=`expr $CSN_ERRORS + 1`;
1468		if test -n "$CSN_VERBOSE"; then
1469			echo "contextCSN on central master:"
1470			echo "$CSN1"
1471			echo "contextCSN on site2 master:"
1472			echo "$CSN3"
1473		fi
1474	fi
1475	if  test -z "$CSN4" -o "$CSN1" != "$CSN4" ; then
1476		echo "ERROR: contextCSN mismatch between central master and central search"
1477		CSN_ERRORS=`expr $CSN_ERRORS + 1`;
1478		if test -n "$CSN_VERBOSE"; then
1479			echo "contextCSN on central master:"
1480			echo "$CSN1"
1481			echo "contextCSN on central search:"
1482			echo "$CSN4"
1483		fi
1484	fi
1485	if  test -z "$CSN5" -o "$CSN2" != "$CSN5" ; then
1486		echo "ERROR: contextCSN mismatch between site1 master and site1 search"
1487		CSN_ERRORS=`expr $CSN_ERRORS + 1`;
1488		if test -n "$CSN_VERBOSE"; then
1489			echo "contextCSN on site1 master:"
1490			echo "$CSN2"
1491			echo "contextCSN on site1 search:"
1492			echo "$CSN5"
1493		fi
1494	fi
1495	if  test -z "$CSN6" -o "$CSN3" != "$CSN6" ; then
1496		echo "ERROR: contextCSN mismatch between site2 master and site2 search:"
1497		CSN_ERRORS=`expr $CSN_ERRORS + 1`;
1498		if test -n "$CSN_VERBOSE"; then
1499			echo "contextCSN on site2 master:"
1500			echo "$CSN3"
1501			echo "contextCSN on site2 search:"
1502			echo "$CSN6"
1503		fi
1504	fi
1505fi
1506
1507ERRORS=`expr $ERRORS + $CSN_ERRORS`
1508
1509EOF
1510
1511test $BACKEND = null && echo : > $TESTDIR/checkcsn.sh
1512
1513chmod +x $TESTDIR/checkcsn.sh
1514
1515
1516echo "Checking contextCSN after initial replication..."
1517. $TESTDIR/checkcsn.sh
1518
1519MNUM=1
1520
1521# TEST:
1522# Test that updates to the first backend on central master, which should
1523# be replicated to all servers actually is so, and that the contextCSN is
1524# updated everywhere:
1525echo "Using ldapmodify to modify first backend on central master..."
1526$LDAPMODIFY -D "$MANAGERDN" -H $URI1 -w $PASSWD <<EOF >> $TESTOUT 2>&1
1527dn: ou=ou1,dc=example,dc=com
1528changetype: modify
1529add: description
1530description: Modify$MNUM
1531
1532EOF
1533RC=$?
1534if test $RC != 0 ; then
1535	echo "ldapmodify failed ($RC)!"
1536	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1537	exit $RC
1538fi
1539sleep 1
1540
1541echo "Using ldapsearch to check replication to central search..."
1542RC=32
1543for i in 1 2 3 4 5; do
1544	RESULT=`$LDAPSEARCH -H $URI4 \
1545		-s base -b "ou=ou1,$BASEDN" \
1546		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "OK"}'`
1547	if test "x$RESULT$nullOK" = "xOK" ; then
1548		RC=0
1549		break
1550	fi
1551	echo "Waiting $i seconds for syncrepl to receive changes..."
1552	sleep $i
1553done
1554if test $RC != 0 ; then
1555	echo "ldapsearch failed ($RC)!"
1556	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1557	exit $RC
1558fi
1559
1560echo "Using ldapsearch to check replication to site1 search..."
1561RC=32
1562for i in 1 2 3 4 5; do
1563	RESULT=`$LDAPSEARCH -H $URI5 \
1564		-s base -b "ou=ou1,$BASEDN" \
1565		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "OK"}'`
1566	if test "x$RESULT$nullOK" = "xOK" ; then
1567		RC=0
1568		break
1569	fi
1570	echo "Waiting $i seconds for syncrepl to receive changes..."
1571	sleep $i
1572done
1573if test $RC != 0 ; then
1574	echo "ldapsearch failed ($RC)!"
1575	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1576	exit $RC
1577fi
1578
1579echo "Using ldapsearch to check replication to site2 search..."
1580RC=32
1581for i in 1 2 3 4 5; do
1582	RESULT=`$LDAPSEARCH -H $URI6 \
1583		-s base -b "ou=ou1,$BASEDN" \
1584		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "OK"}'`
1585	if test "x$RESULT$nullOK" = "xOK" ; then
1586		RC=0
1587		break
1588	fi
1589	echo "Waiting $i seconds for syncrepl to receive changes..."
1590	sleep $i
1591done
1592if test $RC != 0 ; then
1593	echo "ldapsearch failed ($RC)!"
1594	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1595	exit $RC
1596fi
1597
1598echo "Checking contextCSN after modify of first backend on central master..."
1599. $TESTDIR/checkcsn.sh
1600
1601# TEST:
1602# Test that updates to the second backend on central master is only
1603# replicated to those search servers that should receive that backend.
1604# The contextCSN should still be updated everywhere:
1605MNUM=`expr $MNUM + 1`
1606echo "Using ldapmodify to modify second backend on central master..."
1607$LDAPMODIFY -D "$MANAGERDN" -H $URI1 -w $PASSWD <<EOF >> $TESTOUT 2>&1
1608dn: ou=ou2,dc=example,dc=com
1609changetype: modify
1610add: description
1611description: Modify$MNUM
1612
1613EOF
1614RC=$?
1615if test $RC != 0 ; then
1616	echo "ldapmodify failed ($RC)!"
1617	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1618	exit $RC
1619fi
1620sleep 1
1621
1622echo "Using ldapsearch to check replication to site2 search..."
1623RC=32
1624for i in 1 2 3 4 5; do
1625	RESULT=`$LDAPSEARCH -H $URI6 \
1626		-s base -b "ou=ou2,$BASEDN" \
1627		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "OK"}'`
1628	if test "x$RESULT$nullOK" = "xOK" ; then
1629		RC=0
1630		break
1631	fi
1632	echo "Waiting $i seconds for syncrepl to receive changes..."
1633	sleep $i
1634done
1635if test $RC != 0 ; then
1636	echo "ldapsearch failed ($RC)!"
1637	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1638	exit $RC
1639fi
1640
1641echo "Using ldapsearch to check no replication to site1 master..."
1642for i in 1 2 3 4 5; do
1643	RESULT=`$LDAPSEARCH -H $URI2 \
1644		-s base -b "ou=ou2,$BASEDN" \
1645		"(description=Modify$NMUM)" 2>&1 | awk '/^dn:/ {print "NOK"}'`
1646	if test "x$RESULT" = "xNOK" ; then
1647		echo "Change was replicated to site1 search!"
1648		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1649		exit 1
1650	fi
1651	sleep 1
1652done
1653
1654echo "Using ldapsearch to check no replication to central search..."
1655for i in 1 2 3 4 5; do
1656	RESULT=`$LDAPSEARCH -H $URI4 \
1657		-s base -b "ou=ou2,$BASEDN" \
1658		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "NOK"}'`
1659	if test "x$RESULT" = "xNOK" ; then
1660		echo "Change was replicated to central search!"
1661		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1662		exit 1
1663	fi
1664	sleep 1
1665done
1666
1667echo "Checking contextCSN after modify of second backend on central master..."
1668. $TESTDIR/checkcsn.sh
1669
1670# TEST:
1671# Test that updates to the first backend on site1 master, which should be
1672# replicated everywhere except to central and site2 search.  The contextCSN
1673# should be updated on all servers:
1674MNUM=`expr $MNUM + 1`
1675echo "Using ldapmodify to modify first backend on site1 master..."
1676$LDAPMODIFY -D "ou=sm1ou1,$BASEDN" -H $URI2 -w $PASSWD <<EOF >> $TESTOUT 2>&1
1677dn: ou=sm1ou1,dc=example,dc=com
1678changetype: modify
1679add: description
1680description: Modify$MNUM
1681
1682EOF
1683RC=$?
1684if test $RC != 0 ; then
1685	echo "ldapmodify failed ($RC)!"
1686	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1687	exit $RC
1688fi
1689sleep 1
1690
1691echo "Using ldapsearch to check replication to site1 search..."
1692RC=32
1693for i in 1 2 3 4 5; do
1694	RESULT=`$LDAPSEARCH -H $URI5 \
1695		-s base -b "ou=sm1ou1,$BASEDN" \
1696		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "OK"}'`
1697	if test "x$RESULT$nullOK" = "xOK" ; then
1698		RC=0
1699		break
1700	fi
1701	echo "Waiting $i seconds for syncrepl to receive changes..."
1702	sleep $i
1703done
1704if test $RC != 0 ; then
1705	echo "ldapsearch failed ($RC)!"
1706	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1707	exit $RC
1708fi
1709
1710echo "Using ldapsearch to check replication to site2 master..."
1711RC=32
1712for i in 1 2 3 4 5; do
1713	RESULT=`$LDAPSEARCH -H $URI3 \
1714		-s base -b "ou=sm1ou1,$BASEDN" \
1715		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "OK"}'`
1716	if test "x$RESULT$nullOK" = "xOK" ; then
1717		RC=0
1718		break
1719	fi
1720	echo "Waiting $i seconds for syncrepl to receive changes..."
1721	sleep $i
1722done
1723if test $RC != 0 ; then
1724	echo "ldapsearch failed ($RC)!"
1725	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1726	exit $RC
1727fi
1728
1729echo "Using ldapsearch to check no replication to site2 search..."
1730for i in 1 2 3 4 5; do
1731	RESULT=`$LDAPSEARCH -H $URI6 \
1732		-s base -b "ou=sm1ou2,$BASEDN" \
1733		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "NOK"}'`
1734	if test "x$RESULT" = "xNOK" ; then
1735		echo "Change was replicated to central search!"
1736		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1737		exit 1
1738	fi
1739	sleep 1
1740done
1741
1742echo "Using ldapsearch to check no replication to central search..."
1743for i in 1 2 3 4 5; do
1744	RESULT=`$LDAPSEARCH -H $URI4 \
1745		-s base -b "ou=sm1ou2,$BASEDN" \
1746		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "NOK"}'`
1747	if test "x$RESULT" = "xNOK" ; then
1748		echo "Change was replicated to central search!"
1749		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1750		exit 1
1751	fi
1752	sleep 1
1753done
1754
1755echo "Checking contextCSN after modify of first backend on site1 master..."
1756. $TESTDIR/checkcsn.sh
1757
1758
1759# TEST:
1760# Test updates to the second backend on site1 master, which should only be
1761# replicated to site1 search.  The contextCSN should be updated everywhere.
1762MNUM=`expr $MNUM + 1`
1763echo "Using ldapmodify to modify second backend on site1 master..."
1764$LDAPMODIFY -D "ou=sm1ou1,$BASEDN" -H $URI2 -w $PASSWD <<EOF >> $TESTOUT 2>&1
1765dn: ou=sm1ou2,dc=example,dc=com
1766changetype: modify
1767add: description
1768description: Modify$MNUM
1769
1770EOF
1771RC=$?
1772if test $RC != 0 ; then
1773	echo "ldapmodify failed ($RC)!"
1774	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1775	exit $RC
1776fi
1777sleep 1
1778
1779
1780echo "Using ldapsearch to check replication to site1 search..."
1781RC=32
1782for i in 1 2 3 4 5; do
1783	RESULT=`$LDAPSEARCH -H $URI5 \
1784		-s base -b "ou=sm1ou2,$BASEDN" \
1785		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "OK"}'`
1786	if test "x$RESULT$nullOK" = "xOK" ; then
1787		RC=0
1788		break
1789	fi
1790	echo "Waiting $i seconds for syncrepl to receive changes..."
1791	sleep $i
1792done
1793if test $RC != 0 ; then
1794	echo "ldapsearch failed ($RC)!"
1795	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1796	exit $RC
1797fi
1798
1799echo "Using ldapsearch to check no replication to central master..."
1800for i in 1 2 3 4 5; do
1801	RESULT=`$LDAPSEARCH -H $URI1 \
1802		-s base -b "ou=sm1ou2,$BASEDN" \
1803		"(description=Modify$NMUM)" 2>&1 | awk '/^dn:/ {print "NOK"}'`
1804	if test "x$RESULT" = "xNOK" ; then
1805		echo "Change was replicated to site2 search!"
1806		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1807		exit 1
1808	fi
1809	sleep 1
1810done
1811
1812echo "Checking contextCSN after modify of second backend on site1 master..."
1813. $TESTDIR/checkcsn.sh
1814
1815
1816# TEST:
1817# Test updates to first backend on site2 master, which should be
1818# replicated to the central servers, but not site1.  The contextCSN
1819# should be updated everywhere:
1820MNUM=`expr $MNUM + 1`
1821echo "Using ldapmodify to modify first backend on site2 master..."
1822$LDAPMODIFY -D "ou=sm2ou1,$BASEDN" -H $URI3 -w $PASSWD <<EOF >> $TESTOUT 2>&1
1823dn: ou=sm2ou1,dc=example,dc=com
1824changetype: modify
1825add: description
1826description: Modify$MNUM
1827
1828EOF
1829RC=$?
1830if test $RC != 0 ; then
1831	echo "ldapmodify failed ($RC)!"
1832	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1833	exit $RC
1834fi
1835sleep 1
1836
1837echo "Using ldapsearch to check replication to central master..."
1838RC=32
1839for i in 1 2 3 4 5; do
1840	RESULT=`$LDAPSEARCH -H $URI1 \
1841		-s base -b "ou=sm2ou1,$BASEDN" \
1842		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "OK"}'`
1843	if test "x$RESULT$nullOK" = "xOK" ; then
1844		RC=0
1845		break
1846	fi
1847	echo "Waiting $i seconds for syncrepl to receive changes..."
1848	sleep $i
1849done
1850if test $RC != 0 ; then
1851	echo "ldapsearch failed ($RC)!"
1852	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1853	exit $RC
1854fi
1855
1856echo "Using ldapsearch to check replication to site2 search..."
1857RC=32
1858for i in 1 2 3 4 5; do
1859	RESULT=`$LDAPSEARCH -H $URI6 \
1860		-s base -b "ou=sm2ou1,$BASEDN" \
1861		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "OK"}'`
1862	if test "x$RESULT$nullOK" = "xOK" ; then
1863		RC=0
1864		break
1865	fi
1866	echo "Waiting $i seconds for syncrepl to receive changes..."
1867	sleep $i
1868done
1869if test $RC != 0 ; then
1870	echo "ldapsearch failed ($RC)!"
1871	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1872	exit $RC
1873fi
1874
1875echo "Using ldapsearch to check no replication to site1 master..."
1876for i in 1 2 3 4 5; do
1877	RESULT=`$LDAPSEARCH -H $URI2 \
1878		-s base -b "ou=sm2ou1,$BASEDN" \
1879		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "NOK"}'`
1880	if test "x$RESULT" = "xNOK" ; then
1881		echo "Change was replicated to site2 search!"
1882		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1883		exit 1
1884	fi
1885	sleep 1
1886done
1887
1888echo "Using ldapsearch to check no replication to central search..."
1889for i in 1 2 3 4 5; do
1890	RESULT=`$LDAPSEARCH -H $URI4 \
1891		-s base -b "ou=sm2ou1,$BASEDN" \
1892		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "NOK"}'`
1893	if test "x$RESULT" = "xNOK" ; then
1894		echo "Change was replicated to site2 search!"
1895		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1896		exit 1
1897	fi
1898	sleep 1
1899done
1900
1901echo "Checking contextCSN after modify of first backend on site2 master..."
1902. $TESTDIR/checkcsn.sh
1903
1904
1905# TEST:
1906# Test updates to the second backend on site2 master, which should only be
1907# replicated to site2 search.  As always, contextCSN should be updated
1908# everywhere:
1909MNUM=`expr $MNUM + 1`
1910echo "Using ldapmodify to modify second backend on site2 master..."
1911$LDAPMODIFY -D "ou=sm2ou1,$BASEDN" -H $URI3 -w $PASSWD <<EOF >> $TESTOUT 2>&1
1912dn: ou=sm2ou2,dc=example,dc=com
1913changetype: modify
1914add: description
1915description: Modify$MNUM
1916
1917EOF
1918RC=$?
1919if test $RC != 0 ; then
1920	echo "ldapmodify failed ($RC)!"
1921	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1922	exit $RC
1923fi
1924sleep 1
1925
1926echo "Using ldapsearch to check replication to site2 search..."
1927RC=32
1928for i in 1 2 3 4 5; do
1929	RESULT=`$LDAPSEARCH -H $URI6 \
1930		-s base -b "ou=sm2ou2,$BASEDN" \
1931		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "OK"}'`
1932	if test "x$RESULT$nullOK" = "xOK" ; then
1933		RC=0
1934		break
1935	fi
1936	echo "Waiting $i seconds for syncrepl to receive changes..."
1937	sleep $i
1938done
1939if test $RC != 0 ; then
1940	echo "ldapsearch failed ($RC)!"
1941	test $KILLSERVERS != no && kill -HUP $KILLPIDS
1942	exit $RC
1943fi
1944
1945echo "Using ldapsearch to check no replication to central master..."
1946for i in 1 2 3 4 5; do
1947	RESULT=`$LDAPSEARCH -H $URI4 \
1948		-s base -b "ou=sm2ou2,$BASEDN" \
1949		"(description=Modify$MNUM)" 2>&1 | awk '/^dn:/ {print "NOK"}'`
1950	if test "x$RESULT" = "xNOK" ; then
1951		echo "Change was replicated to central search!"
1952		test $KILLSERVERS != no && kill -HUP $KILLPIDS
1953		exit 1
1954	fi
1955	sleep 1
1956done
1957
1958echo "Checking contextCSN after modify of second backend on site2 master..."
1959. $TESTDIR/checkcsn.sh
1960
1961# TEST:
1962# Test that all contextCSN values are updated on the slaves when they
1963# starts with an empty database.  Start site2 master first, then site2
1964# search and finally central master so that the site2 search's syncrepl
1965# connection has been set up when site2 master receives the database:
1966echo "Stopping central master and site2 servers to test start with emtpy db..."
1967kill -HUP  $SMC_PID $SM2_PID $SS2_PID
1968for pid in $SMC_PID $SM2_PID $SS2_PID; do wait $pid ; done
1969KILLPIDS=`echo "$KILLPIDS " | sed -e "s/ $SMC_PID / /"`;
1970KILLPIDS=`echo "$KILLPIDS " | sed -e "s/ $SM2_PID / /"`;
1971KILLPIDS=`echo "$KILLPIDS " | sed -e "s/ $SS2_PID / /"`;
1972SMC_PID=
1973SM2_PID=
1974SS2_PID=
1975rm -rf $SM2_DIR/db/*
1976rm -rf $SS2_DIR/db/*
1977
1978echo "Starting site2 master slapd on TCP/IP port $PORT3..."
1979cd $SM2_DIR
1980$SLAPD -F slapd.d -h $URI3 -d $LVL $TIMING >> $LOG3 2>&1 &
1981SM2_PID=$!
1982if test $WAIT != 0 ; then
1983	echo PID $SM2_PID
1984	read foo
1985fi
1986KILLPIDS="$KILLPIDS $SM2_PID"
1987cd $TESTWD
1988sleep 1
1989echo "Using ldapsearch to check that site2 master slapd is running..."
1990for i in 1 2 3 4 5; do
1991	$LDAPSEARCH -s base -b "" -H $URI3 \
1992		'objectclass=*' > /dev/null 2>&1
1993	RC=$?
1994	test $RC = 0 && break
1995	echo "Waiting $i seconds for slapd to start..."
1996	sleep $i
1997done
1998if test $RC != 0 ; then
1999	echo "ldapsearch failed ($RC)!"
2000	test $KILLSERVERS != no && kill -HUP $KILLPIDS
2001	exit $RC
2002fi
2003
2004echo "Starting site2 search slapd on TCP/IP port $PORT6..."
2005cd $SS2_DIR
2006$SLAPD -F slapd.d -h $URI6 -d $LVL $TIMING >> $LOG6 2>&1 &
2007SS2_PID=$!
2008if test $WAIT != 0 ; then
2009	echo PID $SS2_PID
2010	read foo
2011fi
2012KILLPIDS="$KILLPIDS $SS2_PID"
2013cd $TESTWD
2014sleep 1
2015echo "Using ldapsearch to check that site2 search slapd is running..."
2016for i in 1 2 3 4 5; do
2017	$LDAPSEARCH -s base -b "" -H $URI6 \
2018		'objectclass=*' > /dev/null 2>&1
2019	RC=$?
2020	test $RC = 0 && break
2021	echo "Waiting $i seconds for slapd to start..."
2022	sleep $i
2023done
2024if test $RC != 0 ; then
2025	echo "ldapsearch failed ($RC)!"
2026	test $KILLSERVERS != no && kill -HUP $KILLPIDS
2027	exit $RC
2028fi
2029
2030echo "Starting central master slapd on TCP/IP port $PORT1..."
2031cd $SMC_DIR
2032$SLAPD -F slapd.d -h $URI1 -d $LVL $TIMING >> $LOG1 2>&1 &
2033SMC_PID=$!
2034if test $WAIT != 0 ; then
2035	echo PID $SMC_PID
2036	read foo
2037fi
2038KILLPIDS="$KILLPIDS $SMC_PID"
2039cd $TESTWD
2040sleep 1
2041echo "Using ldapsearch to check that central master slapd is running..."
2042for i in 1 2 3 4 5; do
2043	$LDAPSEARCH -s base -b "" -H $URI1 \
2044		'objectclass=*' > /dev/null 2>&1
2045	RC=$?
2046	test $RC = 0 && break
2047	echo "Waiting $i seconds for slapd to start..."
2048	sleep $i
2049done
2050if test $RC != 0 ; then
2051	echo "ldapsearch failed ($RC)!"
2052	test $KILLSERVERS != no && kill -HUP $KILLPIDS
2053	exit $RC
2054fi
2055
2056echo "Using ldapsearch to check that site2 master received base..."
2057RC=32
2058for i in 1 2 3 4 5; do
2059	RESULT=`$LDAPSEARCH -H $URI3 \
2060		-s base -b "$BASEDN" \
2061		'(objectClass=*)' 2>&1 | awk '/^dn:/ {print "OK"}'`
2062	if test "x$RESULT$nullOK" = "xOK" ; then
2063		RC=0
2064		break
2065	fi
2066	echo "Waiting $i seconds for syncrepl to receive changes..."
2067	sleep $i
2068done
2069if test $RC != 0 ; then
2070	echo "ldapsearch failed ($RC)!"
2071	test $KILLSERVERS != no && kill -HUP $KILLPIDS
2072	exit $RC
2073fi
2074
2075echo "Using ldapsearch to check that site2 search received base..."
2076RC=32
2077for i in 1 2 3 4 5; do
2078	RESULT=`$LDAPSEARCH -H $URI6 \
2079		-s base -b "$BASEDN" \
2080		'(objectClass=*)' 2>&1 | awk '/^dn:/ {print "OK"}'`
2081	if test "x$RESULT$nullOK" = "xOK" ; then
2082		RC=0
2083		break
2084	fi
2085	echo "Waiting $i seconds for syncrepl to receive changes..."
2086	sleep $i
2087done
2088if test $RC != 0 ; then
2089	echo "ldapsearch failed ($RC)!"
2090	test $KILLSERVERS != no && kill -HUP $KILLPIDS
2091	exit $RC
2092fi
2093
2094sleep $SLEEP1
2095
2096echo "Checking contextCSN after site2 servers repopulated..."
2097. $TESTDIR/checkcsn.sh
2098
2099if test $ERRORS -ne 0; then
2100	test $KILLSERVERS != no && kill -HUP $KILLPIDS
2101	echo "Found $ERRORS errors"
2102	exit $ERRORS
2103fi
2104
2105# TEST:
2106# Adding syncrepl of the second site1 master backend on central master
2107# will not initialize the database unless the contextCSN attribute is
2108# stored in the suffix of the database and not the suffix of the glue
2109# database:
2110echo "Adding syncrepl of second site1 master backend on central master..."
2111$LDAPMODIFY -D cn=config -H $URI1 -y $CONFIGPWF <<EOF >> $TESTOUT 2>&1
2112dn: olcDatabase={4}$BACKEND,cn=config
2113changetype: modify
2114add: olcSyncRepl
2115olcSyncRepl: rid=4 provider=$URI2 searchbase="ou=sm1ou2,$BASEDN"
2116  binddn="ou=sm1ou1,$BASEDN" bindmethod=simple credentials=$PASSWD
2117  type=refreshAndPersist retry="$RETRY" timeout=1
2118EOF
2119RC=$?
2120if test $RC != 0 ; then
2121	echo "ldapmodify failed to add syncrepl on central master ($RC)!"
2122	test $KILLSERVERS != no && kill -HUP $KILLPIDS
2123	exit $RC
2124fi
2125sleep 1
2126
2127echo "Using ldapsearch to check that central master received second site1 backend..."
2128RC=32
2129for i in 1 2 3 4 5; do
2130	RESULT=`$LDAPSEARCH -H $URI1 \
2131		-s base -b "ou=sm1ou2,$BASEDN" \
2132		'(objectClass=*)' 2>&1 | awk '/^dn:/ {print "OK"}'`
2133	if test "x$RESULT$nullOK" = "xOK" ; then
2134		RC=0
2135		break
2136	fi
2137	echo "Waiting $i seconds for syncrepl to receive changes..."
2138	sleep $i
2139done
2140if test $RC != 0 ; then
2141	echo "ERROR: Second site1 backend not replicated to central master"
2142	ERRORS=`expr $ERRORS + 1`
2143
2144	echo "Restarting central master slapd on TCP/IP port $PORT1..."
2145	kill -HUP $SMC_PID
2146	wait $SMC_PID
2147	KILLPIDS=`echo "$KILLPIDS " | sed -e "s/ $SMC_PID / /"`;
2148
2149	cd $SMC_DIR
2150	$SLAPD -F slapd.d -h $URI1 -c rid=4,csn=0 -d $LVL $TIMING >> $LOG1 2>&1 &
2151	SMC_PID=$!
2152	if test $WAIT != 0 ; then
2153		echo PID $SMC_PID
2154		read foo
2155	fi
2156	KILLPIDS="$KILLPIDS $SMC_PID"
2157	cd $TESTWD
2158	echo "Using ldapsearch to check that central master slapd is running..."
2159	for i in 1 2 3 4 5; do
2160		$LDAPSEARCH -s base -b "" -H $URI1 \
2161			'objectclass=*' > /dev/null 2>&1
2162		RC=$?
2163		test $RC = 0 && break
2164		echo "Waiting $i seconds for slapd to start..."
2165		sleep $i
2166	done
2167	if test $RC != 0 ; then
2168		echo "ldapsearch failed ($RC)!"
2169		test $KILLSERVERS != no && kill -HUP $KILLPIDS
2170		exit $RC
2171	fi
2172
2173	echo "Using ldapsearch to check that central master received second site1 backend..."
2174	RC=32
2175	for i in 1 2 3 4 5; do
2176		RESULT=`$LDAPSEARCH -H $URI1 \
2177			-s base -b "ou=sm1ou2,$BASEDN" \
2178			'(objectClass=*)' 2>&1 | awk '/^dn:/ {print "OK"}'`
2179		if test "x$RESULT$nullOK" = "xOK" ; then
2180			RC=0
2181			break
2182		fi
2183		echo "Waiting $i seconds for syncrepl to receive changes..."
2184		sleep $i
2185	done
2186	if test $RC != 0 ; then
2187		echo "ldapsearch failed ($RC)!"
2188		test $KILLSERVERS != no && kill -HUP $KILLPIDS
2189		exit $RC
2190	fi
2191fi
2192
2193echo "Using ldapsearch to check that central search received second site1 backend..."
2194RC=32
2195for i in 1 2 3 4 5; do
2196	RESULT=`$LDAPSEARCH -H $URI4 \
2197		-s base -b "ou=sm1ou2,$BASEDN" \
2198		'(objectClass=*)' 2>&1 | awk '/^dn:/ {print "OK"}'`
2199	if test "x$RESULT$nullOK" = "xOK" ; then
2200		RC=0
2201		break
2202	fi
2203	echo "Waiting $i seconds for syncrepl to receive changes..."
2204	sleep $i
2205done
2206if test $RC != 0 ; then
2207	echo "ERROR: Second site1 backend not replicated to central search"
2208	ERRORS=`expr $ERRORS + 1`
2209
2210	echo "Restarting central search slapd on TCP/IP port $PORT4..."
2211	kill -HUP $SSC_PID
2212	wait $SSC_PID
2213	KILLPIDS=`echo "$KILLPIDS " | sed -e "s/ $SSC_PID / /"`;
2214
2215	cd $SSC_DIR
2216	$SLAPD -F slapd.d -h $URI4 -c rid=1,csn=0 -d $LVL $TIMING >> $LOG4 2>&1 &
2217	SSC_PID=$!
2218	if test $WAIT != 0 ; then
2219		echo PID $SSC_PID
2220		read foo
2221	fi
2222	KILLPIDS="$KILLPIDS $SSC_PID"
2223	cd $TESTWD
2224	echo "Using ldapsearch to check that central search slapd is running..."
2225	for i in 1 2 3 4 5; do
2226		$LDAPSEARCH -s base -b "" -H $URI4 \
2227			'objectclass=*' > /dev/null 2>&1
2228		RC=$?
2229		test $RC = 0 && break
2230		echo "Waiting $i seconds for slapd to start..."
2231		sleep $i
2232	done
2233	if test $RC != 0 ; then
2234		echo "ldapsearch failed ($RC)!"
2235		test $KILLSERVERS != no && kill -HUP $KILLPIDS
2236		exit $RC
2237	fi
2238
2239	echo "Using ldapsearch to check that central search received second site1 backend..."
2240	RC=32
2241	for i in 1 2 3 4 5; do
2242		RESULT=`$LDAPSEARCH -H $URI4 \
2243			-s base -b "ou=sm1ou2,$BASEDN" \
2244			'(objectClass=*)' 2>&1 | awk '/^dn:/ {print "OK"}'`
2245		if test "x$RESULT$nullOK" = "xOK" ; then
2246			RC=0
2247			break
2248		fi
2249		echo "Waiting $i seconds for syncrepl to receive changes..."
2250		sleep $i
2251	done
2252	if test $RC != 0 ; then
2253		echo "ldapsearch failed ($RC)!"
2254		test $KILLSERVERS != no && kill -HUP $KILLPIDS
2255		exit $RC
2256	fi
2257fi
2258
2259
2260# TEST:
2261# Run race tests when more than one backend is replicated from the same
2262# producer.  This will usually fail long before 100 iterations unless
2263# syncrepl stores the contextCSN in the suffix of its own database, and
2264# that syncprov follows these rules before updating its own CSN when it
2265# detects updates from syncrepl:
2266# 1) A contextCSN value must have been stored in the suffix of all the
2267#	 syncrepl configured databases within the glued syncprov database.
2268# 2) Of all contextCSN values stored by syncrepl with the same SID,
2269#	 syncprov must always select the one with the lowest csn value.
2270test -z "$RACE_TESTS" && RACE_TESTS=10
2271RACE_NUM=0
2272RACE_ERROR=0
2273
2274SUB_DN=ou=sub,ou=sm1ou2,dc=example,dc=com
2275
2276while test $RACE_ERROR -eq 0 -a $RACE_NUM -lt $RACE_TESTS ; do
2277	RACE_NUM=`expr $RACE_NUM + 1`
2278	echo "Running $RACE_NUM of $RACE_TESTS syncrepl race tests..."
2279
2280	echo "Stopping central master..."
2281	kill -HUP $SMC_PID
2282	wait $SMC_PID
2283	KILLPIDS=`echo "$KILLPIDS " | sed -e "s/ $SMC_PID / /"`;
2284
2285	MNUM=`expr $MNUM + 1`
2286	echo "Using ldapadd to add entry on site1 master..."
2287	$LDAPADD -D "ou=sm1ou1,$BASEDN" -H $URI2 -w $PASSWD <<EOF >> $TESTOUT 2>&1
2288dn: $SUB_DN
2289objectClass: top
2290objectClass: organizationalUnit
2291ou: sub
2292
2293EOF
2294	RC=$?
2295	if test $RC != 0 ; then
2296		echo "ldapadd failed ($RC)!"
2297		test $KILLSERVERS != no && kill -HUP $KILLPIDS
2298		exit $RC
2299	fi
2300
2301	echo "Starting central master again..."
2302	cd $SMC_DIR
2303	$SLAPD -F slapd.d -h $URI1 -d $LVL $TIMING >> $LOG1 2>&1 &
2304	SMC_PID=$!
2305	KILLPIDS="$KILLPIDS $SMC_PID"
2306	cd $TESTWD
2307	echo "Using ldapsearch to check that central master received entry..."
2308	for i in 1 2 3 4 5; do
2309		$LDAPSEARCH -s base -b "$SUB_DN"  -H $URI1 > /dev/null 2>&1
2310		RC=$?
2311		test $RC = 0 && break
2312		sleep $i
2313	done
2314	if test $RC != 0 ; then
2315		echo "ERROR: entry not replicated to central master!"
2316		RACE_ERROR=1
2317		break
2318	fi
2319
2320	echo "Using ldapsearch to check that central search received entry..."
2321	for i in 1 2 3 4 5; do
2322		$LDAPSEARCH -s base -b "$SUB_DN"  -H $URI4 > /dev/null 2>&1
2323		RC=$?
2324		test $RC = 0 && break
2325		sleep $i
2326	done
2327	if test $RC != 0 ; then
2328		echo "ERROR: entry not replicated to central master!"
2329		RACE_ERROR=1
2330		break
2331	fi
2332
2333	echo "Stopping central master..."
2334	kill -HUP $SMC_PID
2335	wait $SMC_PID
2336	KILLPIDS=`echo "$KILLPIDS " | sed -e "s/ $SMC_PID / /"`;
2337
2338	echo "Using ldapdelete to delete entry on site1 master..."
2339	$LDAPDELETE -D "ou=sm1ou1,$BASEDN" -H $URI2 -w $PASSWD "$SUB_DN"
2340	RC=$?
2341	if test $RC != 0 ; then
2342		echo "ldapdelete failed ($RC)!"
2343		test $KILLSERVERS != no && kill -HUP $KILLPIDS
2344		exit $RC
2345	fi
2346
2347	echo "Starting central master again..."
2348	cd $SMC_DIR
2349	$SLAPD -F slapd.d -h $URI1 -d $LVL $TIMING >> $LOG1 2>&1 &
2350	SMC_PID=$!
2351	KILLPIDS="$KILLPIDS $SMC_PID"
2352	cd $TESTWD
2353
2354	echo "Using ldapsearch to check that entry was deleted on central master..."
2355	RC=0
2356	for i in 1 2 3 4 5; do
2357		$LDAPSEARCH -s base -b "$SUB_DN" -H $URI1 > /dev/null 2>&1
2358		RC=$?
2359		if test $RC = $wantNoObj; then break; fi
2360		sleep $i
2361	done
2362
2363	if test $RC != $wantNoObj; then
2364		if test $RC != 0; then
2365			echo "ldapsearch failed ($RC)!"
2366			test $KILLSERVERS != no && kill -HUP $KILLPIDS
2367			exit $RC
2368		fi
2369		echo "ERROR: Entry not removed on central master!"
2370		RACE_ERROR=1
2371		break
2372	fi
2373
2374	echo "Using ldapsearch to check that entry was deleted on central search..."
2375	RC=0
2376	for i in 1 2 3 4 5; do
2377		$LDAPSEARCH -s base -b "$SUB_DN" -H $URI4 > /dev/null 2>&1
2378		RC=$?
2379		if test $RC != 0; then break; fi
2380		sleep $i
2381	done
2382
2383	if test $RC != $wantNoObj; then
2384		echo "ERROR: Entry not removed on central search! (RC=$RC)"
2385		RACE_ERROR=1
2386		break
2387	fi
2388done
2389
2390if test $RACE_ERROR != 0; then
2391	echo "Race error found after $RACE_NUM of $RACE_TESTS iterations"
2392	ERRORS=`expr $ERRORS + $RACE_ERROR`
2393else
2394	echo "No race errors found after $RACE_TESTS iterations"
2395fi
2396
2397test $KILLSERVERS != no && kill -HUP $KILLPIDS
2398
2399if test $ERRORS -ne 0; then
2400	echo "Found $ERRORS errors"
2401	echo ">>>>>> Exiting with a false success status for now"
2402	exit 0
2403fi
2404
2405echo ">>>>> Test succeeded"
2406
2407exit 0
2408