1#! /bin/sh 2# $OpenLDAP$ 3## This work is part of OpenLDAP Software <http://www.openldap.org/>. 4## 5## Copyright 1998-2011 The OpenLDAP Foundation. 6## All rights reserved. 7## 8## Redistribution and use in source and binary forms, with or without 9## modification, are permitted only as authorized by the OpenLDAP 10## Public License. 11## 12## A copy of this license is available in the file LICENSE in the 13## top-level directory of the distribution or, alternatively, at 14## <http://www.OpenLDAP.org/license.html>. 15 16# This script tests race conditions related to setting up the syncrepl 17# refresh phase, especially when the provider is itself a consumer 18# refreshing from its provider again. 19 20# The configuration used is a provider->forwarder->consumer chain, where 21# the forwarder is restarted between add/delete of entries on the provider. 22 23echo "Running defines.sh" 24. $SRCDIR/scripts/defines.sh 25 26test "x$INITIATION_RACE_TESTS" = "x" && INITIATION_RACE_TESTS=1 27 28if test $SYNCPROV = syncprovno; then 29 echo "Syncrepl provider overlay not available, test skipped" 30 exit 0 31fi 32 33RETRY="1 +" 34 35PROV_DIR=$TESTDIR/prov 36CONS_DIR=$TESTDIR/cons 37FWD1_DIR=$TESTDIR/fwd1 38FWD2_DIR=$TESTDIR/fwd2 39 40PROV_URI=$URI1 41CONS_URI=$URI2 42FWD1_URI=$URI3 43 44PROV_LOG=$LOG1 45CONS_LOG=$LOG2 46FWD1_LOG=$LOG3 47 48DIRS="$PROV_DIR $CONS_DIR $FWD1_DIR" 49URIS="$PROV_URI $CONS_URI $FWD1_URI" 50 51noObj=32 52nullExclude="" nullOK="" 53test $BACKEND = null && nullExclude="# " nullOK="OK" noObj=0 54 55mkdir -p $TESTDIR 56 57for dir in $DIRS; do 58 mkdir -p $dir $dir/slapd.d $dir/db 59done 60 61KILLPIDS= 62 63$SLAPPASSWD -g -n >$CONFIGPWF 64 65case "$BACKEND" in 66 bdb|hdb) olcDbCheckpoint="olcDbCheckpoint";; 67 *) olcDbCheckpoint="# olcDbCheckpoint";; 68esac 69 70echo "Initializing server configurations" 71for dir in $DIRS; do 72 $SLAPADD -F $dir/slapd.d -n 0 <<EOF 73dn: cn=config 74objectClass: olcGlobal 75cn: config 76olcServerID: 1 $PROV_URI 77olcServerID: 2 $CONS_URI 78olcServerID: 3 $FWD1_URI 79 80dn: olcDatabase={0}config,cn=config 81objectClass: olcDatabaseConfig 82olcDatabase: {0}config 83olcRootPW:< file://$CONFIGPWF 84 85EOF 86done 87 88echo "Starting provider slapd on $PROV_URI" 89cd $PROV_DIR 90$SLAPD -F slapd.d -h $PROV_URI -d $LVL $TIMING >> $PROV_LOG 2>&1 & 91PROV_PID=$! 92if test $WAIT != 0 ; then 93 echo PID $PROV_PID 94 read foo 95fi 96KILLPIDS="$KILLPIDS $PROV_PID" 97cd $TESTWD 98sleep 1 99for i in 1 2 3 4 5; do 100 $LDAPSEARCH -s base -b "" -H $PROV_URI \ 101 'objectclass=*' > /dev/null 2>&1 102 RC=$? 103 test $RC = 0 && break 104 echo "Waiting $i seconds for slapd to start..." 105 sleep $i 106done 107if test $RC != 0 ; then 108 echo "ldapsearch failed ($RC)!" 109 test $KILLSERVERS != no && kill -HUP $KILLPIDS 110 exit $RC 111fi 112 113echo "Starting forward1 slapd on $FWD1_URI" 114cd $FWD1_DIR 115$SLAPD -F slapd.d -h $FWD1_URI -d $LVL $TIMING >> $FWD1_LOG 2>&1 & 116FWD1_PID=$! 117if test $WAIT != 0 ; then 118 echo PID $FWD1_PID 119 read foo 120fi 121KILLPIDS="$KILLPIDS $FWD1_PID" 122cd $TESTWD 123sleep 1 124for i in 1 2 3 4 5; do 125 $LDAPSEARCH -s base -b "" -H $FWD1_URI \ 126 'objectclass=*' > /dev/null 2>&1 127 RC=$? 128 test $RC = 0 && break 129 echo "Waiting $i seconds for slapd to start..." 130 sleep $i 131done 132if test $RC != 0 ; then 133 echo "ldapsearch failed ($RC)!" 134 test $KILLSERVERS != no && kill -HUP $KILLPIDS 135 exit $RC 136fi 137 138echo "Starting consumer slapd on $CONS_URI" 139cd $CONS_DIR 140$SLAPD -F slapd.d -h $CONS_URI -d $LVL $TIMING >> $CONS_LOG 2>&1 & 141CONS_PID=$! 142if test $WAIT != 0 ; then 143 echo PID $CONS_PID 144 read foo 145fi 146KILLPIDS="$KILLPIDS $CONS_PID" 147cd $TESTWD 148sleep 1 149for i in 1 2 3 4 5; do 150 $LDAPSEARCH -s base -b "" -H $CONS_URI \ 151 'objectclass=*' > /dev/null 2>&1 152 RC=$? 153 test $RC = 0 && break 154 echo "Waiting $i seconds for slapd to start..." 155 sleep $i 156done 157if test $RC != 0 ; then 158 echo "ldapsearch failed ($RC)!" 159 test $KILLSERVERS != no && kill -HUP $KILLPIDS 160 exit $RC 161fi 162 163for uri in $URIS; do 164 echo "Adding schema on $uri" 165 $LDAPADD -D cn=config -H $uri -y $CONFIGPWF <<EOF > $TESTOUT 2>&1 166include: file://$ABS_SCHEMADIR/core.ldif 167 168include: file://$ABS_SCHEMADIR/cosine.ldif 169 170include: file://$ABS_SCHEMADIR/inetorgperson.ldif 171 172include: file://$ABS_SCHEMADIR/openldap.ldif 173 174include: file://$ABS_SCHEMADIR/nis.ldif 175 176EOF 177 RC=$? 178 if test $RC != 0 ; then 179 echo "ldapadd failed ($RC)!" 180 test $KILLSERVERS != no && kill -HUP $KILLPIDS 181 exit $RC 182 fi 183 184 [ "$BACKENDTYPE" = mod ] || continue 185 186 echo "Adding backend module on $uri..." 187 $LDAPADD -D cn=config -H $uri -y $CONFIGPWF <<EOF >>$TESTOUT 2>&1 188dn: cn=module,cn=config 189objectClass: olcModuleList 190cn: module 191olcModulePath: $TESTWD/../servers/slapd/back-$BACKEND 192olcModuleLoad: back_$BACKEND.la 193 194EOF 195 RC=$? 196 if test $RC != 0 ; then 197 echo "ldapadd failed for backend module ($RC)!" 198 test $KILLSERVERS != no && kill -HUP $KILLPIDS 199 exit $RC 200 fi 201done 202 203syncprov_module='' 204[ "$AC_syncprov" = syncprovmod ] && syncprov_module=" 205dn: cn=module,cn=config 206objectClass: olcModuleList 207cn: module 208olcModulePath: $TESTWD/../servers/slapd/overlays 209olcModuleLoad: syncprov.la" 210 211for uri in $PROV_URI; do 212 echo "Adding database configuration on $uri" 213 $LDAPADD -D cn=config -H $uri -y $CONFIGPWF <<EOF > $TESTOUT 2>&1 214dn: olcDatabase={1}$BACKEND,cn=config 215objectClass: olcDatabaseConfig 216objectClass: olc${BACKEND}Config 217olcDatabase: {1}$BACKEND 218${nullExclude}olcDbDirectory: ./db 219$olcDbCheckpoint: 1024 5 220olcSuffix: $BASEDN 221olcRootDN: $MANAGERDN 222olcRootPW: $PASSWD 223 224$syncprov_module 225 226dn: olcOverlay={0}syncprov,olcDatabase={1}$BACKEND,cn=config 227objectClass: olcOverlayConfig 228objectClass: olcSyncProvConfig 229olcOverlay: {0}syncprov 230olcSpCheckpoint: 1 1 231 232EOF 233 RC=$? 234 if test $RC != 0 ; then 235 echo "ldapadd failed ($RC)!" 236 test $KILLSERVERS != no && kill -HUP $KILLPIDS 237 exit $RC 238 fi 239 240 echo "Populating provider on $uri" 241 $LDAPADD -D "$MANAGERDN" -H $PROV_URI -w $PASSWD <<EOF >> $TESTOUT 2>&1 242dn: $BASEDN 243objectClass: top 244objectClass: organization 245objectClass: dcObject 246dc: example 247o: Example, Inc 248 249EOF 250 RC=$? 251 if test $RC != 0 ; then 252 echo "ldapadd failed ($RC)!" 253 test $KILLSERVERS != no && kill -HUP $KILLPIDS 254 exit $RC 255 fi 256done 257 258for uri in $FWD1_URI; do 259 echo "Adding database configuration on $uri" 260 $LDAPADD -D cn=config -H $uri -y $CONFIGPWF <<EOF > $TESTOUT 2>&1 261dn: olcDatabase={1}$BACKEND,cn=config 262objectClass: olcDatabaseConfig 263objectClass: olc${BACKEND}Config 264olcDatabase: {1}$BACKEND 265${nullExclude}olcDbDirectory: ./db 266$olcDbCheckpoint: 1024 5 267olcSuffix: $BASEDN 268olcRootDN: $MANAGERDN 269olcRootPW: $PASSWD 270olcSyncRepl: rid=1 provider=$PROV_URI searchbase="$BASEDN" 271 binddn="$MANAGERDN" bindmethod=simple credentials=$PASSWD 272 type=refreshAndPersist retry="$RETRY" timeout=1 273 274$syncprov_module 275 276dn: olcOverlay={0}syncprov,olcDatabase={1}$BACKEND,cn=config 277objectClass: olcOverlayConfig 278objectClass: olcSyncProvConfig 279olcOverlay: {0}syncprov 280olcSpCheckpoint: 1 1 281 282EOF 283 RC=$? 284 if test $RC != 0 ; then 285 echo "ldapadd failed ($RC)!" 286 test $KILLSERVERS != no && kill -HUP $KILLPIDS 287 exit $RC 288 fi 289done 290 291for uri in $CONS_URI; do 292 echo "Adding database configuration on $uri" 293 $LDAPADD -D cn=config -H $uri -y $CONFIGPWF <<EOF > $TESTOUT 2>&1 294dn: olcDatabase={1}$BACKEND,cn=config 295objectClass: olcDatabaseConfig 296objectClass: olc${BACKEND}Config 297olcDatabase: {1}$BACKEND 298${nullExclude}olcDbDirectory: ./db 299$olcDbCheckpoint: 1024 5 300olcSuffix: $BASEDN 301olcRootDN: $MANAGERDN 302olcRootPW: $PASSWD 303olcSyncRepl: rid=1 provider=$FWD1_URI searchbase="$BASEDN" 304 binddn="$MANAGERDN" bindmethod=simple credentials=$PASSWD 305 type=refreshAndPersist retry="$RETRY" timeout=1 306 307EOF 308 RC=$? 309 if test $RC != 0 ; then 310 echo "ldapadd failed ($RC)!" 311 test $KILLSERVERS != no && kill -HUP $KILLPIDS 312 exit $RC 313 fi 314done 315 316for uri in $FWD1_URI $CONS_URI; do 317 echo "Using ldapsearch to check that $uri received database..." 318 for i in 1 2 3 4 5; do 319 $LDAPSEARCH -s base -b "$BASEDN" -H $uri \ 320 'objectclass=*' > /dev/null 2>&1 321 RC=$? 322 test $RC = 0 && break 323 echo "Waiting $i seconds for slapd to receive database..." 324 sleep $i 325 done 326 if test $RC != 0 ; then 327 echo "ldapsearch failed ($RC)!" 328 test $KILLSERVERS != no && kill -HUP $KILLPIDS 329 exit $RC 330 fi 331done 332 333RACE_NUM=0 334ERROR=0 335 336nEntries=10 337 338addEnd=1 339delEnd=1 340 341addIdx=1 342delIdx=1 343 344while test $ERROR -eq 0 -a $RACE_NUM -lt $INITIATION_RACE_TESTS ; do 345 RACE_NUM=`expr $RACE_NUM + 1` 346 echo "Running $RACE_NUM of $INITIATION_RACE_TESTS syncrepl initiation race tests..." 347 348 echo "Stopping forwarders for add test" 349 for pid in $FWD1_PID; do 350 kill -HUP $pid 351 wait $pid 352 KILLPIDS=`echo "$KILLPIDS " | sed -e "s/ $pid / /"`; 353 done 354 355 addStart=$addEnd 356 addEnd=`expr $addEnd + $nEntries` 357 358 echo "Using ldapadd to add $nEntries entries on provider" 359 while test $addIdx -lt $addEnd; do 360 $LDAPADD -D "$MANAGERDN" -H $PROV_URI -w $PASSWD <<EOF >> $TESTOUT 2>&1 361dn: ou=$addIdx,$BASEDN 362objectClass: top 363objectClass: organizationalUnit 364ou: $addIdx 365 366EOF 367 RC=$? 368 if test $RC != 0 ; then 369 echo "ldapadd failed for entry $addIdx ($RC)!" 370 test $KILLSERVERS != no && kill -HUP $KILLPIDS 371 exit $RC 372 fi 373 addIdx=`expr $addIdx + 1` 374 done 375 376 echo "Starting forwarders again" 377 cd $FWD1_DIR 378 $SLAPD -F slapd.d -h $FWD1_URI -d $LVL $TIMING >> $FWD1_LOG 2>&1 & 379 FWD1_PID=$! 380 KILLPIDS="$KILLPIDS $FWD1_PID" 381 cd $TESTWD 382 383 addEnd=`expr $addEnd + $nEntries` 384 385 echo "Using ldapadd to add $nEntries more entries on provider" 386 while test $addIdx -lt $addEnd; do 387 $LDAPADD -D "$MANAGERDN" -H $PROV_URI -w $PASSWD <<EOF >> $TESTOUT 2>&1 388dn: ou=$addIdx,$BASEDN 389objectClass: top 390objectClass: organizationalUnit 391ou: $addIdx 392 393EOF 394 RC=$? 395 if test $RC != 0 ; then 396 echo "ldapadd failed for entry $addIdx ($RC)!" 397 test $KILLSERVERS != no && kill -HUP $KILLPIDS 398 exit $RC 399 fi 400 addIdx=`expr $addIdx + 1` 401 done 402 403 for uri in $FWD1_URI $CONS_URI; do 404 echo "Checking replication to $uri" 405 RC=32 406 i=$addStart 407 while test $i -lt $addEnd; do 408 for j in 1 2 3 4 5; do 409 RESULT=`$LDAPSEARCH -H $uri -s base -b "ou=$i,$BASEDN" 2>&1 \ 410 | awk '/^dn:/ {print "OK"}'` 411 if test "x$RESULT$nullOK" = "xOK" ; then 412 RC=0 413 break 414 fi 415 echo "Waiting $j seconds for $uri to receive entry $i..." 416 sleep $j 417 done 418 if test $RC != 0 ; then 419 echo "ERROR: Entry $i not replicated to $uri! ($RC)!" 420 ERROR=1 421 break 422 fi 423 i=`expr $i + 1` 424 done 425 if test $ERROR != 0; then break; fi 426 done 427 if test $ERROR != 0; then break; fi 428 429 echo "Stopping forwarders for add/delete test" 430 for pid in $FWD1_PID; do 431 kill -HUP $pid 432 wait $pid 433 KILLPIDS=`echo "$KILLPIDS " | sed -e "s/ $pid / /"`; 434 done 435 436 addStart=$addEnd 437 addEnd=`expr $addEnd + $nEntries` 438 439 echo "Using ldapadd to add $nEntries entries on provider" 440 while test $addIdx -lt $addEnd; do 441 $LDAPADD -D "$MANAGERDN" -H $PROV_URI -w $PASSWD <<EOF >> $TESTOUT 2>&1 442dn: ou=$addIdx,$BASEDN 443objectClass: top 444objectClass: organizationalUnit 445ou: $addIdx 446 447EOF 448 RC=$? 449 if test $RC != 0 ; then 450 echo "ldapadd failed for entry $addIdx ($RC)!" 451 test $KILLSERVERS != no && kill -HUP $KILLPIDS 452 exit $RC 453 fi 454 addIdx=`expr $addIdx + 1` 455 done 456 457 delStart=$delEnd 458 delEnd=`expr $delEnd + $nEntries` 459 460 echo "Using ldapdelete to delete $nEntries entries on provider" 461 while test $delIdx -lt $delEnd; do 462 $LDAPDELETE -D "$MANAGERDN" -H $PROV_URI -w $PASSWD "ou=$delIdx,$BASEDN" 463 RC=$? 464 if test $RC != 0 ; then 465 echo "ldapdelete failed ($RC)!" 466 test $KILLSERVERS != no && kill -HUP $KILLPIDS 467 exit 1 468 fi 469 delIdx=`expr $delIdx + 1` 470 done 471 472 echo "Starting forwarders again" 473 cd $FWD1_DIR 474 $SLAPD -F slapd.d -h $FWD1_URI -d $LVL $TIMING >> $FWD1_LOG 2>&1 & 475 FWD1_PID=$! 476 KILLPIDS="$KILLPIDS $FWD1_PID" 477 cd $TESTWD 478 479 addEnd=`expr $addEnd + $nEntries` 480 delEnd=`expr $delEnd + $nEntries` 481 482 echo "Using ldapadd to add $nEntries more entries on provider" 483 while test $addIdx -lt $addEnd; do 484 $LDAPADD -D "$MANAGERDN" -H $PROV_URI -w $PASSWD <<EOF >> $TESTOUT 2>&1 485dn: ou=$addIdx,$BASEDN 486objectClass: top 487objectClass: organizationalUnit 488ou: $addIdx 489 490EOF 491 RC=$? 492 if test $RC != 0 ; then 493 echo "ldapadd failed for entry $addIdx ($RC)!" 494 test $KILLSERVERS != no && kill -HUP $KILLPIDS 495 exit $RC 496 fi 497 addIdx=`expr $addIdx + 1` 498 done 499 500 echo "Using ldapdelete to delete $nEntries more entries on provider" 501 while test $delIdx -lt $delEnd; do 502 $LDAPDELETE -D "$MANAGERDN" -H $PROV_URI -w $PASSWD "ou=$delIdx,$BASEDN" 503 RC=$? 504 if test $RC != 0 ; then 505 echo "ldapdelete failed ($RC)!" 506 test $KILLSERVERS != no && kill -HUP $KILLPIDS 507 exit 1 508 fi 509 delIdx=`expr $delIdx + 1` 510 done 511 512 for uri in $FWD1_URI $CONS_URI; do 513 echo "Checking replication to $uri" 514 RC=32 515 i=$addStart 516 while test $i -lt $addEnd; do 517 for j in 1 2 3 4 5; do 518 RESULT=`$LDAPSEARCH -H $uri -s base -b "ou=$i,$BASEDN" 2>&1 \ 519 | awk '/^dn:/ {print "OK"}'` 520 if test "x$RESULT$nullOK" = "xOK" ; then 521 RC=0 522 break 523 fi 524 echo "Waiting $j seconds for $uri to receive entry $i..." 525 sleep $j 526 done 527 if test $RC != 0 ; then 528 echo "ERROR: Entry $i not replicated to $uri! ($RC)!" 529 ERROR=1 530 break 531 fi 532 i=`expr $i + 1` 533 done 534 if test $ERROR != 0; then break; fi 535 536 i=$delStart 537 while test $i -lt $delEnd; do 538 for j in 1 2 3 4 5; do 539 $LDAPSEARCH -s base -b "ou=$i,$BASEDN" -H $uri > /dev/null 2>&1 540 RC=$? 541 if test $RC = $noObj; then break; fi 542 echo "Waiting $j seconds for $uri to delete entry $i..." 543 sleep $j 544 done 545 if test $RC != $noObj; then 546 echo "ERROR: Entry $i not removed on $uri! (RC=$RC)" 547 ERROR=1 548 break 549 fi 550 i=`expr $i + 1` 551 done 552 if test $ERROR != 0; then break; fi 553 done 554 if test $ERROR != 0; then break; fi 555 556 echo "Stopping forwarders for delete test" 557 for pid in $FWD1_PID; do 558 kill -HUP $pid 559 wait $pid 560 KILLPIDS=`echo "$KILLPIDS " | sed -e "s/ $pid / /"`; 561 done 562 563 delStart=$delEnd 564 delEnd=`expr $delEnd + $nEntries` 565 566 echo "Using ldapdelete to delete entries on provider" 567 while test $delIdx -lt $delEnd; do 568 $LDAPDELETE -D "$MANAGERDN" -H $PROV_URI -w $PASSWD "ou=$delIdx,$BASEDN" 569 RC=$? 570 if test $RC != 0 ; then 571 echo "ldapdelete failed ($RC)!" 572 test $KILLSERVERS != no && kill -HUP $KILLPIDS 573 exit 1 574 fi 575 delIdx=`expr $delIdx + 1` 576 done 577 578 echo "Starting forwarders again" 579 cd $FWD1_DIR 580 $SLAPD -F slapd.d -h $FWD1_URI -d $LVL $TIMING >> $FWD1_LOG 2>&1 & 581 FWD1_PID=$! 582 KILLPIDS="$KILLPIDS $FWD1_PID" 583 cd $TESTWD 584 585 delEnd=`expr $delEnd + $nEntries` 586 587 echo "Using ldapdelete to delete $nEntries more entries on provider" 588 while test $delIdx -lt $delEnd; do 589 $LDAPDELETE -D "$MANAGERDN" -H $PROV_URI -w $PASSWD "ou=$delIdx,$BASEDN" 590 RC=$? 591 if test $RC != 0 ; then 592 echo "ldapdelete failed ($RC)!" 593 test $KILLSERVERS != no && kill -HUP $KILLPIDS 594 exit 1 595 fi 596 delIdx=`expr $delIdx + 1` 597 done 598 599 for uri in $FWD1_URI $CONS_URI; do 600 echo "Checking replication to $uri" 601 RC=0 602 i=$delStart 603 while test $i -lt $delEnd; do 604 for j in 1 2 3 4 5; do 605 $LDAPSEARCH -s base -b "ou=$i,$BASEDN" -H $uri > /dev/null 2>&1 606 RC=$? 607 if test $RC = $noObj; then break; fi 608 echo "Waiting $j seconds for $uri to delete entry $i..." 609 sleep $j 610 done 611 if test $RC != $noObj; then 612 echo "ERROR: Entry $i not removed on $uri! (RC=$RC)" 613 ERROR=1 614 break 615 fi 616 i=`expr $i + 1` 617 done 618 if test $ERROR != 0; then break; fi 619 done 620 if test $ERROR != 0; then break; fi 621 622 echo "Checking contextCSN" 623 CSN_ERRORS=0 624 CSN1=`$LDAPSEARCH -H $URI1 -b $BASEDN -s base contextCSN | grep contextCSN` 625 CSN2=`$LDAPSEARCH -H $URI2 -b $BASEDN -s base contextCSN | grep contextCSN` 626 CSN3=`$LDAPSEARCH -H $URI3 -b $BASEDN -s base contextCSN | grep contextCSN` 627 628 if test -z "$CSN1" ; then 629 test $BACKEND = null && break 630 echo "ERROR: contextCSN empty on provider" 631 ERROR=1 632 break 633 fi 634 nCSN=`echo "$CSN1" | wc -l` 635 if test "$nCSN" -ne 1 ; then 636 echo "ERROR: Wrong contextCSN count on provder, should be 1" 637 echo "$CSN1" 638 test $KILLSERVERS != no && kill -HUP $KILLPIDS 639 exit 1 640 fi 641 if test -z "$CSN2" -o "$CSN1" != "$CSN2" ; then 642 echo "ERROR: contextCSN mismatch between provider and consumer" 643 echo "contextCSN on provider: $CSN1" 644 echo "contextCSN on consumer: $CSN2" 645 ERROR=1 646 break 647 fi 648 if test -z "$CSN3" -o "$CSN1" != "$CSN3" ; then 649 echo "ERROR: contextCSN mismatch between provider and forward1" 650 echo "contextCSN on provider: $CSN1" 651 echo "contextCSN on forward1: $CSN3" 652 ERROR=1 653 break 654 fi 655done 656 657test $KILLSERVERS != no && kill -HUP $KILLPIDS 658 659if test $ERROR != 0; then 660 echo "Error found after $RACE_NUM of $INITIATION_RACE_TESTS iterations" 661 exit 1 662else 663 echo "No race errors found after $INITIATION_RACE_TESTS iterations" 664fi 665 666echo ">>>>> Test succeeded" 667 668exit 0 669