1# See the file LICENSE for redistribution information. 2# 3# Copyright (c) 2001,2008 Oracle. All rights reserved. 4# 5# $Id: rep021.tcl,v 12.17 2008/04/02 02:46:54 moshen Exp $ 6# 7# TEST rep021 8# TEST Replication and multiple environments. 9# TEST Run similar tests in separate environments, making sure 10# TEST that some data overlaps. Then, "move" one client env 11# TEST from one replication group to another and make sure that 12# TEST we do not get divergent logs. We either match the first 13# TEST record and end up with identical logs or we get an error. 14# TEST Verify all client logs are identical if successful. 15# 16proc rep021 { method { nclients 3 } { tnum "021" } args } { 17 18 source ./include.tcl 19 if { $is_windows9x_test == 1 } { 20 puts "Skipping replication test on Win 9x platform." 21 return 22 } 23 24 # Run for all access methods. 25 if { $checking_valid_methods } { 26 return "ALL" 27 } 28 29 # This test depends on copying logs, so can't be run with 30 # in-memory logging. 31 global mixed_mode_logging 32 if { $mixed_mode_logging > 0 } { 33 puts "Rep$tnum: Skipping for mixed-mode logging." 34 return 35 } 36 37 set args [convert_args $method $args] 38 set logsets [create_logsets [expr $nclients + 1]] 39 40 # Run the body of the test with and without recovery. 41 foreach r $test_recopts { 42 foreach l $logsets { 43 set logindex [lsearch -exact $l "in-memory"] 44 if { $r == "-recover" && $logindex != -1 } { 45 puts "Rep$tnum: Skipping\ 46 for in-memory logs with -recover." 47 continue 48 } 49 puts "Rep$tnum ($method $r):\ 50 Replication and $nclients recovered clients in sync." 51 puts "Rep$tnum: Master logs are [lindex $l 0]" 52 for { set i 0 } { $i < $nclients } { incr i } { 53 puts "Rep$tnum: Client $i logs are\ 54 [lindex $l [expr $i + 1]]" 55 } 56 rep021_sub $method $nclients $tnum $l $r $args 57 } 58 } 59} 60 61proc rep021_sub { method nclients tnum logset recargs largs } { 62 global testdir 63 global util_path 64 global rep_verbose 65 global verbose_type 66 67 set verbargs "" 68 if { $rep_verbose == 1 } { 69 set verbargs " -verbose {$verbose_type on} " 70 } 71 72 set orig_tdir $testdir 73 env_cleanup $testdir 74 75 replsetup $testdir/MSGQUEUEDIR 76 77 set niter 100 78 set offset 5 79 set masterdir $testdir/MASTERDIR 80 set masterdir2 $testdir/MASTERDIR.NEW 81 file mkdir $masterdir 82 file mkdir $masterdir2 83 84 set m_logtype [lindex $logset 0] 85 set m_logargs [adjust_logargs $m_logtype] 86 set m_txnargs [adjust_txnargs $m_logtype] 87 88 # We want to run the test 3 times in 2 separate repl envs. 89 # This is a little bit tricky due to how we manage replication 90 # in Tcl. It assumes one replication group. 91 # This is tricky because we need to manage/clear the repl 92 # message queues for the different groups when running 93 # to one group or the other. 94 # To accomplish this we run entirely in the 2nd group first. 95 # We set it up and then close all its envs. Then we run 96 # to the 1st group, and set it up. Then we add in a client 97 # from the 2nd group into the existing 1st group. 98 # Although we're doing them in separate parts, this is 99 # a description of what we're doing. 100 # 101 # 1. First add divergent data to database: 102 # RepGrp1: Add niter data from 0 to database. 103 # RepGrp2: Add niter data from offset to database. 104 # This gives us overlapping data in the databases, but they're 105 # additions will be at different offsets in the log files. 106 # 107 # 2. Add identical data to both databases. 108 # RepGrp1: Add niter data from niter + offset to database. 109 # RepGrp2: Add niter data from niter + offset to database. 110 # This gives us identical data in the databases and logs. 111 # 112 # 3. Again add divergent data to databases. 113 # RepGrp1: Add niter data from niter*2+offset to database. 114 # RepGrp2: Add niter data from niter*2+offset*2 to database. 115 # This gives us overlapping data in the databases, but they're 116 # additions will be at different offsets in the log files. 117 # 118 # 4. Add a client from one group to the other. Then try 119 # to sync up that client. We should get a failure with 120 # one of the non-matching error messages: 121 # "Too few log files to sync with master" 122 # REP_JOIN_FAILURE 123 124 # Open a 2nd master. Make all the 2nd env ids >= 10. 125 # For the 2nd group, just have 1 master and 1 client. 126 repladd 10 127 set ma2_envcmd "berkdb_env_noerr -create $m_txnargs $verbargs \ 128 $m_logargs -home $masterdir2 \ 129 -rep_master -rep_transport \[list 10 replsend\]" 130 set menv2 [eval $ma2_envcmd $recargs] 131 132 set clientdir2 $testdir/CLIENTDIR.NEW 133 file mkdir $clientdir2 134 set id2 11 135 set c_logtype($id2) [lindex $logset 1] 136 set c_logargs($id2) [adjust_logargs $c_logtype($id2)] 137 set c_txnargs($id2) [adjust_txnargs $c_logtype($id2)] 138 139 set id2 11 140 repladd $id2 141 set cl2_envcmd "berkdb_env_noerr -create $c_txnargs($id2) $verbargs \ 142 $c_logargs($id2) -home $clientdir2 \ 143 -rep_client -rep_transport \[list $id2 replsend\]" 144 set clenv2 [eval $cl2_envcmd $recargs] 145 146 set testfile "test$tnum.db" 147 set omethod [convert_method $method] 148 149 set masterdb2 [eval {berkdb_open_noerr -env $menv2 -auto_commit \ 150 -create -mode 0644} $largs $omethod $testfile] 151 error_check_good dbopen [is_valid_db $masterdb2] TRUE 152 153 # 154 # Process startup messages 155 # 156 set env2list {} 157 lappend env2list "$menv2 10" 158 lappend env2list "$clenv2 $id2" 159 process_msgs $env2list 160 161 # 162 # Set up the three runs of rep_test. We need the starting 163 # point for each phase of the test for each group. 164 # 165 set e1phase1 0 166 set e2phase1 $offset 167 set e1phase2 [expr $niter + $offset] 168 set e2phase2 $e1phase2 169 set e1phase3 [expr $e1phase2 + $niter] 170 set e2phase3 [expr $e2phase2 + $niter + $offset] 171 172 puts "\tRep$tnum.a: Running rep_test in 2nd replicated env." 173 eval rep_test $method $menv2 $masterdb2 $niter $e2phase1 1 1 0 $largs 174 eval rep_test $method $menv2 $masterdb2 $niter $e2phase2 1 1 0 $largs 175 eval rep_test $method $menv2 $masterdb2 $niter $e2phase3 1 1 0 $largs 176 error_check_good mdb_cl [$masterdb2 close] 0 177 process_msgs $env2list 178 179 puts "\tRep$tnum.b: Close 2nd replicated env. Open primary." 180 error_check_good mdb_cl [$clenv2 close] 0 181 error_check_good mdb_cl [$menv2 close] 0 182 replclose $testdir/MSGQUEUEDIR 183 184 # 185 # Run recovery in client now to blow away region files so 186 # that this client comes in as a "new" client and announces itself. 187 # 188 set stat [catch {eval exec $util_path/db_recover -h $clientdir2} result] 189 error_check_good stat $stat 0 190 191 # 192 # Now we've run in the 2nd env. We have everything we need 193 # set up and existing in that env. Now run the test in the 194 # 1st env and then we'll try to add in the client. 195 # 196 replsetup $testdir/MSGQUEUEDIR 197 # Open a master. 198 repladd 1 199 set ma_envcmd "berkdb_env_noerr -create $m_txnargs $verbargs \ 200 $m_logargs -home $masterdir \ 201 -rep_master -rep_transport \[list 1 replsend\]" 202 set menv [eval $ma_envcmd $recargs] 203 204 for {set i 0} {$i < $nclients} {incr i} { 205 set clientdir($i) $testdir/CLIENTDIR.$i 206 file mkdir $clientdir($i) 207 set c_logtype($i) [lindex $logset [expr $i + 1]] 208 set c_logargs($i) [adjust_logargs $c_logtype($i)] 209 set c_txnargs($i) [adjust_txnargs $c_logtype($i)] 210 set id($i) [expr 2 + $i] 211 repladd $id($i) 212 set cl_envcmd($i) "berkdb_env_noerr -create $c_txnargs($i) \ 213 $c_logargs($i) -home $clientdir($i) \ 214 $verbargs \ 215 -rep_client -rep_transport \[list $id($i) replsend\]" 216 set clenv($i) [eval $cl_envcmd($i) $recargs] 217 } 218 219 set masterdb [eval {berkdb_open_noerr -env $menv -auto_commit \ 220 -create -mode 0644} $largs $omethod $testfile] 221 error_check_good dbopen [is_valid_db $masterdb] TRUE 222 223 # Bring the clients online by processing the startup messages. 224 set envlist {} 225 lappend envlist "$menv 1" 226 for { set i 0 } { $i < $nclients } { incr i } { 227 lappend envlist "$clenv($i) $id($i)" 228 } 229 process_msgs $envlist 230 231 # Run a modified test001 in the master (and update clients). 232 puts "\tRep$tnum.c: Running rep_test in primary replicated env." 233 eval rep_test $method $menv $masterdb $niter $e1phase1 1 1 0 $largs 234 eval rep_test $method $menv $masterdb $niter $e1phase2 1 1 0 $largs 235 eval rep_test $method $menv $masterdb $niter $e1phase3 1 1 0 $largs 236 error_check_good mdb_cl [$masterdb close] 0 237 # Process any close messages. 238 process_msgs $envlist 239 240 puts "\tRep$tnum.d: Add unrelated client into replication group." 241 set i $nclients 242 set orig $nclients 243 set nclients [expr $nclients + 1] 244 245 set clientdir($i) $clientdir2 246 set id($i) [expr 2 + $i] 247 repladd $id($i) 248 set cl_envcmd($i) "berkdb_env_noerr -create -txn nosync \ 249 -home $clientdir($i) $verbargs \ 250 -rep_client -rep_transport \[list $id($i) replsend\]" 251 set clenv($i) [eval $cl_envcmd($i) $recargs] 252 # 253 # We'll only catch an error if we turn on no-autoinit. 254 # Otherwise, the system will throw away everything on the 255 # client and resync. 256 # 257 $clenv($i) rep_config {noautoinit on} 258 259 lappend envlist "$clenv($i) $id($i)" 260 261 fileremove -f $clientdir2/prlog.orig 262 set stat [catch {eval exec $util_path/db_printlog \ 263 -h $clientdir2 >> $clientdir2/prlog.orig} result] 264 265 set err 0 266 process_msgs $envlist 0 NONE err 267 268 puts "\tRep$tnum.e: Close all envs and run recovery in clients." 269 error_check_good menv_cl [$menv close] 0 270 for {set i 0} {$i < $nclients} {incr i} { 271 error_check_good cl$i.close [$clenv($i) close] 0 272 set hargs($i) "-h $clientdir($i)" 273 } 274 set i [expr $nclients - 1] 275 fileremove -f $clientdir($i)/prlog 276 set stat [catch {eval exec $util_path/db_printlog \ 277 -h $clientdir($i) >> $clientdir($i)/prlog} result] 278 279 # If we got an error, then the log should match the original 280 # and the error message should tell us the client was never 281 # part of this environment. 282 # 283 if { $err != 0 } { 284 puts "\tRep$tnum.f: Verify client log matches original." 285 error_check_good log_cmp(orig,$i) \ 286 [filecmp $clientdir($i)/prlog.orig $clientdir($i)/prlog] 0 287 puts "\tRep$tnum.g: Verify client error." 288 error_check_good errchk [is_substr $err \ 289 "REP_JOIN_FAILURE"] 1 290 } else { 291 puts "\tRep$tnum.f: Verify client log doesn't match original." 292 error_check_good log_cmp(orig,$i) \ 293 [filecmp $clientdir($i)/prlog.orig $clientdir($i)/prlog] 1 294 puts "\tRep$tnum.g: Verify new client log matches master." 295 set stat [catch {eval exec $util_path/db_printlog \ 296 -h $masterdir >& $masterdir/prlog} result] 297 fileremove -f $clientdir($i)/prlog 298 set stat [catch {eval exec $util_path/db_printlog \ 299 -h $clientdir($i) >> $clientdir($i)/prlog} result] 300 error_check_good stat_prlog $stat 0 301 error_check_good log_cmp(master,$i) \ 302 [filecmp $masterdir/prlog $clientdir($i)/prlog] 0 303 } 304 305 replclose $testdir/MSGQUEUEDIR 306 set testdir $orig_tdir 307 return 308} 309 310 311