1# See the file LICENSE for redistribution information. 2# 3# Copyright (c) 2001-2009 Oracle. All rights reserved. 4# 5# $Id$ 6# 7# TEST rep021 8# TEST Replication and multiple environments. 9# TEST Run similar tests in separate environments, making sure 10# TEST that some data overlaps. Then, "move" one client env 11# TEST from one replication group to another and make sure that 12# TEST we do not get divergent logs. We either match the first 13# TEST record and end up with identical logs or we get an error. 14# TEST Verify all client logs are identical if successful. 15# 16proc rep021 { method { nclients 3 } { tnum "021" } args } { 17 18 source ./include.tcl 19 global repfiles_in_memory 20 21 if { $is_windows9x_test == 1 } { 22 puts "Skipping replication test on Win 9x platform." 23 return 24 } 25 26 # Run for all access methods. 27 if { $checking_valid_methods } { 28 return "ALL" 29 } 30 31 # This test depends on copying logs, so can't be run with 32 # in-memory logging. 33 global mixed_mode_logging 34 if { $mixed_mode_logging > 0 } { 35 puts "Rep$tnum: Skipping for mixed-mode logging." 36 return 37 } 38 39 # This test closes its envs, so it's not appropriate for 40 # testing of in-memory named databases. 41 global databases_in_memory 42 if { $databases_in_memory } { 43 puts "Rep$tnum: Skipping for in-memory databases." 44 return 45 } 46 47 set msg2 "and on-disk replication files" 48 if { $repfiles_in_memory } { 49 set msg2 "and in-memory replication files" 50 } 51 52 set args [convert_args $method $args] 53 set logsets [create_logsets [expr $nclients + 1]] 54 55 # Run the body of the test with and without recovery. 56 foreach r $test_recopts { 57 foreach l $logsets { 58 set logindex [lsearch -exact $l "in-memory"] 59 if { $r == "-recover" && $logindex != -1 } { 60 puts "Rep$tnum: Skipping\ 61 for in-memory logs with -recover." 62 continue 63 } 64 puts "Rep$tnum ($method $r): Replication\ 65 and $nclients recovered clients in sync $msg2." 66 puts "Rep$tnum: Master logs are [lindex $l 0]" 67 for { set i 0 } { $i < $nclients } { incr i } { 68 puts "Rep$tnum: Client $i logs are\ 69 [lindex $l [expr $i + 1]]" 70 } 71 rep021_sub $method $nclients $tnum $l $r $args 72 } 73 } 74} 75 76proc rep021_sub { method nclients tnum logset recargs largs } { 77 global testdir 78 global util_path 79 global repfiles_in_memory 80 global rep_verbose 81 global verbose_type 82 83 set verbargs "" 84 if { $rep_verbose == 1 } { 85 set verbargs " -verbose {$verbose_type on} " 86 } 87 88 set repmemargs "" 89 if { $repfiles_in_memory } { 90 set repmemargs "-rep_inmem_files " 91 } 92 93 set orig_tdir $testdir 94 env_cleanup $testdir 95 96 replsetup $testdir/MSGQUEUEDIR 97 98 set niter 100 99 set offset 5 100 set masterdir $testdir/MASTERDIR 101 set masterdir2 $testdir/MASTERDIR.NEW 102 file mkdir $masterdir 103 file mkdir $masterdir2 104 105 set m_logtype [lindex $logset 0] 106 set m_logargs [adjust_logargs $m_logtype] 107 set m_txnargs [adjust_txnargs $m_logtype] 108 109 # We want to run the test 3 times in 2 separate repl envs. 110 # This is a little bit tricky due to how we manage replication 111 # in Tcl. It assumes one replication group. 112 # This is tricky because we need to manage/clear the repl 113 # message queues for the different groups when running 114 # to one group or the other. 115 # To accomplish this we run entirely in the 2nd group first. 116 # We set it up and then close all its envs. Then we run 117 # to the 1st group, and set it up. Then we add in a client 118 # from the 2nd group into the existing 1st group. 119 # Although we're doing them in separate parts, this is 120 # a description of what we're doing. 121 # 122 # 1. First add divergent data to database: 123 # RepGrp1: Add niter data from 0 to database. 124 # RepGrp2: Add niter data from offset to database. 125 # This gives us overlapping data in the databases, but they're 126 # additions will be at different offsets in the log files. 127 # 128 # 2. Add identical data to both databases. 129 # RepGrp1: Add niter data from niter + offset to database. 130 # RepGrp2: Add niter data from niter + offset to database. 131 # This gives us identical data in the databases and logs. 132 # 133 # 3. Again add divergent data to databases. 134 # RepGrp1: Add niter data from niter*2+offset to database. 135 # RepGrp2: Add niter data from niter*2+offset*2 to database. 136 # This gives us overlapping data in the databases, but they're 137 # additions will be at different offsets in the log files. 138 # 139 # 4. Add a client from one group to the other. Then try 140 # to sync up that client. We should get a failure with 141 # one of the non-matching error messages: 142 # "Too few log files to sync with master" 143 # REP_JOIN_FAILURE 144 145 # Open a 2nd master. Make all the 2nd env ids >= 10. 146 # For the 2nd group, just have 1 master and 1 client. 147 repladd 10 148 set ma2_envcmd "berkdb_env_noerr -create $m_txnargs $verbargs \ 149 $m_logargs -home $masterdir2 $repmemargs \ 150 -rep_master -rep_transport \[list 10 replsend\]" 151 set menv2 [eval $ma2_envcmd $recargs] 152 153 set clientdir2 $testdir/CLIENTDIR.NEW 154 file mkdir $clientdir2 155 set id2 11 156 set c_logtype($id2) [lindex $logset 1] 157 set c_logargs($id2) [adjust_logargs $c_logtype($id2)] 158 set c_txnargs($id2) [adjust_txnargs $c_logtype($id2)] 159 160 set id2 11 161 repladd $id2 162 set cl2_envcmd "berkdb_env_noerr -create $c_txnargs($id2) $verbargs \ 163 $c_logargs($id2) -home $clientdir2 $repmemargs \ 164 -rep_client -rep_transport \[list $id2 replsend\]" 165 set clenv2 [eval $cl2_envcmd $recargs] 166 167 set testfile "test$tnum.db" 168 set omethod [convert_method $method] 169 170 set masterdb2 [eval {berkdb_open_noerr -env $menv2 -auto_commit \ 171 -create -mode 0644} $largs $omethod $testfile] 172 error_check_good dbopen [is_valid_db $masterdb2] TRUE 173 174 # 175 # Process startup messages 176 # 177 set env2list {} 178 lappend env2list "$menv2 10" 179 lappend env2list "$clenv2 $id2" 180 process_msgs $env2list 181 182 # 183 # Set up the three runs of rep_test. We need the starting 184 # point for each phase of the test for each group. 185 # 186 set e1phase1 0 187 set e2phase1 $offset 188 set e1phase2 [expr $niter + $offset] 189 set e2phase2 $e1phase2 190 set e1phase3 [expr $e1phase2 + $niter] 191 set e2phase3 [expr $e2phase2 + $niter + $offset] 192 193 puts "\tRep$tnum.a: Running rep_test in 2nd replicated env." 194 eval rep_test $method $menv2 $masterdb2 $niter $e2phase1 1 1 $largs 195 eval rep_test $method $menv2 $masterdb2 $niter $e2phase2 1 1 $largs 196 eval rep_test $method $menv2 $masterdb2 $niter $e2phase3 1 1 $largs 197 error_check_good mdb_cl [$masterdb2 close] 0 198 process_msgs $env2list 199 200 puts "\tRep$tnum.b: Close 2nd replicated env. Open primary." 201 error_check_good mdb_cl [$clenv2 close] 0 202 error_check_good mdb_cl [$menv2 close] 0 203 replclose $testdir/MSGQUEUEDIR 204 205 # 206 # Run recovery in client now to blow away region files so 207 # that this client comes in as a "new" client and announces itself. 208 # 209 set stat [catch {eval exec $util_path/db_recover -h $clientdir2} result] 210 error_check_good stat $stat 0 211 212 # 213 # Now we've run in the 2nd env. We have everything we need 214 # set up and existing in that env. Now run the test in the 215 # 1st env and then we'll try to add in the client. 216 # 217 replsetup $testdir/MSGQUEUEDIR 218 # Open a master. 219 repladd 1 220 set ma_envcmd "berkdb_env_noerr -create $m_txnargs $verbargs \ 221 $m_logargs -home $masterdir $repmemargs \ 222 -rep_master -rep_transport \[list 1 replsend\]" 223 set menv [eval $ma_envcmd $recargs] 224 225 for {set i 0} {$i < $nclients} {incr i} { 226 set clientdir($i) $testdir/CLIENTDIR.$i 227 file mkdir $clientdir($i) 228 set c_logtype($i) [lindex $logset [expr $i + 1]] 229 set c_logargs($i) [adjust_logargs $c_logtype($i)] 230 set c_txnargs($i) [adjust_txnargs $c_logtype($i)] 231 set id($i) [expr 2 + $i] 232 repladd $id($i) 233 set cl_envcmd($i) "berkdb_env_noerr -create $c_txnargs($i) \ 234 $c_logargs($i) -home $clientdir($i) $repmemargs \ 235 $verbargs \ 236 -rep_client -rep_transport \[list $id($i) replsend\]" 237 set clenv($i) [eval $cl_envcmd($i) $recargs] 238 } 239 240 set masterdb [eval {berkdb_open_noerr -env $menv -auto_commit \ 241 -create -mode 0644} $largs $omethod $testfile] 242 error_check_good dbopen [is_valid_db $masterdb] TRUE 243 244 # Bring the clients online by processing the startup messages. 245 set envlist {} 246 lappend envlist "$menv 1" 247 for { set i 0 } { $i < $nclients } { incr i } { 248 lappend envlist "$clenv($i) $id($i)" 249 } 250 process_msgs $envlist 251 252 # Run a modified test001 in the master (and update clients). 253 puts "\tRep$tnum.c: Running rep_test in primary replicated env." 254 eval rep_test $method $menv $masterdb $niter $e1phase1 1 1 $largs 255 eval rep_test $method $menv $masterdb $niter $e1phase2 1 1 $largs 256 eval rep_test $method $menv $masterdb $niter $e1phase3 1 1 $largs 257 error_check_good mdb_cl [$masterdb close] 0 258 # Process any close messages. 259 process_msgs $envlist 260 261 puts "\tRep$tnum.d: Add unrelated client into replication group." 262 set i $nclients 263 set orig $nclients 264 set nclients [expr $nclients + 1] 265 266 set clientdir($i) $clientdir2 267 set id($i) [expr 2 + $i] 268 repladd $id($i) 269 set cl_envcmd($i) "berkdb_env_noerr -create -txn nosync \ 270 -home $clientdir($i) $verbargs $repmemargs \ 271 -rep_client -rep_transport \[list $id($i) replsend\]" 272 set clenv($i) [eval $cl_envcmd($i) $recargs] 273 # 274 # We'll only catch an error if we turn on no-autoinit. 275 # Otherwise, the system will throw away everything on the 276 # client and resync. 277 # 278 $clenv($i) rep_config {noautoinit on} 279 280 lappend envlist "$clenv($i) $id($i)" 281 282 fileremove -f $clientdir2/prlog.orig 283 set stat [catch {eval exec $util_path/db_printlog \ 284 -h $clientdir2 >> $clientdir2/prlog.orig} result] 285 286 set err 0 287 process_msgs $envlist 0 NONE err 288 289 puts "\tRep$tnum.e: Close all envs and run recovery in clients." 290 error_check_good menv_cl [$menv close] 0 291 for {set i 0} {$i < $nclients} {incr i} { 292 error_check_good cl$i.close [$clenv($i) close] 0 293 set hargs($i) "-h $clientdir($i)" 294 } 295 set i [expr $nclients - 1] 296 fileremove -f $clientdir($i)/prlog 297 set stat [catch {eval exec $util_path/db_printlog \ 298 -h $clientdir($i) >> $clientdir($i)/prlog} result] 299 300 # If we got an error, then the log should match the original 301 # and the error message should tell us the client was never 302 # part of this environment. 303 # 304 if { $err != 0 } { 305 puts "\tRep$tnum.f: Verify client log matches original." 306 error_check_good log_cmp(orig,$i) \ 307 [filecmp $clientdir($i)/prlog.orig $clientdir($i)/prlog] 0 308 puts "\tRep$tnum.g: Verify client error." 309 error_check_good errchk [is_substr $err \ 310 "REP_JOIN_FAILURE"] 1 311 } else { 312 puts "\tRep$tnum.f: Verify client log doesn't match original." 313 error_check_good log_cmp(orig,$i) \ 314 [filecmp $clientdir($i)/prlog.orig $clientdir($i)/prlog] 1 315 puts "\tRep$tnum.g: Verify new client log matches master." 316 set stat [catch {eval exec $util_path/db_printlog \ 317 -h $masterdir >& $masterdir/prlog} result] 318 fileremove -f $clientdir($i)/prlog 319 set stat [catch {eval exec $util_path/db_printlog \ 320 -h $clientdir($i) >> $clientdir($i)/prlog} result] 321 error_check_good stat_prlog $stat 0 322 error_check_good log_cmp(master,$i) \ 323 [filecmp $masterdir/prlog $clientdir($i)/prlog] 0 324 } 325 326 replclose $testdir/MSGQUEUEDIR 327 set testdir $orig_tdir 328 return 329} 330 331