1# See the file LICENSE for redistribution information. 2# 3# Copyright (c) 2002-2009 Oracle. All rights reserved. 4# 5# $Id$ 6# 7# TEST rep002 8# TEST Basic replication election test. 9# TEST 10# TEST Run a modified version of test001 in a replicated master 11# TEST environment; hold an election among a group of clients to 12# TEST make sure they select a proper master from amongst themselves, 13# TEST in various scenarios. 14 15proc rep002 { method { niter 10 } { nclients 3 } { tnum "002" } args } { 16 17 source ./include.tcl 18 global repfiles_in_memory 19 20 if { $is_windows9x_test == 1 } { 21 puts "Skipping replication test on Win 9x platform." 22 return 23 } 24 25 # Skip for record-based methods. 26 if { $checking_valid_methods } { 27 set test_methods {} 28 foreach method $valid_methods { 29 if { [is_record_based $method] != 1 } { 30 lappend test_methods $method 31 } 32 } 33 return $test_methods 34 } 35 if { [is_record_based $method] == 1 } { 36 puts "Rep002: Skipping for method $method." 37 return 38 } 39 40 set msg2 "and on-disk replication files" 41 if { $repfiles_in_memory } { 42 set msg2 "and in-memory replication files" 43 } 44 45 set logsets [create_logsets [expr $nclients + 1]] 46 47 # Run the body of the test with and without recovery. 48 foreach r $test_recopts { 49 foreach l $logsets { 50 set logindex [lsearch -exact $l "in-memory"] 51 if { $r == "-recover" && $logindex != -1 } { 52 puts "Skipping test with -recover for in-memory logs." 53 } 54 puts "Rep$tnum ($method $r): Replication election\ 55 test with $nclients clients $msg2." 56 puts "Rep$tnum: Master logs are [lindex $l 0]" 57 for { set i 0 } { $i < $nclients } { incr i } { 58 puts "Rep$tnum: Client $i logs are\ 59 [lindex $l [expr $i + 1]]" 60 } 61 rep002_sub $method $niter $nclients $tnum $l $r $args 62 } 63 } 64} 65 66proc rep002_sub { method niter nclients tnum logset recargs largs } { 67 source ./include.tcl 68 global repfiles_in_memory 69 global elect_timeout elect_serial 70 set elect_timeout(default) 5000000 71 72 global rep_verbose 73 global verbose_type 74 75 set verbargs "" 76 if { $rep_verbose == 1 } { 77 set verbargs " -verbose {$verbose_type on} " 78 } 79 80 set repmemargs "" 81 if { $repfiles_in_memory } { 82 set repmemargs "-rep_inmem_files " 83 } 84 85 env_cleanup $testdir 86 87 set qdir $testdir/MSGQUEUEDIR 88 replsetup $qdir 89 90 set masterdir $testdir/MASTERDIR 91 file mkdir $masterdir 92 set m_logtype [lindex $logset 0] 93 set m_logargs [adjust_logargs $m_logtype] 94 set m_txnargs [adjust_txnargs $m_logtype] 95 96 for { set i 0 } { $i < $nclients } { incr i } { 97 set clientdir($i) $testdir/CLIENTDIR.$i 98 file mkdir $clientdir($i) 99 set c_logtype($i) [lindex $logset [expr $i + 1]] 100 set c_logargs($i) [adjust_logargs $c_logtype($i)] 101 set c_txnargs($i) [adjust_txnargs $c_logtype($i)] 102 } 103 104 # Open a master. 105 repladd 1 106 set env_cmd(M) "berkdb_env_noerr -create -log_max 1000000 \ 107 -event rep_event $repmemargs \ 108 -home $masterdir $m_logargs -errpfx MASTER $verbargs \ 109 $m_txnargs -rep_master -rep_transport \[list 1 replsend\]" 110 # In an election test, the -recovery arg must not go 111 # in the env_cmd string because that is going to be 112 # passed to a child process. 113 set masterenv [eval $env_cmd(M) $recargs] 114 115 # Open the clients. 116 for { set i 0 } { $i < $nclients } { incr i } { 117 set envid [expr $i + 2] 118 repladd $envid 119 set env_cmd($i) "berkdb_env_noerr -create -home $clientdir($i) \ 120 -event rep_event $repmemargs \ 121 $c_logargs($i) $c_txnargs($i) -rep_client -errpfx CLIENT$i \ 122 $verbargs -rep_transport \[list $envid replsend\]" 123 set clientenv($i) [eval $env_cmd($i) $recargs] 124 } 125 126 # Loop, processing first the master's messages, then the client's, 127 # until both queues are empty. 128 set envlist {} 129 lappend envlist "$masterenv 1" 130 for { set i 0 } { $i < $nclients } { incr i } { 131 lappend envlist "$clientenv($i) [expr $i + 2]" 132 } 133 process_msgs $envlist 134 135 # Run a modified test001 in the master. 136 puts "\tRep$tnum.a: Running test001 in replicated env." 137 eval test001 $method $niter 0 0 $tnum -env $masterenv $largs 138 process_msgs $envlist 139 140 # Verify the database in the client dir. 141 for { set i 0 } { $i < $nclients } { incr i } { 142 puts "\tRep$tnum.b: Verifying contents of client database $i." 143 set testdir [get_home $masterenv] 144 set t1 $testdir/t1 145 set t2 $testdir/t2 146 set t3 $testdir/t3 147 open_and_dump_file test$tnum.db $clientenv($i) $testdir/t1 \ 148 test001.check dump_file_direction "-first" "-next" 149 150 if { [string compare [convert_method $method] -recno] != 0 } { 151 filesort $t1 $t3 152 } 153 error_check_good diff_files($t2,$t3) [filecmp $t2 $t3] 0 154 155 verify_dir $clientdir($i) "\tRep$tnum.c: " 0 0 1 156 } 157 158 # Start an election in the first client. 159 puts "\tRep$tnum.d: Starting election with existing master." 160 # We want to verify that the master declares the election 161 # over by fiat, even if everyone uses a lower priority than 20. 162 # Loop and process all messages, keeping track of which 163 # sites got a HOLDELECTION and checking that the master i.d. is 164 # unchanged after the election. 165 166 set origrole [stat_field $masterenv rep_stat "Role"] 167 error_check_good originally_master $origrole "master" 168 set origgeneration [stat_field $masterenv rep_stat "Generation number"] 169 170 set got_hold_elect(M) 0 171 for { set i 0 } { $i < $nclients } { incr i } { 172 set got_hold_elect($i) 0 173 set elect_pipe($i) INVALID 174 } 175 set elect_pipe(0) [start_election C0 $qdir $env_cmd(0) \ 176 [expr $nclients + 1] $nclients 20 $elect_timeout(default)] 177 178 tclsleep 2 179 180 set got_master 0 181 while { 1 } { 182 set nproced 0 183 set he 0 184 185 incr nproced [replprocessqueue $masterenv 1 0 he] 186 187 if { $he == 1 } { 188 incr elect_serial 189 set elect_pipe(M) [start_election CM $qdir \ 190 $env_cmd(M) [expr $nclients + 1] $nclients \ 191 0 $elect_timeout(default)] 192 set got_hold_elect(M) 1 193 } 194 195 for { set i 0 } { $i < $nclients } { incr i } { 196 set he 0 197 set envid [expr $i + 2] 198 incr nproced \ 199 [replprocessqueue $clientenv($i) $envid 0 he] 200 if { $he == 1 } { 201 # error_check_bad client(0)_in_elect $i 0 202 if { $elect_pipe($i) != "INVALID" } { 203 close_election $elect_pipe($i) 204 } 205 incr elect_serial 206 set pfx CHILD$i.$elect_serial 207 set elect_pipe($i) [start_election $pfx $qdir \ 208 $env_cmd($i) [expr $nclients + 1] \ 209 $nclients 0 \ 210 $elect_timeout(default)] 211 set got_hold_elect($i) 1 212 } 213 } 214 215 if { $nproced == 0 } { 216 break 217 } 218 } 219 set role [stat_field $masterenv rep_stat "Role"] 220 set generation [stat_field $masterenv rep_stat "Generation number"] 221 error_check_good master_unchanged $origrole $role 222 error_check_good gen_unchanged $origgeneration $generation 223 cleanup_elections 224 225 # We need multiple clients to proceed from here. 226 if { $nclients < 2 } { 227 puts "\tRep$tnum: Skipping for less than two clients." 228 error_check_good masterenv_close [$masterenv close] 0 229 for { set i 0 } { $i < $nclients } { incr i } { 230 error_check_good clientenv_close($i) \ 231 [$clientenv($i) close] 0 232 } 233 return 234 } 235 236 # Make sure all the clients are synced up and ready to be good 237 # voting citizens. 238 error_check_good master_flush [$masterenv rep_flush] 0 239 process_msgs $envlist 240 241 # Now hold another election in the first client, this time with 242 # a dead master. 243 puts "\tRep$tnum.e: Starting election with dead master." 244 error_check_good masterenv_close [$masterenv close] 0 245 set envlist [lreplace $envlist 0 0] 246 247 set m "Rep$tnum.e" 248 # We're not going to be using err_cmd, so initialize to "none". 249 # Client #1 has priority 100; everyone else has priority 10. 250 for { set i 0 } { $i < $nclients } { incr i } { 251 set err_cmd($i) "none" 252 set crash($i) 0 253 if { $i == 1 } { 254 set pri($i) 100 255 } else { 256 set pri($i) 10 257 } 258 } 259 set nsites $nclients 260 set nvotes $nclients 261 # The elector calls the first election. The expected winner 262 # is $win. 263 set elector 1 264 set win 1 265 run_election env_cmd envlist err_cmd pri crash $qdir $m \ 266 $elector $nsites $nvotes $nclients $win 1 "test$tnum.db" 267 268 # Hold an election with two clients at the same (winning) priority. 269 # Make sure that the tie gets broken, and that the third client 270 # does not win. 271 puts "\tRep$tnum.f: Election with two clients at same priority." 272 set m "Rep$tnum.f" 273 # Clients 0 and 1 have high, matching priority. 274 for { set i 0 } { $i < $nclients } { incr i } { 275 if { $i >= 2 } { 276 set pri($i) 10 277 } else { 278 set pri($i) 100 279 } 280 } 281 282 # Run several elections. 283 set elections 5 284 for { set i 0 } { $i < $elections } { incr i } { 285 # 286 # The expected winner is 0 or 1. Since run_election can only 287 # handle one expected winner, catch the result and inspect it. 288 # 289 set elector 0 290 set win 1 291 set altwin 0 292 if {[catch {eval run_election \ 293 env_cmd envlist err_cmd pri crash $qdir $m $elector $nsites \ 294 $nvotes $nclients $win 1 "test$tnum.db"} res]} { 295 # 296 # If the primary winner didn't win, make sure 297 # the alternative winner won. Do all the cleanup 298 # for that winner normally done in run_election: 299 # open and close the new master, then reopen as a 300 # client for the next cycle. 301 # 302 error_check_good check_winner [is_substr \ 303 $res "expected 3, got [expr $altwin + 2]"] 1 304 puts "\t$m: Election $i: Alternate winner $altwin won." 305 error_check_good make_master \ 306 [$clientenv($altwin) rep_start -master] 0 307 308 cleanup_elections 309 process_msgs $envlist 310 311 error_check_good newmaster_close \ 312 [$clientenv($altwin) close] 0 313 set clientenv($altwin) [eval $env_cmd($altwin)] 314 error_check_good cl($altwin) \ 315 [is_valid_env $clientenv($altwin)] TRUE 316 set newelector "$clientenv($altwin) [expr $altwin + 2]" 317 set envlist [lreplace $envlist $altwin $altwin $newelector] 318 } else { 319 puts "\t$m: Election $i: Primary winner $win won." 320 } 321 process_msgs $envlist 322 } 323 324 foreach pair $envlist { 325 set cenv [lindex $pair 0] 326 error_check_good cenv_close [$cenv close] 0 327 } 328 329 replclose $testdir/MSGQUEUEDIR 330} 331