1# See the file LICENSE for redistribution information. 2# 3# Copyright (c) 2002,2008 Oracle. All rights reserved. 4# 5# $Id: rep002.tcl,v 12.19 2008/01/08 20:58:53 bostic Exp $ 6# 7# TEST rep002 8# TEST Basic replication election test. 9# TEST 10# TEST Run a modified version of test001 in a replicated master 11# TEST environment; hold an election among a group of clients to 12# TEST make sure they select a proper master from amongst themselves, 13# TEST in various scenarios. 14 15proc rep002 { method { niter 10 } { nclients 3 } { tnum "002" } args } { 16 17 source ./include.tcl 18 if { $is_windows9x_test == 1 } { 19 puts "Skipping replication test on Win 9x platform." 20 return 21 } 22 23 # Skip for record-based methods. 24 if { $checking_valid_methods } { 25 set test_methods {} 26 foreach method $valid_methods { 27 if { [is_record_based $method] != 1 } { 28 lappend test_methods $method 29 } 30 } 31 return $test_methods 32 } 33 if { [is_record_based $method] == 1 } { 34 puts "Rep002: Skipping for method $method." 35 return 36 } 37 38 set logsets [create_logsets [expr $nclients + 1]] 39 40 # Run the body of the test with and without recovery. 41 foreach r $test_recopts { 42 foreach l $logsets { 43 set logindex [lsearch -exact $l "in-memory"] 44 if { $r == "-recover" && $logindex != -1 } { 45 puts "Skipping test with -recover for in-memory logs." 46 } 47 puts "Rep$tnum ($method $r):\ 48 Replication election test with $nclients clients." 49 puts "Rep$tnum: Master logs are [lindex $l 0]" 50 for { set i 0 } { $i < $nclients } { incr i } { 51 puts "Rep$tnum: Client $i logs are\ 52 [lindex $l [expr $i + 1]]" 53 } 54 rep002_sub $method $niter $nclients $tnum $l $r $args 55 } 56 } 57} 58 59proc rep002_sub { method niter nclients tnum logset recargs largs } { 60 source ./include.tcl 61 global elect_timeout elect_serial 62 set elect_timeout(default) 5000000 63 64 global rep_verbose 65 global verbose_type 66 67 set verbargs "" 68 if { $rep_verbose == 1 } { 69 set verbargs " -verbose {$verbose_type on} " 70 } 71 72 env_cleanup $testdir 73 74 set qdir $testdir/MSGQUEUEDIR 75 replsetup $qdir 76 77 set masterdir $testdir/MASTERDIR 78 file mkdir $masterdir 79 set m_logtype [lindex $logset 0] 80 set m_logargs [adjust_logargs $m_logtype] 81 set m_txnargs [adjust_txnargs $m_logtype] 82 83 for { set i 0 } { $i < $nclients } { incr i } { 84 set clientdir($i) $testdir/CLIENTDIR.$i 85 file mkdir $clientdir($i) 86 set c_logtype($i) [lindex $logset [expr $i + 1]] 87 set c_logargs($i) [adjust_logargs $c_logtype($i)] 88 set c_txnargs($i) [adjust_txnargs $c_logtype($i)] 89 } 90 91 # Open a master. 92 repladd 1 93 set env_cmd(M) "berkdb_env_noerr -create -log_max 1000000 \ 94 -event rep_event \ 95 -home $masterdir $m_logargs -errpfx MASTER $verbargs \ 96 $m_txnargs -rep_master -rep_transport \[list 1 replsend\]" 97 # In an election test, the -recovery arg must not go 98 # in the env_cmd string because that is going to be 99 # passed to a child process. 100 set masterenv [eval $env_cmd(M) $recargs] 101 102 # Open the clients. 103 for { set i 0 } { $i < $nclients } { incr i } { 104 set envid [expr $i + 2] 105 repladd $envid 106 set env_cmd($i) "berkdb_env_noerr -create -home $clientdir($i) \ 107 -event rep_event \ 108 $c_logargs($i) $c_txnargs($i) -rep_client -errpfx CLIENT$i \ 109 $verbargs -rep_transport \[list $envid replsend\]" 110 set clientenv($i) [eval $env_cmd($i) $recargs] 111 } 112 113 # Loop, processing first the master's messages, then the client's, 114 # until both queues are empty. 115 set envlist {} 116 lappend envlist "$masterenv 1" 117 for { set i 0 } { $i < $nclients } { incr i } { 118 lappend envlist "$clientenv($i) [expr $i + 2]" 119 } 120 process_msgs $envlist 121 122 # Run a modified test001 in the master. 123 puts "\tRep$tnum.a: Running test001 in replicated env." 124 eval test001 $method $niter 0 0 $tnum -env $masterenv $largs 125 process_msgs $envlist 126 127 # Verify the database in the client dir. 128 for { set i 0 } { $i < $nclients } { incr i } { 129 puts "\tRep$tnum.b: Verifying contents of client database $i." 130 set testdir [get_home $masterenv] 131 set t1 $testdir/t1 132 set t2 $testdir/t2 133 set t3 $testdir/t3 134 open_and_dump_file test$tnum.db $clientenv($i) $testdir/t1 \ 135 test001.check dump_file_direction "-first" "-next" 136 137 if { [string compare [convert_method $method] -recno] != 0 } { 138 filesort $t1 $t3 139 } 140 error_check_good diff_files($t2,$t3) [filecmp $t2 $t3] 0 141 142 verify_dir $clientdir($i) "\tRep$tnum.c: " 0 0 1 143 } 144 145 # Start an election in the first client. 146 puts "\tRep$tnum.d: Starting election with existing master." 147 # We want to verify that the master declares the election 148 # over by fiat, even if everyone uses a lower priority than 20. 149 # Loop and process all messages, keeping track of which 150 # sites got a HOLDELECTION and checking that the master i.d. is 151 # unchanged after the election. 152 153 set origmasterid [stat_field $masterenv rep_stat "Master"] 154 set origgeneration [stat_field $masterenv rep_stat "Generation number"] 155 156 set got_hold_elect(M) 0 157 for { set i 0 } { $i < $nclients } { incr i } { 158 set got_hold_elect($i) 0 159 set elect_pipe($i) INVALID 160 } 161 set elect_pipe(0) [start_election C0 $qdir $env_cmd(0) \ 162 [expr $nclients + 1] $nclients 20 $elect_timeout(default)] 163 164 tclsleep 2 165 166 set got_master 0 167 while { 1 } { 168 set nproced 0 169 set he 0 170 171 incr nproced [replprocessqueue $masterenv 1 0 he] 172 173 if { $he == 1 } { 174 incr elect_serial 175 set elect_pipe(M) [start_election CM $qdir \ 176 $env_cmd(M) [expr $nclients + 1] $nclients \ 177 0 $elect_timeout(default)] 178 set got_hold_elect(M) 1 179 } 180 181 for { set i 0 } { $i < $nclients } { incr i } { 182 set he 0 183 set envid [expr $i + 2] 184 incr nproced \ 185 [replprocessqueue $clientenv($i) $envid 0 he] 186 if { $he == 1 } { 187 # error_check_bad client(0)_in_elect $i 0 188 if { $elect_pipe($i) != "INVALID" } { 189 close_election $elect_pipe($i) 190 } 191 incr elect_serial 192 set pfx CHILD$i.$elect_serial 193 set elect_pipe($i) [start_election $pfx $qdir \ 194 $env_cmd($i) [expr $nclients + 1] \ 195 $nclients 0 \ 196 $elect_timeout(default)] 197 set got_hold_elect($i) 1 198 } 199 } 200 201 if { $nproced == 0 } { 202 break 203 } 204 } 205 set masterid [stat_field $masterenv rep_stat "Master"] 206 set generation [stat_field $masterenv rep_stat "Generation number"] 207 error_check_good master_unchanged $origmasterid $masterid 208 error_check_good gen_unchanged $origgeneration $generation 209 cleanup_elections 210 211 # We need multiple clients to proceed from here. 212 if { $nclients < 2 } { 213 puts "\tRep$tnum: Skipping for less than two clients." 214 error_check_good masterenv_close [$masterenv close] 0 215 for { set i 0 } { $i < $nclients } { incr i } { 216 error_check_good clientenv_close($i) \ 217 [$clientenv($i) close] 0 218 } 219 return 220 } 221 222 # Make sure all the clients are synced up and ready to be good 223 # voting citizens. 224 error_check_good master_flush [$masterenv rep_flush] 0 225 process_msgs $envlist 226 227 # Now hold another election in the first client, this time with 228 # a dead master. 229 puts "\tRep$tnum.e: Starting election with dead master." 230 error_check_good masterenv_close [$masterenv close] 0 231 set envlist [lreplace $envlist 0 0] 232 233 set m "Rep$tnum.e" 234 # We're not going to be using err_cmd, so initialize to "none". 235 # Client #1 has priority 100; everyone else has priority 10. 236 for { set i 0 } { $i < $nclients } { incr i } { 237 set err_cmd($i) "none" 238 set crash($i) 0 239 if { $i == 1 } { 240 set pri($i) 100 241 } else { 242 set pri($i) 10 243 } 244 } 245 set nsites $nclients 246 set nvotes $nclients 247 # The elector calls the first election. The expected winner 248 # is $win. 249 set elector 1 250 set win 1 251 run_election env_cmd envlist err_cmd pri crash $qdir $m \ 252 $elector $nsites $nvotes $nclients $win 1 "test$tnum.db" 253 254 # Hold an election with two clients at the same (winning) priority. 255 # Make sure that the tie gets broken, and that the third client 256 # does not win. 257 puts "\tRep$tnum.f: Election with two clients at same priority." 258 set m "Rep$tnum.f" 259 # Clients 0 and 1 have high, matching priority. 260 for { set i 0 } { $i < $nclients } { incr i } { 261 if { $i >= 2 } { 262 set pri($i) 10 263 } else { 264 set pri($i) 100 265 } 266 } 267 268 # Run several elections. 269 set elections 5 270 for { set i 0 } { $i < $elections } { incr i } { 271 # 272 # The expected winner is 0 or 1. Since run_election can only 273 # handle one expected winner, catch the result and inspect it. 274 # 275 set elector 0 276 set win 1 277 set altwin 0 278 if {[catch {eval run_election \ 279 env_cmd envlist err_cmd pri crash $qdir $m $elector $nsites \ 280 $nvotes $nclients $win 1 "test$tnum.db"} res]} { 281 # 282 # If the primary winner didn't win, make sure 283 # the alternative winner won. Do all the cleanup 284 # for that winner normally done in run_election: 285 # open and close the new master, then reopen as a 286 # client for the next cycle. 287 # 288 error_check_good check_winner [is_substr \ 289 $res "expected 3, got [expr $altwin + 2]"] 1 290 puts "\t$m: Election $i: Alternate winner $altwin won." 291 error_check_good make_master \ 292 [$clientenv($altwin) rep_start -master] 0 293 294 cleanup_elections 295 process_msgs $envlist 296 297 error_check_good newmaster_close \ 298 [$clientenv($altwin) close] 0 299 set clientenv($altwin) [eval $env_cmd($altwin)] 300 error_check_good cl($altwin) \ 301 [is_valid_env $clientenv($altwin)] TRUE 302 set newelector "$clientenv($altwin) [expr $altwin + 2]" 303 set envlist [lreplace $envlist $altwin $altwin $newelector] 304 } else { 305 puts "\t$m: Election $i: Primary winner $win won." 306 } 307 process_msgs $envlist 308 } 309 310 foreach pair $envlist { 311 set cenv [lindex $pair 0] 312 error_check_good cenv_close [$cenv close] 0 313 } 314 315 replclose $testdir/MSGQUEUEDIR 316} 317