1# See the file LICENSE for redistribution information. 2# 3# Copyright (c) 2004,2008 Oracle. All rights reserved. 4# 5# $Id: rep022.tcl,v 12.19 2008/01/08 20:58:53 bostic Exp $ 6# 7# TEST rep022 8# TEST Replication elections - test election generation numbers 9# TEST during simulated network partition. 10# TEST 11proc rep022 { method args } { 12 13 source ./include.tcl 14 if { $is_windows9x_test == 1 } { 15 puts "Skipping replication test on Win 9x platform." 16 return 17 } 18 global rand_init 19 global mixed_mode_logging 20 set tnum "022" 21 22 # Run for btree only. 23 if { $checking_valid_methods } { 24 set test_methods { btree } 25 return $test_methods 26 } 27 if { [is_btree $method] == 0 } { 28 puts "Rep$tnum: Skipping for method $method." 29 return 30 } 31 32 if { $mixed_mode_logging > 0 } { 33 puts "Rep$tnum: Skipping for mixed-mode logging." 34 return 35 } 36 37 error_check_good set_random_seed [berkdb srand $rand_init] 0 38 set nclients 5 39 set logsets [create_logsets [expr $nclients + 1]] 40 foreach l $logsets { 41 puts "Rep$tnum ($method): Election generation test\ 42 with simulated network partition." 43 puts "Rep$tnum: Master logs are [lindex $l 0]" 44 for { set i 0 } { $i < $nclients } { incr i } { 45 puts "Rep$tnum: Client $i logs are\ 46 [lindex $l [expr $i + 1]]" 47 } 48 rep022_sub $method $nclients $tnum $l $args 49 } 50} 51 52proc rep022_sub { method nclients tnum logset largs } { 53 source ./include.tcl 54 global rep_verbose 55 global verbose_type 56 57 set verbargs "" 58 if { $rep_verbose == 1 } { 59 set verbargs " -verbose {$verbose_type on} " 60 } 61 62 env_cleanup $testdir 63 64 set qdir $testdir/MSGQUEUEDIR 65 replsetup $qdir 66 67 set masterdir $testdir/MASTERDIR 68 file mkdir $masterdir 69 set m_logtype [lindex $logset 0] 70 set m_logargs [adjust_logargs $m_logtype] 71 set m_txnargs [adjust_txnargs $m_logtype] 72 73 for { set i 0 } { $i < $nclients } { incr i } { 74 set clientdir($i) $testdir/CLIENTDIR.$i 75 file mkdir $clientdir($i) 76 set c_logtype($i) [lindex $logset [expr $i + 1]] 77 set c_logargs($i) [adjust_logargs $c_logtype($i)] 78 set c_txnargs($i) [adjust_txnargs $c_logtype($i)] 79 } 80 81 # Open a master. 82 set envlist {} 83 repladd 1 84 set env_cmd(M) "berkdb_env_noerr -create -log_max 1000000 $verbargs \ 85 -event rep_event \ 86 -home $masterdir $m_txnargs $m_logargs -rep_master \ 87 -errpfx MASTER -rep_transport \[list 1 replsend\]" 88 set masterenv [eval $env_cmd(M)] 89 lappend envlist "$masterenv 1" 90 91 # Open the clients. 92 for { set i 0 } { $i < $nclients } { incr i } { 93 set envid [expr $i + 2] 94 repladd $envid 95 set env_cmd($i) "berkdb_env_noerr -create $verbargs \ 96 -errpfx CLIENT.$i -event rep_event \ 97 -home $clientdir($i) $c_txnargs($i) $c_logargs($i) \ 98 -rep_client -rep_transport \[list $envid replsend\]" 99 set clientenv($i) [eval $env_cmd($i)] 100 lappend envlist "$clientenv($i) $envid" 101 } 102 103 # Bring the clients online by processing the startup messages. 104 process_msgs $envlist 105 106 # Run a modified test001 in the master. 107 puts "\tRep$tnum.a: Running rep_test in replicated env." 108 set niter 10 109 eval rep_test $method $masterenv NULL $niter 0 0 0 0 $largs 110 process_msgs $envlist 111 error_check_good masterenv_close [$masterenv close] 0 112 set envlist [lreplace $envlist 0 0] 113 114 foreach pair $envlist { 115 set i [expr [lindex $pair 1] - 2] 116 replclear [expr $i + 2] 117 set err_cmd($i) "none" 118 set pri($i) 10 119 set crash($i) 0 120 if { $rep_verbose == 1 } { 121 $clientenv($i) errpfx CLIENT$i 122 $clientenv($i) verbose $verbose_type on 123 $clientenv($i) errfile /dev/stderr 124 set env_cmd($i) [concat $env_cmd($i) \ 125 "-errpfx CLIENT$i -errfile /dev/stderr"] 126 } 127 } 128 129 set msg "Rep$tnum.b" 130 puts "\t$msg: Run election for clients 0,1,2." 131 # 132 # Run an election with clients 0, 1, and 2. 133 # Make client 0 be the winner, and let it stay master. 134 # 135 set origlist $envlist 136 set orignclients $nclients 137 set envlist [lrange $origlist 0 2] 138 set nclients 3 139 set nsites 3 140 set nvotes 3 141 set winner 0 142 setpriority pri $nclients $winner 143 set elector [berkdb random_int 0 [expr $nclients - 1]] 144 run_election env_cmd envlist err_cmd pri crash \ 145 $qdir $msg $elector $nsites $nvotes $nclients $winner 0 test.db 146 147 set msg "Rep$tnum.c" 148 puts "\t$msg: Close and reopen client 2 with recovery." 149 # 150 # Now close and reopen 2 with recovery. Update the 151 # list of all client envs with the new information. 152 # 153 replclear 5 154 replclear 6 155 error_check_good flush [$clientenv(2) log_flush] 0 156 error_check_good clientenv_close(2) [$clientenv(2) close] 0 157 set clientenv(2) [eval $env_cmd(2) -recover] 158 set origlist [lreplace $origlist 2 2 "$clientenv(2) 4"] 159 160 # Get last LSN for client 2. 161 set logc [$clientenv(2) log_cursor] 162 error_check_good logc \ 163 [is_valid_logc $logc $clientenv(2)] TRUE 164 set lastlsn2 [lindex [lindex [$logc get -last] 0] 1] 165 error_check_good close_cursor [$logc close] 0 166 167 set msg "Rep$tnum.d" 168 puts "\t$msg: Close and reopen client 4 with recovery." 169 # 170 # This forces the last LSN for client 4 up to the last 171 # LSN for client 2 so client 4 can be elected. 172 # 173 set lastlsn4 0 174 while { $lastlsn4 < $lastlsn2 } { 175 error_check_good clientenv_close(4) [$clientenv(4) close] 0 176 set clientenv(4) [eval $env_cmd(4) -recover] 177 error_check_good flush [$clientenv(4) log_flush] 0 178 set origlist [lreplace $origlist 4 4 "$clientenv(4) 6"] 179 set logc [$clientenv(4) log_cursor] 180 error_check_good logc \ 181 [is_valid_logc $logc $clientenv(4)] TRUE 182 set lastlsn4 [lindex [lindex [$logc get -last] 0] 1] 183 error_check_good close_cursor [$logc close] 0 184 } 185 186 set msg "Rep$tnum.e" 187 puts "\t$msg: Run election for clients 2,3,4." 188 # 189 # Run an election with clients 2, 3, 4. 190 # Make last client be the winner, and let it stay master. 191 # Need to process messages before running election so 192 # that clients 2 and 4 update to the right gen with 193 # client 3. 194 # 195 set envlist [lrange $origlist 2 4] 196 process_msgs $envlist 197 foreach pair $envlist { 198 set i [expr [lindex $pair 1] - 2] 199 set clientenv($i) [lindex $pair 0] 200 set egen($i) [stat_field \ 201 $clientenv($i) rep_stat "Election generation number"] 202 } 203 set winner 4 204 setpriority pri $nclients $winner 2 205 set elector [berkdb random_int 2 4] 206 run_election env_cmd envlist err_cmd pri crash \ 207 $qdir $msg $elector $nsites $nvotes $nclients $winner 0 test.db 208 209 # Note egens for all the clients. 210 set envlist $origlist 211 foreach pair $envlist { 212 set i [expr [lindex $pair 1] - 2] 213 set clientenv($i) [lindex $pair 0] 214 set egen($i) [stat_field \ 215 $clientenv($i) rep_stat "Election generation number"] 216 } 217 218 # Have client 4 (currently a master) run an operation. 219 eval rep_test $method $clientenv(4) NULL $niter 0 0 0 0 $largs 220 221 # Check that clients 0 and 4 get DUPMASTER messages and 222 # restart them as clients. 223 # 224 puts "\tRep$tnum.f: Check for DUPMASTER" 225 set envlist0 [lrange $envlist 0 0] 226 process_msgs $envlist0 0 dup err 227 error_check_good is_dupmaster0 [lindex $dup 0] 1 228 error_check_good downgrade0 [$clientenv(0) rep_start -client] 0 229 230 set envlist4 [lrange $envlist 4 4] 231 process_msgs $envlist4 0 dup err 232 error_check_good is_dupmaster4 [lindex $dup 0] 1 233 error_check_good downgrade4 [$clientenv(4) rep_start -client] 0 234 235 # All DUPMASTER messages are now gone. 236 # We might get residual errors however because client 4 237 # responded as a master to client 0 and then became a 238 # client immediately. Therefore client 4 might get some 239 # "master-only" records and return EINVAL. We want to 240 # ignore those and process records until calm is restored. 241 set err 1 242 while { $err == 1 } { 243 process_msgs $envlist 0 dup err 244 error_check_good no_dupmaster $dup 0 245 } 246 247 # Check LSNs before new election. 248 foreach pair $envlist { 249 set i [expr [lindex $pair 1] - 2] 250 set logc [$clientenv($i) log_cursor] 251 error_check_good logc \ 252 [is_valid_logc $logc $clientenv($i)] TRUE 253 set lastlsn [lindex [lindex [$logc get -last] 0] 1] 254 error_check_good cursor_close [$logc close] 0 255 } 256 257 set msg "Rep$tnum.g" 258 puts "\t$msg: Run election for all clients after DUPMASTER." 259 260 # Call a new election with all participants. Make 4 the 261 # winner, since it should have a high enough LSN to win. 262 set nclients $orignclients 263 set nsites $nclients 264 set nvotes $nclients 265 set winner 4 266 setpriority pri $nclients $winner 267 set elector [berkdb random_int 0 [expr $nclients - 1]] 268 run_election env_cmd envlist err_cmd pri crash \ 269 $qdir $msg $elector $nsites $nvotes $nclients $winner 0 test.db 270 271 # Pull out new egens. 272 foreach pair $envlist { 273 set i [expr [lindex $pair 1] - 2] 274 set clientenv($i) [lindex $pair 0] 275 set newegen($i) [stat_field \ 276 $clientenv($i) rep_stat "Election generation number"] 277 } 278 279 # Egen numbers should all be the same now, and all greater than 280 # they were before the election. 281 set currentegen $newegen(0) 282 for { set i 0 } { $i < $nclients } { incr i } { 283 set egen_diff [expr $newegen($i) - $egen($i)] 284 error_check_good egen_increased [expr $egen_diff > 0] 1 285 error_check_good newegens_match $currentegen $newegen($i) 286 } 287 288 # Clean up. 289 foreach pair $envlist { 290 set cenv [lindex $pair 0] 291 error_check_good cenv_close [$cenv close] 0 292 } 293 294 replclose $testdir/MSGQUEUEDIR 295} 296 297 298