1# See the file LICENSE for redistribution information. 2# 3# Copyright (c) 2004-2009 Oracle. All rights reserved. 4# 5# $Id$ 6# 7# TEST rep022 8# TEST Replication elections - test election generation numbers 9# TEST during simulated network partition. 10# TEST 11proc rep022 { method args } { 12 13 source ./include.tcl 14 if { $is_windows9x_test == 1 } { 15 puts "Skipping replication test on Win 9x platform." 16 return 17 } 18 global rand_init 19 global mixed_mode_logging 20 global databases_in_memory 21 global repfiles_in_memory 22 23 set tnum "022" 24 25 # Run for btree only. 26 if { $checking_valid_methods } { 27 set test_methods { btree } 28 return $test_methods 29 } 30 if { [is_btree $method] == 0 } { 31 puts "Rep$tnum: Skipping for method $method." 32 return 33 } 34 35 if { $mixed_mode_logging > 0 } { 36 puts "Rep$tnum: Skipping for mixed-mode logging." 37 return 38 } 39 40 if { $databases_in_memory > 0 } { 41 puts "Rep$tnum: Skipping for in-memory databases." 42 return 43 } 44 45 set msg2 "and on-disk replication files" 46 if { $repfiles_in_memory } { 47 set msg2 "and in-memory replication files" 48 } 49 50 error_check_good set_random_seed [berkdb srand $rand_init] 0 51 set nclients 5 52 set logsets [create_logsets [expr $nclients + 1]] 53 foreach l $logsets { 54 puts "Rep$tnum ($method): Election generation test\ 55 with simulated network partition $msg2." 56 puts "Rep$tnum: Master logs are [lindex $l 0]" 57 for { set i 0 } { $i < $nclients } { incr i } { 58 puts "Rep$tnum: Client $i logs are\ 59 [lindex $l [expr $i + 1]]" 60 } 61 rep022_sub $method $nclients $tnum $l $args 62 } 63} 64 65proc rep022_sub { method nclients tnum logset largs } { 66 source ./include.tcl 67 global repfiles_in_memory 68 global rep_verbose 69 global verbose_type 70 71 set verbargs "" 72 if { $rep_verbose == 1 } { 73 set verbargs " -verbose {$verbose_type on} " 74 } 75 76 set repmemargs "" 77 if { $repfiles_in_memory } { 78 set repmemargs "-rep_inmem_files " 79 } 80 81 env_cleanup $testdir 82 83 set qdir $testdir/MSGQUEUEDIR 84 replsetup $qdir 85 86 set masterdir $testdir/MASTERDIR 87 file mkdir $masterdir 88 set m_logtype [lindex $logset 0] 89 set m_logargs [adjust_logargs $m_logtype] 90 set m_txnargs [adjust_txnargs $m_logtype] 91 92 for { set i 0 } { $i < $nclients } { incr i } { 93 set clientdir($i) $testdir/CLIENTDIR.$i 94 file mkdir $clientdir($i) 95 set c_logtype($i) [lindex $logset [expr $i + 1]] 96 set c_logargs($i) [adjust_logargs $c_logtype($i)] 97 set c_txnargs($i) [adjust_txnargs $c_logtype($i)] 98 } 99 100 # Open a master. 101 set envlist {} 102 repladd 1 103 set env_cmd(M) "berkdb_env_noerr -create -log_max 1000000 $verbargs \ 104 -event rep_event $repmemargs \ 105 -home $masterdir $m_txnargs $m_logargs -rep_master \ 106 -errpfx MASTER -rep_transport \[list 1 replsend\]" 107 set masterenv [eval $env_cmd(M)] 108 lappend envlist "$masterenv 1" 109 110 # Open the clients. 111 for { set i 0 } { $i < $nclients } { incr i } { 112 set envid [expr $i + 2] 113 repladd $envid 114 set env_cmd($i) "berkdb_env_noerr -create $verbargs \ 115 -errpfx CLIENT.$i -event rep_event $repmemargs \ 116 -home $clientdir($i) $c_txnargs($i) $c_logargs($i) \ 117 -rep_client -rep_transport \[list $envid replsend\]" 118 set clientenv($i) [eval $env_cmd($i)] 119 lappend envlist "$clientenv($i) $envid" 120 } 121 122 # Bring the clients online by processing the startup messages. 123 process_msgs $envlist 124 125 # Run a modified test001 in the master. 126 puts "\tRep$tnum.a: Running rep_test in replicated env." 127 set niter 10 128 eval rep_test $method $masterenv NULL $niter 0 0 0 $largs 129 process_msgs $envlist 130 error_check_good masterenv_close [$masterenv close] 0 131 set envlist [lreplace $envlist 0 0] 132 133 foreach pair $envlist { 134 set i [expr [lindex $pair 1] - 2] 135 replclear [expr $i + 2] 136 set err_cmd($i) "none" 137 set pri($i) 10 138 set crash($i) 0 139 if { $rep_verbose == 1 } { 140 $clientenv($i) errpfx CLIENT$i 141 $clientenv($i) verbose $verbose_type on 142 $clientenv($i) errfile /dev/stderr 143 set env_cmd($i) [concat $env_cmd($i) \ 144 "-errpfx CLIENT$i -errfile /dev/stderr"] 145 } 146 } 147 148 set msg "Rep$tnum.b" 149 puts "\t$msg: Run election for clients 0,1,2." 150 # 151 # Run an election with clients 0, 1, and 2. 152 # Make client 0 be the winner, and let it stay master. 153 # 154 set origlist $envlist 155 set orignclients $nclients 156 set envlist [lrange $origlist 0 2] 157 set nclients 3 158 set nsites 3 159 set nvotes 3 160 set winner 0 161 setpriority pri $nclients $winner 162 set elector [berkdb random_int 0 [expr $nclients - 1]] 163 run_election env_cmd envlist err_cmd pri crash \ 164 $qdir $msg $elector $nsites $nvotes $nclients $winner 0 test.db 165 166 set msg "Rep$tnum.c" 167 puts "\t$msg: Close and reopen client 2 with recovery." 168 # 169 # Now close and reopen 2 with recovery. Update the 170 # list of all client envs with the new information. 171 # 172 replclear 5 173 replclear 6 174 error_check_good flush [$clientenv(2) log_flush] 0 175 error_check_good clientenv_close(2) [$clientenv(2) close] 0 176 set clientenv(2) [eval $env_cmd(2) -recover] 177 set origlist [lreplace $origlist 2 2 "$clientenv(2) 4"] 178 179 # Get last LSN for client 2. 180 set logc [$clientenv(2) log_cursor] 181 error_check_good logc \ 182 [is_valid_logc $logc $clientenv(2)] TRUE 183 set lastlsn2 [lindex [lindex [$logc get -last] 0] 1] 184 error_check_good close_cursor [$logc close] 0 185 186 set msg "Rep$tnum.d" 187 puts "\t$msg: Close and reopen client 4 with recovery." 188 # 189 # This forces the last LSN for client 4 up to the last 190 # LSN for client 2 so client 4 can be elected. 191 # 192 set lastlsn4 0 193 while { $lastlsn4 < $lastlsn2 } { 194 error_check_good clientenv_close(4) [$clientenv(4) close] 0 195 set clientenv(4) [eval $env_cmd(4) -recover] 196 error_check_good flush [$clientenv(4) log_flush] 0 197 set origlist [lreplace $origlist 4 4 "$clientenv(4) 6"] 198 set logc [$clientenv(4) log_cursor] 199 error_check_good logc \ 200 [is_valid_logc $logc $clientenv(4)] TRUE 201 set lastlsn4 [lindex [lindex [$logc get -last] 0] 1] 202 error_check_good close_cursor [$logc close] 0 203 } 204 205 set msg "Rep$tnum.e" 206 puts "\t$msg: Run election for clients 2,3,4." 207 # 208 # Run an election with clients 2, 3, 4. 209 # Make last client be the winner, and let it stay master. 210 # Need to process messages before running election so 211 # that clients 2 and 4 update to the right gen with 212 # client 3. 213 # 214 set envlist [lrange $origlist 2 4] 215 process_msgs $envlist 216 foreach pair $envlist { 217 set i [expr [lindex $pair 1] - 2] 218 set clientenv($i) [lindex $pair 0] 219 set egen($i) [stat_field \ 220 $clientenv($i) rep_stat "Election generation number"] 221 } 222 set winner 4 223 setpriority pri $nclients $winner 2 224 set elector [berkdb random_int 2 4] 225 run_election env_cmd envlist err_cmd pri crash \ 226 $qdir $msg $elector $nsites $nvotes $nclients $winner 0 test.db 227 228 # Note egens for all the clients. 229 set envlist $origlist 230 foreach pair $envlist { 231 set i [expr [lindex $pair 1] - 2] 232 set clientenv($i) [lindex $pair 0] 233 set egen($i) [stat_field \ 234 $clientenv($i) rep_stat "Election generation number"] 235 } 236 237 # Have client 4 (currently a master) run an operation. 238 eval rep_test $method $clientenv(4) NULL $niter 0 0 0 $largs 239 240 # Check that clients 0 and 4 get DUPMASTER messages and 241 # restart them as clients. 242 # 243 puts "\tRep$tnum.f: Check for DUPMASTER" 244 set envlist0 [lrange $envlist 0 0] 245 process_msgs $envlist0 0 dup err 246 error_check_good is_dupmaster0 [lindex $dup 0] 1 247 error_check_good downgrade0 [$clientenv(0) rep_start -client] 0 248 249 set envlist4 [lrange $envlist 4 4] 250 process_msgs $envlist4 0 dup err 251 error_check_good is_dupmaster4 [lindex $dup 0] 1 252 error_check_good downgrade4 [$clientenv(4) rep_start -client] 0 253 254 # All DUPMASTER messages are now gone. 255 # We might get residual errors however because client 4 256 # responded as a master to client 0 and then became a 257 # client immediately. Therefore client 4 might get some 258 # "master-only" records and return EINVAL. We want to 259 # ignore those and process records until calm is restored. 260 set err 1 261 while { $err == 1 } { 262 process_msgs $envlist 0 dup err 263 error_check_good no_dupmaster $dup 0 264 } 265 266 # Check LSNs before new election. 267 foreach pair $envlist { 268 set i [expr [lindex $pair 1] - 2] 269 set logc [$clientenv($i) log_cursor] 270 error_check_good logc \ 271 [is_valid_logc $logc $clientenv($i)] TRUE 272 set lastlsn [lindex [lindex [$logc get -last] 0] 1] 273 error_check_good cursor_close [$logc close] 0 274 } 275 276 set msg "Rep$tnum.g" 277 puts "\t$msg: Run election for all clients after DUPMASTER." 278 279 # Call a new election with all participants. Make 4 the 280 # winner, since it should have a high enough LSN to win. 281 set nclients $orignclients 282 set nsites $nclients 283 set nvotes $nclients 284 set winner 4 285 setpriority pri $nclients $winner 286 set elector [berkdb random_int 0 [expr $nclients - 1]] 287 run_election env_cmd envlist err_cmd pri crash \ 288 $qdir $msg $elector $nsites $nvotes $nclients $winner 0 test.db 289 290 # Pull out new egens. 291 foreach pair $envlist { 292 set i [expr [lindex $pair 1] - 2] 293 set clientenv($i) [lindex $pair 0] 294 set newegen($i) [stat_field \ 295 $clientenv($i) rep_stat "Election generation number"] 296 } 297 298 # Egen numbers should all be the same now, and all greater than 299 # they were before the election. 300 set currentegen $newegen(0) 301 for { set i 0 } { $i < $nclients } { incr i } { 302 set egen_diff [expr $newegen($i) - $egen($i)] 303 error_check_good egen_increased [expr $egen_diff > 0] 1 304 error_check_good newegens_match $currentegen $newegen($i) 305 } 306 307 # Clean up. 308 foreach pair $envlist { 309 set cenv [lindex $pair 0] 310 error_check_good cenv_close [$cenv close] 0 311 } 312 313 replclose $testdir/MSGQUEUEDIR 314} 315 316 317