1# See the file LICENSE for redistribution information. 2# 3# Copyright (c) 2004-2009 Oracle. All rights reserved. 4# 5# $Id$ 6# 7# TEST rep026 8# TEST Replication elections - simulate a crash after sending 9# TEST a vote. 10 11proc rep026 { method args } { 12 source ./include.tcl 13 14 global mixed_mode_logging 15 global databases_in_memory 16 global repfiles_in_memory 17 18 set tnum "026" 19 if { $is_windows9x_test == 1 } { 20 puts "Skipping replication test on Win 9x platform." 21 return 22 } 23 24 # Run for btree only. 25 if { $checking_valid_methods } { 26 set test_methods { btree } 27 return $test_methods 28 } 29 if { [is_btree $method] == 0 } { 30 puts "Rep$tnum: Skipping for method $method." 31 return 32 } 33 34 # This test uses recovery, so mixed-mode testing and in-memory 35 # database testing aren't appropriate. 36 if { $mixed_mode_logging > 0 } { 37 puts "Rep$tnum: Skipping for mixed-mode logging." 38 return 39 } 40 if { $databases_in_memory == 1 } { 41 puts "Rep$tnum: Skipping for in-memory databases." 42 return 43 } 44 45 set msg2 "and on-disk replication files" 46 if { $repfiles_in_memory } { 47 set msg2 "and in-memory replication files" 48 } 49 50 global rand_init 51 error_check_good set_random_seed [berkdb srand $rand_init] 0 52 53 set nclients 5 54 set logsets [create_logsets [expr $nclients + 1]] 55 foreach l $logsets { 56 puts "Rep$tnum ($method): Election generations -\ 57 simulate crash after sending a vote $msg2." 58 puts "Rep$tnum: Master logs are [lindex $l 0]" 59 for { set i 0 } { $i < $nclients } { incr i } { 60 puts "Rep$tnum: Client $i logs are\ 61 [lindex $l [expr $i + 1]]" 62 } 63 rep026_sub $method $nclients $tnum $l $args 64 } 65} 66 67proc rep026_sub { method nclients tnum logset largs } { 68 source ./include.tcl 69 global machids 70 global repfiles_in_memory 71 global rep_verbose 72 global verbose_type 73 74 set verbargs "" 75 if { $rep_verbose == 1 } { 76 set verbargs " -verbose {$verbose_type on} " 77 } 78 79 set repmemargs "" 80 if { $repfiles_in_memory } { 81 set repmemargs "-rep_inmem_files " 82 } 83 84 env_cleanup $testdir 85 86 set qdir $testdir/MSGQUEUEDIR 87 replsetup $qdir 88 89 set masterdir $testdir/MASTERDIR 90 file mkdir $masterdir 91 set m_logtype [lindex $logset 0] 92 set m_logargs [adjust_logargs $m_logtype] 93 set m_txnargs [adjust_txnargs $m_logtype] 94 95 for { set i 0 } { $i < $nclients } { incr i } { 96 set clientdir($i) $testdir/CLIENTDIR.$i 97 file mkdir $clientdir($i) 98 set c_logtype($i) [lindex $logset [expr $i + 1]] 99 set c_logargs($i) [adjust_logargs $c_logtype($i)] 100 set c_txnargs($i) [adjust_txnargs $c_logtype($i)] 101 } 102 103 # Open a master. 104 set envlist {} 105 repladd 1 106 set env_cmd(M) "berkdb_env -create -log_max 1000000 $verbargs \ 107 -event rep_event $repmemargs \ 108 -home $masterdir $m_txnargs $m_logargs -rep_master \ 109 -errpfx MASTER -rep_transport \[list 1 replsend\]" 110 set masterenv [eval $env_cmd(M)] 111 lappend envlist "$masterenv 1" 112 113 # Open the clients. 114 for { set i 0 } { $i < $nclients } { incr i } { 115 set envid [expr $i + 2] 116 repladd $envid 117 set env_cmd($i) "berkdb_env_noerr -create $verbargs \ 118 -event rep_event $repmemargs \ 119 -home $clientdir($i) $c_txnargs($i) $c_logargs($i) \ 120 -rep_client -rep_transport \[list $envid replsend\]" 121 set clientenv($i) [eval $env_cmd($i)] 122 error_check_good \ 123 client_env($i) [is_valid_env $clientenv($i)] TRUE 124 lappend envlist "$clientenv($i) $envid" 125 } 126 # Bring the clients online by processing the startup messages. 127 process_msgs $envlist 128 129 # Run a modified test001 in the master. 130 puts "\tRep$tnum.a: Running rep_test in replicated env." 131 set niter 10 132 eval rep_test $method $masterenv NULL $niter 0 0 0 $largs 133 process_msgs $envlist 134 error_check_good masterenv_close [$masterenv close] 0 135 set envlist [lreplace $envlist 0 0] 136 137 foreach pair $envlist { 138 set i [expr [lindex $pair 1] - 2] 139 replclear [expr $i + 2] 140 set err_cmd($i) "none" 141 set crash($i) 0 142 set pri($i) 10 143 if { $rep_verbose == 1 } { 144 $clientenv($i) errpfx CLIENT$i 145 $clientenv($i) verbose $verbose_type on 146 $clientenv($i) errfile /dev/stderr 147 set env_cmd($i) [concat $env_cmd($i) \ 148 "-errpfx CLIENT$i -errfile /dev/stderr"] 149 } 150 } 151 152 # In each case we simulate a crash in client C, recover, and 153 # call a second election. We vary the caller of the second 154 # election (C or some other client) and when the election 155 # messages from before the crash are processed - before or 156 # after the second election. 157 # 158 foreach option { "1 b before" "2 c before" "1 d after" "2 e after"} { 159 # Elector 1 calls the first election, elector 2 160 # calls the second election. 161 set elector1 1 162 set elector2 [lindex $option 0] 163 set let [lindex $option 1] 164 set restore [lindex $option 2] 165 166 if { $elector1 == $elector2 } { 167 puts "\tRep$tnum.$let: Simulated crash and recovery\ 168 (crashing client calls second election)." 169 } else { 170 puts "\tRep$tnum.$let: Simulated crash and recovery\ 171 (non-crashing client calls second election)." 172 } 173 174 puts "\tRep$tnum.$let: Process messages from crasher\ 175 $restore 2nd election." 176 177 puts "\t\tRep$tnum.$let.1: Note egens for all clients." 178 # Note egens for all the clients. 179 foreach pair $envlist { 180 set i [expr [lindex $pair 1] - 2] 181 set clientenv($i) [lindex $pair 0] 182 set egen($i) [stat_field \ 183 $clientenv($i) rep_stat "Election generation number"] 184 } 185 186 # Call an election which simulates a crash after sending 187 # its VOTE1. 188 set msg "\tRep$tnum.$let.2" 189 puts "\t$msg: Start election, simulate a crash." 190 set nsites $nclients 191 set nvotes $nclients 192 # Make the winner the crashing client, since the 193 # crashing client will have the biggest LSN. 194 set elector 1 195 set winner $elector 196 set crash($elector) 1 197 setpriority pri $nclients $winner 198 set err_cmd($elector) "electvote1" 199 run_election env_cmd envlist err_cmd pri crash $qdir \ 200 $msg $elector $nsites $nvotes $nclients $winner 0 test.db 201 set msg "\tRep$tnum.$let.3" 202 puts "\t$msg: Close and reopen elector with recovery." 203 error_check_good \ 204 clientenv_close($elector) [$clientenv($elector) close] 0 205 206 # Have other clients SKIP the election messages and process 207 # only C's startup messages. We'll do it by copying the files 208 # and emptying the originals. 209 set cwd [pwd] 210 foreach machid $machids { 211 file copy -force $qdir/repqueue$machid.db $qdir/save$machid.db 212 replclear $machid 213 } 214 215 # Reopen C and process messages. Only the startup messages 216 # will be available. 217 set clientenv($elector) [eval $env_cmd($elector) -recover] 218 set envlist [lreplace $envlist \ 219 $elector $elector "$clientenv($elector) [expr $elector + 2]"] 220 process_msgs $envlist 221 222 # Verify egens (should be +1 in C, and unchanged 223 # in other clients). 224 foreach pair $envlist { 225 set i [expr [lindex $pair 1] - 2] 226 set clientenv($i) [lindex $pair 0] 227 set newegen($i) [stat_field $clientenv($i) \ 228 rep_stat "Election generation number"] 229 if { $i == $elector && $repfiles_in_memory == 0 } { 230 error_check_good \ 231 egen+1 $newegen($i) [expr $egen($i) + 1] 232 } else { 233 error_check_good \ 234 egen_unchanged $newegen($i) $egen($i) 235 } 236 } 237 238 # First chance to restore messages. 239 if { $restore == "before" } { 240 restore_messages $qdir 241 } 242 243 # Have C call an election (no crash simulation) and process 244 # all the messages. 245 set msg "\tRep$tnum.$let.4" 246 puts "\t$msg: Call second election." 247 set err_cmd($elector) "none" 248 set crash($elector) 0 249 run_election env_cmd envlist err_cmd pri crash $qdir \ 250 $msg $elector2 $nsites $nvotes $nclients $winner 1 test.db 251 252 # Second chance to restore messages. 253 if { $restore == "after" } { 254 restore_messages $qdir 255 } 256 process_msgs $envlist 257 258 # Verify egens (should be +2 or more in all clients). 259 puts "\t\tRep$tnum.$let.5: Check egens." 260 foreach pair $envlist { 261 set i [expr [lindex $pair 1] - 2] 262 set clientenv($i) [lindex $pair 0] 263 set newegen($i) [stat_field \ 264 $clientenv($i) rep_stat "Election generation number"] 265 266 # If rep files are in-memory, egen value must come 267 # from other sites instead of the egen file, and 268 # will not increase as quickly. 269 if { $repfiles_in_memory } { 270 set mingen [expr $egen($i) + 1] 271 } else { 272 set mingen [expr $egen($i) + 2] 273 } 274 error_check_good egen+more($i) \ 275 [expr $newegen($i) >= $mingen] 1 276 } 277 } 278 279 # Clean up. 280 foreach pair $envlist { 281 set cenv [lindex $pair 0] 282 error_check_good cenv_close [$cenv close] 0 283 } 284 replclose $testdir/MSGQUEUEDIR 285} 286 287proc restore_messages { qdir } { 288 global machids 289 set cwd [pwd] 290 foreach machid $machids { 291 file copy -force $qdir/save$machid.db $qdir/repqueue$machid.db 292 } 293} 294 295