1# See the file LICENSE for redistribution information. 2# 3# Copyright (c) 1996-2009 Oracle. All rights reserved. 4# 5# The procs in this file are used for replication messaging 6# ONLY when the default mechanism of setting up a queue of 7# messages in a environment is not possible. This situation 8# is fairly rare, but it is necessary when a replication 9# test simultaneously runs different versions of Berkeley DB, 10# because different versions cannot share an env. 11# 12# Note, all procs should be named with the suffix _noenv 13# so it's explicit that we are using them. 14# 15# Close up a replication group - close all message dbs. 16proc replclose_noenv { queuedir } { 17 global queuedbs machids 18 19 set dbs [array names queuedbs] 20 foreach tofrom $dbs { 21 set handle $queuedbs($tofrom) 22 error_check_good db_close [$handle close] 0 23 unset queuedbs($tofrom) 24 } 25 26 set machids {} 27} 28 29# Create a replication group for testing. 30proc replsetup_noenv { queuedir } { 31 global queuedbs machids 32 33 file mkdir $queuedir 34 35 # If there are any leftover handles, get rid of them. 36 set dbs [array names queuedbs] 37 foreach tofrom $dbs { 38 unset queuedbs($tofrom) 39 } 40 set machids {} 41} 42 43# Send function for replication. 44proc replsend_noenv { control rec fromid toid flags lsn } { 45 global is_repchild 46 global queuedbs machids 47 global drop drop_msg 48 global perm_sent_list 49 global anywhere 50 global qtestdir testdir 51 52 if { ![info exists qtestdir] } { 53 set qtestdir $testdir 54 } 55 set queuedir $qtestdir/MSGQUEUEDIR 56 set permflags [lsearch $flags "perm"] 57 if { [llength $perm_sent_list] != 0 && $permflags != -1 } { 58# puts "replsend_noenv sent perm message, LSN $lsn" 59 lappend perm_sent_list $lsn 60 } 61 62 # 63 # If we are testing with dropped messages, then we drop every 64 # $drop_msg time. If we do that just return 0 and don't do 65 # anything. 66 # 67 if { $drop != 0 } { 68 incr drop 69 if { $drop == $drop_msg } { 70 set drop 1 71 return 0 72 } 73 } 74 # XXX 75 # -1 is DB_BROADCAST_EID 76 if { $toid == -1 } { 77 set machlist $machids 78 } else { 79 set m NULL 80 # If we can send this anywhere, send it to the first id 81 # we find that is neither toid or fromid. If we don't 82 # find any other candidates, this falls back to the 83 # original toid. 84 if { $anywhere != 0 } { 85 set anyflags [lsearch $flags "any"] 86 if { $anyflags != -1 } { 87 foreach m $machids { 88 if { $m == $fromid || $m == $toid } { 89 continue 90 } 91 set machlist [list $m] 92 break 93 } 94 } 95 } 96 # 97 # If we didn't find a different site, fall back 98 # to the toid. 99 # 100 if { $m == "NULL" } { 101 set machlist [list $toid] 102 } 103 } 104 foreach m $machlist { 105 # Do not broadcast to self. 106 if { $m == $fromid } { 107 continue 108 } 109 # Find the handle for the right message file. 110 set pid [pid] 111 set db $queuedbs($m.$fromid.$pid) 112 set stat [catch {$db put -append [list $control $rec $fromid]} ret] 113 } 114 if { $is_repchild } { 115 replready_noenv $fromid from 116 } 117 118 return 0 119} 120 121# 122proc replmsglen_noenv { machid {tf "to"}} { 123 global queuedbs qtestdir testdir 124 125 if { ![info exists qtestdir] } { 126 set qtestdir $testdir 127 } 128 set queuedir $qtestdir/MSGQUEUEDIR 129 set orig [pwd] 130 131 cd $queuedir 132 if { $tf == "to" } { 133 set msgdbs [glob -nocomplain ready.$machid.*] 134 } else { 135 set msgdbs [glob -nocomplain ready.*.$machid.*] 136 } 137 cd $orig 138 return [llength $msgdbs] 139} 140 141# Discard all the pending messages for a particular site. 142proc replclear_noenv { machid {tf "to"}} { 143 global queuedbs qtestdir testdir 144 145 if { ![info exists qtestdir] } { 146 set qtestdir $testdir 147 } 148 set queuedir $qtestdir/MSGQUEUEDIR 149 set orig [pwd] 150 151 cd $queuedir 152 if { $tf == "to" } { 153 set msgdbs [glob -nocomplain ready.$machid.*] 154 } else { 155 set msgdbs [glob -nocomplain ready.*.$machid.*] 156 } 157 foreach m $msgdbs { 158 file delete -force $m 159 } 160 cd $orig 161 set dbs [array names queuedbs] 162 foreach tofrom $dbs { 163 # Process only messages _to_ the specified machid. 164 if { [string match $machid.* $tofrom] == 1 } { 165 set db $queuedbs($tofrom) 166 set dbc [$db cursor] 167 for { set dbt [$dbc get -first] } \ 168 { [llength $dbt] > 0 } \ 169 { set dbt [$dbc get -next] } { 170 error_check_good \ 171 replclear($machid)_del [$dbc del] 0 172 } 173 error_check_good replclear($db)_dbc_close [$dbc close] 0 174 } 175 } 176 cd $queuedir 177 if { $tf == "to" } { 178 set msgdbs [glob -nocomplain temp.$machid.*] 179 } else { 180 set msgdbs [glob -nocomplain temp.*.$machid.*] 181 } 182 foreach m $msgdbs { 183# file delete -force $m 184 } 185 cd $orig 186} 187 188# Makes messages available to replprocessqueue by closing and 189# renaming the message files. We ready the files for one machine 190# ID at a time -- just those "to" or "from" the machine we want to 191# process, depending on 'tf'. 192proc replready_noenv { machid tf } { 193 global queuedbs machids 194 global counter 195 global qtestdir testdir 196 197 if { ![info exists qtestdir] } { 198 set qtestdir $testdir 199 } 200 set queuedir $qtestdir/MSGQUEUEDIR 201 202 set pid [pid] 203 # 204 # Close the temporary message files for the specified machine. 205 # Only close it if there are messages available. 206 # 207 set dbs [array names queuedbs] 208 set closed {} 209 foreach tofrom $dbs { 210 set toidx [string first . $tofrom] 211 set toid [string replace $tofrom $toidx end] 212 set fidx [expr $toidx + 1] 213 set fromidx [string first . $tofrom $fidx] 214 # 215 # First chop off the end, then chop off the toid 216 # in the beginning. 217 # 218 set fromid [string replace $tofrom $fromidx end] 219 set fromid [string replace $fromid 0 $toidx] 220 if { ($tf == "to" && $machid == $toid) || \ 221 ($tf == "from" && $machid == $fromid) } { 222 set nkeys [stat_field $queuedbs($tofrom) \ 223 stat "Number of keys"] 224 if { $nkeys != 0 } { 225 lappend closed \ 226 [list $toid $fromid temp.$tofrom] 227 error_check_good temp_close \ 228 [$queuedbs($tofrom) close] 0 229 } 230 } 231 } 232 233 # Rename the message files. 234 set cwd [pwd] 235 foreach filename $closed { 236 set toid [lindex $filename 0] 237 set fromid [lindex $filename 1] 238 set fname [lindex $filename 2] 239 set tofrom [string replace $fname 0 4] 240 incr counter($machid) 241 cd $queuedir 242# puts "$queuedir: Msg ready $fname to ready.$tofrom.$counter($machid)" 243 file rename -force $fname ready.$tofrom.$counter($machid) 244 cd $cwd 245 replsetuptempfile_noenv $toid $fromid $queuedir 246 247 } 248} 249 250# Add a machine to a replication environment. This checks 251# that we have not already established that machine id, and 252# adds the machid to the list of ids. 253proc repladd_noenv { machid } { 254 global queuedbs machids counter qtestdir testdir 255 256 if { ![info exists qtestdir] } { 257 set qtestdir $testdir 258 } 259 set queuedir $qtestdir/MSGQUEUEDIR 260 if { [info exists machids] } { 261 if { [lsearch -exact $machids $machid] >= 0 } { 262 error "FAIL: repladd_noenv: machid $machid already exists." 263 } 264 } 265 266 set counter($machid) 0 267 lappend machids $machid 268 269 # Create all the databases that receive messages sent _to_ 270 # the new machid. 271 replcreatetofiles_noenv $machid $queuedir 272 273 # Create all the databases that receive messages sent _from_ 274 # the new machid. 275 replcreatefromfiles_noenv $machid $queuedir 276} 277 278# Creates all the databases that a machid needs for receiving messages 279# from other participants in a replication group. Used when first 280# establishing the temp files, but also used whenever replready_noenv moves 281# the temp files away, because we'll need new files for any future messages. 282proc replcreatetofiles_noenv { toid queuedir } { 283 global machids 284 285 foreach m $machids { 286 # We don't need a file for a machid to send itself messages. 287 if { $m == $toid } { 288 continue 289 } 290 replsetuptempfile_noenv $toid $m $queuedir 291 } 292} 293 294# Creates all the databases that a machid needs for sending messages 295# to other participants in a replication group. Used when first 296# establishing the temp files only. Replready moves files based on 297# recipient, so we recreate files based on the recipient, also. 298proc replcreatefromfiles_noenv { fromid queuedir } { 299 global machids 300 301 foreach m $machids { 302 # We don't need a file for a machid to send itself messages. 303 if { $m == $fromid } { 304 continue 305 } 306 replsetuptempfile_noenv $m $fromid $queuedir 307 } 308} 309 310proc replsetuptempfile_noenv { to from queuedir } { 311 global queuedbs 312 313 set pid [pid] 314# puts "Open new temp.$to.$from.$pid" 315 set queuedbs($to.$from.$pid) [berkdb_open -create -excl -recno\ 316 -renumber $queuedir/temp.$to.$from.$pid] 317 error_check_good open_queuedbs [is_valid_db $queuedbs($to.$from.$pid)] TRUE 318} 319 320# Process a queue of messages, skipping every "skip_interval" entry. 321# We traverse the entire queue, but since we skip some messages, we 322# may end up leaving things in the queue, which should get picked up 323# on a later run. 324proc replprocessqueue_noenv { dbenv machid { skip_interval 0 } { hold_electp NONE } \ 325 { dupmasterp NONE } { errp NONE } } { 326 global errorCode 327 global perm_response_list 328 global qtestdir testdir 329 330 # hold_electp is a call-by-reference variable which lets our caller 331 # know we need to hold an election. 332 if { [string compare $hold_electp NONE] != 0 } { 333 upvar $hold_electp hold_elect 334 } 335 set hold_elect 0 336 337 # dupmasterp is a call-by-reference variable which lets our caller 338 # know we have a duplicate master. 339 if { [string compare $dupmasterp NONE] != 0 } { 340 upvar $dupmasterp dupmaster 341 } 342 set dupmaster 0 343 344 # errp is a call-by-reference variable which lets our caller 345 # know we have gotten an error (that they expect). 346 if { [string compare $errp NONE] != 0 } { 347 upvar $errp errorp 348 } 349 set errorp 0 350 351 set nproced 0 352 353 set queuedir $qtestdir/MSGQUEUEDIR 354# puts "replprocessqueue_noenv: Make ready messages to eid $machid" 355 356 # Change directories temporarily so we get just the msg file name. 357 set cwd [pwd] 358 cd $queuedir 359 set msgdbs [glob -nocomplain ready.$machid.*] 360# puts "$queuedir.$machid: My messages: $msgdbs" 361 cd $cwd 362 363 foreach msgdb $msgdbs { 364 set db [berkdb_open $queuedir/$msgdb] 365 set dbc [$db cursor] 366 367 error_check_good process_dbc($machid) \ 368 [is_valid_cursor $dbc $db] TRUE 369 370 for { set dbt [$dbc get -first] } \ 371 { [llength $dbt] != 0 } \ 372 { set dbt [$dbc get -next] } { 373 set data [lindex [lindex $dbt 0] 1] 374 set recno [lindex [lindex $dbt 0] 0] 375 376 # If skip_interval is nonzero, we want to process 377 # messages out of order. We do this in a simple but 378 # slimy way -- continue walking with the cursor 379 # without processing the message or deleting it from 380 # the queue, but do increment "nproced". The way 381 # this proc is normally used, the precise value of 382 # nproced doesn't matter--we just don't assume the 383 # queues are empty if it's nonzero. Thus, if we 384 # contrive to make sure it's nonzero, we'll always 385 # come back to records we've skipped on a later call 386 # to replprocessqueue. (If there really are no records, 387 # we'll never get here.) 388 # 389 # Skip every skip_interval'th record (and use a 390 # remainder other than zero so that we're guaranteed 391 # to really process at least one record on every call). 392 if { $skip_interval != 0 } { 393 if { $nproced % $skip_interval == 1 } { 394 incr nproced 395 set dbt [$dbc get -next] 396 continue 397 } 398 } 399 400 # We need to remove the current message from the 401 # queue, because we're about to end the transaction 402 # and someone else processing messages might come in 403 # and reprocess this message which would be bad. 404 # 405 error_check_good queue_remove [$dbc del] 0 406 407 # We have to play an ugly cursor game here: we 408 # currently hold a lock on the page of messages, but 409 # rep_process_message might need to lock the page with 410 # a different cursor in order to send a response. So 411 # save the next recno, close the cursor, and then 412 # reopen and reset the cursor. If someone else is 413 # processing this queue, our entry might have gone 414 # away, and we need to be able to handle that. 415 # 416# error_check_good dbc_process_close [$dbc close] 0 417 418 set ret [catch {$dbenv rep_process_message \ 419 [lindex $data 2] [lindex $data 0] \ 420 [lindex $data 1]} res] 421 422 # Save all ISPERM and NOTPERM responses so we can 423 # compare their LSNs to the LSN in the log. The 424 # variable perm_response_list holds the entire 425 # response so we can extract responses and LSNs as 426 # needed. 427 # 428 if { [llength $perm_response_list] != 0 && \ 429 ([is_substr $res ISPERM] || [is_substr $res NOTPERM]) } { 430 lappend perm_response_list $res 431 } 432 433 if { $ret != 0 } { 434 if { [string compare $errp NONE] != 0 } { 435 set errorp "$dbenv $machid $res" 436 } else { 437 error "FAIL:[timestamp]\ 438 rep_process_message returned $res" 439 } 440 } 441 442 incr nproced 443 if { $ret == 0 } { 444 set rettype [lindex $res 0] 445 set retval [lindex $res 1] 446 # 447 # Do nothing for 0 and NEWSITE 448 # 449 if { [is_substr $rettype HOLDELECTION] } { 450 set hold_elect 1 451 } 452 if { [is_substr $rettype DUPMASTER] } { 453 set dupmaster "1 $dbenv $machid" 454 } 455 if { [is_substr $rettype NOTPERM] || \ 456 [is_substr $rettype ISPERM] } { 457 set lsnfile [lindex $retval 0] 458 set lsnoff [lindex $retval 1] 459 } 460 } 461 462 if { $errorp != 0 } { 463 # Break on an error, caller wants to handle it. 464 break 465 } 466 if { $hold_elect == 1 } { 467 # Break on a HOLDELECTION, for the same reason. 468 break 469 } 470 if { $dupmaster == 1 } { 471 # Break on a DUPMASTER, for the same reason. 472 break 473 } 474 475 } 476 error_check_good dbc_close [$dbc close] 0 477 478 # 479 # Check the number of keys remaining because we only 480 # want to rename to done, message file that are 481 # fully processed. Some message types might break 482 # out of the loop early and we want to process 483 # the remaining messages the next time through. 484 # 485 set nkeys [stat_field $db stat "Number of keys"] 486 error_check_good db_close [$db close] 0 487 488 if { $nkeys == 0 } { 489 set dbname [string replace $msgdb 0 5 done.] 490 # 491 # We have to do a special dance to get rid of the 492 # empty messaging files because of the way Windows 493 # handles open files marked for deletion. 494 # On Windows, a file is marked for deletion but 495 # does not actually get deleted until the last handle 496 # is closed. This causes a problem when a test tries 497 # to create a new file with a previously-used name, 498 # and Windows believes the old file still exists. 499 # Therefore, we rename the files before deleting them, 500 # to guarantee they are out of the way. 501 # 502 file rename -force $queuedir/$msgdb $queuedir/$dbname 503 file delete -force $queuedir/$dbname 504 } 505 } 506 # Return the number of messages processed. 507 return $nproced 508} 509 510