1# See the file LICENSE for redistribution information. 2# 3# Copyright (c) 2001-2009 Oracle. All rights reserved. 4# 5# $Id$ 6# 7# TEST rep047 8# TEST Replication and log gap bulk transfers. 9# TEST Set bulk transfer replication option. 10# TEST Run test. Start a new client (to test ALL_REQ and bulk). 11# TEST Run small test again. Clear messages for 1 client. 12# TEST Run small test again to test LOG_REQ gap processing and bulk. 13# TEST Process and verify on clients. 14# 15proc rep047 { method { nentries 200 } { tnum "047" } args } { 16 source ./include.tcl 17 global databases_in_memory 18 global repfiles_in_memory 19 20 if { $is_windows9x_test == 1 } { 21 puts "Skipping replication test on Win9x platform." 22 return 23 } 24 25 # Valid for all access methods. 26 if { $checking_valid_methods } { 27 return "ALL" 28 } 29 30 set args [convert_args $method $args] 31 set logsets [create_logsets 3] 32 33 # Set up for on-disk or in-memory databases. 34 set msg "using on-disk databases" 35 if { $databases_in_memory } { 36 set msg "using named in-memory databases" 37 if { [is_queueext $method] } { 38 puts -nonewline "Skipping rep$tnum for method " 39 puts "$method with named in-memory databases." 40 return 41 } 42 } 43 44 set msg2 "and on-disk replication files" 45 if { $repfiles_in_memory } { 46 set msg2 "and in-memory replication files" 47 } 48 49 # Run the body of the test with and without recovery, 50 # and with and without cleaning. Skip recovery with in-memory 51 # logging - it doesn't make sense. 52 foreach r $test_recopts { 53 foreach l $logsets { 54 set logindex [lsearch -exact $l "in-memory"] 55 if { $r == "-recover" && $logindex != -1 } { 56 puts "Skipping rep$tnum for -recover\ 57 with in-memory logs." 58 continue 59 } 60 puts "Rep$tnum ($method $r):\ 61 Replication and resend bulk transfer $msg" 62 puts "Rep$tnum: Master logs are [lindex $l 0]" 63 puts "Rep$tnum: Client logs are [lindex $l 1]" 64 puts "Rep$tnum: Client 2 logs are [lindex $l 2]" 65 rep047_sub $method $nentries $tnum $l $r $args 66 } 67 } 68} 69 70proc rep047_sub { method niter tnum logset recargs largs } { 71 global testdir 72 global util_path 73 global overflowword1 overflowword2 74 global databases_in_memory 75 global repfiles_in_memory 76 global rep_verbose 77 global verbose_type 78 79 set verbargs "" 80 if { $rep_verbose == 1 } { 81 set verbargs " -verbose {$verbose_type on} " 82 } 83 84 set repmemargs "" 85 if { $repfiles_in_memory } { 86 set repmemargs "-rep_inmem_files " 87 } 88 89 set overflowword1 "0" 90 set overflowword2 "0" 91 set orig_tdir $testdir 92 env_cleanup $testdir 93 94 replsetup $testdir/MSGQUEUEDIR 95 96 set masterdir $testdir/MASTERDIR 97 set clientdir $testdir/CLIENTDIR 98 set clientdir2 $testdir/CLIENTDIR2 99 100 file mkdir $masterdir 101 file mkdir $clientdir 102 file mkdir $clientdir2 103 104 set m_logtype [lindex $logset 0] 105 set c_logtype [lindex $logset 1] 106 set c2_logtype [lindex $logset 2] 107 108 # In-memory logs cannot be used with -txn nosync. 109 set m_logargs [adjust_logargs $m_logtype] 110 set c_logargs [adjust_logargs $c_logtype] 111 set c2_logargs [adjust_logargs $c2_logtype] 112 set m_txnargs [adjust_txnargs $m_logtype] 113 set c_txnargs [adjust_txnargs $c_logtype] 114 set c2_txnargs [adjust_txnargs $c2_logtype] 115 116 set in_memory_log \ 117 [expr { $m_logtype == "in-memory" || $c_logtype == "in-memory" || \ 118 $c2_logtype == "in-memory" }] 119 120 # Open a master. 121 repladd 1 122 set ma_envcmd "berkdb_env -create $m_txnargs $m_logargs \ 123 $verbargs -errpfx MASTER -home $masterdir $repmemargs \ 124 -rep_master -rep_transport \[list 1 replsend\]" 125 set masterenv [eval $ma_envcmd $recargs] 126 error_check_good master_env [is_valid_env $masterenv] TRUE 127 128 # Open two clients. 129 repladd 2 130 set cl_envcmd "berkdb_env -create $c_txnargs $c_logargs \ 131 $verbargs -errpfx CLIENT -home $clientdir $repmemargs \ 132 -rep_client -rep_transport \[list 2 replsend\]" 133 set clientenv [eval $cl_envcmd $recargs] 134 error_check_good client_env [is_valid_env $clientenv] TRUE 135 136 repladd 3 137 set cl2_envcmd "berkdb_env -create $c2_txnargs $c2_logargs \ 138 $verbargs -errpfx CLIENT2 -home $clientdir2 $repmemargs \ 139 -rep_client -rep_transport \[list 3 replsend\]" 140 141 # Bring the client online by processing the startup messages. 142 set envlist "{$masterenv 1} {$clientenv 2}" 143 process_msgs $envlist 144 145 error_check_good set_bulk [$masterenv rep_config {bulk on}] 0 146 147 puts "\tRep$tnum.a: Create and open master database" 148 149 # Set up databases as in-memory or on-disk. 150 if { $databases_in_memory } { 151 set dbname { "" "test.db" } 152 } else { 153 set dbname "test.db" 154 } 155 156 set omethod [convert_method $method] 157 set masterdb [eval {berkdb_open -env $masterenv -auto_commit \ 158 -create -mode 0644} $largs $omethod $dbname] 159 error_check_good dbopen [is_valid_db $masterdb] TRUE 160 161 # Bring the client online by processing the startup messages. 162 set envlist "{$masterenv 1} {$clientenv 2}" 163 process_msgs $envlist 164 165 # Run a modified test001 in the master (and update clients). 166 puts "\tRep$tnum.b: Basic long running txn" 167 rep_test_bulk $method $masterenv $masterdb $niter 0 0 0 168 process_msgs $envlist 169 rep_verify $masterdir $masterenv\ 170 $clientdir $clientenv $in_memory_log 1 1 171 172 # Clean up after rep_verify: remove the temporary "prlog" file. Now 173 # that a newly joining client uses internal init, when the master scans 174 # its directory for database files it complains about prlog not looking 175 # like a proper db. This is harmless, but it does put a distracting 176 # error message into the test output. 177 # 178 file delete $masterdir/prlog 179 180 puts "\tRep$tnum.c: Bring new client online" 181 replclear 3 182 set bulkrec1 [stat_field $masterenv rep_stat "Bulk records stored"] 183 set bulkxfer1 [stat_field $masterenv rep_stat "Bulk buffer transfers"] 184 set clientenv2 [eval $cl2_envcmd $recargs] 185 error_check_good client_env [is_valid_env $clientenv2] TRUE 186 set envlist "{$masterenv 1} {$clientenv 2} {$clientenv2 3}" 187 process_msgs $envlist 188 189 # 190 # We know we added $niter items to the database so there should be 191 # at least $niter records stored to the log. Verify that 192 # when we brought client 2 online, we sent at least $niter more 193 # records via bulk. 194 # 195 set bulkrec2 [stat_field $masterenv rep_stat "Bulk records stored"] 196 set bulkxfer2 [stat_field $masterenv rep_stat "Bulk buffer transfers"] 197 set recstat [expr $bulkrec2 > [expr $bulkrec1 + $niter]] 198 error_check_good recstat $recstat 1 199 error_check_good xferstat [expr $bulkxfer2 > $bulkxfer1] 1 200 puts "\tRep$tnum.c.0: Take new client offline" 201 202 puts "\tRep$tnum.d: Run small test creating a log gap" 203 set skip $niter 204 set start $niter 205 set niter 10 206 rep_test_bulk $method $masterenv $masterdb $niter $start $skip 0 207 # 208 # Skip and clear messages for client 2. 209 # 210 replclear 3 211 set envlist "{$masterenv 1} {$clientenv 2}" 212 process_msgs $envlist 213 214 puts "\tRep$tnum.e: Bring new client online again" 215 set envlist "{$masterenv 1} {$clientenv 2} {$clientenv2 3}" 216 # 217 # Since we're relying on the client to detect a gap and request missing 218 # records, reset gap parameters to small values. Otherwise, 219 # "wait_recs" is still set at its maximum "high" value, due to this 220 # client having been through an internal init. Send a record to 221 # create a real gap and not an end-of-record stream pause. 222 # 223 $masterenv txn_checkpoint -force 224 $clientenv2 rep_request 4000 128000 225 process_msgs $envlist 226 tclsleep 1 227 set bulkrec1 [stat_field $masterenv rep_stat "Bulk records stored"] 228 set bulkxfer1 [stat_field $masterenv rep_stat "Bulk buffer transfers"] 229 set skip [expr $skip + $niter] 230 set start $skip 231 rep_test_bulk $method $masterenv $masterdb $niter $start $skip 0 232 233 process_msgs $envlist 234 # 235 # We know we added 2*$niter items to the database so there should be 236 # at least 2*$niter records stored to the log. Verify that 237 # when we brought client 2 online, we sent at least 2*$niter more 238 # records via bulk. 239 # 240 set bulkrec2 [stat_field $masterenv rep_stat "Bulk records stored"] 241 set bulkxfer2 [stat_field $masterenv rep_stat "Bulk buffer transfers"] 242 set recstat [expr $bulkrec2 > [expr $bulkrec1 + [expr 2 * $niter]]] 243 error_check_good recstat $recstat 1 244 error_check_good xferstat [expr $bulkxfer2 > $bulkxfer1] 1 245 246 # Turn off bulk processing now on the master. We need to do 247 # this because some configurations (like debug_rop/wop) will 248 # generate log records when verifying the logs and databases. 249 # We want to control processing those messages. 250 # 251 error_check_good set_bulk [$masterenv rep_config {bulk off}] 0 252 253 rep_verify $masterdir $masterenv\ 254 $clientdir $clientenv $in_memory_log 1 1 255 256 # Process messages again in case we are running with debug_rop. 257 process_msgs $envlist 258 rep_verify $masterdir $masterenv\ 259 $clientdir2 $clientenv2 $in_memory_log 1 1 260 261 error_check_good dbclose [$masterdb close] 0 262 error_check_good mclose [$masterenv close] 0 263 error_check_good cclose [$clientenv close] 0 264 error_check_good c2close [$clientenv2 close] 0 265 replclose $testdir/MSGQUEUEDIR 266} 267