1# See the file LICENSE for redistribution information. 2# 3# Copyright (c) 2001,2008 Oracle. All rights reserved. 4# 5# $Id: rep046.tcl,v 12.24 2008/01/08 20:58:53 bostic Exp $ 6# 7# TEST rep046 8# TEST Replication and basic bulk transfer. 9# TEST Set bulk transfer replication option. 10# TEST Run long txns on master and then commit. Process on client 11# TEST and verify contents. Run a very long txn so that logging 12# TEST must send the log. Process and verify on client. 13# 14proc rep046 { method { nentries 200 } { tnum "046" } args } { 15 global mixed_mode_logging 16 source ./include.tcl 17 18 if { $is_windows9x_test == 1 } { 19 puts "Skipping replication test on Win9x platform." 20 return 21 } 22 23 if { $checking_valid_methods } { 24 return "ALL" 25 } 26 27 set args [convert_args $method $args] 28 set logsets [create_logsets 3] 29 30 # Run the body of the test with and without recovery. 31 set throttle { "throttle" "" } 32 foreach r $test_recopts { 33 foreach l $logsets { 34 set logindex [lsearch -exact $l "in-memory"] 35 if { $r == "-recover" && $logindex != -1 } { 36 puts "Skipping test with -recover for \ 37 in-memory logs." 38 continue 39 } 40 foreach t $throttle { 41 puts "Rep$tnum ($method $r $t):\ 42 Replication and bulk transfer." 43 puts "Rep$tnum: Master logs are [lindex $l 0]" 44 puts "Rep$tnum: Client 0 logs are [lindex $l 1]" 45 puts "Rep$tnum: Client 1 logs are [lindex $l 2]" 46 rep046_sub $method $nentries $tnum $l $r \ 47 $t $args 48 } 49 } 50 } 51} 52 53proc rep046_sub { method niter tnum logset recargs throttle largs } { 54 global overflowword1 55 global overflowword2 56 global testdir 57 global util_path 58 global rep_verbose 59 global verbose_type 60 61 set verbargs "" 62 if { $rep_verbose == 1 } { 63 set verbargs " -verbose {$verbose_type on} " 64 } 65 66 set orig_tdir $testdir 67 env_cleanup $testdir 68 69 replsetup $testdir/MSGQUEUEDIR 70 71 set masterdir $testdir/MASTERDIR 72 set clientdir $testdir/CLIENTDIR 73 file mkdir $masterdir 74 file mkdir $clientdir 75 76 set m_logtype [lindex $logset 0] 77 set c_logtype [lindex $logset 1] 78 set c2_logtype [lindex $logset 2] 79 80 set in_memory_log \ 81 [expr { $m_logtype == "in-memory" || $c_logtype == "in-memory" || \ 82 $c2_logtype == "in-memory" }] 83 84 # In-memory logs require a large log buffer, and can not 85 # be used with -txn nosync. Adjust the args for master 86 # and client. 87 # This test has a long transaction, allocate a larger log 88 # buffer for in-memory test. 89 set m_logargs [adjust_logargs $m_logtype [expr 20 * 1024 * 1024]] 90 set c_logargs [adjust_logargs $c_logtype [expr 20 * 1024 * 1024]] 91 set c2_logargs [adjust_logargs $c2_logtype [expr 20 * 1024 * 1024]] 92 set m_txnargs [adjust_txnargs $m_logtype] 93 set c_txnargs [adjust_txnargs $c_logtype] 94 set c2_txnargs [adjust_txnargs $c2_logtype] 95 96 97 set bigniter [expr 10000 - [expr 2 * $niter]] 98 set lkmax [expr $bigniter * 2] 99 100 # Open a master. 101 repladd 1 102 set ma_envcmd "berkdb_env_noerr -create $m_txnargs $m_logargs \ 103 $verbargs -lock_max_locks 10000 -lock_max_objects 10000 \ 104 -errpfx MASTER -home $masterdir -rep_master -rep_transport \ 105 \[list 1 replsend\]" 106 set masterenv [eval $ma_envcmd $recargs] 107 error_check_good master_env [is_valid_env $masterenv] TRUE 108 109 repladd 2 110 set cl_envcmd "berkdb_env_noerr -create $c_txnargs $c_logargs \ 111 $verbargs -home $clientdir -errpfx CLIENT \ 112 -lock_max_locks 10000 -lock_max_objects 10000 \ 113 -rep_client -rep_transport \[list 2 replsend\]" 114 set clientenv [eval $cl_envcmd $recargs] 115 116 if { $throttle == "throttle" } { 117 set clientdir2 $testdir/CLIENTDIR2 118 file mkdir $clientdir2 119 repladd 3 120 set cl2_envcmd "berkdb_env_noerr -create $c2_txnargs $verbargs \ 121 $c2_logargs -home $clientdir2 -errpfx CLIENT2 \ 122 -lock_max_locks 10000 -lock_max_objects 10000 \ 123 -rep_client -rep_transport \[list 3 replsend\]" 124 set cl2env [eval $cl2_envcmd $recargs] 125 set envlist "{$masterenv 1} {$clientenv 2} {$cl2env 3}" 126 # 127 # Turn throttling on in master 128 # 129 error_check_good thr [$masterenv rep_limit 0 [expr 32 * 1024]] 0 130 } else { 131 set envlist "{$masterenv 1} {$clientenv 2}" 132 } 133 # Bring the client online by processing the startup messages. 134 process_msgs $envlist 135 136 # 137 # Turn on bulk processing now on the master. 138 # 139 error_check_good set_bulk [$masterenv rep_config {bulk on}] 0 140 141 puts "\tRep$tnum.a: Create and open master database" 142 set testfile "test.db" 143 set omethod [convert_method $method] 144 set masterdb [eval {berkdb_open_noerr -env $masterenv -auto_commit \ 145 -create -mode 0644} $largs $omethod $testfile] 146 error_check_good dbopen [is_valid_db $masterdb] TRUE 147 148 # Process database. 149 process_msgs $envlist 150 151 # Run a modified test001 in the master (and update clients). 152 puts "\tRep$tnum.b: Basic long running txn" 153 set bulkrec1 [stat_field $masterenv rep_stat "Bulk records stored"] 154 set bulkxfer1 [stat_field $masterenv rep_stat "Bulk buffer transfers"] 155 156 set overflowword1 "0" 157 set overflowword2 "0" 158 rep_test_bulk $method $masterenv $masterdb $niter 0 0 159 process_msgs $envlist 160 set bulkrec2 [stat_field $masterenv rep_stat "Bulk records stored"] 161 set bulkxfer2 [stat_field $masterenv rep_stat "Bulk buffer transfers"] 162 error_check_good recstat [expr $bulkrec2 > $bulkrec1] 1 163 error_check_good xferstat [expr $bulkxfer2 > $bulkxfer1] 1 164 rep_verify $masterdir $masterenv $clientdir $clientenv $in_memory_log 165 166 puts "\tRep$tnum.c: Very long txn" 167 # Determine whether this build is configured with --enable-debug_rop 168 # or --enable-debug_wop. 169 set conf [berkdb getconfig] 170 set debug_rop_wop 0 171 if { [is_substr $conf "debug_rop"] == 1 || \ 172 [is_substr $conf "debug_wop"] == 1 } { 173 set debug_rop_wop 1 174 } 175 176 # If debug_rop/wop is set test will write more info to log. 177 # An in-memory log has a smaller "file" size than the large 178 # items written in this part of the test, so skip this section 179 # if any in-memory logs and debug_rop/wop is set. 180 if { $in_memory_log == 1 && $debug_rop_wop == 1 } { 181 puts "\t\tSkipping for in-memory log and debug_rop/wop" 182 } else { 183 set skip $niter 184 set start $niter 185 set orig $niter 186 set bulkfill1 [stat_field $masterenv rep_stat \ 187 "Bulk buffer fills"] 188 rep_test_bulk $method $masterenv $masterdb $bigniter \ 189 $start $skip 190 set start [expr $niter + $bigniter] 191 if { $throttle == "throttle" } { 192 # 193 # If we're throttling clear all messages from client 3 194 # so that we force a huge gap that the client will have 195 # to ask for to invoke a rerequest that throttles. 196 # 197 replclear 3 198 set old_thr \ 199 [stat_field $masterenv rep_stat \ 200 "Transmission limited"] 201 } 202 process_msgs $envlist 203 set bulkfill2 [stat_field $masterenv rep_stat \ 204 "Bulk buffer fills"] 205 error_check_good fillstat [expr $bulkfill2 > $bulkfill1] 1 206 rep_verify $masterdir $masterenv $clientdir $clientenv \ 207 $in_memory_log 208 } 209 210 puts "\tRep$tnum.d: Very large data" 211 212 # If debug_rop/wop is set test will write entire item to log. 213 # An in-memory log has a smaller "file" size than the large 214 # items written in this part of the test, so skip this section 215 # if any in-memory logs and debug_rop/wop is set. 216 if { $in_memory_log == 1 && $debug_rop_wop == 1 } { 217 puts "\t\tSkipping for in-memory log and debug_rop/wop" 218 } else { 219 set bulkovf1 [stat_field $masterenv rep_stat \ 220 "Bulk buffer overflows"] 221 set bulkfill1 [stat_field $masterenv rep_stat \ 222 "Bulk buffer fills"] 223 # 224 # Send in '2' exactly because we're sending in the flag to use 225 # the overflow entries. We have 2 overflow entries. 226 # If it's fixed length, we can't overflow. Induce throttling 227 # by putting in a bunch more entries. 228 if { [is_fixed_length $method] == 1 } { 229 rep_test_bulk $method $masterenv $masterdb $niter \ 230 $start $start 0 231 } else { 232 rep_test_bulk $method $masterenv $masterdb 2 0 0 1 233 } 234 process_msgs $envlist 235 236 # Generally overflows cannot happen because large data gets 237 # broken up into overflow pages, and none will be larger than 238 # the buffer. However, if we're configured for debug_rop/wop 239 # then we record the data as is and will overflow. 240 # 241 set bulkovf2 [stat_field $masterenv rep_stat \ 242 "Bulk buffer overflows"] 243 set bulkfill2 [stat_field $masterenv rep_stat \ 244 "Bulk buffer fills"] 245 if { [is_fixed_length $method] == 0 } { 246 error_check_good fillstat1 \ 247 [expr $bulkfill2 > $bulkfill1] 1 248 if { $debug_rop_wop == 1 } { 249 error_check_good overflows \ 250 [expr $bulkovf2 > $bulkovf1] 1 251 } else { 252 error_check_good no_overflows $bulkovf2 0 253 } 254 } 255 } 256 257 # !!! 258 # Turn off bulk processing now on the master. We need to do 259 # this because some configurations (like debug_rop/wop) will 260 # generate log records when verifying the logs and databases. 261 # We want to control processing those messages. 262 # 263 error_check_good set_bulk [$masterenv rep_config {bulk off}] 0 264 265 if { $in_memory_log == 1 && $debug_rop_wop == 1 } { 266 puts "\t\tSkipping for in-memory log and debug_rop/wop" 267 } else { 268 rep_verify $masterdir $masterenv $clientdir $clientenv \ 269 $in_memory_log 270 271 if { $throttle == "throttle" } { 272 puts "\tRep$tnum.e: Verify throttling." 273 set new_thr \ 274 [stat_field $masterenv rep_stat \ 275 "Transmission limited"] 276 error_check_bad nthrottles1 $new_thr -1 277 error_check_bad nthrottles0 $new_thr 0 278 error_check_good nthrottles \ 279 [expr $old_thr < $new_thr] 1 280 process_msgs $envlist 281 rep_verify $masterdir $masterenv $clientdir2 $cl2env \ 282 $in_memory_log 283 } 284 } 285 286 if { $throttle == "throttle" } { 287 error_check_good cclose [$cl2env close] 0 288 } 289 290 error_check_good dbclose [$masterdb close] 0 291 error_check_good mclose [$masterenv close] 0 292 error_check_good cclose [$clientenv close] 0 293 replclose $testdir/MSGQUEUEDIR 294} 295