1# See the file LICENSE for redistribution information. 2# 3# Copyright (c) 2001-2009 Oracle. All rights reserved. 4# 5# $Id$ 6# 7# TEST rep046 8# TEST Replication and basic bulk transfer. 9# TEST Set bulk transfer replication option. 10# TEST Run long txns on master and then commit. Process on client 11# TEST and verify contents. Run a very long txn so that logging 12# TEST must send the log. Process and verify on client. 13# 14proc rep046 { method { nentries 200 } { tnum "046" } args } { 15 source ./include.tcl 16 global databases_in_memory 17 global repfiles_in_memory 18 19 if { $is_windows9x_test == 1 } { 20 puts "Skipping replication test on Win9x platform." 21 return 22 } 23 24 if { $checking_valid_methods } { 25 return "ALL" 26 } 27 28 set args [convert_args $method $args] 29 set logsets [create_logsets 3] 30 31 # Set up for on-disk or in-memory databases. 32 set msg "using on-disk databases" 33 if { $databases_in_memory } { 34 set msg "using named in-memory databases" 35 if { [is_queueext $method] } { 36 puts -nonewline "Skipping rep$tnum for method " 37 puts "$method with named in-memory databases." 38 return 39 } 40 } 41 42 set msg2 "and on-disk replication files" 43 if { $repfiles_in_memory } { 44 set msg2 "and in-memory replication files" 45 } 46 47 # Run the body of the test with and without recovery. 48 set throttle { "throttle" "" } 49 foreach r $test_recopts { 50 foreach l $logsets { 51 set logindex [lsearch -exact $l "in-memory"] 52 if { $r == "-recover" && $logindex != -1 } { 53 puts "Skipping test with -recover for \ 54 in-memory logs." 55 continue 56 } 57 foreach t $throttle { 58 puts "Rep$tnum ($method $r $t):\ 59 Replication and bulk transfer $msg $msg2." 60 puts "Rep$tnum: Master logs are [lindex $l 0]" 61 puts "Rep$tnum: Client 0 logs are [lindex $l 1]" 62 puts "Rep$tnum: Client 1 logs are [lindex $l 2]" 63 rep046_sub $method $nentries $tnum $l $r \ 64 $t $args 65 } 66 } 67 } 68} 69 70proc rep046_sub { method niter tnum logset recargs throttle largs } { 71 global overflowword1 72 global overflowword2 73 global databases_in_memory 74 global repfiles_in_memory 75 global testdir 76 global util_path 77 global rep_verbose 78 global verbose_type 79 80 set verbargs "" 81 if { $rep_verbose == 1 } { 82 set verbargs " -verbose {$verbose_type on} " 83 } 84 85 set repmemargs "" 86 if { $repfiles_in_memory } { 87 set repmemargs "-rep_inmem_files " 88 } 89 90 set orig_tdir $testdir 91 env_cleanup $testdir 92 93 replsetup $testdir/MSGQUEUEDIR 94 95 set masterdir $testdir/MASTERDIR 96 set clientdir $testdir/CLIENTDIR 97 file mkdir $masterdir 98 file mkdir $clientdir 99 100 set m_logtype [lindex $logset 0] 101 set c_logtype [lindex $logset 1] 102 set c2_logtype [lindex $logset 2] 103 104 set in_memory_log \ 105 [expr { $m_logtype == "in-memory" || $c_logtype == "in-memory" || \ 106 $c2_logtype == "in-memory" }] 107 108 # In-memory logs require a large log buffer, and can not 109 # be used with -txn nosync. Adjust the args for master 110 # and client. 111 # This test has a long transaction, allocate a larger log 112 # buffer for in-memory test. 113 set m_logargs [adjust_logargs $m_logtype [expr 20 * 1024 * 1024]] 114 set c_logargs [adjust_logargs $c_logtype [expr 20 * 1024 * 1024]] 115 set c2_logargs [adjust_logargs $c2_logtype [expr 20 * 1024 * 1024]] 116 set m_txnargs [adjust_txnargs $m_logtype] 117 set c_txnargs [adjust_txnargs $c_logtype] 118 set c2_txnargs [adjust_txnargs $c2_logtype] 119 120 # If replication files are in-memory we'll need a bigger cache. 121 set cacheargs "" 122 if { $repfiles_in_memory } { 123 set cachesize [expr 8 * (1024 * 1024)] 124 set cacheargs "-cachesize {0 $cachesize 1} " 125 } 126 127 set bigniter [expr 10000 - [expr 2 * $niter]] 128 set lkmax [expr $bigniter * 2] 129 130 # Open a master. 131 repladd 1 132 set ma_envcmd "berkdb_env_noerr -create $m_txnargs $m_logargs \ 133 $repmemargs $cacheargs \ 134 $verbargs -lock_max_locks 10000 -lock_max_objects 10000 \ 135 -errpfx MASTER -home $masterdir -rep_master -rep_transport \ 136 \[list 1 replsend\]" 137 set masterenv [eval $ma_envcmd $recargs] 138 error_check_good master_env [is_valid_env $masterenv] TRUE 139 140 repladd 2 141 set cl_envcmd "berkdb_env_noerr -create $c_txnargs $c_logargs \ 142 $repmemargs $cacheargs \ 143 $verbargs -home $clientdir -errpfx CLIENT \ 144 -lock_max_locks 10000 -lock_max_objects 10000 \ 145 -rep_client -rep_transport \[list 2 replsend\]" 146 set clientenv [eval $cl_envcmd $recargs] 147 148 if { $throttle == "throttle" } { 149 set clientdir2 $testdir/CLIENTDIR2 150 file mkdir $clientdir2 151 repladd 3 152 set cl2_envcmd "berkdb_env_noerr -create $c2_txnargs $verbargs \ 153 $repmemargs $cacheargs \ 154 $c2_logargs -home $clientdir2 -errpfx CLIENT2 \ 155 -lock_max_locks 10000 -lock_max_objects 10000 \ 156 -rep_client -rep_transport \[list 3 replsend\]" 157 set cl2env [eval $cl2_envcmd $recargs] 158 set envlist "{$masterenv 1} {$clientenv 2} {$cl2env 3}" 159 # 160 # Turn throttling on in master 161 # 162 error_check_good thr [$masterenv rep_limit 0 [expr 32 * 1024]] 0 163 } else { 164 set envlist "{$masterenv 1} {$clientenv 2}" 165 } 166 # Bring the client online by processing the startup messages. 167 process_msgs $envlist 168 169 # 170 # Turn on bulk processing now on the master. 171 # 172 error_check_good set_bulk [$masterenv rep_config {bulk on}] 0 173 174 puts "\tRep$tnum.a: Create and open master database" 175 # Set up databases as in-memory or on-disk. 176 if { $databases_in_memory } { 177 set dbname { "" "test.db" } 178 } else { 179 set dbname "test.db" 180 } 181 182 set omethod [convert_method $method] 183 set masterdb [eval {berkdb_open_noerr -env $masterenv -auto_commit \ 184 -create -mode 0644} $largs $omethod $dbname] 185 error_check_good dbopen [is_valid_db $masterdb] TRUE 186 187 # Process database. 188 process_msgs $envlist 189 190 # Run a modified test001 in the master (and update clients). 191 puts "\tRep$tnum.b: Basic long running txn" 192 set bulkrec1 [stat_field $masterenv rep_stat "Bulk records stored"] 193 set bulkxfer1 [stat_field $masterenv rep_stat "Bulk buffer transfers"] 194 195 set overflowword1 "0" 196 set overflowword2 "0" 197 rep_test_bulk $method $masterenv $masterdb $niter 0 0 198 process_msgs $envlist 199 set bulkrec2 [stat_field $masterenv rep_stat "Bulk records stored"] 200 set bulkxfer2 [stat_field $masterenv rep_stat "Bulk buffer transfers"] 201 error_check_good recstat [expr $bulkrec2 > $bulkrec1] 1 202 error_check_good xferstat [expr $bulkxfer2 > $bulkxfer1] 1 203 rep_verify $masterdir $masterenv\ 204 $clientdir $clientenv $in_memory_log 1 1 205 206 puts "\tRep$tnum.c: Very long txn" 207 # Determine whether this build is configured with --enable-debug_rop 208 # or --enable-debug_wop. 209 set conf [berkdb getconfig] 210 set debug_rop_wop 0 211 if { [is_substr $conf "debug_rop"] == 1 || \ 212 [is_substr $conf "debug_wop"] == 1 } { 213 set debug_rop_wop 1 214 } 215 216 # If debug_rop/wop is set test will write more info to log. 217 # An in-memory log has a smaller "file" size than the large 218 # items written in this part of the test, so skip this section 219 # if any in-memory logs and debug_rop/wop is set. 220 if { $in_memory_log == 1 && $debug_rop_wop == 1 } { 221 puts "\t\tSkipping for in-memory log and debug_rop/wop" 222 } else { 223 set skip $niter 224 set start $niter 225 set orig $niter 226 set bulkfill1 [stat_field $masterenv rep_stat \ 227 "Bulk buffer fills"] 228 rep_test_bulk $method $masterenv $masterdb $bigniter \ 229 $start $skip 230 set start [expr $niter + $bigniter] 231 if { $throttle == "throttle" } { 232 # 233 # If we're throttling clear all messages from client 3 234 # so that we force a huge gap that the client will have 235 # to ask for to invoke a rerequest that throttles. 236 # 237 replclear 3 238 set old_thr \ 239 [stat_field $masterenv rep_stat \ 240 "Transmission limited"] 241 } 242 process_msgs $envlist 243 set bulkfill2 [stat_field $masterenv rep_stat \ 244 "Bulk buffer fills"] 245 error_check_good fillstat [expr $bulkfill2 > $bulkfill1] 1 246 rep_verify $masterdir $masterenv $clientdir $clientenv \ 247 $in_memory_log 1 1 248 } 249 250 puts "\tRep$tnum.d: Very large data" 251 252 # If debug_rop/wop is set test will write entire item to log. 253 # An in-memory log has a smaller "file" size than the large 254 # items written in this part of the test, so skip this section 255 # if any in-memory logs and debug_rop/wop is set. 256 if { $in_memory_log == 1 && $debug_rop_wop == 1 } { 257 puts "\t\tSkipping for in-memory log and debug_rop/wop" 258 } else { 259 set bulkovf1 [stat_field $masterenv rep_stat \ 260 "Bulk buffer overflows"] 261 set bulkfill1 [stat_field $masterenv rep_stat \ 262 "Bulk buffer fills"] 263 # 264 # Send in '2' exactly because we're sending in the flag to use 265 # the overflow entries. We have 2 overflow entries. 266 # If it's fixed length, we can't overflow. Induce throttling 267 # by putting in a bunch more entries. Create a gap by 268 # forcing a checkpoint record. 269 # 270 $masterenv txn_checkpoint -force 271 process_msgs $envlist 272 tclsleep 1 273 if { [is_fixed_length $method] == 1 } { 274 rep_test_bulk $method $masterenv $masterdb $niter \ 275 $start $start 0 276 } else { 277 rep_test_bulk $method $masterenv $masterdb 2 0 0 1 278 } 279 process_msgs $envlist 280 281 # Generally overflows cannot happen because large data gets 282 # broken up into overflow pages, and none will be larger than 283 # the buffer. However, if we're configured for debug_rop/wop 284 # then we record the data as is and will overflow. 285 # 286 set bulkovf2 [stat_field $masterenv rep_stat \ 287 "Bulk buffer overflows"] 288 set bulkfill2 [stat_field $masterenv rep_stat \ 289 "Bulk buffer fills"] 290 if { [is_fixed_length $method] == 0 } { 291 error_check_good fillstat1 \ 292 [expr $bulkfill2 > $bulkfill1] 1 293 if { $debug_rop_wop == 1 } { 294 error_check_good overflows \ 295 [expr $bulkovf2 > $bulkovf1] 1 296 } else { 297 error_check_good no_overflows $bulkovf2 0 298 } 299 } 300 } 301 302 # !!! 303 # Turn off bulk processing now on the master. We need to do 304 # this because some configurations (like debug_rop/wop) will 305 # generate log records when verifying the logs and databases. 306 # We want to control processing those messages. 307 # 308 error_check_good set_bulk [$masterenv rep_config {bulk off}] 0 309 310 if { $in_memory_log == 1 && $debug_rop_wop == 1 } { 311 puts "\t\tSkipping for in-memory log and debug_rop/wop" 312 } else { 313 rep_verify $masterdir $masterenv $clientdir $clientenv \ 314 $in_memory_log 315 316 if { $throttle == "throttle" } { 317 puts "\tRep$tnum.e: Verify throttling." 318 set new_thr \ 319 [stat_field $masterenv rep_stat \ 320 "Transmission limited"] 321 error_check_bad nthrottles1 $new_thr -1 322 error_check_bad nthrottles0 $new_thr 0 323 error_check_good nthrottles \ 324 [expr $old_thr < $new_thr] 1 325 process_msgs $envlist 326 rep_verify $masterdir $masterenv $clientdir2 $cl2env \ 327 $in_memory_log 328 } 329 } 330 331 if { $throttle == "throttle" } { 332 error_check_good cclose [$cl2env close] 0 333 } 334 335 error_check_good dbclose [$masterdb close] 0 336 error_check_good mclose [$masterenv close] 0 337 error_check_good cclose [$clientenv close] 0 338 replclose $testdir/MSGQUEUEDIR 339} 340