1# See the file LICENSE for redistribution information.
2#
3# Copyright (c) 2001-2009 Oracle.  All rights reserved.
4#
5# $Id$
6#
7# TEST  rep046
8# TEST	Replication and basic bulk transfer.
9# TEST	Set bulk transfer replication option.
10# TEST	Run long txns on master and then commit.  Process on client
11# TEST	and verify contents.  Run a very long txn so that logging
12# TEST	must send the log.  Process and verify on client.
13#
14proc rep046 { method { nentries 200 } { tnum "046" } args } {
15	source ./include.tcl
16	global databases_in_memory
17	global repfiles_in_memory
18
19	if { $is_windows9x_test == 1 } {
20		puts "Skipping replication test on Win9x platform."
21		return
22	}
23
24	if { $checking_valid_methods } {
25		return "ALL"
26	}
27
28	set args [convert_args $method $args]
29	set logsets [create_logsets 3]
30
31	# Set up for on-disk or in-memory databases.
32	set msg "using on-disk databases"
33	if { $databases_in_memory } {
34		set msg "using named in-memory databases"
35		if { [is_queueext $method] } {
36			puts -nonewline "Skipping rep$tnum for method "
37			puts "$method with named in-memory databases."
38			return
39		}
40	}
41
42	set msg2 "and on-disk replication files"
43	if { $repfiles_in_memory } {
44		set msg2 "and in-memory replication files"
45	}
46
47	# Run the body of the test with and without recovery.
48	set throttle { "throttle" "" }
49	foreach r $test_recopts {
50		foreach l $logsets {
51			set logindex [lsearch -exact $l "in-memory"]
52			if { $r == "-recover" && $logindex != -1 } {
53				puts "Skipping test with -recover for \
54				    in-memory logs."
55				continue
56			}
57			foreach t $throttle {
58				puts "Rep$tnum ($method $r $t):\
59				    Replication and bulk transfer $msg $msg2."
60				puts "Rep$tnum: Master logs are [lindex $l 0]"
61				puts "Rep$tnum: Client 0 logs are [lindex $l 1]"
62				puts "Rep$tnum: Client 1 logs are [lindex $l 2]"
63				rep046_sub $method $nentries $tnum $l $r \
64				    $t $args
65			}
66		}
67	}
68}
69
70proc rep046_sub { method niter tnum logset recargs throttle largs } {
71	global overflowword1
72	global overflowword2
73	global databases_in_memory
74	global repfiles_in_memory
75	global testdir
76	global util_path
77	global rep_verbose
78	global verbose_type
79
80	set verbargs ""
81	if { $rep_verbose == 1 } {
82		set verbargs " -verbose {$verbose_type on} "
83	}
84
85	set repmemargs ""
86	if { $repfiles_in_memory } {
87		set repmemargs "-rep_inmem_files "
88	}
89
90	set orig_tdir $testdir
91	env_cleanup $testdir
92
93	replsetup $testdir/MSGQUEUEDIR
94
95	set masterdir $testdir/MASTERDIR
96	set clientdir $testdir/CLIENTDIR
97	file mkdir $masterdir
98	file mkdir $clientdir
99
100	set m_logtype [lindex $logset 0]
101	set c_logtype [lindex $logset 1]
102	set c2_logtype [lindex $logset 2]
103
104	set in_memory_log \
105	    [expr { $m_logtype == "in-memory" || $c_logtype == "in-memory" || \
106	    $c2_logtype == "in-memory" }]
107
108	# In-memory logs require a large log buffer, and can not
109	# be used with -txn nosync.  Adjust the args for master
110	# and client.
111	# This test has a long transaction, allocate a larger log
112	# buffer for in-memory test.
113	set m_logargs [adjust_logargs $m_logtype [expr 20 * 1024 * 1024]]
114	set c_logargs [adjust_logargs $c_logtype [expr 20 * 1024 * 1024]]
115	set c2_logargs [adjust_logargs $c2_logtype [expr 20 * 1024 * 1024]]
116	set m_txnargs [adjust_txnargs $m_logtype]
117	set c_txnargs [adjust_txnargs $c_logtype]
118	set c2_txnargs [adjust_txnargs $c2_logtype]
119
120	# If replication files are in-memory we'll need a bigger cache.
121	set cacheargs ""
122	if { $repfiles_in_memory } {
123		set cachesize [expr 8 * (1024 * 1024)]
124		set cacheargs "-cachesize {0 $cachesize 1} "
125	}
126
127	set bigniter [expr 10000 - [expr 2 * $niter]]
128	set lkmax [expr $bigniter * 2]
129
130	# Open a master.
131	repladd 1
132	set ma_envcmd "berkdb_env_noerr -create $m_txnargs $m_logargs \
133	    $repmemargs $cacheargs \
134	    $verbargs -lock_max_locks 10000 -lock_max_objects 10000 \
135	    -errpfx MASTER -home $masterdir -rep_master -rep_transport \
136	    \[list 1 replsend\]"
137	set masterenv [eval $ma_envcmd $recargs]
138	error_check_good master_env [is_valid_env $masterenv] TRUE
139
140	repladd 2
141	set cl_envcmd "berkdb_env_noerr -create $c_txnargs $c_logargs \
142	    $repmemargs $cacheargs \
143	    $verbargs -home $clientdir -errpfx CLIENT \
144	    -lock_max_locks 10000 -lock_max_objects 10000 \
145	    -rep_client -rep_transport \[list 2 replsend\]"
146	set clientenv [eval $cl_envcmd $recargs]
147
148	if { $throttle == "throttle" } {
149		set clientdir2 $testdir/CLIENTDIR2
150		file mkdir $clientdir2
151		repladd 3
152		set cl2_envcmd "berkdb_env_noerr -create $c2_txnargs $verbargs \
153		    $repmemargs $cacheargs \
154		    $c2_logargs -home $clientdir2 -errpfx CLIENT2 \
155	    	    -lock_max_locks 10000 -lock_max_objects 10000 \
156		    -rep_client -rep_transport \[list 3 replsend\]"
157		set cl2env [eval $cl2_envcmd $recargs]
158		set envlist "{$masterenv 1} {$clientenv 2} {$cl2env 3}"
159		#
160		# Turn throttling on in master
161		#
162		error_check_good thr [$masterenv rep_limit 0 [expr 32 * 1024]] 0
163	} else {
164		set envlist "{$masterenv 1} {$clientenv 2}"
165	}
166	# Bring the client online by processing the startup messages.
167	process_msgs $envlist
168
169	#
170	# Turn on bulk processing now on the master.
171	#
172	error_check_good set_bulk [$masterenv rep_config {bulk on}] 0
173
174	puts "\tRep$tnum.a: Create and open master database"
175	# Set up databases as in-memory or on-disk.
176	if { $databases_in_memory } {
177		set dbname { "" "test.db" }
178	} else {
179		set dbname "test.db"
180	}
181
182	set omethod [convert_method $method]
183	set masterdb [eval {berkdb_open_noerr -env $masterenv -auto_commit \
184	    -create -mode 0644} $largs $omethod $dbname]
185	error_check_good dbopen [is_valid_db $masterdb] TRUE
186
187	# Process database.
188	process_msgs $envlist
189
190	# Run a modified test001 in the master (and update clients).
191	puts "\tRep$tnum.b: Basic long running txn"
192	set bulkrec1 [stat_field $masterenv rep_stat "Bulk records stored"]
193	set bulkxfer1 [stat_field $masterenv rep_stat "Bulk buffer transfers"]
194
195	set overflowword1 "0"
196	set overflowword2 "0"
197	rep_test_bulk $method $masterenv $masterdb $niter 0 0
198	process_msgs $envlist
199	set bulkrec2 [stat_field $masterenv rep_stat "Bulk records stored"]
200	set bulkxfer2 [stat_field $masterenv rep_stat "Bulk buffer transfers"]
201	error_check_good recstat [expr $bulkrec2 > $bulkrec1] 1
202	error_check_good xferstat [expr $bulkxfer2 > $bulkxfer1] 1
203	rep_verify $masterdir $masterenv\
204	    $clientdir $clientenv $in_memory_log 1 1
205
206	puts "\tRep$tnum.c: Very long txn"
207	# Determine whether this build is configured with --enable-debug_rop
208	# or --enable-debug_wop.
209	set conf [berkdb getconfig]
210	set debug_rop_wop 0
211	if { [is_substr $conf "debug_rop"] == 1 || \
212	    [is_substr $conf "debug_wop"] == 1 } {
213		set debug_rop_wop 1
214	}
215
216	# If debug_rop/wop is set test will write more info to log.
217	# An in-memory log has a smaller "file" size than the large
218	# items written in this part of the test, so skip this section
219	# if any in-memory logs and debug_rop/wop is set.
220	if { $in_memory_log == 1 && $debug_rop_wop == 1 } {
221		puts "\t\tSkipping for in-memory log and debug_rop/wop"
222	} else {
223		set skip $niter
224		set start $niter
225		set orig $niter
226		set bulkfill1 [stat_field $masterenv rep_stat \
227		    "Bulk buffer fills"]
228		rep_test_bulk $method $masterenv $masterdb $bigniter \
229		    $start $skip
230		set start [expr $niter + $bigniter]
231		if { $throttle == "throttle" } {
232			#
233			# If we're throttling clear all messages from client 3
234			# so that we force a huge gap that the client will have
235			# to ask for to invoke a rerequest that throttles.
236			#
237			replclear 3
238			set old_thr \
239			    [stat_field $masterenv rep_stat \
240			    "Transmission limited"]
241		}
242		process_msgs $envlist
243		set bulkfill2 [stat_field $masterenv rep_stat \
244		    "Bulk buffer fills"]
245		error_check_good fillstat [expr $bulkfill2 > $bulkfill1] 1
246		rep_verify $masterdir $masterenv $clientdir $clientenv \
247		    $in_memory_log 1 1
248        }
249
250	puts "\tRep$tnum.d: Very large data"
251
252	# If debug_rop/wop is set test will write entire item to log.
253	# An in-memory log has a smaller "file" size than the large
254	# items written in this part of the test, so skip this section
255	# if any in-memory logs and debug_rop/wop is set.
256	if { $in_memory_log == 1 && $debug_rop_wop == 1 } {
257		puts "\t\tSkipping for in-memory log and debug_rop/wop"
258	} else {
259		set bulkovf1 [stat_field $masterenv rep_stat \
260		    "Bulk buffer overflows"]
261		set bulkfill1 [stat_field $masterenv rep_stat \
262		    "Bulk buffer fills"]
263		#
264		# Send in '2' exactly because we're sending in the flag to use
265		# the overflow entries.  We have 2 overflow entries.
266		# If it's fixed length, we can't overflow.  Induce throttling
267		# by putting in a bunch more entries.  Create a gap by
268		# forcing a checkpoint record.
269		#
270		$masterenv txn_checkpoint -force
271		process_msgs $envlist
272		tclsleep 1
273		if { [is_fixed_length $method] == 1 } {
274			rep_test_bulk $method $masterenv $masterdb $niter \
275			    $start $start 0
276		} else {
277			rep_test_bulk $method $masterenv $masterdb 2 0 0 1
278		}
279		process_msgs $envlist
280
281		# Generally overflows cannot happen because large data gets
282		# broken up into overflow pages, and none will be larger than
283		# the buffer.  However, if we're configured for debug_rop/wop
284		# then we record the data as is and will overflow.
285		#
286		set bulkovf2 [stat_field $masterenv rep_stat \
287		    "Bulk buffer overflows"]
288		set bulkfill2 [stat_field $masterenv rep_stat \
289		    "Bulk buffer fills"]
290		if { [is_fixed_length $method] == 0 } {
291			error_check_good fillstat1 \
292			    [expr $bulkfill2 > $bulkfill1] 1
293			if { $debug_rop_wop == 1 } {
294				error_check_good overflows \
295				    [expr $bulkovf2 > $bulkovf1] 1
296			} else {
297				error_check_good no_overflows $bulkovf2 0
298			}
299		}
300	}
301
302	# !!!
303	# Turn off bulk processing now on the master.  We need to do
304	# this because some configurations (like debug_rop/wop) will
305	# generate log records when verifying the logs and databases.
306	# We want to control processing those messages.
307	#
308	error_check_good set_bulk [$masterenv rep_config {bulk off}] 0
309
310	if { $in_memory_log == 1 && $debug_rop_wop == 1 } {
311		puts "\t\tSkipping for in-memory log and debug_rop/wop"
312	} else {
313		rep_verify $masterdir $masterenv $clientdir $clientenv \
314		    $in_memory_log
315
316		if { $throttle == "throttle" } {
317			puts "\tRep$tnum.e: Verify throttling."
318			set new_thr \
319			    [stat_field $masterenv rep_stat \
320			    "Transmission limited"]
321			error_check_bad nthrottles1 $new_thr -1
322			error_check_bad nthrottles0 $new_thr 0
323			error_check_good nthrottles \
324			    [expr $old_thr < $new_thr] 1
325			process_msgs $envlist
326			rep_verify $masterdir $masterenv $clientdir2 $cl2env \
327			    $in_memory_log
328		}
329	}
330
331	if { $throttle == "throttle" } {
332		error_check_good cclose [$cl2env close] 0
333	}
334
335	error_check_good dbclose [$masterdb close] 0
336	error_check_good mclose [$masterenv close] 0
337	error_check_good cclose [$clientenv close] 0
338	replclose $testdir/MSGQUEUEDIR
339}
340