1# See the file LICENSE for redistribution information.
2#
3# Copyright (c) 2004-2009 Oracle.  All rights reserved.
4#
5# $Id$
6#
7# TEST  	rep035
8# TEST	Test sync-up recovery in replication.
9# TEST
10# TEST	We need to fork off 3 child tclsh processes to operate
11# TEST	on Site 3's (client always) home directory:
12# TEST	Process 1 continually calls lock_detect.
13# TEST	Process 2 continually calls txn_checkpoint.
14# TEST	Process 3 continually calls memp_trickle.
15# TEST	Process 4 continually calls log_archive.
16# TEST	Sites 1 and 2 will continually swap being master
17# TEST	(forcing site 3 to continually run sync-up recovery)
18# TEST 	New master performs 1 operation, replicates and downgrades.
19
20proc rep035 { method { niter 100 } { tnum "035" } args } {
21
22	source ./include.tcl
23	global databases_in_memory
24	global repfiles_in_memory
25
26	if { $is_windows9x_test == 1 } {
27		puts "Skipping replication test on Win 9x platform."
28		return
29	}
30
31	# Valid for all access methods.
32	if { $checking_valid_methods } {
33		return "ALL"
34	}
35
36	set saved_args $args
37	set logsets [create_logsets 3]
38
39	# Set up for on-disk or in-memory databases.
40	set msg "using on-disk databases"
41	if { $databases_in_memory } {
42		set msg "using named in-memory databases"
43		if { [is_queueext $method] } {
44			puts -nonewline "Skipping rep$tnum for method "
45			puts "$method with named in-memory databases."
46			return
47		}
48	}
49
50	set msg2 "and on-disk replication files"
51	if { $repfiles_in_memory } {
52		set msg2 "and in-memory replication files"
53	}
54
55	foreach l $logsets {
56		set envargs ""
57		set args $saved_args
58		puts "Rep$tnum: Test sync-up recovery ($method) $msg $msg2."
59		puts "Rep$tnum: Master logs are [lindex $l 0]"
60		puts "Rep$tnum: Client 0 logs are [lindex $l 1]"
61		puts "Rep$tnum: Client 1 logs are [lindex $l 2]"
62		rep035_sub $method $niter $tnum $envargs $l $args
63	}
64}
65
66proc rep035_sub { method niter tnum envargs logset largs } {
67	source ./include.tcl
68	global testdir
69	global databases_in_memory
70	global repfiles_in_memory
71	global rep_verbose
72	global verbose_type
73
74	set verbargs ""
75	if { $rep_verbose == 1 } {
76		set verbargs " -verbose {$verbose_type on} "
77	}
78
79	set repmemargs ""
80	if { $repfiles_in_memory } {
81		set repmemargs "-rep_inmem_files "
82	}
83
84	env_cleanup $testdir
85
86	replsetup $testdir/MSGQUEUEDIR
87
88	set masterdir $testdir/MASTERDIR
89	set clientdir1 $testdir/CLIENTDIR1
90	set clientdir2 $testdir/CLIENTDIR2
91
92	file mkdir $masterdir
93	file mkdir $clientdir1
94	file mkdir $clientdir2
95
96	set m_logtype [lindex $logset 0]
97	set c_logtype [lindex $logset 1]
98	set c2_logtype [lindex $logset 2]
99
100	# In-memory logs require a large log buffer, and cannot
101	# be used with -txn nosync.
102	set m_logargs [adjust_logargs $m_logtype]
103	set c_logargs [adjust_logargs $c_logtype]
104	set c2_logargs [adjust_logargs $c2_logtype]
105	set m_txnargs [adjust_txnargs $m_logtype]
106	set c_txnargs [adjust_txnargs $c_logtype]
107	set c2_txnargs [adjust_txnargs $c2_logtype]
108
109	# Open a master.
110	repladd 1
111	set env_cmd(M) "berkdb_env_noerr -create $verbargs $repmemargs \
112	    -log_max 1000000 $envargs -home $masterdir $m_logargs \
113	    -errpfx MASTER -errfile /dev/stderr $m_txnargs -rep_master \
114	    -rep_transport \[list 1 replsend\]"
115	set env1 [eval $env_cmd(M)]
116
117	# Open two clients
118	repladd 2
119	set env_cmd(C1) "berkdb_env_noerr -create $verbargs $repmemargs \
120	    -log_max 1000000 $envargs -home $clientdir1 $c_logargs \
121	    -errfile /dev/stderr -errpfx CLIENT $c_txnargs -rep_client \
122	    -rep_transport \[list 2 replsend\]"
123	set env2 [eval $env_cmd(C1)]
124
125	# Second client needs lock_detect flag.
126	repladd 3
127	set env_cmd(C2) "berkdb_env_noerr -create $verbargs $repmemargs \
128	    -log_max 1000000 $envargs -home $clientdir2 $c2_logargs \
129	    -errpfx CLIENT2 -errfile /dev/stderr $c2_txnargs -rep_client \
130	    -lock_detect default -rep_transport \[list 3 replsend\]"
131	set env3 [eval $env_cmd(C2)]
132	error_check_good client_env [is_valid_env $env3] TRUE
133
134	# Bring the client online by processing the startup messages.
135	set envlist "{$env1 1} {$env2 2} {$env3 3}"
136	process_msgs $envlist
137
138	# Clobber replication's 30-second anti-archive timer, which will have
139	# been started by client sync-up internal init, so that we can do a
140	# log_archive in a moment.
141	#
142	$env1 test force noarchive_timeout
143
144	# We need to fork off 3 child tclsh processes to operate
145	# on Site 3's (client always) home directory:
146	#	Process 1 continually calls lock_detect (DB_LOCK_DEFAULT)
147	#	Process 2 continually calls txn_checkpoint (DB_FORCE)
148	#	Process 3 continually calls memp_trickle (large % like 90)
149	# 	Process 4 continually calls log_archive.
150
151	puts "\tRep$tnum.a: Fork child process running lock_detect on client2."
152	set pid1 [exec $tclsh_path $test_path/wrap.tcl \
153	    rep035script.tcl $testdir/lock_detect.log \
154	    $clientdir2 detect &]
155
156	puts "\tRep$tnum.b:\
157	    Fork child process running txn_checkpoint on client2."
158	set pid2 [exec $tclsh_path $test_path/wrap.tcl \
159	    rep035script.tcl $testdir/txn_checkpoint.log \
160	    $clientdir2 checkpoint &]
161
162	puts "\tRep$tnum.c: Fork child process running memp_trickle on client2."
163	set pid3 [exec $tclsh_path $test_path/wrap.tcl \
164	    rep035script.tcl $testdir/memp_trickle.log \
165	    $clientdir2 trickle &]
166
167	puts "\tRep$tnum.d: Fork child process running log_archive on client2."
168	set pid4 [exec $tclsh_path $test_path/wrap.tcl \
169	    rep035script.tcl $testdir/log_archive.log \
170	    $clientdir2 archive &]
171
172	# Pause a bit to let the children get going.
173	tclsleep 5
174
175	set logfilelist [list lock_detect.log \
176	    txn_checkpoint.log memp_trickle.log log_archive.log]
177	set pidlist [list $pid1 $pid2 $pid3 $pid4]
178
179	#
180	# Sites 1 and 2 will continually swap being master
181	# forcing site 3 to continually run sync-up recovery.
182	# New master performs 1 operation, replicates and downgrades.
183	# Site 3 will always stay a client.
184	#
185	# Set up all the master/client data we're going to need
186	# to keep track of and swap.  Set up the handles for rep_test.
187	#
188
189	set masterenv $env1
190	set mid 1
191	set clientenv $env2
192	set cid 2
193
194	# Set up databases as in-memory or on-disk as specified.
195	if { $databases_in_memory } {
196		set testfile { "" "test$tnum.db" }
197	} else {
198		set testfile "test$tnum.db"
199	}
200
201	set args [convert_args $method]
202	set omethod [convert_method $method]
203	set mdb_cmd "{berkdb_open_noerr} -env $masterenv -auto_commit \
204	     -create $omethod $args -mode 0644 $testfile"
205	set cdb_cmd "{berkdb_open_noerr} -env $clientenv -auto_commit \
206	     $omethod $args -mode 0644 $testfile"
207
208	set masterdb [eval $mdb_cmd]
209	error_check_good dbopen [is_valid_db $masterdb] TRUE
210	process_msgs $envlist
211
212	set clientdb [eval $cdb_cmd]
213	error_check_good dbopen [is_valid_db $clientdb] TRUE
214
215	tclsleep 2
216	puts "\tRep$tnum.e: Swap master and client $niter times."
217	for { set i 0 } { $i < $niter } { incr i } {
218
219		# Do a few ops
220		eval rep_test $method $masterenv $masterdb 2 $i $i 0 $largs
221		set envlist "{$masterenv $mid} {$clientenv $cid} {$env3 3}"
222		process_msgs $envlist
223
224		# Do one op on master and process messages and drop
225		# to clientenv to force sync-up recovery next time.
226		eval rep_test $method $masterenv $masterdb 1 $i $i 0 $largs
227		set envlist "{$masterenv $mid} {$env3 3}"
228		replclear $cid
229		process_msgs $envlist
230
231		# Swap all the info we need.
232		set tmp $masterenv
233		set masterenv $clientenv
234		set clientenv $tmp
235
236		set tmp $masterdb
237		set masterdb $clientdb
238		set clientdb $tmp
239
240		set tmp $mid
241		set mid $cid
242		set cid $tmp
243
244		set tmp $mdb_cmd
245		set mdb_cmd $cdb_cmd
246		set cdb_cmd $tmp
247
248		puts "\tRep$tnum.e.$i: Swap: master $mid, client $cid"
249		error_check_good downgrade [$clientenv rep_start -client] 0
250		error_check_good upgrade [$masterenv rep_start -master] 0
251		set envlist "{$masterenv $mid} {$clientenv $cid} {$env3 3}"
252		process_msgs $envlist
253
254		# Close old and reopen since we will get HANDLE_DEAD
255		# otherwise because we dropped messages to the new master.
256		error_check_good masterdb [$masterdb close] 0
257		error_check_good clientdb [$clientdb close] 0
258
259		set masterdb [eval $mdb_cmd]
260		error_check_good dbopen [is_valid_db $masterdb] TRUE
261
262		set clientdb [eval $cdb_cmd]
263		error_check_good dbopen [is_valid_db $clientdb] TRUE
264		process_msgs $envlist
265	}
266
267	# Communicate with child processes by creating a marker file.
268	set markerenv [berkdb_env_noerr -create -home $testdir -txn]
269	error_check_good markerenv_open [is_valid_env $markerenv] TRUE
270	set marker [eval "berkdb_open_noerr \
271	    -create -btree -auto_commit -env $markerenv marker.db"]
272	error_check_good marker_close [$marker close] 0
273
274	# Wait for child processes; they should shut down quickly.
275	watch_procs $pidlist 1
276
277	# There should not be any messages in the log files.
278	# If there are, print them out.
279	foreach file $logfilelist {
280		puts "\tRep$tnum.f: Checking $file for errors."
281		set fd [open $testdir/$file r]
282		while { [gets $fd str] != -1 } {
283			error "FAIL: found message $str"
284		}
285	}
286
287	error_check_good masterdb [$masterdb close] 0
288	error_check_good clientdb [$clientdb close] 0
289	error_check_good env1_close [$env1 close] 0
290	error_check_good env2_close [$env2 close] 0
291	error_check_good env3_close [$env3 close] 0
292	error_check_good markerenv_close [$markerenv close] 0
293	replclose $testdir/MSGQUEUEDIR
294}
295