1# See the file LICENSE for redistribution information.
2#
3# Copyright (c) 2004,2008 Oracle.  All rights reserved.
4#
5# $Id: rep035.tcl,v 12.19 2008/01/08 20:58:53 bostic Exp $
6#
7# TEST  	rep035
8# TEST	Test sync-up recovery in replication.
9# TEST
10# TEST	We need to fork off 3 child tclsh processes to operate
11# TEST	on Site 3's (client always) home directory:
12# TEST	Process 1 continually calls lock_detect.
13# TEST	Process 2 continually calls txn_checkpoint.
14# TEST	Process 3 continually calls memp_trickle.
15# TEST	Process 4 continually calls log_archive.
16# TEST	Sites 1 and 2 will continually swap being master
17# TEST	(forcing site 3 to continually run sync-up recovery)
18# TEST 	New master performs 1 operation, replicates and downgrades.
19
20proc rep035 { method { niter 100 } { tnum "035" } args } {
21
22	source ./include.tcl
23	if { $is_windows9x_test == 1 } {
24		puts "Skipping replication test on Win 9x platform."
25		return
26	}
27
28	# Valid for all access methods.
29	if { $checking_valid_methods } {
30		return "ALL"
31	}
32
33	set saved_args $args
34	set logsets [create_logsets 3]
35
36	foreach l $logsets {
37		set envargs ""
38		set args $saved_args
39		puts "Rep$tnum: Test sync-up recovery ($method)."
40		puts "Rep$tnum: Master logs are [lindex $l 0]"
41		puts "Rep$tnum: Client 0 logs are [lindex $l 1]"
42		puts "Rep$tnum: Client 1 logs are [lindex $l 2]"
43		rep035_sub $method $niter $tnum $envargs $l $args
44	}
45}
46
47proc rep035_sub { method niter tnum envargs logset largs } {
48	source ./include.tcl
49	global testdir
50	global rep_verbose
51	global verbose_type
52
53	set verbargs ""
54	if { $rep_verbose == 1 } {
55		set verbargs " -verbose {$verbose_type on} "
56	}
57
58	env_cleanup $testdir
59
60	replsetup $testdir/MSGQUEUEDIR
61
62	set masterdir $testdir/MASTERDIR
63	set clientdir1 $testdir/CLIENTDIR1
64	set clientdir2 $testdir/CLIENTDIR2
65
66	file mkdir $masterdir
67	file mkdir $clientdir1
68	file mkdir $clientdir2
69
70	set m_logtype [lindex $logset 0]
71	set c_logtype [lindex $logset 1]
72	set c2_logtype [lindex $logset 2]
73
74	# In-memory logs require a large log buffer, and cannot
75	# be used with -txn nosync.
76	set m_logargs [adjust_logargs $m_logtype]
77	set c_logargs [adjust_logargs $c_logtype]
78	set c2_logargs [adjust_logargs $c2_logtype]
79	set m_txnargs [adjust_txnargs $m_logtype]
80	set c_txnargs [adjust_txnargs $c_logtype]
81	set c2_txnargs [adjust_txnargs $c2_logtype]
82
83	# Open a master.
84	repladd 1
85	set env_cmd(M) "berkdb_env_noerr -create $verbargs \
86	    -log_max 1000000 $envargs -home $masterdir $m_logargs \
87	    -errpfx MASTER -errfile /dev/stderr $m_txnargs -rep_master \
88	    -rep_transport \[list 1 replsend\]"
89	set env1 [eval $env_cmd(M)]
90
91	# Open two clients
92	repladd 2
93	set env_cmd(C1) "berkdb_env_noerr -create $verbargs \
94	    -log_max 1000000 $envargs -home $clientdir1 $c_logargs \
95	    -errfile /dev/stderr -errpfx CLIENT $c_txnargs -rep_client \
96	    -rep_transport \[list 2 replsend\]"
97	set env2 [eval $env_cmd(C1)]
98
99	# Second client needs lock_detect flag.
100	repladd 3
101	set env_cmd(C2) "berkdb_env_noerr -create $verbargs \
102	    -log_max 1000000 $envargs -home $clientdir2 $c2_logargs \
103	    -errpfx CLIENT2 -errfile /dev/stderr $c2_txnargs -rep_client \
104	    -lock_detect default -rep_transport \[list 3 replsend\]"
105	set env3 [eval $env_cmd(C2)]
106	error_check_good client_env [is_valid_env $env3] TRUE
107
108	# Bring the client online by processing the startup messages.
109	set envlist "{$env1 1} {$env2 2} {$env3 3}"
110	process_msgs $envlist
111
112	# Clobber replication's 30-second anti-archive timer, which will have
113	# been started by client sync-up internal init, so that we can do a
114	# log_archive in a moment.
115	#
116	$env1 test force noarchive_timeout
117
118	# We need to fork off 3 child tclsh processes to operate
119	# on Site 3's (client always) home directory:
120	#	Process 1 continually calls lock_detect (DB_LOCK_DEFAULT)
121	#	Process 2 continually calls txn_checkpoint (DB_FORCE)
122	#	Process 3 continually calls memp_trickle (large % like 90)
123	# 	Process 4 continually calls log_archive.
124
125	puts "\tRep$tnum.a: Fork child process running lock_detect on client2."
126	set pid1 [exec $tclsh_path $test_path/wrap.tcl \
127	    rep035script.tcl $testdir/lock_detect.log \
128	    $clientdir2 detect &]
129
130	puts "\tRep$tnum.b: Fork child process running txn_checkpoint on client2."
131	set pid2 [exec $tclsh_path $test_path/wrap.tcl \
132	    rep035script.tcl $testdir/txn_checkpoint.log \
133	    $clientdir2 checkpoint &]
134
135	puts "\tRep$tnum.c: Fork child process running memp_trickle on client2."
136	set pid3 [exec $tclsh_path $test_path/wrap.tcl \
137	    rep035script.tcl $testdir/memp_trickle.log \
138	    $clientdir2 trickle &]
139
140	puts "\tRep$tnum.d: Fork child process running log_archive on client2."
141	set pid4 [exec $tclsh_path $test_path/wrap.tcl \
142	    rep035script.tcl $testdir/log_archive.log \
143	    $clientdir2 archive &]
144
145	# Pause a bit to let the children get going.
146	tclsleep 5
147
148	set logfilelist [list lock_detect.log \
149	    txn_checkpoint.log memp_trickle.log log_archive.log]
150	set pidlist [list $pid1 $pid2 $pid3 $pid4]
151
152	#
153	# Sites 1 and 2 will continually swap being master
154	# forcing site 3 to continually run sync-up recovery.
155	# New master performs 1 operation, replicates and downgrades.
156	# Site 3 will always stay a client.
157	#
158	# Set up all the master/client data we're going to need
159	# to keep track of and swap.  Set up the handles for rep_test.
160	#
161
162	set masterenv $env1
163	set mid 1
164	set clientenv $env2
165	set cid 2
166	set testfile "test$tnum.db"
167	set args [convert_args $method]
168	set omethod [convert_method $method]
169	set mdb_cmd "{berkdb_open_noerr} -env $masterenv -auto_commit \
170	     -create $omethod $args -mode 0644 $testfile"
171	set cdb_cmd "{berkdb_open_noerr} -env $clientenv -auto_commit \
172	     $omethod $args -mode 0644 $testfile"
173
174	set masterdb [eval $mdb_cmd]
175	error_check_good dbopen [is_valid_db $masterdb] TRUE
176	process_msgs $envlist
177
178	set clientdb [eval $cdb_cmd]
179	error_check_good dbopen [is_valid_db $clientdb] TRUE
180
181	tclsleep 2
182	puts "\tRep$tnum.e: Swap master and client $niter times."
183	for { set i 0 } { $i < $niter } { incr i } {
184
185		# Do a few ops
186		eval rep_test $method $masterenv $masterdb 2 $i $i 0 0 $largs
187		set envlist "{$masterenv $mid} {$clientenv $cid} {$env3 3}"
188		process_msgs $envlist
189
190		# Do one op on master and process messages and drop
191		# to clientenv to force sync-up recovery next time.
192		eval rep_test $method $masterenv $masterdb 1 $i $i 0 0 $largs
193		set envlist "{$masterenv $mid} {$env3 3}"
194		replclear $cid
195		process_msgs $envlist
196
197		# Swap all the info we need.
198		set tmp $masterenv
199		set masterenv $clientenv
200		set clientenv $tmp
201
202		set tmp $masterdb
203		set masterdb $clientdb
204		set clientdb $tmp
205
206		set tmp $mid
207		set mid $cid
208		set cid $tmp
209
210		set tmp $mdb_cmd
211		set mdb_cmd $cdb_cmd
212		set cdb_cmd $tmp
213
214		puts "\tRep$tnum.e.$i: Swap: master $mid, client $cid"
215		error_check_good downgrade [$clientenv rep_start -client] 0
216		error_check_good upgrade [$masterenv rep_start -master] 0
217		set envlist "{$masterenv $mid} {$clientenv $cid} {$env3 3}"
218		process_msgs $envlist
219
220		# Close old and reopen since we will get HANDLE_DEAD
221		# otherwise because we dropped messages to the new master.
222		error_check_good masterdb [$masterdb close] 0
223		error_check_good clientdb [$clientdb close] 0
224
225		set masterdb [eval $mdb_cmd]
226		error_check_good dbopen [is_valid_db $masterdb] TRUE
227
228		set clientdb [eval $cdb_cmd]
229		error_check_good dbopen [is_valid_db $clientdb] TRUE
230		process_msgs $envlist
231	}
232
233	# Communicate with child processes by creating a marker file.
234	set markerenv [berkdb_env_noerr -create -home $testdir -txn]
235	error_check_good markerenv_open [is_valid_env $markerenv] TRUE
236	set marker [eval "berkdb_open_noerr \
237	    -create -btree -auto_commit -env $markerenv marker.db"]
238	error_check_good marker_close [$marker close] 0
239
240	# Wait for child processes; they should shut down quickly.
241	watch_procs $pidlist 1
242
243	# There should not be any messages in the log files.
244	# If there are, print them out.
245	foreach file $logfilelist {
246		puts "\tRep$tnum.f: Checking $file for errors."
247		set fd [open $testdir/$file r]
248		while { [gets $fd str] != -1 } {
249			error "FAIL: found message $str"
250		}
251	}
252
253	error_check_good masterdb [$masterdb close] 0
254	error_check_good clientdb [$clientdb close] 0
255	error_check_good env1_close [$env1 close] 0
256	error_check_good env2_close [$env2 close] 0
257	error_check_good env3_close [$env3 close] 0
258	error_check_good markerenv_close [$markerenv close] 0
259	replclose $testdir/MSGQUEUEDIR
260}
261