1# See the file LICENSE for redistribution information.
2#
3# Copyright (c) 2004-2009 Oracle.  All rights reserved.
4#
5# $Id$
6#
7# TEST  rep025
8# TEST  Test of DB_REP_JOIN_FAILURE.
9# TEST
10# TEST  One master, one client.
11# TEST  Generate several log files.
12# TEST  Remove old master log files.
13# TEST  Delete client files and restart client.
14# TEST  Put one more record to the master.  At the next
15# TEST  processing of messages, the client should get JOIN_FAILURE.
16# TEST  Recover with a hot failover.
17#
18proc rep025 { method { niter 200 } { tnum "025" } args } {
19	source ./include.tcl
20	global databases_in_memory
21	global repfiles_in_memory
22
23	# Run for all access methods.
24	if { $checking_valid_methods } {
25		return "ALL"
26	}
27
28	set args [convert_args $method $args]
29
30	# This test needs to set its own pagesize.
31	set pgindex [lsearch -exact $args "-pagesize"]
32	if { $pgindex != -1 } {
33		puts "Rep$tnum: skipping for specific pagesizes"
34		return
35	}
36
37	set logsets [create_logsets 2]
38
39	# Set up for on-disk or in-memory databases.
40	set msg "using on-disk databases"
41	if { $databases_in_memory } {
42		set msg "using named in-memory databases"
43		if { [is_queueext $method] } {
44			puts -nonewline "Skipping rep$tnum for method "
45			puts "$method with named in-memory databases."
46			return
47		}
48	}
49
50	set msg2 "and on-disk replication files"
51	if { $repfiles_in_memory } {
52		set msg2 "and in-memory replication files"
53	}
54
55	# Run the body of the test with and without recovery,
56	# and with and without cleaning.  Skip recovery with in-memory
57	# logging - it doesn't make sense.
58	foreach r $test_recopts {
59		foreach l $logsets {
60			set logindex [lsearch -exact $l "in-memory"]
61			if { $r == "-recover" && $logindex != -1 } {
62				puts "Skipping rep$tnum for -recover\
63				    with in-memory logs."
64				continue
65			}
66			puts "Rep$tnum ($method $r): Test of manual\
67			    initialization and join failure $msg $msg2."
68			puts "Rep$tnum: Master logs are [lindex $l 0]"
69			puts "Rep$tnum: Client logs are [lindex $l 1]"
70			rep025_sub $method $niter $tnum $l $r $args
71		}
72	}
73}
74
75proc rep025_sub { method niter tnum logset recargs largs } {
76	global testdir
77	global util_path
78	global databases_in_memory
79	global repfiles_in_memory
80	global rep_verbose
81	global verbose_type
82
83	set verbargs ""
84	if { $rep_verbose == 1 } {
85		set verbargs " -verbose {$verbose_type on} "
86	}
87
88	set repmemargs ""
89	if { $repfiles_in_memory } {
90		set repmemargs "-rep_inmem_files "
91	}
92
93	env_cleanup $testdir
94
95	replsetup $testdir/MSGQUEUEDIR
96
97	set masterdir $testdir/MASTERDIR
98	set clientdir $testdir/CLIENTDIR
99
100	file mkdir $masterdir
101	file mkdir $clientdir
102
103	# Log size is small so we quickly create more than one.
104	# The documentation says that the log file must be at least
105	# four times the size of the in-memory log buffer.
106	set pagesize 4096
107	append largs " -pagesize $pagesize "
108	set log_max [expr $pagesize * 8]
109
110	set m_logtype [lindex $logset 0]
111	set c_logtype [lindex $logset 1]
112
113	# In-memory logs cannot be used with -txn nosync.
114	set m_logargs [adjust_logargs $m_logtype]
115	set c_logargs [adjust_logargs $c_logtype]
116	set m_txnargs [adjust_txnargs $m_logtype]
117	set c_txnargs [adjust_txnargs $c_logtype]
118
119	# Open a master.
120	repladd 1
121	set ma_envcmd "berkdb_env_noerr -create $m_txnargs $repmemargs \
122	    $m_logargs -log_max $log_max $verbargs -errpfx MASTER \
123	    -home $masterdir -rep_transport \[list 1 replsend\]"
124	set masterenv [eval $ma_envcmd $recargs -rep_master]
125
126	# Open a client
127	repladd 2
128	set cl_envcmd "berkdb_env_noerr -create $c_txnargs $repmemargs \
129	    $c_logargs -log_max $log_max $verbargs -errpfx CLIENT \
130	    -home $clientdir -rep_transport \[list 2 replsend\]"
131	set clientenv [eval $cl_envcmd $recargs -rep_client]
132
133	# Bring the clients online by processing the startup messages.
134	set envlist "{$masterenv 1} {$clientenv 2}"
135	process_msgs $envlist
136
137	# Clobber replication's 30-second anti-archive timer, which will have
138	# been started by client sync-up internal init, so that we can do a
139	# log_archive in a moment.
140	#
141	$masterenv test force noarchive_timeout
142
143	# Run a modified test001 in the master (and update client).
144	puts "\tRep$tnum.a: Running rep_test in replicated env."
145	set start 0
146	eval rep_test $method $masterenv NULL $niter $start $start 0 $largs
147	incr start $niter
148	process_msgs $envlist
149
150	# Find out what exists on the client.  We need to loop until
151	# the first master log file > last client log file.
152	puts "\tRep$tnum.b: Close client."
153	if { $c_logtype != "in-memory" } {
154		set res [eval exec $util_path/db_archive -l -h $clientdir]
155	}
156	set last_client_log [get_logfile $clientenv last]
157	error_check_good client_close [$clientenv close] 0
158
159	set stop 0
160	while { $stop == 0 } {
161		# Run rep_test in the master (don't update client).
162		puts "\tRep$tnum.c: Running rep_test in replicated env."
163	 	eval rep_test \
164		    $method $masterenv NULL $niter $start $start 0 $largs
165		incr start $niter
166		replclear 2
167
168		puts "\tRep$tnum.d: Run db_archive on master."
169		if { $m_logtype != "in-memory"} {
170			set res [eval exec $util_path/db_archive -d -h $masterdir]
171		}
172		set first_master_log [get_logfile $masterenv first]
173		if { $first_master_log > $last_client_log } {
174			set stop 1
175		}
176	}
177
178	puts "\tRep$tnum.e: Clean client and reopen."
179	env_cleanup $clientdir
180	set clientenv [eval $cl_envcmd $recargs -rep_client]
181	error_check_good client_env [is_valid_env $clientenv] TRUE
182	set envlist "{$masterenv 1} {$clientenv 2}"
183
184	# Set initialization to manual.
185	$clientenv rep_config {noautoinit on}
186	process_msgs $envlist 0 NONE err
187	error_check_good error_on_right_env [lindex $err 0] $clientenv
188	error_check_good right_error [is_substr $err DB_REP_JOIN_FAILURE] 1
189
190	# Add records to the master and update client.
191	puts "\tRep$tnum.f: Update master; client should return error."
192	#
193	# Force a log record to create a gap to force rerequest.
194	#
195	$masterenv txn_checkpoint -force
196	process_msgs $envlist 0 NONE err
197	tclsleep 1
198	set entries 100
199	eval rep_test $method $masterenv NULL $entries $start $start 0 $largs
200	incr start $entries
201	process_msgs $envlist 0 NONE err
202	error_check_good error_on_right_env [lindex $err 0] $clientenv
203	error_check_good right_error [is_substr $err DB_REP_JOIN_FAILURE] 1
204
205	# If the master logs and the databases are on-disk, copy from master
206	# to client and restart with recovery.  If the logs or databases are
207	# in-memory, we'll have to re-enable internal initialization and
208	# restart the client.
209	if { $m_logtype == "on-disk" && $databases_in_memory == 0 } {
210		puts "\tRep$tnum.g: Hot failover and catastrophic recovery."
211		error_check_good client_close [$clientenv close] 0
212		env_cleanup $clientdir
213		set files [glob $masterdir/log.* $masterdir/*.db]
214		foreach f $files {
215			set filename [file tail $f]
216			file copy -force $f $clientdir/$filename
217		}
218		set clientenv [eval $cl_envcmd -recover_fatal -rep_client]
219	} else {
220		puts "\tRep$tnum.g: Restart client forcing internal init."
221		set clientenv [eval $cl_envcmd -rep_client]
222		$clientenv rep_config {noautoinit off}
223	}
224	error_check_good client_env [is_valid_env $clientenv] TRUE
225	set envlist "{$masterenv 1} {$clientenv 2}"
226	process_msgs $envlist 0 NONE err
227	error_check_good no_errors1 $err 0
228
229	# Adding another entry should not flush out an error.
230	eval rep_test $method $masterenv NULL $entries $start $start 0 $largs
231	process_msgs $envlist 0 NONE err
232	error_check_good no_errors2 $err 0
233
234	error_check_good masterenv_close [$masterenv close] 0
235	error_check_good clientenv_close [$clientenv close] 0
236	replclose $testdir/MSGQUEUEDIR
237}
238