1# See the file LICENSE for redistribution information.
2#
3# Copyright (c) 2004-2009 Oracle.  All rights reserved.
4#
5# $Id$
6#
7# TEST	rep026
8# TEST	Replication elections - simulate a crash after sending
9# TEST	a vote.
10
11proc rep026 { method args } {
12	source ./include.tcl
13
14	global mixed_mode_logging
15	global databases_in_memory
16	global repfiles_in_memory
17
18	set tnum "026"
19	if { $is_windows9x_test == 1 } {
20		puts "Skipping replication test on Win 9x platform."
21		return
22	}
23
24	# Run for btree only.
25	if { $checking_valid_methods } {
26		set test_methods { btree }
27		return $test_methods
28	}
29	if { [is_btree $method] == 0 } {
30		puts "Rep$tnum: Skipping for method $method."
31		return
32	}
33
34	# This test uses recovery, so mixed-mode testing and in-memory
35	# database testing aren't appropriate.
36	if { $mixed_mode_logging > 0  } {
37		puts "Rep$tnum: Skipping for mixed-mode logging."
38		return
39	}
40	if { $databases_in_memory == 1 } {
41		puts "Rep$tnum: Skipping for in-memory databases."
42		return
43	}
44
45	set msg2 "and on-disk replication files"
46	if { $repfiles_in_memory } {
47		set msg2 "and in-memory replication files"
48	}
49
50	global rand_init
51	error_check_good set_random_seed [berkdb srand $rand_init] 0
52
53	set nclients 5
54	set logsets [create_logsets [expr $nclients + 1]]
55	foreach l $logsets {
56		puts "Rep$tnum ($method): Election generations -\
57		    simulate crash after sending a vote $msg2."
58		puts "Rep$tnum: Master logs are [lindex $l 0]"
59		for { set i 0 } { $i < $nclients } { incr i } {
60			puts "Rep$tnum: Client $i logs are\
61			    [lindex $l [expr $i + 1]]"
62		}
63		rep026_sub $method $nclients $tnum $l $args
64	}
65}
66
67proc rep026_sub { method nclients tnum logset largs } {
68	source ./include.tcl
69	global machids
70	global repfiles_in_memory
71	global rep_verbose
72	global verbose_type
73
74	set verbargs ""
75	if { $rep_verbose == 1 } {
76		set verbargs " -verbose {$verbose_type on} "
77	}
78
79	set repmemargs ""
80	if { $repfiles_in_memory } {
81		set repmemargs "-rep_inmem_files "
82	}
83
84	env_cleanup $testdir
85
86	set qdir $testdir/MSGQUEUEDIR
87	replsetup $qdir
88
89	set masterdir $testdir/MASTERDIR
90	file mkdir $masterdir
91	set m_logtype [lindex $logset 0]
92	set m_logargs [adjust_logargs $m_logtype]
93	set m_txnargs [adjust_txnargs $m_logtype]
94
95	for { set i 0 } { $i < $nclients } { incr i } {
96		set clientdir($i) $testdir/CLIENTDIR.$i
97		file mkdir $clientdir($i)
98		set c_logtype($i) [lindex $logset [expr $i + 1]]
99		set c_logargs($i) [adjust_logargs $c_logtype($i)]
100		set c_txnargs($i) [adjust_txnargs $c_logtype($i)]
101	}
102
103	# Open a master.
104	set envlist {}
105	repladd 1
106	set env_cmd(M) "berkdb_env -create -log_max 1000000 $verbargs \
107	    -event rep_event $repmemargs \
108	    -home $masterdir $m_txnargs $m_logargs -rep_master \
109	    -errpfx MASTER -rep_transport \[list 1 replsend\]"
110	set masterenv [eval $env_cmd(M)]
111	lappend envlist "$masterenv 1"
112
113	# Open the clients.
114	for { set i 0 } { $i < $nclients } { incr i } {
115		set envid [expr $i + 2]
116		repladd $envid
117		set env_cmd($i) "berkdb_env_noerr -create $verbargs \
118		    -event rep_event $repmemargs \
119		    -home $clientdir($i) $c_txnargs($i) $c_logargs($i) \
120		    -rep_client -rep_transport \[list $envid replsend\]"
121		set clientenv($i) [eval $env_cmd($i)]
122		error_check_good \
123		    client_env($i) [is_valid_env $clientenv($i)] TRUE
124		lappend envlist "$clientenv($i) $envid"
125	}
126	# Bring the clients online by processing the startup messages.
127	process_msgs $envlist
128
129	# Run a modified test001 in the master.
130	puts "\tRep$tnum.a: Running rep_test in replicated env."
131	set niter 10
132	eval rep_test $method $masterenv NULL $niter 0 0 0 $largs
133	process_msgs $envlist
134	error_check_good masterenv_close [$masterenv close] 0
135	set envlist [lreplace $envlist 0 0]
136
137	foreach pair $envlist {
138		set i [expr [lindex $pair 1] - 2]
139		replclear [expr $i + 2]
140		set err_cmd($i) "none"
141		set crash($i) 0
142		set pri($i) 10
143		if { $rep_verbose == 1 } {
144			$clientenv($i) errpfx CLIENT$i
145			$clientenv($i) verbose $verbose_type on
146			$clientenv($i) errfile /dev/stderr
147			set env_cmd($i) [concat $env_cmd($i) \
148			    "-errpfx CLIENT$i -errfile /dev/stderr"]
149		}
150	}
151
152	# In each case we simulate a crash in client C, recover, and
153	# call a second election.  We vary the caller of the second
154	# election (C or some other client) and when the election
155	# messages from before the crash are processed - before or
156	# after the second election.
157	#
158	foreach option { "1 b before" "2 c before" "1 d after" "2 e after"} {
159		# Elector 1 calls the first election, elector 2
160		# calls the second election.
161		set elector1 1
162		set elector2 [lindex $option 0]
163		set let [lindex $option 1]
164		set restore [lindex $option 2]
165
166		if { $elector1 == $elector2 } {
167			puts "\tRep$tnum.$let: Simulated crash and recovery\
168			    (crashing client calls second election)."
169		} else {
170			puts "\tRep$tnum.$let: Simulated crash and recovery\
171			    (non-crashing client calls second election)."
172		}
173
174		puts "\tRep$tnum.$let: Process messages from crasher\
175		    $restore 2nd election."
176
177		puts "\t\tRep$tnum.$let.1: Note egens for all clients."
178		# Note egens for all the clients.
179		foreach pair $envlist {
180			set i [expr [lindex $pair 1] - 2]
181			set clientenv($i) [lindex $pair 0]
182			set egen($i) [stat_field \
183			    $clientenv($i) rep_stat "Election generation number"]
184		}
185
186		# Call an election which simulates a crash after sending
187		# its VOTE1.
188		set msg "\tRep$tnum.$let.2"
189		puts "\t$msg: Start election, simulate a crash."
190		set nsites $nclients
191		set nvotes $nclients
192		# Make the winner the crashing client, since the
193		# crashing client will have the biggest LSN.
194		set elector 1
195		set winner $elector
196		set crash($elector) 1
197		setpriority pri $nclients $winner
198		set err_cmd($elector) "electvote1"
199		run_election env_cmd envlist err_cmd pri crash $qdir \
200		    $msg $elector $nsites $nvotes $nclients $winner 0 test.db
201		set msg "\tRep$tnum.$let.3"
202		puts "\t$msg: Close and reopen elector with recovery."
203		error_check_good \
204		    clientenv_close($elector) [$clientenv($elector) close] 0
205
206		# Have other clients SKIP the election messages and process
207		# only C's startup messages.  We'll do it by copying the files
208		# and emptying the originals.
209		set cwd [pwd]
210		foreach machid $machids {
211			file copy -force $qdir/repqueue$machid.db $qdir/save$machid.db
212			replclear $machid
213		}
214
215		# Reopen C and process messages.  Only the startup messages
216		# will be available.
217		set clientenv($elector) [eval $env_cmd($elector) -recover]
218		set envlist [lreplace $envlist \
219		    $elector $elector "$clientenv($elector) [expr $elector + 2]"]
220		process_msgs $envlist
221
222		# Verify egens (should be +1 in C, and unchanged
223		# in other clients).
224		foreach pair $envlist {
225			set i [expr [lindex $pair 1] - 2]
226			set clientenv($i) [lindex $pair 0]
227			set newegen($i) [stat_field $clientenv($i) \
228			    rep_stat "Election generation number"]
229			if { $i == $elector && $repfiles_in_memory == 0 } {
230				error_check_good \
231				    egen+1 $newegen($i) [expr $egen($i) + 1]
232			} else {
233				error_check_good \
234				    egen_unchanged $newegen($i) $egen($i)
235			}
236		}
237
238		# First chance to restore messages.
239		if { $restore == "before" } {
240			restore_messages $qdir
241		}
242
243		# Have C call an election (no crash simulation) and process
244		# all the messages.
245		set msg "\tRep$tnum.$let.4"
246		puts "\t$msg: Call second election."
247		set err_cmd($elector) "none"
248		set crash($elector) 0
249		run_election env_cmd envlist err_cmd pri crash $qdir \
250		    $msg $elector2 $nsites $nvotes $nclients $winner 1 test.db
251
252		# Second chance to restore messages.
253		if { $restore == "after" } {
254			restore_messages $qdir
255		}
256		process_msgs $envlist
257
258		# Verify egens (should be +2 or more in all clients).
259		puts "\t\tRep$tnum.$let.5: Check egens."
260		foreach pair $envlist {
261			set i [expr [lindex $pair 1] - 2]
262			set clientenv($i) [lindex $pair 0]
263			set newegen($i) [stat_field \
264			    $clientenv($i) rep_stat "Election generation number"]
265
266			# If rep files are in-memory, egen value must come
267			# from other sites instead of the egen file, and
268			# will not increase as quickly.
269			if { $repfiles_in_memory } {
270				set mingen [expr $egen($i) + 1]
271			} else {
272				set mingen [expr $egen($i) + 2]
273			}
274			error_check_good egen+more($i) \
275			    [expr $newegen($i) >= $mingen] 1
276		}
277	}
278
279	# Clean up.
280	foreach pair $envlist {
281		set cenv [lindex $pair 0]
282		error_check_good cenv_close [$cenv close] 0
283	}
284	replclose $testdir/MSGQUEUEDIR
285}
286
287proc restore_messages { qdir } {
288	global machids
289	set cwd [pwd]
290	foreach machid $machids {
291		file copy -force $qdir/save$machid.db $qdir/repqueue$machid.db
292	}
293}
294
295