1# See the file LICENSE for redistribution information.
2#
3# Copyright (c) 2007,2008 Oracle.  All rights reserved.
4#
5# $Id: rep074.tcl,v 12.8 2008/04/09 16:26:32 carol Exp $
6#
7# TEST	rep074
8# TEST	Verify replication withstands send errors processing requests.
9# TEST
10# TEST	Run for btree only because access method shouldn't matter.
11# TEST
12proc rep074 { method { niter 20 } { tnum "074" } args } {
13
14	source ./include.tcl
15
16	if { $is_windows9x_test == 1 } {
17		puts "Skipping replication test on Win9x platform."
18		return
19	}
20
21	# Skip for all methods except btree.
22	if { $checking_valid_methods } {
23		return btree
24	}
25	if { [is_btree $method] == 0 } {
26		puts "Rep$tnum: skipping for non-btree method $method."
27		return
28	}
29
30	set args [convert_args $method $args]
31
32	set logsets [create_logsets 2]
33
34	foreach l $logsets {
35		puts "Rep$tnum ($method): Test of send errors processing \
36		    requests"
37		puts "Rep$tnum: Master logs are [lindex $l 0]"
38		puts "Rep$tnum: Client logs are [lindex $l 1]"
39		rep074_sub $method $niter $tnum $l $args
40	}
41}
42
43proc rep074_sub { method niter tnum logset largs } {
44	global testdir rep074_failure_count
45	global rep_verbose
46	global verbose_type
47
48	set rep074_failure_count -1
49	set verbargs ""
50	if { $rep_verbose == 1 } {
51		set verbargs " -verbose {$verbose_type on} "
52	}
53
54	env_cleanup $testdir
55
56	replsetup $testdir/MSGQUEUEDIR
57
58	set masterdir $testdir/MASTERDIR
59	set clientdir $testdir/CLIENTDIR
60
61	file mkdir $masterdir
62	file mkdir $clientdir
63
64	set m_logtype [lindex $logset 0]
65	set c_logtype [lindex $logset 1]
66
67	# In-memory logs require a large log buffer, and cannot
68	# be used with -txn nosync.  Adjust the args for master
69	# and client.
70	set m_logargs [adjust_logargs $m_logtype]
71	set c_logargs [adjust_logargs $c_logtype]
72	set m_txnargs [adjust_txnargs $m_logtype]
73	set c_txnargs [adjust_txnargs $c_logtype]
74
75	# Open a master.
76	repladd 1
77	set ma_envcmd "berkdb_env_noerr -create $verbargs -errpfx MASTER \
78	    -home $masterdir $m_logargs $m_txnargs \
79	    -rep_transport \[list 1 rep074_replsend\]"
80	set masterenv [eval $ma_envcmd -rep_master]
81
82	# Create some new records, so that the master will have something
83	# substantial to say when asked for LOG_REQ.
84	#
85	puts "\tRep$tnum.a: Running rep_test in replicated env."
86	eval rep_test $method $masterenv NULL $niter 0 0 0 0 $largs
87
88	# Open a client
89	repladd 2
90	set cl_envcmd "berkdb_env_noerr -create $verbargs -errpfx CLIENT \
91	    -home $clientdir $c_logargs $c_txnargs \
92	    -rep_transport \[list 2 replsend\]"
93	set clientenv [eval $cl_envcmd -rep_client]
94	set envlist "{$masterenv 1} {$clientenv 2}"
95
96	# Bring the client online by processing the startup messages.  This will
97	# cause the client to send a request to the master.
98	#
99	# In the first cycle, the client gets NEWMASTER and sends an UPDATE_REQ.
100	# In the second cycle, the master answers the UPDATE_REQ with an UPDATE,
101	# and the client sends a PAGE_REQ.  Third, once we've gotten pages, we
102	# send a LOG_REQ.
103	#
104	# 1. NEWCLIENT -> NEWMASTER -> UPDATE_REQ
105	# 2.              UPDATE -> PAGE_REQ
106	# 3.              PAGE -> LOG_REQ
107	#
108	puts "\tRep$tnum.b: NEWMASTER -> UPDATE_REQ"
109	proc_msgs_once $envlist
110	puts "\tRep$tnum.c: UPDATE -> PAGE_REQ"
111	proc_msgs_once $envlist
112	puts "\tRep$tnum.d: PAGE -> LOG_REQ"
113	proc_msgs_once $envlist
114
115	# Force a sending error at the master while processing the LOG_REQ.
116	# We should ignore it, and return success to rep_proces_message
117	#
118	puts "\tRep$tnum.e: Simulate a send error."
119	set rep074_failure_count [expr $niter / 2]
120	proc_msgs_once $envlist NONE errorp
121
122	puts "\tRep$tnum.f: Check for good return from rep_process_msg."
123	error_check_good rep_resilient $errorp 0
124
125	# Since we interrupted the flow with the simulated error, we don't have
126	# the log records we need yet.
127	#
128	error_check_bad startupdone \
129	    [stat_field $clientenv rep_stat "Startup complete"] 1
130
131	# Run some more new txns at the master, so that the client eventually
132	# decides to request the remainder of the LOG_REQ response that it's
133	# missing.  Pause for a second to make sure we reach the lower
134	# threshold for re-request on fast machines.
135	#
136	set rep074_failure_count -1
137	tclsleep 1
138	eval rep_test $method $masterenv NULL $niter 0 0 0 0 $largs
139	process_msgs $envlist
140
141	error_check_good startupdone \
142	    [stat_field $clientenv rep_stat "Startup complete"] 1
143
144	$masterenv close
145	$clientenv close
146	replclose $testdir/MSGQUEUEDIR
147}
148
149# Failure count < 0 turns off any special failure simulation processing.
150# When the count is > 0, it means we should process that many messages normally,
151# before invoking a failure.
152#
153proc rep074_replsend { control rec fromid toid flags lsn } {
154	global rep074_failure_count
155
156	if { $rep074_failure_count < 0 } {
157		return [replsend $control $rec $fromid $toid $flags $lsn]
158	}
159
160	if { $rep074_failure_count > 0 } {
161		incr rep074_failure_count -1
162		return [replsend $control $rec $fromid $toid $flags $lsn]
163	}
164
165	# Return an arbitrary non-zero value to indicate an error.
166	return 1
167}
168