1# See the file LICENSE for redistribution information.
2#
3# Copyright (c) 1996-2009 Oracle.  All rights reserved.
4#
5# $Id$
6#
7# TEST	test017
8# TEST	Basic offpage duplicate test.
9# TEST
10# TEST	Run duplicates with small page size so that we test off page duplicates.
11# TEST	Then after we have an off-page database, test with overflow pages too.
12proc test017 { method {contents 0} {ndups 19} {tnum "017"} args } {
13	source ./include.tcl
14
15	set args [convert_args $method $args]
16	set omethod [convert_method $method]
17
18	# Btree with compression does not support unsorted duplicates.
19	if { [is_compressed $args] == 1 } {
20		puts "Test$tnum skipping for btree with compression."
21		return
22	}
23
24	if { [is_record_based $method] == 1 || [is_rbtree $method] == 1 } {
25		puts "Test$tnum skipping for method $method"
26		return
27	}
28	set pgindex [lsearch -exact $args "-pagesize"]
29	if { $pgindex != -1 } {
30		incr pgindex
31		if { [lindex $args $pgindex] > 8192 } {
32			puts "Test$tnum: Skipping for large pagesizes"
33			return
34		}
35	}
36
37	# Create the database and open the dictionary
38	set limit 0
39	set txnenv 0
40	set eindex [lsearch -exact $args "-env"]
41	#
42	# If we are using an env, then testfile should just be the db name.
43	# Otherwise it is the test directory and the name.
44	if { $eindex == -1 } {
45		set testfile $testdir/test$tnum.db
46		set env NULL
47	} else {
48		set testfile test$tnum.db
49		incr eindex
50		set env [lindex $args $eindex]
51		set txnenv [is_txnenv $env]
52		if { $txnenv == 1 } {
53			append args " -auto_commit "
54			set limit 100
55		}
56		set testdir [get_home $env]
57	}
58	set t1 $testdir/t1
59	set t2 $testdir/t2
60	set t3 $testdir/t3
61	set t4 $testdir/t4
62
63	cleanup $testdir $env
64
65	set db [eval {berkdb_open \
66	     -create -mode 0644 -dup} $args {$omethod $testfile}]
67	error_check_good dbopen [is_valid_db $db] TRUE
68
69	set pflags ""
70	set gflags ""
71	set txn ""
72	set count 0
73
74	set file_list [get_file_list 1]
75	if { $txnenv == 1 } {
76		if { [llength $file_list] > $limit } {
77			set file_list [lrange $file_list 0 $limit]
78		}
79		set flen [llength $file_list]
80		reduce_dups flen ndups
81	}
82	puts "Test$tnum: $method ($args) Off page duplicate tests\
83	    with $ndups duplicates"
84
85	set ovfl ""
86	# Here is the loop where we put and get each key/data pair
87	puts -nonewline "\tTest$tnum.a: Creating duplicates with "
88	if { $contents != 0 } {
89		puts "file contents as key/data"
90	} else {
91		puts "file name as key/data"
92	}
93	foreach f $file_list {
94		if { $contents != 0 } {
95			set fid [open $f r]
96			fconfigure $fid -translation binary
97			#
98			# Prepend file name to guarantee uniqueness
99			set filecont [read $fid]
100			set str $f:$filecont
101			close $fid
102		} else {
103			set str $f
104		}
105		for { set i 1 } { $i <= $ndups } { incr i } {
106			set datastr $i:$str
107			if { $txnenv == 1 } {
108				set t [$env txn]
109				error_check_good txn [is_valid_txn $t $env] TRUE
110				set txn "-txn $t"
111			}
112			set ret [eval {$db put} \
113			    $txn $pflags {$str [chop_data $method $datastr]}]
114			error_check_good put $ret 0
115			if { $txnenv == 1 } {
116				error_check_good txn [$t commit] 0
117			}
118		}
119
120		#
121		# Save 10% files for overflow test
122		#
123		if { $contents == 0 && [expr $count % 10] == 0 } {
124			lappend ovfl $f
125		}
126		# Now retrieve all the keys matching this key
127		set ret [$db get $str]
128		error_check_bad $f:dbget_dups [llength $ret] 0
129		error_check_good $f:dbget_dups1 [llength $ret] $ndups
130		set x 1
131		if { $txnenv == 1 } {
132			set t [$env txn]
133			error_check_good txn [is_valid_txn $t $env] TRUE
134			set txn "-txn $t"
135		}
136		set dbc [eval {$db cursor} $txn]
137		for {set ret [$dbc get "-set" $str]} \
138		    {[llength $ret] != 0} \
139		    {set ret [$dbc get "-next"] } {
140			set k [lindex [lindex $ret 0] 0]
141			if { [string compare $k $str] != 0 } {
142				break
143			}
144			set datastr [lindex [lindex $ret 0] 1]
145			set d [data_of $datastr]
146			if {[string length $d] == 0} {
147				break
148			}
149			error_check_good "Test$tnum:get" $d $str
150			set id [ id_of $datastr ]
151			error_check_good "Test$tnum:$f:dup#" $id $x
152			incr x
153		}
154		error_check_good "Test$tnum:ndups:$str" [expr $x - 1] $ndups
155		error_check_good cursor_close [$dbc close] 0
156		if { $txnenv == 1 } {
157			error_check_good txn [$t commit] 0
158		}
159		incr count
160	}
161
162	# Now we will get each key from the DB and compare the results
163	# to the original.
164	puts "\tTest$tnum.b: Checking file for correct duplicates"
165	set dlist ""
166	for { set i 1 } { $i <= $ndups } {incr i} {
167		lappend dlist $i
168	}
169	set oid [open $t2.tmp w]
170	set o1id [open $t4.tmp w]
171	foreach f $file_list {
172		for {set i 1} {$i <= $ndups} {incr i} {
173			puts $o1id $f
174		}
175		puts $oid $f
176	}
177	close $oid
178	close $o1id
179	filesort $t2.tmp $t2
180	filesort $t4.tmp $t4
181	fileremove $t2.tmp
182	fileremove $t4.tmp
183
184	if { $txnenv == 1 } {
185		set t [$env txn]
186		error_check_good txn [is_valid_txn $t $env] TRUE
187		set txn "-txn $t"
188	}
189	dup_check $db $txn $t1 $dlist
190	if { $txnenv == 1 } {
191		error_check_good txn [$t commit] 0
192	}
193	if {$contents == 0} {
194		filesort $t1 $t3
195
196		error_check_good Test$tnum:diff($t3,$t2) [filecmp $t3 $t2] 0
197
198		# Now compare the keys to see if they match the file names
199		if { $txnenv == 1 } {
200			set t [$env txn]
201			error_check_good txn [is_valid_txn $t $env] TRUE
202			set txn "-txn $t"
203		}
204		dump_file $db $txn $t1 test017.check
205		if { $txnenv == 1 } {
206			error_check_good txn [$t commit] 0
207		}
208		filesort $t1 $t3
209
210		error_check_good Test$tnum:diff($t3,$t4) [filecmp $t3 $t4] 0
211	}
212
213	error_check_good db_close [$db close] 0
214	set db [eval {berkdb_open} $args $testfile]
215	error_check_good dbopen [is_valid_db $db] TRUE
216
217	puts "\tTest$tnum.c: Checking file for correct duplicates after close"
218	if { $txnenv == 1 } {
219		set t [$env txn]
220		error_check_good txn [is_valid_txn $t $env] TRUE
221		set txn "-txn $t"
222	}
223	dup_check $db $txn $t1 $dlist
224	if { $txnenv == 1 } {
225		error_check_good txn [$t commit] 0
226	}
227
228	if {$contents == 0} {
229		# Now compare the keys to see if they match the filenames
230		filesort $t1 $t3
231		error_check_good Test$tnum:diff($t3,$t2) [filecmp $t3 $t2] 0
232	}
233	error_check_good db_close [$db close] 0
234
235	puts "\tTest$tnum.d: Verify off page duplicates and overflow status"
236	set db [eval {berkdb_open} $args $testfile]
237	error_check_good dbopen [is_valid_db $db] TRUE
238	set stat [$db stat]
239	if { [is_btree $method] } {
240		error_check_bad stat:offpage \
241		    [is_substr $stat "{{Internal pages} 0}"] 1
242	}
243	if {$contents == 0} {
244		# This check doesn't work in hash, since overflow
245		# pages count extra pages in buckets as well as true
246		# P_OVERFLOW pages.
247		if { [is_hash $method] == 0 } {
248			error_check_good overflow \
249			    [is_substr $stat "{{Overflow pages} 0}"] 1
250		}
251	} else {
252		if { [is_hash $method] } {
253			error_check_bad overflow \
254			    [is_substr $stat "{{Number of big pages} 0}"] 1
255		} else {
256			error_check_bad overflow \
257			    [is_substr $stat "{{Overflow pages} 0}"] 1
258		}
259	}
260
261	#
262	# If doing overflow test, do that now.  Else we are done.
263	# Add overflow pages by adding a large entry to a duplicate.
264	#
265	if { [llength $ovfl] == 0} {
266		error_check_good db_close [$db close] 0
267		return
268	}
269
270	puts "\tTest$tnum.e: Add overflow duplicate entries"
271	set ovfldup [expr $ndups + 1]
272	foreach f $ovfl {
273		#
274		# This is just like put_file, but prepends the dup number
275		#
276		set fid [open $f r]
277		fconfigure $fid -translation binary
278		set fdata [read $fid]
279		close $fid
280		set data $ovfldup:$fdata:$fdata:$fdata:$fdata
281
282		if { $txnenv == 1 } {
283			set t [$env txn]
284			error_check_good txn [is_valid_txn $t $env] TRUE
285			set txn "-txn $t"
286		}
287		set ret [eval {$db put} $txn $pflags {$f $data}]
288		error_check_good ovfl_put $ret 0
289		if { $txnenv == 1 } {
290			error_check_good txn [$t commit] 0
291		}
292	}
293
294	puts "\tTest$tnum.f: Verify overflow duplicate entries"
295	if { $txnenv == 1 } {
296		set t [$env txn]
297		error_check_good txn [is_valid_txn $t $env] TRUE
298		set txn "-txn $t"
299	}
300	dup_check $db $txn $t1 $dlist $ovfldup
301	if { $txnenv == 1 } {
302		error_check_good txn [$t commit] 0
303	}
304	filesort $t1 $t3
305	error_check_good Test$tnum:diff($t3,$t2) [filecmp $t3 $t2] 0
306
307	set stat [$db stat]
308	if { [is_hash [$db get_type]] } {
309		error_check_bad overflow1_hash [is_substr $stat \
310		    "{{Number of big pages} 0}"] 1
311	} else {
312		error_check_bad \
313		    overflow1 [is_substr $stat "{{Overflow pages} 0}"] 1
314	}
315	error_check_good db_close [$db close] 0
316}
317
318# Check function; verify data contains key
319proc test017.check { key data } {
320	error_check_good "data mismatch for key $key" $key [data_of $data]
321}
322