1# See the file LICENSE for redistribution information.
2#
3# Copyright (c) 2005-2009 Oracle.  All rights reserved.
4#
5# $Id$
6#
7# TEST	test115
8# TEST	Test database compaction with user-specified btree sort.
9# TEST
10# TEST	This is essentially test111 with the user-specified sort.
11# TEST	Populate a database.  Remove a high proportion of entries.
12# TEST	Dump and save contents.  Compact the database, dump again,
13# TEST	and make sure we still have the same contents.
14# TEST	Add back some entries, delete more entries (this time by
15# TEST	cursor), dump, compact, and do the before/after check again.
16
17proc test115 { method {nentries 10000} {tnum "115"} args } {
18	source ./include.tcl
19	global btvals
20	global btvalsck
21	global encrypt
22	global passwd
23
24	if { [is_btree $method] != 1 } {
25		puts "Skipping test$tnum for method $method."
26		return
27	}
28
29	# If a page size was specified, find out what it is.  Pages
30	# might not be freed in the case of really large pages (64K)
31	# but we still want to run this test just to make sure
32	# nothing funny happens.
33	set pagesize 0
34	set pgindex [lsearch -exact $args "-pagesize"]
35	if { $pgindex != -1 } {
36		incr pgindex
37		set pagesize [lindex $args $pgindex]
38	}
39
40	set args [convert_args $method $args]
41	set encargs ""
42	set args [split_encargs $args encargs]
43	set omethod [convert_method $method]
44
45	# If we are using an env, then testfile should just be the db name.
46	# Otherwise it is the test directory and the name.
47	set txnenv 0
48	set eindex [lsearch -exact $args "-env"]
49	if { $eindex == -1 } {
50		set basename $testdir/test$tnum
51		set env NULL
52		set envargs ""
53	} else {
54		set basename test$tnum
55		incr eindex
56		set env [lindex $args $eindex]
57		set envargs " -env $env "
58		set rpcenv [is_rpcenv $env]
59		if { $rpcenv == 1 } {
60			puts "Test$tnum: skipping for RPC."
61			return
62		}
63		set txnenv [is_txnenv $env]
64		if { $txnenv == 1 } {
65			append args " -auto_commit "
66		}
67		set testdir [get_home $env]
68	}
69
70	puts "Test$tnum:\
71	    ($method $args $encargs) Database compaction with user-specified sort."
72
73	cleanup $testdir $env
74	set t1 $testdir/t1
75	set t2 $testdir/t2
76	set splitopts { "" "-revsplitoff" }
77	set txn ""
78
79	set checkfunc test093_check
80
81	foreach splitopt $splitopts {
82		set testfile $basename.db
83		if { $splitopt == "-revsplitoff" } {
84			set testfile $basename.rev.db
85		}
86		set did [open $dict]
87
88		puts "\tTest$tnum.a: Create and populate database ($splitopt)."
89		set db [eval {berkdb_open -create -btcompare test093_cmp1 \
90		    -mode 0644} $splitopt $args $encargs $omethod $testfile]
91		error_check_good dbopen [is_valid_db $db] TRUE
92
93		set count 0
94		set btvals {}
95		set btvalsck {}
96		if { $txnenv == 1 } {
97			set t [$env txn]
98			error_check_good txn [is_valid_txn $t $env] TRUE
99			set txn "-txn $t"
100		}
101		while { [gets $did str] != -1 && $count < $nentries } {
102			set key $str
103			set str [reverse $str]
104
105			set ret [eval \
106			    {$db put} $txn {$key [chop_data $method $str]}]
107			error_check_good put $ret 0
108			lappend btvals $key
109			incr count
110
111		}
112		if { $txnenv == 1 } {
113			error_check_good txn_commit [$t commit] 0
114		}
115		close $did
116		error_check_good db_sync [$db sync] 0
117
118		if { $env != "NULL" } {
119			set testdir [get_home $env]
120			set filename $testdir/$testfile
121		} else {
122			set filename $testfile
123		}
124		set size1 [file size $filename]
125		set free1 [stat_field $db stat "Pages on freelist"]
126		set leaf1 [stat_field $db stat "Leaf pages"]
127		set internal1 [stat_field $db stat "Internal pages"]
128
129		puts "\tTest$tnum.b: Delete most entries from database."
130		set did [open $dict]
131		set count [expr $nentries - 1]
132		set n 14
133
134		# Leave every nth item.  Since rrecno renumbers, we
135		# delete starting at nentries and working down to 0.
136		if { $txnenv == 1 } {
137			set t [$env txn]
138			error_check_good txn [is_valid_txn $t $env] TRUE
139			set txn "-txn $t"
140		}
141		while { [gets $did str] != -1 && $count > 0 } {
142			set key $str
143
144			if { [expr $count % $n] != 0 } {
145				set ret [eval {$db del} $txn {$key}]
146				error_check_good del $ret 0
147			}
148			incr count -1
149		}
150		if { $txnenv == 1 } {
151			error_check_good t_commit [$t commit] 0
152		}
153		close $did
154		error_check_good db_sync [$db sync] 0
155
156		puts "\tTest$tnum.c: Do a dump_file on contents."
157		if { $txnenv == 1 } {
158			set t [$env txn]
159			error_check_good txn [is_valid_txn $t $env] TRUE
160			set txn "-txn $t"
161		}
162
163		dump_file $db $txn $t1 $checkfunc
164		if { $txnenv == 1 } {
165			error_check_good txn_commit [$t commit] 0
166		}
167
168		puts "\tTest$tnum.d: Compact and verify database."
169		for {set commit 0} {$commit <= $txnenv} {incr commit} {
170			if { $txnenv == 1 } {
171				set t [$env txn]
172				error_check_good txn [is_valid_txn $t $env] TRUE
173				set txn "-txn $t"
174			}
175			set ret [eval $db compact $txn -freespace]
176			if { $txnenv == 1 } {
177				if { $commit == 0 } {
178					puts "\tTest$tnum.d: Aborting."
179					error_check_good txn_abort [$t abort] 0
180				} else {
181					puts "\tTest$tnum.d: Committing."
182					error_check_good txn_commit [$t commit] 0
183				}
184			}
185			error_check_good db_sync [$db sync] 0
186			if { [catch {eval \
187			    {berkdb dbverify -btcompare test093_cmp1}\
188			    $envargs $encargs {$testfile}} res] } {
189				puts "FAIL: Verification failed with $res"
190			}
191
192		}
193
194		set size2 [file size $filename]
195		set free2 [stat_field $db stat "Pages on freelist"]
196		set leaf2 [stat_field $db stat "Leaf pages"]
197		set internal2 [stat_field $db stat "Internal pages"]
198
199		# The sum of internal pages, leaf pages, and pages freed
200		# should decrease on compaction, indicating that pages
201		# have been freed to the file system.
202		set sum1 [expr $free1 + $leaf1 + $internal1]
203		set sum2 [expr $free2 + $leaf2 + $internal2]
204		error_check_good pages_freed [expr $sum1 > $sum2] 1
205
206		# Check for reduction in file size.
207#### We should look at the partitioned files #####
208if { [is_partitioned $args] == 0 } {
209		set reduction .95
210		error_check_good \
211		    file_size [expr [expr $size1 * $reduction] > $size2] 1
212}
213		puts "\tTest$tnum.e: Contents are the same after compaction."
214		if { $txnenv == 1 } {
215			set t [$env txn]
216			error_check_good txn [is_valid_txn $t $env] TRUE
217			set txn "-txn $t"
218		}
219		dump_file $db $txn $t2 $checkfunc
220		if { $txnenv == 1 } {
221			error_check_good txn_commit [$t commit] 0
222		}
223
224		error_check_good filecmp [filecmp $t1 $t2] 0
225
226		puts "\tTest$tnum.f: Add more entries to database."
227		set did [open $dict]
228		if { $txnenv == 1 } {
229			set t [$env txn]
230			error_check_good txn [is_valid_txn $t $env] TRUE
231			set txn "-txn $t"
232		}
233		while { [gets $did str] != -1 && $count < $nentries } {
234			set key $str
235			set str [reverse $str]
236
237			set ret [eval \
238			    {$db put} $txn {$key [chop_data $method $str]}]
239			error_check_good put $ret 0
240			lappend btvals $key
241			incr count
242		}
243		if { $txnenv == 1 } {
244			error_check_good txn_commit [$t commit] 0
245		}
246		close $did
247		error_check_good db_sync [$db sync] 0
248
249		set size3 [file size $filename]
250		set free3 [stat_field $db stat "Pages on freelist"]
251		set leaf3 [stat_field $db stat "Leaf pages"]
252		set internal3 [stat_field $db stat "Internal pages"]
253
254		puts "\tTest$tnum.g: Remove more entries, this time by cursor."
255		set count 0
256		if { $txnenv == 1 } {
257			set t [$env txn]
258			error_check_good txn [is_valid_txn $t $env] TRUE
259			set txn "-txn $t"
260		}
261		set dbc [eval {$db cursor} $txn]
262
263		# Leave every nth item.
264		for { set dbt [$dbc get -first] } { [llength $dbt] > 0 }\
265		    { set dbt [$dbc get -next] ; incr count } {
266			if { [expr $count % $n] != 0 } {
267				error_check_good dbc_del [$dbc del] 0
268			}
269		}
270
271		error_check_good cursor_close [$dbc close] 0
272		if { $txnenv == 1 } {
273			error_check_good t_commit [$t commit] 0
274		}
275		error_check_good db_sync [$db sync] 0
276
277		puts "\tTest$tnum.h: Save contents."
278		if { $txnenv == 1 } {
279			set t [$env txn]
280			error_check_good txn [is_valid_txn $t $env] TRUE
281			set txn "-txn $t"
282		}
283		dump_file $db $txn $t1 $checkfunc
284		if { $txnenv == 1 } {
285			error_check_good t_commit [$t commit] 0
286		}
287
288		puts "\tTest$tnum.i: Compact and verify database again."
289		for {set commit 0} {$commit <= $txnenv} {incr commit} {
290			if { $txnenv == 1 } {
291				set t [$env txn]
292				error_check_good txn [is_valid_txn $t $env] TRUE
293				set txn "-txn $t"
294			}
295			set ret [eval $db compact $txn -freespace]
296			if { $txnenv == 1 } {
297				if { $commit == 0 } {
298					puts "\tTest$tnum.d: Aborting."
299					error_check_good txn_abort [$t abort] 0
300				} else {
301					puts "\tTest$tnum.d: Committing."
302					error_check_good txn_commit [$t commit] 0
303				}
304			}
305			error_check_good db_sync [$db sync] 0
306			if { [catch {eval \
307			    {berkdb dbverify -btcompare test093_cmp1}\
308			    $envargs $encargs {$testfile}} res] } {
309				puts "FAIL: Verification failed with $res"
310			}
311		}
312
313		set size4 [file size $filename]
314		set free4 [stat_field $db stat "Pages on freelist"]
315		set leaf4 [stat_field $db stat "Leaf pages"]
316		set internal4 [stat_field $db stat "Internal pages"]
317
318		# The sum of internal pages, leaf pages, and pages freed
319		# should decrease on compaction, indicating that pages
320		# have been freed to the file system.
321		set sum3 [expr $free3 + $leaf3 + $internal3]
322		set sum4 [expr $free4 + $leaf4 + $internal4]
323		error_check_good pages_freed [expr $sum3 > $sum4] 1
324
325		# Check for file size reduction.
326#### We should look at the partitioned files #####
327if { [is_partitioned $args] == 0 } {
328		error_check_good\
329		    file_size [expr [expr $size3 * $reduction] > $size4] 1
330}
331
332		puts "\tTest$tnum.j: Contents are the same after compaction."
333		if { $txnenv == 1 } {
334			set t [$env txn]
335			error_check_good txn [is_valid_txn $t $env] TRUE
336			set txn "-txn $t"
337		}
338		dump_file $db $txn $t2 $checkfunc
339		if { $txnenv == 1 } {
340			error_check_good t_commit [$t commit] 0
341		}
342		error_check_good filecmp [filecmp $t1 $t2] 0
343
344		error_check_good db_close [$db close] 0
345		if { $env != "NULL" } {
346			set testdir [get_home $env]
347		}
348		cleanup $testdir $env
349	}
350
351	# Clean up so the general verification (without the custom comparator)
352	# doesn't fail.
353	set eindex [lsearch -exact $args "-env"]
354	if { $eindex == -1 } {
355		set env NULL
356	} else {
357		incr eindex
358		set env [lindex $args $eindex]
359		set testdir [get_home $env]
360	}
361	cleanup $testdir $env
362}
363