1# See the file LICENSE for redistribution information.
2#
3# Copyright (c) 2005-2009 Oracle.  All rights reserved.
4#
5# $Id$
6#
7# TEST	test114
8# TEST	Test database compaction with overflows.
9# TEST
10# TEST	Populate a database.  Remove a high proportion of entries.
11# TEST	Dump and save contents.  Compact the database, dump again,
12# TEST	and make sure we still have the same contents.
13# TEST  Add back some entries, delete more entries (this time by
14# TEST	cursor), dump, compact, and do the before/after check again.
15
16proc test114 { method {nentries 10000} {tnum "114"} args } {
17	source ./include.tcl
18	global alphabet
19
20	# Compaction is an option for btree and recno databases only.
21	if { [is_hash $method] == 1 || [is_queue $method] == 1 } {
22		puts "Skipping test$tnum for method $method."
23		return
24	}
25
26	# We run with a small page size to force overflows.  Skip
27	# testing for specified page size.
28	set pgindex [lsearch -exact $args "-pagesize"]
29	if { $pgindex != -1 } {
30		puts "Test$tnum: Skipping for specific pagesize."
31		return
32	}
33
34	set args [convert_args $method $args]
35	set omethod [convert_method $method]
36	if  { [is_partition_callback $args] == 1 } {
37		set nodump 1
38	} else {
39		set nodump 0
40	}
41
42	# If we are using an env, then testfile should just be the db name.
43	# Otherwise it is the test directory and the name.
44	set txnenv 0
45	set eindex [lsearch -exact $args "-env"]
46	if { $eindex == -1 } {
47		set basename $testdir/test$tnum
48		set env NULL
49	} else {
50		set basename test$tnum
51		incr eindex
52		set env [lindex $args $eindex]
53		set rpcenv [is_rpcenv $env]
54		if { $rpcenv == 1 } {
55			puts "Test$tnum: skipping for RPC"
56			return
57		}
58		set txnenv [is_txnenv $env]
59		if { $txnenv == 1 } {
60			append args " -auto_commit"
61		}
62		set testdir [get_home $env]
63	}
64
65	puts "Test$tnum: ($method $args) Database compaction with overflows."
66	set t1 $testdir/t1
67	set t2 $testdir/t2
68	set splitopts { "" "-revsplitoff" }
69	set txn ""
70
71	if { [is_record_based $method] == 1 } {
72		set checkfunc test001_recno.check
73	} else {
74		set checkfunc test001.check
75	}
76
77	foreach splitopt $splitopts {
78		set testfile $basename.db
79		if { $splitopt == "-revsplitoff" } {
80			set testfile $basename.rev.db
81			if { [is_record_based $method] == 1 } {
82				puts "Skipping\
83				    -revsplitoff option for method $method."
84				continue
85			}
86		}
87		set did [open $dict]
88		if { $env != "NULL" } {
89			set testdir [get_home $env]
90		}
91		cleanup $testdir $env
92
93		puts "\tTest$tnum.a: Create and populate database ($splitopt)."
94		set pagesize 512
95		set db [eval {berkdb_open -create -pagesize $pagesize \
96		    -mode 0644} $splitopt $args $omethod $testfile]
97		error_check_good dbopen [is_valid_db $db] TRUE
98
99		set count 0
100		if { $txnenv == 1 } {
101			set t [$env txn]
102			error_check_good txn [is_valid_txn $t $env] TRUE
103			set txn "-txn $t"
104		}
105		while { [gets $did str] != -1 && $count < $nentries } {
106			if { [is_record_based $method] == 1 } {
107				set key [expr $count + 1]
108			} else {
109				set key $str
110			}
111			set str [repeat $alphabet 100]
112
113			set ret [eval \
114			    {$db put} $txn {$key [chop_data $method $str]}]
115			error_check_good put $ret 0
116			incr count
117
118		}
119		if { $txnenv == 1 } {
120			error_check_good txn_commit [$t commit] 0
121		}
122		close $did
123		error_check_good db_sync [$db sync] 0
124
125		if { $env != "NULL" } {
126			set testdir [get_home $env]
127			set filename $testdir/$testfile
128		} else {
129			set filename $testfile
130		}
131		set size1 [file size $filename]
132		set free1 [stat_field $db stat "Pages on freelist"]
133
134		puts "\tTest$tnum.b: Delete most entries from database."
135		set did [open $dict]
136		set count [expr $nentries - 1]
137		set n 57
138
139		# Leave every nth item.  Since rrecno renumbers, we
140		# delete starting at nentries and working down to 0.
141		if { $txnenv == 1 } {
142			set t [$env txn]
143			error_check_good txn [is_valid_txn $t $env] TRUE
144			set txn "-txn $t"
145		}
146		while { [gets $did str] != -1 && $count > 0 } {
147			if { [is_record_based $method] == 1 } {
148				set key [expr $count + 1]
149			} else {
150				set key $str
151			}
152
153			if { [expr $count % $n] != 0 } {
154				set ret [eval {$db del} $txn {$key}]
155				error_check_good del $ret 0
156			}
157			incr count -1
158		}
159		if { $txnenv == 1 } {
160			error_check_good t_commit [$t commit] 0
161		}
162		error_check_good db_sync [$db sync] 0
163
164		puts "\tTest$tnum.c: Do a dump_file on contents."
165		if { $txnenv == 1 } {
166			set t [$env txn]
167			error_check_good txn [is_valid_txn $t $env] TRUE
168			set txn "-txn $t"
169		}
170		dump_file $db $txn $t1
171		if { $txnenv == 1 } {
172			error_check_good txn_commit [$t commit] 0
173		}
174
175		puts "\tTest$tnum.d: Compact and verify database."
176		for {set commit 0} {$commit <= $txnenv} {incr commit} {
177			if { $txnenv == 1 } {
178				set t [$env txn]
179				error_check_good txn [is_valid_txn $t $env] TRUE
180				set txn "-txn $t"
181			}
182			set ret [eval $db compact $txn -freespace]
183			if { $txnenv == 1 } {
184				if { $commit == 0 } {
185					puts "\tTest$tnum.d: Aborting."
186					error_check_good txn_abort [$t abort] 0
187				} else {
188					puts "\tTest$tnum.d: Committing."
189					error_check_good txn_commit [$t commit] 0
190				}
191			}
192			error_check_good db_sync [$db sync] 0
193			error_check_good verify_dir \
194			    [ verify_dir $testdir "" 0 0 $nodump] 0
195		}
196
197		set size2 [file size $filename]
198		set free2 [stat_field $db stat "Pages on freelist"]
199
200		# Reduction in on-disk size should be substantial.
201#### We should look at the partitioned files #####
202if { [is_partitioned $args] == 0 } {
203		set reduction .80
204		error_check_good \
205		    file_size [expr [expr $size1 * $reduction] > $size2] 1
206}
207
208		# Pages should be freed for all methods except maybe
209		# record-based non-queue methods.  Even with recno, the
210		# number of free pages may not decline.
211		if { [is_record_based $method] == 1 } {
212			error_check_good pages_freed [expr $free2 >= $free1] 1
213		} else {
214			error_check_good pages_freed [expr $free2 > $free1] 1
215		}
216
217		puts "\tTest$tnum.e: Contents are the same after compaction."
218		if { $txnenv == 1 } {
219			set t [$env txn]
220			error_check_good txn [is_valid_txn $t $env] TRUE
221			set txn "-txn $t"
222		}
223		dump_file $db $txn $t2
224		if { $txnenv == 1 } {
225			error_check_good txn_commit [$t commit] 0
226		}
227
228		error_check_good filecmp [filecmp $t1 $t2] 0
229
230		puts "\tTest$tnum.f: Add more entries to database."
231		# Use integers as keys instead of strings, just to mix it up
232		# a little.
233		if { $txnenv == 1 } {
234			set t [$env txn]
235			error_check_good txn [is_valid_txn $t $env] TRUE
236			set txn "-txn $t"
237		}
238		for { set i 1 } { $i < $nentries } { incr i } {
239			set key $i
240			set str [repeat $alphabet 100]
241			set ret [eval \
242			    {$db put} $txn {$key [chop_data $method $str]}]
243			error_check_good put $ret 0
244		}
245		if { $txnenv == 1 } {
246			error_check_good t_commit [$t commit] 0
247		}
248		error_check_good db_sync [$db sync] 0
249
250		set size3 [file size $filename]
251		set free3 [stat_field $db stat "Pages on freelist"]
252
253		puts "\tTest$tnum.g: Remove more entries, this time by cursor."
254		set count 0
255		if { $txnenv == 1 } {
256			set t [$env txn]
257			error_check_good txn [is_valid_txn $t $env] TRUE
258			set txn "-txn $t"
259		}
260		set dbc [eval {$db cursor} $txn]
261
262		# Leave every nth item.
263		for { set dbt [$dbc get -first] } { [llength $dbt] > 0 }\
264		    { set dbt [$dbc get -next] ; incr count } {
265			if { [expr $count % $n] != 0 } {
266				error_check_good dbc_del [$dbc del] 0
267			}
268		}
269
270		error_check_good cursor_close [$dbc close] 0
271		if { $txnenv == 1 } {
272			error_check_good t_commit [$t commit] 0
273		}
274		error_check_good db_sync [$db sync] 0
275
276		puts "\tTest$tnum.h: Save contents."
277		if { $txnenv == 1 } {
278			set t [$env txn]
279			error_check_good txn [is_valid_txn $t $env] TRUE
280			set txn "-txn $t"
281		}
282		dump_file $db $txn $t1
283		if { $txnenv == 1 } {
284			error_check_good t_commit [$t commit] 0
285		}
286
287		puts "\tTest$tnum.i: Compact and verify database again."
288		for {set commit 0} {$commit <= $txnenv} {incr commit} {
289			if { $txnenv == 1 } {
290				set t [$env txn]
291				error_check_good txn [is_valid_txn $t $env] TRUE
292				set txn "-txn $t"
293			}
294			set ret [eval $db compact $txn -freespace]
295			if { $txnenv == 1 } {
296				if { $commit == 0 } {
297					puts "\tTest$tnum.d: Aborting."
298					error_check_good txn_abort [$t abort] 0
299				} else {
300					puts "\tTest$tnum.d: Committing."
301					error_check_good txn_commit [$t commit] 0
302				}
303			}
304			error_check_good db_sync [$db sync] 0
305			error_check_good verify_dir \
306			    [ verify_dir $testdir "" 0 0 $nodump] 0
307		}
308
309		set size4 [file size $filename]
310		set free4 [stat_field $db stat "Pages on freelist"]
311
312#### We should look at the partitioned files #####
313if { [is_partitioned $args] == 0 } {
314		error_check_good \
315		    file_size [expr [expr $size3 * $reduction] > $size4] 1
316#### We are specifying -freespace why should there be more things on the free list? #######
317		if { [is_record_based $method] == 1 } {
318			error_check_good pages_freed [expr $free4 >= $free3] 1
319		} else {
320			error_check_good pages_freed [expr $free4 > $free3] 1
321		}
322}
323
324		puts "\tTest$tnum.j: Contents are the same after compaction."
325		if { $txnenv == 1 } {
326			set t [$env txn]
327			error_check_good txn [is_valid_txn $t $env] TRUE
328			set txn "-txn $t"
329		}
330		dump_file $db $txn $t2
331		if { $txnenv == 1 } {
332			error_check_good t_commit [$t commit] 0
333		}
334		error_check_good filecmp [filecmp $t1 $t2] 0
335
336		error_check_good db_close [$db close] 0
337		close $did
338	}
339}
340