1# See the file LICENSE for redistribution information. 2# 3# Copyright (c) 2005-2009 Oracle. All rights reserved. 4# 5# $Id$ 6# 7# TEST test115 8# TEST Test database compaction with user-specified btree sort. 9# TEST 10# TEST This is essentially test111 with the user-specified sort. 11# TEST Populate a database. Remove a high proportion of entries. 12# TEST Dump and save contents. Compact the database, dump again, 13# TEST and make sure we still have the same contents. 14# TEST Add back some entries, delete more entries (this time by 15# TEST cursor), dump, compact, and do the before/after check again. 16 17proc test115 { method {nentries 10000} {tnum "115"} args } { 18 source ./include.tcl 19 global btvals 20 global btvalsck 21 global encrypt 22 global passwd 23 24 if { [is_btree $method] != 1 } { 25 puts "Skipping test$tnum for method $method." 26 return 27 } 28 29 # If a page size was specified, find out what it is. Pages 30 # might not be freed in the case of really large pages (64K) 31 # but we still want to run this test just to make sure 32 # nothing funny happens. 33 set pagesize 0 34 set pgindex [lsearch -exact $args "-pagesize"] 35 if { $pgindex != -1 } { 36 incr pgindex 37 set pagesize [lindex $args $pgindex] 38 } 39 40 set args [convert_args $method $args] 41 set encargs "" 42 set args [split_encargs $args encargs] 43 set omethod [convert_method $method] 44 45 # If we are using an env, then testfile should just be the db name. 46 # Otherwise it is the test directory and the name. 47 set txnenv 0 48 set eindex [lsearch -exact $args "-env"] 49 if { $eindex == -1 } { 50 set basename $testdir/test$tnum 51 set env NULL 52 set envargs "" 53 } else { 54 set basename test$tnum 55 incr eindex 56 set env [lindex $args $eindex] 57 set envargs " -env $env " 58 set rpcenv [is_rpcenv $env] 59 if { $rpcenv == 1 } { 60 puts "Test$tnum: skipping for RPC." 61 return 62 } 63 set txnenv [is_txnenv $env] 64 if { $txnenv == 1 } { 65 append args " -auto_commit " 66 } 67 set testdir [get_home $env] 68 } 69 70 puts "Test$tnum:\ 71 ($method $args $encargs) Database compaction with user-specified sort." 72 73 cleanup $testdir $env 74 set t1 $testdir/t1 75 set t2 $testdir/t2 76 set splitopts { "" "-revsplitoff" } 77 set txn "" 78 79 set checkfunc test093_check 80 81 foreach splitopt $splitopts { 82 set testfile $basename.db 83 if { $splitopt == "-revsplitoff" } { 84 set testfile $basename.rev.db 85 } 86 set did [open $dict] 87 88 puts "\tTest$tnum.a: Create and populate database ($splitopt)." 89 set db [eval {berkdb_open -create -btcompare test093_cmp1 \ 90 -mode 0644} $splitopt $args $encargs $omethod $testfile] 91 error_check_good dbopen [is_valid_db $db] TRUE 92 93 set count 0 94 set btvals {} 95 set btvalsck {} 96 if { $txnenv == 1 } { 97 set t [$env txn] 98 error_check_good txn [is_valid_txn $t $env] TRUE 99 set txn "-txn $t" 100 } 101 while { [gets $did str] != -1 && $count < $nentries } { 102 set key $str 103 set str [reverse $str] 104 105 set ret [eval \ 106 {$db put} $txn {$key [chop_data $method $str]}] 107 error_check_good put $ret 0 108 lappend btvals $key 109 incr count 110 111 } 112 if { $txnenv == 1 } { 113 error_check_good txn_commit [$t commit] 0 114 } 115 close $did 116 error_check_good db_sync [$db sync] 0 117 118 if { $env != "NULL" } { 119 set testdir [get_home $env] 120 set filename $testdir/$testfile 121 } else { 122 set filename $testfile 123 } 124 set size1 [file size $filename] 125 set free1 [stat_field $db stat "Pages on freelist"] 126 set leaf1 [stat_field $db stat "Leaf pages"] 127 set internal1 [stat_field $db stat "Internal pages"] 128 129 puts "\tTest$tnum.b: Delete most entries from database." 130 set did [open $dict] 131 set count [expr $nentries - 1] 132 set n 14 133 134 # Leave every nth item. Since rrecno renumbers, we 135 # delete starting at nentries and working down to 0. 136 if { $txnenv == 1 } { 137 set t [$env txn] 138 error_check_good txn [is_valid_txn $t $env] TRUE 139 set txn "-txn $t" 140 } 141 while { [gets $did str] != -1 && $count > 0 } { 142 set key $str 143 144 if { [expr $count % $n] != 0 } { 145 set ret [eval {$db del} $txn {$key}] 146 error_check_good del $ret 0 147 } 148 incr count -1 149 } 150 if { $txnenv == 1 } { 151 error_check_good t_commit [$t commit] 0 152 } 153 close $did 154 error_check_good db_sync [$db sync] 0 155 156 puts "\tTest$tnum.c: Do a dump_file on contents." 157 if { $txnenv == 1 } { 158 set t [$env txn] 159 error_check_good txn [is_valid_txn $t $env] TRUE 160 set txn "-txn $t" 161 } 162 163 dump_file $db $txn $t1 $checkfunc 164 if { $txnenv == 1 } { 165 error_check_good txn_commit [$t commit] 0 166 } 167 168 puts "\tTest$tnum.d: Compact and verify database." 169 for {set commit 0} {$commit <= $txnenv} {incr commit} { 170 if { $txnenv == 1 } { 171 set t [$env txn] 172 error_check_good txn [is_valid_txn $t $env] TRUE 173 set txn "-txn $t" 174 } 175 set ret [eval $db compact $txn -freespace] 176 if { $txnenv == 1 } { 177 if { $commit == 0 } { 178 puts "\tTest$tnum.d: Aborting." 179 error_check_good txn_abort [$t abort] 0 180 } else { 181 puts "\tTest$tnum.d: Committing." 182 error_check_good txn_commit [$t commit] 0 183 } 184 } 185 error_check_good db_sync [$db sync] 0 186 if { [catch {eval \ 187 {berkdb dbverify -btcompare test093_cmp1}\ 188 $envargs $encargs {$testfile}} res] } { 189 puts "FAIL: Verification failed with $res" 190 } 191 192 } 193 194 set size2 [file size $filename] 195 set free2 [stat_field $db stat "Pages on freelist"] 196 set leaf2 [stat_field $db stat "Leaf pages"] 197 set internal2 [stat_field $db stat "Internal pages"] 198 199 # The sum of internal pages, leaf pages, and pages freed 200 # should decrease on compaction, indicating that pages 201 # have been freed to the file system. 202 set sum1 [expr $free1 + $leaf1 + $internal1] 203 set sum2 [expr $free2 + $leaf2 + $internal2] 204 error_check_good pages_freed [expr $sum1 > $sum2] 1 205 206 # Check for reduction in file size. 207#### We should look at the partitioned files ##### 208if { [is_partitioned $args] == 0 } { 209 set reduction .95 210 error_check_good \ 211 file_size [expr [expr $size1 * $reduction] > $size2] 1 212} 213 puts "\tTest$tnum.e: Contents are the same after compaction." 214 if { $txnenv == 1 } { 215 set t [$env txn] 216 error_check_good txn [is_valid_txn $t $env] TRUE 217 set txn "-txn $t" 218 } 219 dump_file $db $txn $t2 $checkfunc 220 if { $txnenv == 1 } { 221 error_check_good txn_commit [$t commit] 0 222 } 223 224 error_check_good filecmp [filecmp $t1 $t2] 0 225 226 puts "\tTest$tnum.f: Add more entries to database." 227 set did [open $dict] 228 if { $txnenv == 1 } { 229 set t [$env txn] 230 error_check_good txn [is_valid_txn $t $env] TRUE 231 set txn "-txn $t" 232 } 233 while { [gets $did str] != -1 && $count < $nentries } { 234 set key $str 235 set str [reverse $str] 236 237 set ret [eval \ 238 {$db put} $txn {$key [chop_data $method $str]}] 239 error_check_good put $ret 0 240 lappend btvals $key 241 incr count 242 } 243 if { $txnenv == 1 } { 244 error_check_good txn_commit [$t commit] 0 245 } 246 close $did 247 error_check_good db_sync [$db sync] 0 248 249 set size3 [file size $filename] 250 set free3 [stat_field $db stat "Pages on freelist"] 251 set leaf3 [stat_field $db stat "Leaf pages"] 252 set internal3 [stat_field $db stat "Internal pages"] 253 254 puts "\tTest$tnum.g: Remove more entries, this time by cursor." 255 set count 0 256 if { $txnenv == 1 } { 257 set t [$env txn] 258 error_check_good txn [is_valid_txn $t $env] TRUE 259 set txn "-txn $t" 260 } 261 set dbc [eval {$db cursor} $txn] 262 263 # Leave every nth item. 264 for { set dbt [$dbc get -first] } { [llength $dbt] > 0 }\ 265 { set dbt [$dbc get -next] ; incr count } { 266 if { [expr $count % $n] != 0 } { 267 error_check_good dbc_del [$dbc del] 0 268 } 269 } 270 271 error_check_good cursor_close [$dbc close] 0 272 if { $txnenv == 1 } { 273 error_check_good t_commit [$t commit] 0 274 } 275 error_check_good db_sync [$db sync] 0 276 277 puts "\tTest$tnum.h: Save contents." 278 if { $txnenv == 1 } { 279 set t [$env txn] 280 error_check_good txn [is_valid_txn $t $env] TRUE 281 set txn "-txn $t" 282 } 283 dump_file $db $txn $t1 $checkfunc 284 if { $txnenv == 1 } { 285 error_check_good t_commit [$t commit] 0 286 } 287 288 puts "\tTest$tnum.i: Compact and verify database again." 289 for {set commit 0} {$commit <= $txnenv} {incr commit} { 290 if { $txnenv == 1 } { 291 set t [$env txn] 292 error_check_good txn [is_valid_txn $t $env] TRUE 293 set txn "-txn $t" 294 } 295 set ret [eval $db compact $txn -freespace] 296 if { $txnenv == 1 } { 297 if { $commit == 0 } { 298 puts "\tTest$tnum.d: Aborting." 299 error_check_good txn_abort [$t abort] 0 300 } else { 301 puts "\tTest$tnum.d: Committing." 302 error_check_good txn_commit [$t commit] 0 303 } 304 } 305 error_check_good db_sync [$db sync] 0 306 if { [catch {eval \ 307 {berkdb dbverify -btcompare test093_cmp1}\ 308 $envargs $encargs {$testfile}} res] } { 309 puts "FAIL: Verification failed with $res" 310 } 311 } 312 313 set size4 [file size $filename] 314 set free4 [stat_field $db stat "Pages on freelist"] 315 set leaf4 [stat_field $db stat "Leaf pages"] 316 set internal4 [stat_field $db stat "Internal pages"] 317 318 # The sum of internal pages, leaf pages, and pages freed 319 # should decrease on compaction, indicating that pages 320 # have been freed to the file system. 321 set sum3 [expr $free3 + $leaf3 + $internal3] 322 set sum4 [expr $free4 + $leaf4 + $internal4] 323 error_check_good pages_freed [expr $sum3 > $sum4] 1 324 325 # Check for file size reduction. 326#### We should look at the partitioned files ##### 327if { [is_partitioned $args] == 0 } { 328 error_check_good\ 329 file_size [expr [expr $size3 * $reduction] > $size4] 1 330} 331 332 puts "\tTest$tnum.j: Contents are the same after compaction." 333 if { $txnenv == 1 } { 334 set t [$env txn] 335 error_check_good txn [is_valid_txn $t $env] TRUE 336 set txn "-txn $t" 337 } 338 dump_file $db $txn $t2 $checkfunc 339 if { $txnenv == 1 } { 340 error_check_good t_commit [$t commit] 0 341 } 342 error_check_good filecmp [filecmp $t1 $t2] 0 343 344 error_check_good db_close [$db close] 0 345 if { $env != "NULL" } { 346 set testdir [get_home $env] 347 } 348 cleanup $testdir $env 349 } 350 351 # Clean up so the general verification (without the custom comparator) 352 # doesn't fail. 353 set eindex [lsearch -exact $args "-env"] 354 if { $eindex == -1 } { 355 set env NULL 356 } else { 357 incr eindex 358 set env [lindex $args $eindex] 359 set testdir [get_home $env] 360 } 361 cleanup $testdir $env 362} 363