1# See the file LICENSE for redistribution information. 2# 3# Copyright (c) 2005-2009 Oracle. All rights reserved. 4# 5# $Id$ 6# 7# TEST test114 8# TEST Test database compaction with overflows. 9# TEST 10# TEST Populate a database. Remove a high proportion of entries. 11# TEST Dump and save contents. Compact the database, dump again, 12# TEST and make sure we still have the same contents. 13# TEST Add back some entries, delete more entries (this time by 14# TEST cursor), dump, compact, and do the before/after check again. 15 16proc test114 { method {nentries 10000} {tnum "114"} args } { 17 source ./include.tcl 18 global alphabet 19 20 # Compaction is an option for btree and recno databases only. 21 if { [is_hash $method] == 1 || [is_queue $method] == 1 } { 22 puts "Skipping test$tnum for method $method." 23 return 24 } 25 26 # We run with a small page size to force overflows. Skip 27 # testing for specified page size. 28 set pgindex [lsearch -exact $args "-pagesize"] 29 if { $pgindex != -1 } { 30 puts "Test$tnum: Skipping for specific pagesize." 31 return 32 } 33 34 set args [convert_args $method $args] 35 set omethod [convert_method $method] 36 if { [is_partition_callback $args] == 1 } { 37 set nodump 1 38 } else { 39 set nodump 0 40 } 41 42 # If we are using an env, then testfile should just be the db name. 43 # Otherwise it is the test directory and the name. 44 set txnenv 0 45 set eindex [lsearch -exact $args "-env"] 46 if { $eindex == -1 } { 47 set basename $testdir/test$tnum 48 set env NULL 49 } else { 50 set basename test$tnum 51 incr eindex 52 set env [lindex $args $eindex] 53 set rpcenv [is_rpcenv $env] 54 if { $rpcenv == 1 } { 55 puts "Test$tnum: skipping for RPC" 56 return 57 } 58 set txnenv [is_txnenv $env] 59 if { $txnenv == 1 } { 60 append args " -auto_commit" 61 } 62 set testdir [get_home $env] 63 } 64 65 puts "Test$tnum: ($method $args) Database compaction with overflows." 66 set t1 $testdir/t1 67 set t2 $testdir/t2 68 set splitopts { "" "-revsplitoff" } 69 set txn "" 70 71 if { [is_record_based $method] == 1 } { 72 set checkfunc test001_recno.check 73 } else { 74 set checkfunc test001.check 75 } 76 77 foreach splitopt $splitopts { 78 set testfile $basename.db 79 if { $splitopt == "-revsplitoff" } { 80 set testfile $basename.rev.db 81 if { [is_record_based $method] == 1 } { 82 puts "Skipping\ 83 -revsplitoff option for method $method." 84 continue 85 } 86 } 87 set did [open $dict] 88 if { $env != "NULL" } { 89 set testdir [get_home $env] 90 } 91 cleanup $testdir $env 92 93 puts "\tTest$tnum.a: Create and populate database ($splitopt)." 94 set pagesize 512 95 set db [eval {berkdb_open -create -pagesize $pagesize \ 96 -mode 0644} $splitopt $args $omethod $testfile] 97 error_check_good dbopen [is_valid_db $db] TRUE 98 99 set count 0 100 if { $txnenv == 1 } { 101 set t [$env txn] 102 error_check_good txn [is_valid_txn $t $env] TRUE 103 set txn "-txn $t" 104 } 105 while { [gets $did str] != -1 && $count < $nentries } { 106 if { [is_record_based $method] == 1 } { 107 set key [expr $count + 1] 108 } else { 109 set key $str 110 } 111 set str [repeat $alphabet 100] 112 113 set ret [eval \ 114 {$db put} $txn {$key [chop_data $method $str]}] 115 error_check_good put $ret 0 116 incr count 117 118 } 119 if { $txnenv == 1 } { 120 error_check_good txn_commit [$t commit] 0 121 } 122 close $did 123 error_check_good db_sync [$db sync] 0 124 125 if { $env != "NULL" } { 126 set testdir [get_home $env] 127 set filename $testdir/$testfile 128 } else { 129 set filename $testfile 130 } 131 set size1 [file size $filename] 132 set free1 [stat_field $db stat "Pages on freelist"] 133 134 puts "\tTest$tnum.b: Delete most entries from database." 135 set did [open $dict] 136 set count [expr $nentries - 1] 137 set n 57 138 139 # Leave every nth item. Since rrecno renumbers, we 140 # delete starting at nentries and working down to 0. 141 if { $txnenv == 1 } { 142 set t [$env txn] 143 error_check_good txn [is_valid_txn $t $env] TRUE 144 set txn "-txn $t" 145 } 146 while { [gets $did str] != -1 && $count > 0 } { 147 if { [is_record_based $method] == 1 } { 148 set key [expr $count + 1] 149 } else { 150 set key $str 151 } 152 153 if { [expr $count % $n] != 0 } { 154 set ret [eval {$db del} $txn {$key}] 155 error_check_good del $ret 0 156 } 157 incr count -1 158 } 159 if { $txnenv == 1 } { 160 error_check_good t_commit [$t commit] 0 161 } 162 error_check_good db_sync [$db sync] 0 163 164 puts "\tTest$tnum.c: Do a dump_file on contents." 165 if { $txnenv == 1 } { 166 set t [$env txn] 167 error_check_good txn [is_valid_txn $t $env] TRUE 168 set txn "-txn $t" 169 } 170 dump_file $db $txn $t1 171 if { $txnenv == 1 } { 172 error_check_good txn_commit [$t commit] 0 173 } 174 175 puts "\tTest$tnum.d: Compact and verify database." 176 for {set commit 0} {$commit <= $txnenv} {incr commit} { 177 if { $txnenv == 1 } { 178 set t [$env txn] 179 error_check_good txn [is_valid_txn $t $env] TRUE 180 set txn "-txn $t" 181 } 182 set ret [eval $db compact $txn -freespace] 183 if { $txnenv == 1 } { 184 if { $commit == 0 } { 185 puts "\tTest$tnum.d: Aborting." 186 error_check_good txn_abort [$t abort] 0 187 } else { 188 puts "\tTest$tnum.d: Committing." 189 error_check_good txn_commit [$t commit] 0 190 } 191 } 192 error_check_good db_sync [$db sync] 0 193 error_check_good verify_dir \ 194 [ verify_dir $testdir "" 0 0 $nodump] 0 195 } 196 197 set size2 [file size $filename] 198 set free2 [stat_field $db stat "Pages on freelist"] 199 200 # Reduction in on-disk size should be substantial. 201#### We should look at the partitioned files ##### 202if { [is_partitioned $args] == 0 } { 203 set reduction .80 204 error_check_good \ 205 file_size [expr [expr $size1 * $reduction] > $size2] 1 206} 207 208 # Pages should be freed for all methods except maybe 209 # record-based non-queue methods. Even with recno, the 210 # number of free pages may not decline. 211 if { [is_record_based $method] == 1 } { 212 error_check_good pages_freed [expr $free2 >= $free1] 1 213 } else { 214 error_check_good pages_freed [expr $free2 > $free1] 1 215 } 216 217 puts "\tTest$tnum.e: Contents are the same after compaction." 218 if { $txnenv == 1 } { 219 set t [$env txn] 220 error_check_good txn [is_valid_txn $t $env] TRUE 221 set txn "-txn $t" 222 } 223 dump_file $db $txn $t2 224 if { $txnenv == 1 } { 225 error_check_good txn_commit [$t commit] 0 226 } 227 228 error_check_good filecmp [filecmp $t1 $t2] 0 229 230 puts "\tTest$tnum.f: Add more entries to database." 231 # Use integers as keys instead of strings, just to mix it up 232 # a little. 233 if { $txnenv == 1 } { 234 set t [$env txn] 235 error_check_good txn [is_valid_txn $t $env] TRUE 236 set txn "-txn $t" 237 } 238 for { set i 1 } { $i < $nentries } { incr i } { 239 set key $i 240 set str [repeat $alphabet 100] 241 set ret [eval \ 242 {$db put} $txn {$key [chop_data $method $str]}] 243 error_check_good put $ret 0 244 } 245 if { $txnenv == 1 } { 246 error_check_good t_commit [$t commit] 0 247 } 248 error_check_good db_sync [$db sync] 0 249 250 set size3 [file size $filename] 251 set free3 [stat_field $db stat "Pages on freelist"] 252 253 puts "\tTest$tnum.g: Remove more entries, this time by cursor." 254 set count 0 255 if { $txnenv == 1 } { 256 set t [$env txn] 257 error_check_good txn [is_valid_txn $t $env] TRUE 258 set txn "-txn $t" 259 } 260 set dbc [eval {$db cursor} $txn] 261 262 # Leave every nth item. 263 for { set dbt [$dbc get -first] } { [llength $dbt] > 0 }\ 264 { set dbt [$dbc get -next] ; incr count } { 265 if { [expr $count % $n] != 0 } { 266 error_check_good dbc_del [$dbc del] 0 267 } 268 } 269 270 error_check_good cursor_close [$dbc close] 0 271 if { $txnenv == 1 } { 272 error_check_good t_commit [$t commit] 0 273 } 274 error_check_good db_sync [$db sync] 0 275 276 puts "\tTest$tnum.h: Save contents." 277 if { $txnenv == 1 } { 278 set t [$env txn] 279 error_check_good txn [is_valid_txn $t $env] TRUE 280 set txn "-txn $t" 281 } 282 dump_file $db $txn $t1 283 if { $txnenv == 1 } { 284 error_check_good t_commit [$t commit] 0 285 } 286 287 puts "\tTest$tnum.i: Compact and verify database again." 288 for {set commit 0} {$commit <= $txnenv} {incr commit} { 289 if { $txnenv == 1 } { 290 set t [$env txn] 291 error_check_good txn [is_valid_txn $t $env] TRUE 292 set txn "-txn $t" 293 } 294 set ret [eval $db compact $txn -freespace] 295 if { $txnenv == 1 } { 296 if { $commit == 0 } { 297 puts "\tTest$tnum.d: Aborting." 298 error_check_good txn_abort [$t abort] 0 299 } else { 300 puts "\tTest$tnum.d: Committing." 301 error_check_good txn_commit [$t commit] 0 302 } 303 } 304 error_check_good db_sync [$db sync] 0 305 error_check_good verify_dir \ 306 [ verify_dir $testdir "" 0 0 $nodump] 0 307 } 308 309 set size4 [file size $filename] 310 set free4 [stat_field $db stat "Pages on freelist"] 311 312#### We should look at the partitioned files ##### 313if { [is_partitioned $args] == 0 } { 314 error_check_good \ 315 file_size [expr [expr $size3 * $reduction] > $size4] 1 316#### We are specifying -freespace why should there be more things on the free list? ####### 317 if { [is_record_based $method] == 1 } { 318 error_check_good pages_freed [expr $free4 >= $free3] 1 319 } else { 320 error_check_good pages_freed [expr $free4 > $free3] 1 321 } 322} 323 324 puts "\tTest$tnum.j: Contents are the same after compaction." 325 if { $txnenv == 1 } { 326 set t [$env txn] 327 error_check_good txn [is_valid_txn $t $env] TRUE 328 set txn "-txn $t" 329 } 330 dump_file $db $txn $t2 331 if { $txnenv == 1 } { 332 error_check_good t_commit [$t commit] 0 333 } 334 error_check_good filecmp [filecmp $t1 $t2] 0 335 336 error_check_good db_close [$db close] 0 337 close $did 338 } 339} 340