1# Code to load up the tests in to the Queue database 2# $Id: parallel.tcl,v 12.6 2007/06/05 20:00:46 carol Exp $ 3proc load_queue { file {dbdir RUNQUEUE} nitems } { 4 global serial_tests 5 global num_serial 6 global num_parallel 7 8 puts -nonewline "Loading run queue with $nitems items..." 9 flush stdout 10 11 set env [berkdb_env -create -lock -home $dbdir] 12 error_check_good dbenv [is_valid_env $env] TRUE 13 14 # Open two databases, one for tests that may be run 15 # in parallel, the other for tests we want to run 16 # while only a single process is testing. 17 set db [eval {berkdb_open -env $env -create \ 18 -mode 0644 -len 200 -queue queue.db} ] 19 error_check_good dbopen [is_valid_db $db] TRUE 20 set serialdb [eval {berkdb_open -env $env -create \ 21 -mode 0644 -len 200 -queue serialqueue.db} ] 22 error_check_good dbopen [is_valid_db $serialdb] TRUE 23 24 set fid [open $file] 25 26 set count 0 27 28 while { [gets $fid str] != -1 } { 29 set testarr($count) $str 30 incr count 31 } 32 33 # Randomize array of tests. 34 set rseed [pid] 35 berkdb srand $rseed 36 puts -nonewline "randomizing..." 37 flush stdout 38 for { set i 0 } { $i < $count } { incr i } { 39 set tmp $testarr($i) 40 41 # RPC test is very long so force it to run first 42 # in full runs. If we find 'r rpc' as we walk the 43 # array, arrange to put it in slot 0 ... 44 if { [is_substr $tmp "r rpc"] == 1 && \ 45 [string match $nitems ALL] } { 46 set j 0 47 } else { 48 set j [berkdb random_int $i [expr $count - 1]] 49 } 50 # ... and if 'r rpc' is selected to be swapped with the 51 # current item in the array, skip the swap. If we 52 # did the swap and moved to the next item, "r rpc" would 53 # never get moved to slot 0. 54 if { [is_substr $testarr($j) "r rpc"] && \ 55 [string match $nitems ALL] } { 56 continue 57 } 58 59 set testarr($i) $testarr($j) 60 set testarr($j) $tmp 61 } 62 63 if { [string compare ALL $nitems] != 0 } { 64 set maxload $nitems 65 } else { 66 set maxload $count 67 } 68 69 puts "loading..." 70 flush stdout 71 set num_serial 0 72 set num_parallel 0 73 for { set i 0 } { $i < $maxload } { incr i } { 74 set str $testarr($i) 75 # Push serial tests into serial testing db, others 76 # into parallel db. 77 if { [is_serial $str] } { 78 set ret [eval {$serialdb put -append $str}] 79 error_check_good put:serialdb [expr $ret > 0] 1 80 incr num_serial 81 } else { 82 set ret [eval {$db put -append $str}] 83 error_check_good put:paralleldb [expr $ret > 0] 1 84 incr num_parallel 85 } 86 } 87 88 error_check_good maxload $maxload [expr $num_serial + $num_parallel] 89 puts "Loaded $maxload records: $num_serial in serial,\ 90 $num_parallel in parallel." 91 close $fid 92 $db close 93 $serialdb close 94 $env close 95} 96 97proc init_runqueue { {dbdir RUNQUEUE} nitems list} { 98 99 if { [file exists $dbdir] != 1 } { 100 file mkdir $dbdir 101 } 102 puts "Creating test list..." 103 $list ALL -n 104 load_queue ALL.OUT $dbdir $nitems 105 file delete TEST.LIST 106 file rename ALL.OUT TEST.LIST 107} 108 109proc run_parallel { nprocs {list run_all} {nitems ALL} } { 110 global num_serial 111 global num_parallel 112 113 # Forcibly remove stuff from prior runs, if it's still there. 114 fileremove -f ./RUNQUEUE 115 set dirs [glob -nocomplain ./PARALLEL_TESTDIR.*] 116 set files [glob -nocomplain ALL.OUT.*] 117 foreach file $files { 118 fileremove -f $file 119 } 120 foreach dir $dirs { 121 fileremove -f $dir 122 } 123 124 set basename ./PARALLEL_TESTDIR 125 set queuedir ./RUNQUEUE 126 source ./include.tcl 127 128 mkparalleldirs $nprocs $basename $queuedir 129 130 init_runqueue $queuedir $nitems $list 131 132 set basedir [pwd] 133 set queuedir ../../[string range $basedir \ 134 [string last "/" $basedir] end]/$queuedir 135 136 # Run serial tests in parallel testdir 0. 137 run_queue 0 $basename.0 $queuedir serial $num_serial 138 139 set pidlist {} 140 # Run parallel tests in testdirs 1 through n. 141 for { set i 1 } { $i <= $nprocs } { incr i } { 142 set ret [catch { 143 set p [exec $tclsh_path << \ 144 "source $test_path/test.tcl; run_queue $i \ 145 $basename.$i $queuedir parallel $num_parallel" &] 146 lappend pidlist $p 147 set f [open $testdir/begin.$p w] 148 close $f 149 } res] 150 } 151 watch_procs $pidlist 300 1000000 152 153 set failed 0 154 for { set i 0 } { $i <= $nprocs } { incr i } { 155 if { [file exists ALL.OUT.$i] == 1 } { 156 puts -nonewline "Checking output from ALL.OUT.$i ... " 157 if { [check_output ALL.OUT.$i] == 1 } { 158 set failed 1 159 } 160 puts " done." 161 } 162 } 163 if { $failed == 0 } { 164 puts "Regression tests succeeded." 165 } else { 166 puts "Regression tests failed." 167 puts "Review UNEXPECTED OUTPUT lines above for errors." 168 puts "Complete logs found in ALL.OUT.x files" 169 } 170} 171 172proc run_queue { i rundir queuedir {qtype parallel} {nitems 0} } { 173 set builddir [pwd] 174 file delete $builddir/ALL.OUT.$i 175 cd $rundir 176 177 puts "Starting $qtype run_queue process $i (pid [pid])." 178 179 source ./include.tcl 180 global env 181 182 set dbenv [berkdb_env -create -lock -home $queuedir] 183 error_check_good dbenv [is_valid_env $dbenv] TRUE 184 185 if { $qtype == "parallel" } { 186 set db [eval {berkdb_open -env $dbenv \ 187 -mode 0644 -queue queue.db} ] 188 error_check_good dbopen [is_valid_db $db] TRUE 189 } elseif { $qtype == "serial" } { 190 set db [eval {berkdb_open -env $dbenv \ 191 -mode 0644 -queue serialqueue.db} ] 192 error_check_good serialdbopen [is_valid_db $db] TRUE 193 } else { 194 puts "FAIL: queue type $qtype not recognized" 195 } 196 197 set dbc [eval $db cursor] 198 error_check_good cursor [is_valid_cursor $dbc $db] TRUE 199 200 set count 0 201 set waitcnt 0 202 set starttime [timestamp -r] 203 204 while { $waitcnt < 5 } { 205 set line [$db get -consume] 206 if { [ llength $line ] > 0 } { 207 set cmd [lindex [lindex $line 0] 1] 208 set num [lindex [lindex $line 0] 0] 209 set o [open $builddir/ALL.OUT.$i a] 210 puts $o "\nExecuting record $num ([timestamp -w]):\n" 211 set tdir "TESTDIR.$i" 212 regsub -all {TESTDIR} $cmd $tdir cmd 213 puts $o $cmd 214 close $o 215 if { [expr {$num % 10} == 0] && $nitems != 0 } { 216 puts -nonewline \ 217 "Starting test $num of $nitems $qtype items. " 218 set now [timestamp -r] 219 set elapsed_secs [expr $now - $starttime] 220 set secs_per_test [expr $elapsed_secs / $num] 221 set esttotal [expr $nitems * $secs_per_test] 222 set remaining [expr $esttotal - $elapsed_secs] 223 if { $remaining < 3600 } { 224 puts "\tRough guess: less than 1\ 225 hour left." 226 } else { 227 puts "\tRough guess: \ 228 [expr $remaining / 3600] hour(s) left." 229 } 230 } 231# puts "Process $i, record $num:\n$cmd" 232 set env(PURIFYOPTIONS) \ 233 "-log-file=./test$num.%p -follow-child-processes -messages=first" 234 set env(PURECOVOPTIONS) \ 235 "-counts-file=./cov.pcv -log-file=./cov.log -follow-child-processes" 236 if [catch {exec $tclsh_path \ 237 << "source $test_path/test.tcl; $cmd" \ 238 >>& $builddir/ALL.OUT.$i } res] { 239 set o [open $builddir/ALL.OUT.$i a] 240 puts $o "FAIL: '$cmd': $res" 241 close $o 242 } 243 env_cleanup $testdir 244 set o [open $builddir/ALL.OUT.$i a] 245 puts $o "\nEnding record $num ([timestamp])\n" 246 close $o 247 incr count 248 } else { 249 incr waitcnt 250 tclsleep 1 251 } 252 } 253 254 set now [timestamp -r] 255 set elapsed [expr $now - $starttime] 256 puts "Process $i: $count commands executed in [format %02u:%02u \ 257 [expr $elapsed / 3600] [expr ($elapsed % 3600) / 60]]" 258 259 error_check_good close_parallel_cursor_$i [$dbc close] 0 260 error_check_good close_parallel_db_$i [$db close] 0 261 error_check_good close_parallel_env_$i [$dbenv close] 0 262 263 # 264 # We need to put the pid file in the builddir's idea 265 # of testdir, not this child process' local testdir. 266 # Therefore source builddir's include.tcl to get its 267 # testdir. 268 # !!! This resets testdir, so don't do anything else 269 # local to the child after this. 270 source $builddir/include.tcl 271 272 set f [open $builddir/$testdir/end.[pid] w] 273 close $f 274 cd $builddir 275} 276 277proc mkparalleldirs { nprocs basename queuedir } { 278 source ./include.tcl 279 set dir [pwd] 280 281 if { $is_windows_test != 1 } { 282 set EXE "" 283 } else { 284 set EXE ".exe" 285 } 286 for { set i 0 } { $i <= $nprocs } { incr i } { 287 set destdir $basename.$i 288 catch {file mkdir $destdir} 289 puts "Created $destdir" 290 if { $is_windows_test == 1 } { 291 catch {file mkdir $destdir/Debug} 292 catch {eval file copy \ 293 [eval glob {$dir/Debug/*.dll}] $destdir/Debug} 294 } 295 catch {eval file copy \ 296 [eval glob {$dir/{.libs,include.tcl}}] $destdir} 297 # catch {eval file copy $dir/$queuedir $destdir} 298 catch {eval file copy \ 299 [eval glob {$dir/db_{checkpoint,deadlock}$EXE} \ 300 {$dir/db_{dump,load,printlog,recover,stat,upgrade}$EXE} \ 301 {$dir/db_{archive,verify,hotbackup}$EXE}] \ 302 $destdir} 303 304 # Create modified copies of include.tcl in parallel 305 # directories so paths still work. 306 307 set infile [open ./include.tcl r] 308 set d [read $infile] 309 close $infile 310 311 regsub {test_path } $d {test_path ../} d 312 regsub {src_root } $d {src_root ../} d 313 set tdir "TESTDIR.$i" 314 regsub -all {TESTDIR} $d $tdir d 315 regsub {KILL \.} $d {KILL ..} d 316 set outfile [open $destdir/include.tcl w] 317 puts $outfile $d 318 close $outfile 319 320 global svc_list 321 foreach svc_exe $svc_list { 322 if { [file exists $dir/$svc_exe] } { 323 catch {eval file copy $dir/$svc_exe $destdir} 324 } 325 } 326 } 327} 328 329proc run_ptest { nprocs test args } { 330 global parms 331 global valid_methods 332 set basename ./PARALLEL_TESTDIR 333 set queuedir NULL 334 source ./include.tcl 335 336 mkparalleldirs $nprocs $basename $queuedir 337 338 if { [info exists parms($test)] } { 339 foreach method $valid_methods { 340 if { [eval exec_ptest $nprocs $basename \ 341 $test $method $args] != 0 } { 342 break 343 } 344 } 345 } else { 346 eval exec_ptest $nprocs $basename $test $args 347 } 348} 349 350proc exec_ptest { nprocs basename test args } { 351 source ./include.tcl 352 353 set basedir [pwd] 354 set pidlist {} 355 puts "Running $nprocs parallel runs of $test" 356 for { set i 1 } { $i <= $nprocs } { incr i } { 357 set outf ALL.OUT.$i 358 fileremove -f $outf 359 set ret [catch { 360 set p [exec $tclsh_path << \ 361 "cd $basename.$i;\ 362 source ../$test_path/test.tcl;\ 363 $test $args" >& $outf &] 364 lappend pidlist $p 365 set f [open $testdir/begin.$p w] 366 close $f 367 } res] 368 } 369 watch_procs $pidlist 30 36000 370 set failed 0 371 for { set i 1 } { $i <= $nprocs } { incr i } { 372 if { [check_output ALL.OUT.$i] == 1 } { 373 set failed 1 374 puts "Test $test failed in process $i." 375 } 376 } 377 if { $failed == 0 } { 378 puts "Test $test succeeded all processes" 379 return 0 380 } else { 381 puts "Test failed: stopping" 382 return 1 383 } 384} 385