regress-sb.rb revision 147495
10SN/A#!/usr/local/bin/ruby 22362SN/A# -------+---------+---------+-------- + --------+---------+---------+---------+ 30SN/A# Copyright (c) 2005 - Garance Alistair Drosehn <gad@FreeBSD.org>. 40SN/A# All rights reserved. 50SN/A# 60SN/A# Redistribution and use in source and binary forms, with or without 72362SN/A# modification, are permitted provided that the following conditions 80SN/A# are met: 92362SN/A# 1. Redistributions of source code must retain the above copyright 100SN/A# notice, this list of conditions and the following disclaimer. 110SN/A# 2. Redistributions in binary form must reproduce the above copyright 120SN/A# notice, this list of conditions and the following disclaimer in the 130SN/A# documentation and/or other materials provided with the distribution. 140SN/A# 150SN/A# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 160SN/A# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 170SN/A# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 180SN/A# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 190SN/A# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 200SN/A# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 212362SN/A# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 222362SN/A# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 232362SN/A# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 240SN/A# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 250SN/A# SUCH DAMAGE. 260SN/A# -------+---------+---------+-------- + --------+---------+---------+---------+ 270SN/A# $FreeBSD: head/tools/regression/usr.bin/env/regress-sb.rb 147495 2005-06-20 04:17:12Z gad $ 280SN/A# -------+---------+---------+-------- + --------+---------+---------+---------+ 290SN/A# This script was written to provide a battery of regression-tests for some 300SN/A# changes I am making to the `env' command. I wrote a new script for this 310SN/A# for several reasons. 1) I needed to test all kinds of special-character 320SN/A# combinations, and I wanted to be able to type those in exactly as they would 330SN/A# would be in real-life situations. 2) I wanted to set environment variables 340SN/A# before executing a test, 3) I had many different details to test, so I wanted 350SN/A# to write up dozens of tests, without needing to create a hundred separate 360SN/A# little tiny files, 4) I wanted to test *failure* conditions, where I expected 370SN/A# the test would fail but I wanted to be sure that it failed the way I intended 380SN/A# it to fail. 390SN/A# This script was written for the special "shebang-line" testing that I 400SN/A# wanted for my changes to `env', but I expect it could be turned into a 410SN/A# general-purpose test-suite with a little more work. 420SN/A# Garance/June 12/2005 430SN/A# -------+---------+---------+-------- + --------+---------+---------+---------+ 440SN/A 450SN/A 460SN/A# -------+---------+---------+-------- + --------+---------+---------+---------+ 470SN/Aclass ExpectedResult 480SN/A attr_writer :cmdvalue, :shebang_args, :user_args 490SN/A @@gbl_envs = Hash.new 500SN/A 510SN/A def ExpectedResult.add_gblenv(avar, avalue) 520SN/A @@gbl_envs[avar] = avalue 530SN/A end 540SN/A 550SN/A def initialize 560SN/A @shebang_args = "" 570SN/A @cmdvalue = 0 580SN/A @clear_envs = Hash.new 590SN/A @new_envs = Hash.new 600SN/A @old_envs = Hash.new 610SN/A @script_lines = "" 620SN/A @expect_err = Array.new 630SN/A @expect_out = Array.new 640SN/A @symlinks = Array.new 650SN/A @user_args = nil 660SN/A end 670SN/A 680SN/A def add_expecterr(aline) 690SN/A @expect_err << aline 700SN/A end 710SN/A 720SN/A def add_expectout(aline) 730SN/A @expect_out << aline 740SN/A end 750SN/A 760SN/A def add_script(aline) 770SN/A @script_lines += aline 780SN/A @script_lines += "\n" if aline[-1] != "\n" 790SN/A end 800SN/A 810SN/A def add_clearenv(avar) 820SN/A @clear_envs[avar] = true 830SN/A end 840SN/A 850SN/A def add_setenv(avar, avalue) 860SN/A @new_envs[avar] = avalue 870SN/A end 880SN/A 890SN/A def add_symlink(srcf, newf) 900SN/A @symlinks << Array.[](srcf, newf) 910SN/A end 920SN/A 930SN/A def check_out(name, fname, expect_arr) 940SN/A idx = -1 950SN/A all_matched = true 960SN/A extra_lines = 0 970SN/A rdata = File.open(fname) 980SN/A rdata.each_line { |rline| 990SN/A rline.chomp! 1000SN/A idx += 1 1010SN/A if idx > expect_arr.length - 1 1020SN/A if extra_lines == 0 and $verbose >= 1 1030SN/A printf "-- Extra line(s) on %s:\n", name 1040SN/A end 1050SN/A printf "-- [%d] > %s\n", idx, rline if $verbose >= 1 1060SN/A extra_lines += 1 1070SN/A elsif rline != expect_arr[idx] 1080SN/A if all_matched and $verbose >= 1 1090SN/A printf "-- Mismatched line(s) on %s:\n", name 1100SN/A end 1110SN/A printf "-- [%d] < %s\n", idx, expect_arr[idx] if $verbose >= 2 1120SN/A printf "-- > %s\n", rline if $verbose >= 1 1130SN/A all_matched = false 1140SN/A else 1150SN/A printf "-- %s[%d] = %s\n", name, idx, rline if $verbose >= 5 1160SN/A end 1170SN/A } 1180SN/A rdata.close 1190SN/A if extra_lines > 0 1200SN/A printf "-- %d extra line(s) found on %s\n", extra_lines, 1210SN/A name if $verbose == 0 1220SN/A return false 1230SN/A end 1240SN/A if not all_matched 1250SN/A printf "-- Mismatched line(s) found on %s\n", 1260SN/A name if $verbose == 0 1270SN/A return false 1280SN/A end 1290SN/A return true 1300SN/A end 1310SN/A 1320SN/A def create_links 1330SN/A @symlinks.each { |fnames| 1340SN/A if $verbose >= 2 1350SN/A printf "-- Creating: symlink %s %s\n", fnames[0], fnames[1] 1360SN/A end 1370SN/A symres = File.symlink(fnames[0], fnames[1]) 1380SN/A return false if symres == nil 1390SN/A return false unless File.symlink?(fnames[1]) 1400SN/A } 1410SN/A return true 1420SN/A end 1430SN/A 1440SN/A def destroy_links 1450SN/A @symlinks.each { |fnames| 1460SN/A if $verbose >= 2 1470SN/A printf "-- Removing: %s (symlink)\n", fnames[1] 1480SN/A end 14911882Savstepan if File.symlink?(fnames[1]) 1500SN/A if File.delete(fnames[1]) != 1 1510SN/A $stderr.printf "Warning: problem removing symlink '%s'\n", 1520SN/A fnames[1] 1530SN/A end 1540SN/A else 1550SN/A $stderr.printf "Warning: Symlink '%s' does not exist?!?\n", 1560SN/A fnames[1] 1570SN/A end 15811882Savstepan } 1590SN/A return true 1600SN/A end 1610SN/A 1620SN/A def init_io_files 1630SN/A @stderr = $scriptfile + ".stderr" 1640SN/A @stdout = $scriptfile + ".stdout" 1650SN/A File.delete(@stderr) if File.exists?(@stderr) 1660SN/A File.delete(@stdout) if File.exists?(@stdout) 1670SN/A @stdin = "/dev/null" 1680SN/A 16911882Savstepan @redirs = " <" + @stdin 1700SN/A @redirs += " >" + @stdout 1710SN/A @redirs += " 2>" + @stderr 1720SN/A 1730SN/A end 1740SN/A 1750SN/A def pop_envs 1760SN/A @new_envs.each_key { |evar| 1770SN/A if @old_envs.has_key?(evar) 1780SN/A ENV[evar] = @old_envs[evar] 1790SN/A else 1800SN/A ENV.delete(evar) 1810SN/A end 1820SN/A } 1830SN/A end 1840SN/A 1850SN/A def push_envs 1860SN/A @@gbl_envs.each_pair { |evar, eval| 1870SN/A ENV[evar] = eval 1880SN/A } 1890SN/A @new_envs.each_pair { |evar, eval| 1900SN/A if ENV.has_key?(evar) 1910SN/A @old_envs[evar] = ENV[evar] 1920SN/A end 1930SN/A ENV[evar] = eval 1940SN/A } 1950SN/A end 1960SN/A 1970SN/A def run_test 1980SN/A tscript = File.open($scriptfile, "w") 1990SN/A tscript.printf "#!%s", $testpgm 20013513Sigerasim tscript.printf " %s", @shebang_args if @shebang_args != "" 2010SN/A tscript.printf "\n" 2020SN/A tscript.printf "%s", @script_lines if @script_lines != "" 2030SN/A tscript.close 2040SN/A File.chmod(0755, $scriptfile) 2050SN/A 2060SN/A usercmd = $scriptfile 2070SN/A usercmd += " " + @user_args if @user_args != nil 2080SN/A init_io_files 2090SN/A 2100SN/A push_envs 2110SN/A return 0 unless create_links 21211975Sigerasim printf "- Executing: %s\n", usercmd if $verbose >= 1 21311975Sigerasim printf "----- with: %s\n", @redirs if $verbose >= 6 21411975Sigerasim sys_ok = system(usercmd + @redirs) 21511975Sigerasim if sys_ok 21611975Sigerasim @sav_cmdvalue = 0 21711975Sigerasim elsif $?.exited? 21811975Sigerasim @sav_cmdvalue = $?.exitstatus 21911975Sigerasim else 22011975Sigerasim @sav_cmdvalue = 125 2210SN/A end 2220SN/A destroy_links 2230SN/A pop_envs 2240SN/A sys_ok = true 2250SN/A if @sav_cmdvalue != @cmdvalue 2260SN/A printf "-- Expecting cmdvalue of %d, but $? == %d\n", @cmdvalue, 2270SN/A @sav_cmdvalue 2280SN/A sys_ok = false 2290SN/A end 2300SN/A sys_ok = false unless check_out("stdout", @stdout, @expect_out) 2310SN/A sys_ok = false unless check_out("stderr", @stderr, @expect_err) 2320SN/A return 1 if sys_ok 2330SN/A return 0 23411882Savstepan end 2350SN/Aend 2360SN/A 2370SN/A# -------+---------+---------+-------- + --------+---------+---------+---------+ 2380SN/A# Processing of the command-line options given to the regress-sb.rb script. 2390SN/A# 2400SN/Aclass CommandOptions 2410SN/A def CommandOptions.parse(command_args) 2420SN/A parse_ok = true 2430SN/A command_args.each { |userarg| 2440SN/A case userarg 2450SN/A when /^--rgdata=(\S+)$/ 2460SN/A parse_ok = false unless set_rgdatafile($1) 2470SN/A when /^--testpgm=(\S+)$/ 2480SN/A parse_ok = false unless set_testpgm($1) 2490SN/A $cmdopt_testpgm = $testpgm 2500SN/A when "--stop-on-error", "--stop_on_error" 2510SN/A $stop_on_error = true 2520SN/A when /^--/ 2530SN/A $stderr.printf "Error: Invalid long option: %s\n", userarg 2540SN/A parse_ok = false 25511882Savstepan when /^-/ 2560SN/A userarg = userarg[1...userarg.length] 2570SN/A userarg.each_byte { |byte| 2580SN/A char = byte.chr 2590SN/A case char 2600SN/A when "v" 2610SN/A $verbose += 1 2620SN/A else 263 $stderr.printf "Error: Invalid short option: -%s\n", char 264 parse_ok = false 265 end 266 } 267 else 268 $stderr.printf "Error: Invalid request: %s\n", userarg 269 parse_ok = false 270 end 271 } 272 if $rgdatafile == nil 273 rgmatch = Dir.glob("regress*.rgdata") 274 if rgmatch.length == 1 275 $rgdatafile = rgmatch[0] 276 printf "Assuming --rgdata=%s\n", $rgdatafile 277 else 278 $stderr.printf "Error: The --rgdata file was not specified\n" 279 parse_ok = false 280 end 281 end 282 return parse_ok 283 end 284 285 def CommandOptions.set_rgdatafile(fname) 286 if not File.exists?(fname) 287 $stderr.printf "Error: Rgdata file '%s' does not exist\n", fname 288 return false 289 elsif not File.readable?(fname) 290 $stderr.printf "Error: Rgdata file '%s' is not readable\n", fname 291 return false 292 end 293 $rgdatafile = File.expand_path(fname) 294 return true 295 end 296 297 def CommandOptions.set_testpgm(fname) 298 if not File.exists?(fname) 299 $stderr.printf "Error: Testpgm file '%s' does not exist\n", fname 300 return false 301 elsif not File.executable?(fname) 302 $stderr.printf "Error: Testpgm file '%s' is not executable\n", fname 303 return false 304 end 305 $testpgm = File.expand_path(fname) 306 return true 307 end 308end 309 310# -------+---------+---------+-------- + --------+---------+---------+---------+ 311# Processing of the test-specific options specifed in each [test]/[run] 312# section of the regression-data file. This will set values in the 313# global $testdata object. 314# 315class RGTestOptions 316 @@rgtest_opts = nil; 317 318 def RGTestOptions.init_rgtopts 319 @@rgtest_opts = Hash.new 320 @@rgtest_opts["$?"] = true 321 @@rgtest_opts["clearenv"] = true 322 @@rgtest_opts["sb_args"] = true 323 @@rgtest_opts["script"] = true 324 @@rgtest_opts["setenv"] = true 325 @@rgtest_opts["stderr"] = true 326 @@rgtest_opts["stdout"] = true 327 @@rgtest_opts["symlink"] = true 328 @@rgtest_opts["user_args"] = true 329 end 330 331 def RGTestOptions.parse(optname, optval) 332 init_rgtopts unless @@rgtest_opts 333 334 if not @@rgtest_opts.has_key?(optname) 335 $stderr.printf "Error: Invalid test-option in rgdata file: %s\n", 336 optname 337 return false 338 end 339 340 # Support a few very specific substitutions in values specified 341 # for test data. Format of all recognized values should be: 342 # [%-object.value-%] 343 # which is hopefully distinctive-enough that they will never 344 # conflict with any naturally-occuring string. Also note that 345 # we only match the specific values that we recognize, and not 346 # "just anything" that matches the general pattern. There are 347 # no blanks in the recognized values, but I use an x-tended 348 # regexp and then add blanks to make it more readable. 349 optval.gsub!(/\[%- testpgm\.basename -%\]/x, File.basename($testpgm)) 350 optval.gsub!(/\[%- script\.pathname -%\]/x, $scriptfile) 351 352 invalid_value = false 353 case optname 354 when "$?" 355 if optval =~ /^\d+$/ 356 $testdata.cmdvalue = optval.to_i 357 else 358 invalid_value = true 359 end 360 when "clearenv" 361 if optval =~ /^\s*([A-Za-z]\w*)\s*$/ 362 $testdata.add_clearenv($1) 363 else 364 invalid_value = true 365 end 366 when "sb_args" 367 $testdata.shebang_args = optval 368 when "script" 369 $testdata.add_script(optval) 370 when "setenv" 371 if optval =~ /^\s*([A-Za-z]\w*)=(.*)$/ 372 $testdata.add_setenv($1, $2) 373 else 374 invalid_value = true 375 end 376 when "stderr" 377 $testdata.add_expecterr(optval) 378 when "stdout" 379 $testdata.add_expectout(optval) 380 when "symlink" 381 if optval =~ /^\s*(\S+)\s+(\S+)\s*$/ 382 srcfile = $1 383 newfile = $2 384 if not File.exists?(srcfile) 385 $stderr.printf "Error: source file '%s' does not exist.\n", 386 srcfile 387 invalid_value = true 388 elsif File.exists?(newfile) 389 $stderr.printf "Error: new file '%s' already exists.\n", 390 newfile 391 invalid_value = true 392 else 393 $testdata.add_symlink(srcfile, newfile) 394 end 395 else 396 invalid_value = true 397 end 398 when "user_args" 399 $testdata.user_args = optval 400 else 401 $stderr.printf "InternalError: Invalid test-option in rgdata file: %s\n", 402 optname 403 return false 404 end 405 406 if invalid_value 407 $stderr.printf "Error: Invalid value(s) for %s: %s\n", 408 optname, optval 409 return false 410 end 411 return true 412 end 413end 414 415# -------+---------+---------+-------- + --------+---------+---------+---------+ 416# Here's where the "main" routine begins... 417# 418 419$cmdopt_testpgm = nil 420$testpgm = nil 421$rgdatafile = nil 422$scriptfile = "/tmp/env-regress" 423$stop_on_error = false 424$verbose = 0 425 426exit 1 unless CommandOptions.parse(ARGV) 427 428errline = nil 429test_count = 0 430testok_count = 0 431test_lineno = -1 432max_test = -1 433regress_data = File.open($rgdatafile) 434regress_data.each_line { |dline| 435 case dline 436 when /^\s*#/, /^\s*$/ 437 # Just a comment line, ignore it. 438 when /^\s*gblenv=\s*(.+)$/ 439 if test_lineno > 0 440 $stderr.printf "Error: Cannot define a global-value in the middle of a test (#5d)\n", test_lineno 441 errline = regress_data.lineno 442 break; 443 end 444 tempval = $1 445 if tempval !~ /^([A-Za-z]\w*)=(.*)$/ 446 $stderr.printf "Error: Invalid value for 'gblenv=' request: %s\n", 447 tempval 448 errline = regress_data.lineno 449 break; 450 end 451 ExpectedResult.add_gblenv($1, $2) 452 when /^testpgm=\s*(\S+)\s*/ 453 # Set the location of the program to be tested, if it wasn't set 454 # on the command-line processing. 455 if $cmdopt_testpgm == nil 456 if not CommandOptions.set_testpgm($1) 457 errline = regress_data.lineno 458 break; 459 end 460 end 461 when /^\[test\]$/ 462 if test_lineno > 0 463 $stderr.printf "Error: Request to define a [test], but we are still defining\n" 464 $stderr.printf " the [test] at line #%s\n", test_lineno 465 errline = regress_data.lineno 466 break; 467 end 468 test_lineno = regress_data.lineno 469 max_test = test_lineno 470 printf "- Defining test at line #%s\n", test_lineno if $verbose >= 6 471 $testdata = ExpectedResult.new 472 when /^\[end\]$/ 473 # User wants us to ignore the remainder of the rgdata file... 474 break; 475 when /^\[run\]$/ 476 if test_lineno < 0 477 $stderr.printf "Error: Request to [run] a test, but no test is presently defined\n" 478 errline = regress_data.lineno 479 break; 480 end 481 printf "- Running test at line #%s\n", test_lineno if $verbose >= 1 482 run_result = $testdata.run_test 483 test_count += 1 484 printf "[Test #%3d: ", test_count 485 case run_result 486 when 0 487 # Test failed 488 printf "Failed! (line %4d)]\n", test_lineno 489 break if $stop_on_error 490 when 1 491 # Test ran as expected 492 testok_count += 1 493 printf "OK]\n" 494 else 495 # Internal error of some sort 496 printf "InternalError! (line %4d)]\n", test_lineno 497 errline = regress_data.lineno 498 break; 499 end 500 test_lineno = -1 501 502 when /^(\s*)([^\s:]+)\s*:(.+)$/ 503 blankpfx = $1 504 test_lhs = $2 505 test_rhs = $3 506 if test_lineno < 0 507 $stderr.printf "Error: No test is presently being defined\n" 508 errline = regress_data.lineno 509 break; 510 end 511 # All the real work happens in RGTestOptions.parse 512 if not RGTestOptions.parse(test_lhs, test_rhs) 513 errline = regress_data.lineno 514 break; 515 end 516 if blankpfx.length == 0 517 $stderr.printf "Note: You should at least one blank before:%s\n", 518 dline.chomp 519 $stderr.printf " at line %d of rgdata file %s\n", 520 regress_data.lineno, $rgdatafile 521 end 522 523 else 524 $stderr.printf "Error: Invalid line: %s\n", dline.chomp 525 errline = regress_data.lineno 526 break; 527 end 528} 529regress_data.close 530if errline != nil 531 $stderr.printf " at line %d of rgdata file %s\n", errline, $rgdatafile 532 exit 2 533end 534if testok_count != test_count 535 printf "%d of %d tests were successful.\n", testok_count, test_count 536 exit 1 537end 538 539printf "All %d tests were successful!\n", testok_count 540exit 0 541