1#!/usr/bin/env python3 2 3from termcolor import colored 4from ruamel.yaml import YAML 5import os 6import subprocess 7import sys 8import importlib 9import re 10import shutil 11import argparse 12from collections import OrderedDict 13from pathlib import Path 14 15# Check version is at least python 3.7 16if sys.version_info[0] < 3: 17 print(">= python 3.6 is required to run the testing script") 18 print("Your version: {0}.{1}.{2}".format( 19 sys.version_info[0], sys.version_info[1], sys.version_info[2])) 20 sys.exit(1) 21 22 23PYTHON_V37 = sys.version_info[0] == 3 and sys.version_info[1] >= 7 24 25# Check all our dependancies are installed 26 27 28def check_import(name): 29 try: 30 importlib.import_module(name) 31 return True 32 except ImportError as exc: 33 print("Dependency module '{}' not installed - please install via pip3".format(name)) 34 return False 35 36 37importok = [check_import("ruamel.yaml"), check_import("termcolor")] 38 39if not all(importok): 40 sys.exit(1) 41 42# 43# Script Starts here 44# 45 46class PreconditionViolation(Exception): 47 pass 48 49 50CONFIG_FILE_NAME = "config.yaml" 51TEST_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) 52TEST_DIST_DIR = Path(TEST_SCRIPT_DIR, "dist") 53 54def clean_dist(): 55 # rm -rf 56 shutil.rmtree(TEST_DIST_DIR, ignore_errors=True) 57 58 59def setup_dist(): 60 if os.path.exists(TEST_DIST_DIR): 61 clean_dist() 62 os.mkdir(TEST_DIST_DIR) 63 64 65# Dodgy: A global list of test names that should be verbose 66# if it is None, then verbose is off 67# if it is empty, that means all tests should be verbose 68# if it is nonempty, only the test names inside should be verbose 69verbose_test_names = None 70 71# Represents the result of a test 72# Takes in a function which returns 73# (status, errormsg, expected) 74# where status, expected :: "pass" | "fail" | "error" | "wip-pass" | "wip-fail" | "wip" 75 76def is_wip(expected): 77 return (expected == "wip" or 78 expected == "wip-pass" or 79 expected == "wip-fail") 80 81class TestContext: 82 def __init__(self, repo, cogent, dist_dir, script_dir, phases, ignore_phases): 83 self.repo = repo 84 self.cogent = cogent 85 self.dist_dir = dist_dir 86 self.script_dir = script_dir 87 self.phases = phases 88 self.ignore_phases = ignore_phases 89 90class Phase: 91 def __init__(self, phase_path): 92 self.name = phase_path.stem 93 self.phase_file = phase_path.resolve() 94 95 def run(self, context, fname, test): 96 return subprocess.run([self.phase_file] + [fname], 97 stderr = subprocess.STDOUT, 98 stdout = subprocess.PIPE, 99 cwd = context.script_dir, 100 env = { "COGENT_REPO": context.repo 101 , "TEST_DIST_DIR": context.dist_dir 102 , "HOME": os.environ['HOME'] 103 }) 104 105class TestResult: 106 107 block_len = 15 108 109 def __init__(self, test, fullname, test_name): 110 self.test = test 111 self.fullname = fullname 112 self.test_name = test_name 113 self.display() 114 115 def result(self): 116 (status, _, expected) = self.test 117 return status == expected or is_wip(expected) 118 119 # Printing test results 120 def display(self): 121 122 acc = "" 123 print("{}: ".format(os.path.relpath(self.fullname)), end="") 124 (status, output, expected) = self.test 125 126 be_verbose = (verbose_test_names is not None and 127 (verbose_test_names == [] or 128 self.test_name in verbose_test_names)) 129 130 131 if expected == "wip-pass": 132 if status == "pass": 133 acc += colored("WIP (Passed as expected)\n", "green") 134 else: 135 acc += colored("WIP (expected pass, got " + status + ")\n", "yellow") 136 elif expected == "wip-fail": 137 if status == "fail": 138 acc += colored("WIP (Failed as expected)\n", "green") 139 else: 140 acc += colored("WIP (expected fail, got " + status + ")\n", "yellow") 141 142 elif expected == "wip": 143 acc += colored("WIP (got " + status + ")\n", "yellow") 144 145 elif status == "error" and expected != "error": 146 acc += colored("Error? ", "yellow") + "\nReason:\n" 147 elif status == expected: 148 if status == "pass": 149 acc += colored("Passed\n", "green") 150 elif status == "fail": 151 acc += colored("Failed (as expected)\n", "green") 152 elif status == "error": 153 acc += colored("Error (as expected)\n", "green") 154 else: 155 if expected == "error": 156 acc += coloured("Test ran but was expected to error", "red") 157 elif expected == "pass": 158 acc += colored("Failed", "red") + "\n" 159 elif expected == "fail": 160 acc += colored("Failed (expected fail, got pass)", 161 "red") + "\n" 162 163 if be_verbose or (status != expected and expected != "wip"): 164 acc += ("=" * self.block_len + "Test Output" + 165 "=" * self.block_len) + "\n" 166 acc += output 167 acc += ("=" * self.block_len + len("Test Output") 168 * "=" + "=" * self.block_len) + "\n" 169 170 print(acc, end="") 171 return (status == expected) or expected == "wip" 172 173# For validating configurations 174 175 176class InvalidConfigError(Exception): 177 pass 178 179# Represents a test configuration file 180# Can perform multiple actions according to the layout of the file 181 182 183class TestConfiguration: 184 185 valid_test_fields = ["files", "flags", "expected_result", "test_name", "phase"] 186 187 header_block_len = 20 188 189 # file path must be ABSOLUTE 190 def __init__(self, filepath): 191 with open(filepath, 'r') as f: 192 self.settings = YAML().load(f.read()) 193 self.filepath = filepath 194 self.relpath = os.path.relpath(self.filepath) 195 self.dir = os.path.dirname(filepath) 196 self.validate_config() 197 198 # Checks a given config file is valid 199 def validate_config(self): 200 if (not isinstance(self.settings, list)): 201 raise InvalidConfigError( 202 "{}: Config files must be a list of test objects".format(self.relpath)) 203 204 i = 1 205 for f in self.settings: 206 if not "test_name" in f.keys(): 207 raise InvalidConfigError( 208 "Test {0} in {1} must contain mandatory field 'test_name', specifying the (unique) name of the test".format(i, self.relpath)) 209 if not "files" in f.keys(): 210 raise InvalidConfigError( 211 "Test {0} in {1} must contain mandatory field 'files', a list with at least 1 test".format(i, self.relpath)) 212 if not "expected_result" in f.keys(): 213 raise InvalidConfigError( 214 "Test {0} in {1} must contain mandatory field 'expected_result'".format(i, self.relpath)) 215 216 if len(f["files"]) == 0: 217 raise InvalidConfigError( 218 "Test {0} in {1} must have at least 1 test file".format(i, self.relpath)) 219 try: 220 if len(f["flags"]) == 0: 221 raise InvalidConfigError( 222 "Test {0} in {1} must have at least 1 compiler flag".format(i, self.relpath)) 223 except KeyError: 224 pass 225 if f["expected_result"] not in ["error", "pass", "fail", "wip", "wip-pass", "wip-fail"]: 226 raise InvalidConfigError("""Field 'expected_result' must be one of 'pass', 'fail', 'error' or 'wip' in test {0} in {1}\n. Actual value: {2}""" 227 .format(i, self.relpath, str(f["expected_result"]))) 228 229 for k in f.keys(): 230 if k not in self.valid_test_fields: 231 raise InvalidConfigError( 232 "Field '{0}' not a valid field in test {1} in {2}".format(k, i, self.relpath)) 233 234 try: 235 for flag in f["flags"]: 236 if re.compile(r'^\s*--dist-dir').match(flag): 237 raise InvalidConfigError( 238 "The use of the '--dist-dir' flag is prohibited in test flags (test {}, in {})".format(i, self.relpath)) 239 except KeyError: 240 pass 241 242 i += 1 243 244 def get_all_test_names(self): 245 return [x['test_name'] for x in self.settings ] 246 247class Test: 248 def __init__(self, testConfig): 249 self.config = testConfig 250 251 # Run the cogent compiler with the given flags, under test schema d 252 def run_cogent(self, context, filename, test_info): 253 fname = os.path.join(self.config.dir, filename) 254 # Check file exists and error gracefully 255 if not os.path.exists(fname): 256 return TestResult( 257 ("error", "Source file '{}' not found".format(fname), test_info['expected_result']), 258 fname, 259 test_info['test_name'] 260 ) 261 262 # run our test 263 setup_dist() 264 265 try: 266 flags = test_info['flags'] 267 except KeyError: 268 flags = [] 269 270 res = subprocess.run([context.cogent] + flags + ["--dist-dir={}".format(TEST_DIST_DIR)] + [fname], 271 stderr=subprocess.STDOUT, 272 stdout=subprocess.PIPE, 273 cwd=self.config.dir) 274 275 status = "pass" 276 277 # The compiler returns an error code 278 if res.returncode == 134: 279 status = "fail" 280 # The haskell process crashes/errors 281 elif res.returncode != 0: 282 status = "error" 283 284 result = (status, res.stdout.decode("utf-8"), test_info["expected_result"]) 285 286 return TestResult(result, fname, test_info['test_name']) 287 288 def run_phase(self, context, filename, phase, test_info): 289 fname = os.path.join(self.config.dir, filename) 290 # Check file exists and error gracefully 291 if not os.path.exists(fname): 292 return TestResult( 293 ("error", "Source file '{}' not found".format(fname), test_info['expected_result']), 294 fname, 295 test_info['test_name'] 296 ) 297 298 if not context.repo: 299 return TestResult( 300 ("error", "repo not found", test_info['expected_result']), 301 fname, 302 test_info['test_name'] 303 ) 304 305 # runs the test 306 setup_dist() 307 308 res = phase.run(context, fname, test_info) 309 310 status = "pass" 311 312 # The compiler returns an error code 313 if res.returncode == 134: 314 status = "fail" 315 # The haskell process crashes/errors 316 elif res.returncode != 0: 317 status = "error" 318 319 result = (status, res.stdout.decode("utf-8"), test_info["expected_result"]) 320 321 return TestResult(result, fname, test_info['test_name']) 322 323 def print_test_header(self, test_name): 324 print("-" * self.config.header_block_len, 325 " {} ".format(test_name), 326 "-" * self.config.header_block_len) 327 328 # Run one test by name 329 def run_one(self, context, test_name): 330 return self.run_tests(context, filter(lambda t: test_name == t['test_name'], self.config.settings)) 331 332 # Run all tests in the configuration file 333 def run_all(self, context): 334 return self.run_tests(context, self.config.settings) 335 336 def run_tests(self, context, tests): 337 results = [] 338 for test in tests: 339 self.print_test_header(test['test_name']) 340 for f in test['files']: 341 try: 342 phasename = test['phase'] 343 except KeyError: 344 phasename = "cogent" 345 346 if phasename == "cogent" and phasename not in context.ignore_phases: 347 results.append( self.run_cogent(context, f, test) ) 348 349 elif context.phases is None or phasename in context.ignore_phases: 350 continue 351 352 else: 353 try: 354 results.append( self.run_phase(context, f, context.phases[phasename], test) ) 355 except KeyError: 356 results.append( TestResult( 357 ("error", "phase not found: {}\n".format(phasename), test['expected_result']), 358 f, 359 test['test_name'] 360 )) 361 print() 362 return results 363 364# a collection of configurations 365class Configurations: 366 def __init__(self, path): 367 self.files = [f.resolve() for f in path.rglob(CONFIG_FILE_NAME)] 368 self.configs = [] 369 self.errConfigs = [] 370 for f in self.files: 371 try: 372 self.configs.append(TestConfiguration(f)) 373 except InvalidConfigError as e: 374 self.errConfigs.append(e) 375 except OSError as e: 376 self.errConfigs.append(e) 377 378 def has_erroring_configs(self): 379 return self.errConfigs != [] 380 381 def print_errs(self): 382 print(self.errConfigs) 383 for e in self.errConfigs: 384 if type(e) is InvalidConfigError: 385 print(colored("Config error: ", "red"), e) 386 elif type(e) is OSError: 387 print(colored("error - could not find config file for test file {}".format(e), "red")) 388 389 # Based on an asbolute path for a test file, get it's configuration 390 def get_cfg_from_test_name(self, f): 391 cfgs = self.get_configs() 392 for c in cfgs: 393 if f in c.get_all_test_names(): 394 return c 395 return None 396 397 def get_configs(self): 398 return self.configs 399 400 401 402# 403# Main script 404# 405 406def main(): 407 ap = argparse.ArgumentParser( 408 description="Cogent Testing Framework", 409 epilog="Test configurations must be stored in a '{}' file".format( 410 CONFIG_FILE_NAME), 411 allow_abbrev=False 412 ) 413 ap.add_argument("--only", "-o", dest="only_test", 414 help="only run specified tests", 415 metavar="TEST_NAME") 416 ap.add_argument("--verbose", "-v", 417 dest="verbose", 418 help="print output for given tests even if they pass (none supplied = all tests)", 419 metavar="TEST_NAME", 420 nargs='*') 421 ap.add_argument("--validate", "-t", 422 dest="validate", 423 action="store_true", 424 help="Check the format of all config files is correct") 425 ap.add_argument("--extra-phases", "-p", 426 dest="phase_dir", 427 default=None, 428 help="set the location of the additional phase directory") 429 ap.add_argument("--ignore-phases", 430 dest="ignore_phases", 431 action="store", 432 nargs="+", 433 default=[], 434 help="ignore the tests for the specified phases") 435 ap.add_argument("--repo", 436 dest="repo", 437 help="set the location of the repository root") 438 ap.add_argument("--cogent", 439 dest="cogent", 440 default="cogent", 441 help="specify the location of the cogent compiler") 442 ap.add_argument("--ignore-errors", 443 dest="ignore_errors", 444 action="store_true", 445 help="if enabled, a test error does not cause the script to exit with an error") 446 args = ap.parse_args() 447 448 cogent = shutil.which(args.cogent) 449 450 if args.repo is not None: 451 repo = os.path.abspath(args.repo) 452 else: 453 repo = None 454 print("Warning: repository directory not set; use --repo") 455 456 if args.phase_dir is not None: 457 phase_dir = os.path.abspath(args.phase_dir) 458 else: 459 phase_dir = None 460 461 # Check if cogent is installed 462 if cogent is None: 463 print("Could not find cogent compiler - Please either add it to your PATH or set --cogent") 464 sys.exit(1) 465 466 print("Using repository: " + str(repo)) 467 print("Using cogent: " + cogent) 468 print("Using phase dir: " + str(phase_dir)) 469 470 if phase_dir is not None and Path(phase_dir).exists(): 471 files = Path(phase_dir).glob("*.sh") 472 phases = dict(map(lambda p: (p.stem,Phase(p)), files)) 473 else: 474 phases = None 475 476 context = TestContext(repo, cogent, TEST_DIST_DIR, TEST_SCRIPT_DIR, phases, args.ignore_phases) 477 478 # find all config files 479 configs = Configurations(Path(".")) 480 481 # Validate all config files 482 if args.validate: 483 isErr = configs.has_erroring_configs() 484 if isErr: 485 configs.print_errs() 486 print(colored("Errors found in above configuration files", "red")) 487 else: 488 print(colored("All configuration files okay!", "green")) 489 sys.exit((1 if isErr else 0)) 490 elif configs.has_erroring_configs(): 491 print(colored("Errors found in above configuration files:", "red")) 492 print(colored(" call with --validate for more info", "red")) 493 sys.exit(1) 494 495 if args.verbose is not None: 496 verbose_test_names = args.verbose 497 498 results = [] 499 # If we're only running specific tests 500 if args.only_test is not None: 501 test_name = args.only_test 502 conf = configs.get_cfg_from_test_name(test_name) 503 if conf is None: 504 print(colored("Cannot find config file containing test name {}".format(test_name), "red")) 505 sys.exit(1) 506 test = Test(conf) 507 results = test.run_one(context, test_name) 508 # Otherwise, run all possible tests 509 else: 510 tests = list(map(lambda config: Test(config), configs.get_configs())) 511 512 for test in tests: 513 subresults = test.run_all(context) 514 results.extend(subresults) 515 516 setup_dist() 517 518 errs = 0 519 passes = 0 520 fails = 0 521 wips = 0 522 523 for res in results: 524 (status, _, expected) = res.test 525 526 if is_wip(expected): 527 wips += 1 528 elif status == "error": 529 errs += 1 530 elif res.result(): 531 passes += 1 532 else: 533 fails += 1 534 535 print("-"*15 + " Final results: " + "-" * 15) 536 print() 537 538 print("{:>16}{:>16}".format("Result", "Amount")) 539 print("{:>16}{:>16}".format("Errors", errs)) 540 print("{:>16}{:>16}".format("Passes", passes)) 541 print("{:>16}{:>16}".format("Fails", fails)) 542 print("{:>16}{:>16}".format("Work In Progress", wips)) 543 print() 544 545 if fails != 0 or (not args.ignore_errors and errs != 0): 546 sys.exit(1) 547 548 549if __name__ == "__main__": 550 main() 551 552