1#!/usr/bin/env python3
2
3from termcolor import colored
4from ruamel.yaml import YAML
5import os
6import subprocess
7import sys
8import importlib
9import re
10import shutil
11import argparse
12from collections import OrderedDict
13from pathlib import Path
14
15# Check version is at least python 3.7
16if sys.version_info[0] < 3:
17    print(">= python 3.6 is required to run the testing script")
18    print("Your version: {0}.{1}.{2}".format(
19        sys.version_info[0], sys.version_info[1], sys.version_info[2]))
20    sys.exit(1)
21
22
23PYTHON_V37 = sys.version_info[0] == 3 and sys.version_info[1] >= 7
24
25# Check all our dependancies are installed
26
27
28def check_import(name):
29    try:
30        importlib.import_module(name)
31        return True
32    except ImportError as exc:
33        print("Dependency module '{}' not installed - please install via pip3".format(name))
34        return False
35
36
37importok = [check_import("ruamel.yaml"), check_import("termcolor")]
38
39if not all(importok):
40    sys.exit(1)
41
42#
43# Script Starts here
44#
45
46class PreconditionViolation(Exception):
47    pass
48
49
50CONFIG_FILE_NAME = "config.yaml"
51TEST_SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
52TEST_DIST_DIR = Path(TEST_SCRIPT_DIR, "dist")
53
54def clean_dist():
55    # rm -rf
56    shutil.rmtree(TEST_DIST_DIR, ignore_errors=True)
57
58
59def setup_dist():
60    if os.path.exists(TEST_DIST_DIR):
61        clean_dist()
62    os.mkdir(TEST_DIST_DIR)
63
64
65# Dodgy: A global list of test names that should be verbose
66# if it is None, then verbose is off
67# if it is empty, that means all tests should be verbose
68# if it is nonempty, only the test names inside should be verbose
69verbose_test_names = None
70
71# Represents the result of a test
72# Takes in a function which returns
73#   (status, errormsg, expected)
74# where status, expected :: "pass" | "fail" | "error" | "wip-pass" | "wip-fail" | "wip"
75
76def is_wip(expected):
77    return (expected == "wip" or
78            expected == "wip-pass" or
79            expected == "wip-fail")
80
81class TestContext:
82    def __init__(self, repo, cogent, dist_dir, script_dir, phases, ignore_phases):
83        self.repo = repo
84        self.cogent = cogent
85        self.dist_dir = dist_dir
86        self.script_dir = script_dir
87        self.phases = phases
88        self.ignore_phases = ignore_phases
89
90class Phase:
91    def __init__(self, phase_path):
92        self.name = phase_path.stem
93        self.phase_file = phase_path.resolve()
94
95    def run(self, context, fname, test):
96        return subprocess.run([self.phase_file] + [fname],
97                                stderr = subprocess.STDOUT,
98                                stdout = subprocess.PIPE,
99                                cwd = context.script_dir,
100                                env = { "COGENT_REPO": context.repo
101                                      , "TEST_DIST_DIR": context.dist_dir
102                                      , "HOME": os.environ['HOME']
103                                      })
104
105class TestResult:
106
107    block_len = 15
108
109    def __init__(self, test, fullname, test_name):
110        self.test = test
111        self.fullname = fullname
112        self.test_name = test_name
113        self.display()
114
115    def result(self):
116        (status, _, expected) = self.test
117        return status == expected or is_wip(expected)
118
119    # Printing test results
120    def display(self):
121
122        acc = ""
123        print("{}: ".format(os.path.relpath(self.fullname)), end="")
124        (status, output, expected) = self.test
125
126        be_verbose = (verbose_test_names is not None and
127                                (verbose_test_names == [] or
128                                 self.test_name in verbose_test_names))
129
130
131        if expected == "wip-pass":
132            if status == "pass":
133                acc += colored("WIP (Passed as expected)\n", "green")
134            else:
135                acc += colored("WIP (expected pass, got " + status + ")\n", "yellow")
136        elif expected == "wip-fail":
137            if status == "fail":
138                acc += colored("WIP (Failed as expected)\n", "green")
139            else:
140                acc += colored("WIP (expected fail, got " + status + ")\n", "yellow")
141
142        elif expected == "wip":
143            acc += colored("WIP (got " + status + ")\n", "yellow")
144
145        elif status == "error" and expected != "error":
146            acc += colored("Error? ", "yellow") + "\nReason:\n"
147        elif status == expected:
148            if status == "pass":
149                acc += colored("Passed\n", "green")
150            elif status == "fail":
151                acc += colored("Failed (as expected)\n", "green")
152            elif status == "error":
153                acc += colored("Error (as expected)\n", "green")
154        else:
155            if expected == "error":
156                acc += coloured("Test ran but was expected to error", "red")
157            elif expected == "pass":
158                acc += colored("Failed", "red") + "\n"
159            elif expected == "fail":
160                acc += colored("Failed (expected fail, got pass)",
161                               "red") + "\n"
162
163        if be_verbose or (status != expected and expected != "wip"):
164            acc += ("=" * self.block_len + "Test Output" +
165                    "=" * self.block_len) + "\n"
166            acc += output
167            acc += ("=" * self.block_len + len("Test Output")
168                    * "=" + "=" * self.block_len) + "\n"
169
170        print(acc, end="")
171        return (status == expected) or expected == "wip"
172
173# For validating configurations
174
175
176class InvalidConfigError(Exception):
177    pass
178
179# Represents a test configuration file
180# Can perform multiple actions according to the layout of the file
181
182
183class TestConfiguration:
184
185    valid_test_fields = ["files", "flags", "expected_result", "test_name", "phase"]
186
187    header_block_len = 20
188
189    # file path must be ABSOLUTE
190    def __init__(self, filepath):
191        with open(filepath, 'r') as f:
192            self.settings = YAML().load(f.read())
193            self.filepath = filepath
194            self.relpath  = os.path.relpath(self.filepath)
195            self.dir = os.path.dirname(filepath)
196            self.validate_config()
197
198    # Checks a given config file is valid
199    def validate_config(self):
200        if (not isinstance(self.settings, list)):
201            raise InvalidConfigError(
202                "{}: Config files must be a list of test objects".format(self.relpath))
203
204        i = 1
205        for f in self.settings:
206            if not "test_name" in f.keys():
207                raise InvalidConfigError(
208                    "Test {0} in {1} must contain mandatory field 'test_name', specifying the (unique) name of the test".format(i, self.relpath))
209            if not "files" in f.keys():
210                raise InvalidConfigError(
211                    "Test {0} in {1} must contain mandatory field 'files', a list with at least 1 test".format(i, self.relpath))
212            if not "expected_result" in f.keys():
213                raise InvalidConfigError(
214                    "Test {0} in {1} must contain mandatory field 'expected_result'".format(i, self.relpath))
215
216            if len(f["files"]) == 0:
217                raise InvalidConfigError(
218                    "Test {0} in {1} must have at least 1 test file".format(i, self.relpath))
219            try:
220                if len(f["flags"]) == 0:
221                    raise InvalidConfigError(
222                        "Test {0} in {1} must have at least 1 compiler flag".format(i, self.relpath))
223            except KeyError:
224                pass
225            if f["expected_result"] not in ["error", "pass", "fail", "wip", "wip-pass", "wip-fail"]:
226                raise InvalidConfigError("""Field 'expected_result' must be one of 'pass', 'fail', 'error' or 'wip' in test {0} in {1}\n. Actual value: {2}"""
227                                         .format(i, self.relpath, str(f["expected_result"])))
228
229            for k in f.keys():
230                if k not in self.valid_test_fields:
231                    raise InvalidConfigError(
232                        "Field '{0}' not a valid field in test {1} in {2}".format(k, i, self.relpath))
233
234            try:
235                for flag in f["flags"]:
236                    if re.compile(r'^\s*--dist-dir').match(flag):
237                        raise InvalidConfigError(
238                            "The use of the '--dist-dir' flag is prohibited in test flags (test {}, in {})".format(i, self.relpath))
239            except KeyError:
240                pass
241
242            i += 1
243
244    def get_all_test_names(self):
245        return [x['test_name'] for x in self.settings ]
246
247class Test:
248    def __init__(self, testConfig):
249        self.config = testConfig
250
251    # Run the cogent compiler with the given flags, under test schema d
252    def run_cogent(self, context, filename, test_info):
253        fname = os.path.join(self.config.dir, filename)
254        # Check file exists and error gracefully
255        if not os.path.exists(fname):
256            return TestResult(
257                        ("error", "Source file '{}' not found".format(fname), test_info['expected_result']),
258                        fname,
259                        test_info['test_name']
260                    )
261
262        # run our test
263        setup_dist()
264
265        try:
266            flags = test_info['flags']
267        except KeyError:
268            flags = []
269
270        res = subprocess.run([context.cogent] + flags + ["--dist-dir={}".format(TEST_DIST_DIR)] + [fname],
271                                stderr=subprocess.STDOUT,
272                                stdout=subprocess.PIPE,
273                                cwd=self.config.dir)
274
275        status = "pass"
276
277        # The compiler returns an error code
278        if res.returncode == 134:
279            status = "fail"
280        # The haskell process crashes/errors
281        elif res.returncode != 0:
282            status = "error"
283
284        result = (status, res.stdout.decode("utf-8"), test_info["expected_result"])
285
286        return TestResult(result, fname, test_info['test_name'])
287
288    def run_phase(self, context, filename, phase, test_info):
289        fname = os.path.join(self.config.dir, filename)
290        # Check file exists and error gracefully
291        if not os.path.exists(fname):
292            return TestResult(
293                        ("error", "Source file '{}' not found".format(fname), test_info['expected_result']),
294                        fname,
295                        test_info['test_name']
296                    )
297
298        if not context.repo:
299            return TestResult(
300                        ("error", "repo not found", test_info['expected_result']),
301                        fname,
302                        test_info['test_name']
303                    )
304
305        # runs the test
306        setup_dist()
307
308        res = phase.run(context, fname, test_info)
309
310        status = "pass"
311
312        # The compiler returns an error code
313        if res.returncode == 134:
314            status = "fail"
315        # The haskell process crashes/errors
316        elif res.returncode != 0:
317            status = "error"
318
319        result = (status, res.stdout.decode("utf-8"), test_info["expected_result"])
320
321        return TestResult(result, fname, test_info['test_name'])
322
323    def print_test_header(self, test_name):
324        print("-" * self.config.header_block_len,
325              " {} ".format(test_name),
326              "-" * self.config.header_block_len)
327
328    # Run one test by name
329    def run_one(self, context, test_name):
330        return self.run_tests(context, filter(lambda t: test_name == t['test_name'], self.config.settings))
331
332    # Run all tests in the configuration file
333    def run_all(self, context):
334        return self.run_tests(context, self.config.settings)
335
336    def run_tests(self, context, tests):
337        results = []
338        for test in tests:
339            self.print_test_header(test['test_name'])
340            for f in test['files']:
341                try:
342                    phasename = test['phase']
343                except KeyError:
344                    phasename = "cogent"
345
346                if phasename == "cogent" and phasename not in context.ignore_phases:
347                    results.append( self.run_cogent(context, f, test) )
348
349                elif context.phases is None or phasename in context.ignore_phases:
350                    continue
351
352                else:
353                    try:
354                        results.append( self.run_phase(context, f, context.phases[phasename], test) )
355                    except KeyError:
356                        results.append( TestResult(
357                            ("error", "phase not found: {}\n".format(phasename), test['expected_result']),
358                            f,
359                            test['test_name']
360                        ))
361            print()
362        return results
363
364# a collection of configurations
365class Configurations:
366    def __init__(self, path):
367        self.files = [f.resolve() for f in path.rglob(CONFIG_FILE_NAME)]
368        self.configs = []
369        self.errConfigs = []
370        for f in self.files:
371            try:
372                self.configs.append(TestConfiguration(f))
373            except InvalidConfigError as e:
374                self.errConfigs.append(e)
375            except OSError as e:
376                self.errConfigs.append(e)
377
378    def has_erroring_configs(self):
379        return self.errConfigs != []
380
381    def print_errs(self):
382        print(self.errConfigs)
383        for e in self.errConfigs:
384            if type(e) is InvalidConfigError:
385                print(colored("Config error: ", "red"), e)
386            elif type(e) is OSError:
387                print(colored("error - could not find config file for test file {}".format(e), "red"))
388
389    # Based on an asbolute path for a test file, get it's configuration
390    def get_cfg_from_test_name(self, f):
391        cfgs = self.get_configs()
392        for c in cfgs:
393            if f in c.get_all_test_names():
394                return c
395        return None
396
397    def get_configs(self):
398        return self.configs
399
400
401
402#
403# Main script
404#
405
406def main():
407    ap = argparse.ArgumentParser(
408        description="Cogent Testing Framework",
409        epilog="Test configurations must be stored in a '{}' file".format(
410            CONFIG_FILE_NAME),
411        allow_abbrev=False
412    )
413    ap.add_argument("--only", "-o", dest="only_test",
414                    help="only run specified tests",
415                    metavar="TEST_NAME")
416    ap.add_argument("--verbose", "-v",
417                    dest="verbose",
418                    help="print output for given tests even if they pass (none supplied = all tests)",
419                    metavar="TEST_NAME",
420                    nargs='*')
421    ap.add_argument("--validate", "-t",
422                    dest="validate",
423                    action="store_true",
424                    help="Check the format of all config files is correct")
425    ap.add_argument("--extra-phases", "-p",
426                    dest="phase_dir",
427                    default=None,
428                    help="set the location of the additional phase directory")
429    ap.add_argument("--ignore-phases",
430                    dest="ignore_phases",
431                    action="store",
432                    nargs="+",
433                    default=[],
434                    help="ignore the tests for the specified phases")
435    ap.add_argument("--repo",
436                    dest="repo",
437                    help="set the location of the repository root")
438    ap.add_argument("--cogent",
439                    dest="cogent",
440                    default="cogent",
441                    help="specify the location of the cogent compiler")
442    ap.add_argument("--ignore-errors",
443                    dest="ignore_errors",
444                    action="store_true",
445                    help="if enabled, a test error does not cause the script to exit with an error")
446    args = ap.parse_args()
447
448    cogent = shutil.which(args.cogent)
449
450    if args.repo is not None:
451      repo = os.path.abspath(args.repo)
452    else:
453      repo = None
454      print("Warning: repository directory not set; use --repo")
455
456    if args.phase_dir is not None:
457      phase_dir = os.path.abspath(args.phase_dir)
458    else:
459      phase_dir = None
460
461    # Check if cogent is installed
462    if cogent is None:
463        print("Could not find cogent compiler - Please either add it to your PATH or set --cogent")
464        sys.exit(1)
465
466    print("Using repository: " + str(repo))
467    print("Using cogent: " + cogent)
468    print("Using phase dir: " + str(phase_dir))
469
470    if phase_dir is not None and Path(phase_dir).exists():
471      files = Path(phase_dir).glob("*.sh")
472      phases = dict(map(lambda p: (p.stem,Phase(p)), files))
473    else:
474      phases = None
475
476    context = TestContext(repo, cogent, TEST_DIST_DIR, TEST_SCRIPT_DIR, phases, args.ignore_phases)
477
478    # find all config files
479    configs = Configurations(Path("."))
480
481    # Validate all config files
482    if args.validate:
483        isErr = configs.has_erroring_configs()
484        if isErr:
485            configs.print_errs()
486            print(colored("Errors found in above configuration files", "red"))
487        else:
488            print(colored("All configuration files okay!", "green"))
489        sys.exit((1 if isErr else 0))
490    elif configs.has_erroring_configs():
491        print(colored("Errors found in above configuration files:", "red"))
492        print(colored("  call with --validate for more info", "red"))
493        sys.exit(1)
494
495    if args.verbose is not None:
496        verbose_test_names = args.verbose
497
498    results = []
499    # If we're only running specific tests
500    if args.only_test is not None:
501        test_name = args.only_test
502        conf = configs.get_cfg_from_test_name(test_name)
503        if conf is None:
504            print(colored("Cannot find config file containing test name {}".format(test_name), "red"))
505            sys.exit(1)
506        test = Test(conf)
507        results = test.run_one(context, test_name)
508    # Otherwise, run all possible tests
509    else:
510        tests = list(map(lambda config: Test(config), configs.get_configs()))
511
512        for test in tests:
513            subresults = test.run_all(context)
514            results.extend(subresults)
515
516    setup_dist()
517
518    errs   = 0
519    passes = 0
520    fails  = 0
521    wips   = 0
522
523    for res in results:
524        (status, _, expected) = res.test
525
526        if is_wip(expected):
527            wips += 1
528        elif status == "error":
529            errs += 1
530        elif res.result():
531            passes += 1
532        else:
533            fails += 1
534
535    print("-"*15 + " Final results: " + "-" * 15)
536    print()
537
538    print("{:>16}{:>16}".format("Result", "Amount"))
539    print("{:>16}{:>16}".format("Errors", errs))
540    print("{:>16}{:>16}".format("Passes", passes))
541    print("{:>16}{:>16}".format("Fails", fails))
542    print("{:>16}{:>16}".format("Work In Progress", wips))
543    print()
544
545    if fails != 0 or (not args.ignore_errors and errs != 0):
546        sys.exit(1)
547
548
549if __name__ == "__main__":
550    main()
551
552