1#!/usr/bin/python
2#
3# Copyright (C) 2014 Free Software Foundation, Inc.
4#
5# This script is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation; either version 3, or (at your option)
8# any later version.
9
10import sys
11import getopt
12import re
13import io
14from datetime import datetime
15from operator import attrgetter
16
17# True if unrecognised lines should cause a fatal error.  Might want to turn
18# this on by default later.
19strict = False
20
21# True if the order of .log segments should match the .sum file, false if
22# they should keep the original order.
23sort_logs = True
24
25# A version of open() that is safe against whatever binary output
26# might be added to the log.
27def safe_open (filename):
28    if sys.version_info >= (3, 0):
29        return open (filename, 'r', errors = 'surrogateescape')
30    return open (filename, 'r')
31
32# Force stdout to handle escape sequences from a safe_open file.
33if sys.version_info >= (3, 0):
34    sys.stdout = io.TextIOWrapper (sys.stdout.buffer,
35                                   errors = 'surrogateescape')
36
37class Named:
38    def __init__ (self, name):
39        self.name = name
40
41class ToolRun (Named):
42    def __init__ (self, name):
43        Named.__init__ (self, name)
44        # The variations run for this tool, mapped by --target_board name.
45        self.variations = dict()
46
47    # Return the VariationRun for variation NAME.
48    def get_variation (self, name):
49        if name not in self.variations:
50            self.variations[name] = VariationRun (name)
51        return self.variations[name]
52
53class VariationRun (Named):
54    def __init__ (self, name):
55        Named.__init__ (self, name)
56        # A segment of text before the harness runs start, describing which
57        # baseboard files were loaded for the target.
58        self.header = None
59        # The harnesses run for this variation, mapped by filename.
60        self.harnesses = dict()
61        # A list giving the number of times each type of result has
62        # been seen.
63        self.counts = []
64
65    # Return the HarnessRun for harness NAME.
66    def get_harness (self, name):
67        if name not in self.harnesses:
68            self.harnesses[name] = HarnessRun (name)
69        return self.harnesses[name]
70
71class HarnessRun (Named):
72    def __init__ (self, name):
73        Named.__init__ (self, name)
74        # Segments of text that make up the harness run, mapped by a test-based
75        # key that can be used to order them.
76        self.segments = dict()
77        # Segments of text that make up the harness run but which have
78        # no recognized test results.  These are typically harnesses that
79        # are completely skipped for the target.
80        self.empty = []
81        # A list of results.  Each entry is a pair in which the first element
82        # is a unique sorting key and in which the second is the full
83        # PASS/FAIL line.
84        self.results = []
85
86    # Add a segment of text to the harness run.  If the segment includes
87    # test results, KEY is an example of one of them, and can be used to
88    # combine the individual segments in order.  If the segment has no
89    # test results (e.g. because the harness doesn't do anything for the
90    # current configuration) then KEY is None instead.  In that case
91    # just collect the segments in the order that we see them.
92    def add_segment (self, key, segment):
93        if key:
94            assert key not in self.segments
95            self.segments[key] = segment
96        else:
97            self.empty.append (segment)
98
99class Segment:
100    def __init__ (self, filename, start):
101        self.filename = filename
102        self.start = start
103        self.lines = 0
104
105class Prog:
106    def __init__ (self):
107        # The variations specified on the command line.
108        self.variations = []
109        # The variations seen in the input files.
110        self.known_variations = set()
111        # The tools specified on the command line.
112        self.tools = []
113        # Whether to create .sum rather than .log output.
114        self.do_sum = True
115        # Regexps used while parsing.
116        self.test_run_re = re.compile (r'^Test Run By (\S+) on (.*)$')
117        self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$')
118        self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED'
119                                     r'|WARNING|ERROR|UNSUPPORTED|UNTESTED'
120                                     r'|KFAIL|KPASS):\s*(.+)')
121        self.completed_re = re.compile (r'.* completed at (.*)')
122        # Pieces of text to write at the head of the output.
123        # start_line is a pair in which the first element is a datetime
124        # and in which the second is the associated 'Test Run By' line.
125        self.start_line = None
126        self.native_line = ''
127        self.target_line = ''
128        self.host_line = ''
129        self.acats_premable = ''
130        # Pieces of text to write at the end of the output.
131        # end_line is like start_line but for the 'runtest completed' line.
132        self.acats_failures = []
133        self.version_output = ''
134        self.end_line = None
135        # Known summary types.
136        self.count_names = [
137            '# of DejaGnu errors\t\t',
138            '# of expected passes\t\t',
139            '# of unexpected failures\t',
140            '# of unexpected successes\t',
141            '# of expected failures\t\t',
142            '# of unknown successes\t\t',
143            '# of known failures\t\t',
144            '# of untested testcases\t\t',
145            '# of unresolved testcases\t',
146            '# of unsupported tests\t\t'
147        ]
148        self.runs = dict()
149
150    def usage (self):
151        name = sys.argv[0]
152        sys.stderr.write ('Usage: ' + name
153                          + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ...
154
155    tool           The tool (e.g. g++, libffi) for which to create a
156                   new test summary file.  If not specified then output
157                   is created for all tools.
158    variant-list   One or more test variant names.  If the list is
159                   not specified then one is constructed from all
160                   variants in the files for <tool>.
161    sum-file       A test summary file with the format of those
162                   created by runtest from DejaGnu.
163    If -L is used, merge *.log files instead of *.sum.  In this
164    mode the exact order of lines may not be preserved, just different
165    Running *.exp chunks should be in correct order.
166''')
167        sys.exit (1)
168
169    def fatal (self, what, string):
170        if not what:
171            what = sys.argv[0]
172        sys.stderr.write (what + ': ' + string + '\n')
173        sys.exit (1)
174
175    # Parse the command-line arguments.
176    def parse_cmdline (self):
177        try:
178            (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L')
179            if len (self.files) == 0:
180                self.usage()
181            for (option, value) in options:
182                if option == '-l':
183                    self.variations.append (value)
184                elif option == '-t':
185                    self.tools.append (value)
186                else:
187                    self.do_sum = False
188        except getopt.GetoptError as e:
189            self.fatal (None, e.msg)
190
191    # Try to parse time string TIME, returning an arbitrary time on failure.
192    # Getting this right is just a nice-to-have so failures should be silent.
193    def parse_time (self, time):
194        try:
195            return datetime.strptime (time, '%c')
196        except ValueError:
197            return datetime.now()
198
199    # Parse an integer and abort on failure.
200    def parse_int (self, filename, value):
201        try:
202            return int (value)
203        except ValueError:
204            self.fatal (filename, 'expected an integer, got: ' + value)
205
206    # Return a list that represents no test results.
207    def zero_counts (self):
208        return [0 for x in self.count_names]
209
210    # Return the ToolRun for tool NAME.
211    def get_tool (self, name):
212        if name not in self.runs:
213            self.runs[name] = ToolRun (name)
214        return self.runs[name]
215
216    # Add the result counts in list FROMC to TOC.
217    def accumulate_counts (self, toc, fromc):
218        for i in range (len (self.count_names)):
219            toc[i] += fromc[i]
220
221    # Parse the list of variations after 'Schedule of variations:'.
222    # Return the number seen.
223    def parse_variations (self, filename, file):
224        num_variations = 0
225        while True:
226            line = file.readline()
227            if line == '':
228                self.fatal (filename, 'could not parse variation list')
229            if line == '\n':
230                break
231            self.known_variations.add (line.strip())
232            num_variations += 1
233        return num_variations
234
235    # Parse from the first line after 'Running target ...' to the end
236    # of the run's summary.
237    def parse_run (self, filename, file, tool, variation, num_variations):
238        header = None
239        harness = None
240        segment = None
241        final_using = 0
242        has_warning = 0
243
244        # If this is the first run for this variation, add any text before
245        # the first harness to the header.
246        if not variation.header:
247            segment = Segment (filename, file.tell())
248            variation.header = segment
249
250        # Parse the rest of the summary (the '# of ' lines).
251        if len (variation.counts) == 0:
252            variation.counts = self.zero_counts()
253
254        # Parse up until the first line of the summary.
255        if num_variations == 1:
256            end = '\t\t=== ' + tool.name + ' Summary ===\n'
257        else:
258            end = ('\t\t=== ' + tool.name + ' Summary for '
259                   + variation.name + ' ===\n')
260        while True:
261            line = file.readline()
262            if line == '':
263                self.fatal (filename, 'no recognised summary line')
264            if line == end:
265                break
266
267            # Look for the start of a new harness.
268            if line.startswith ('Running ') and line.endswith (' ...\n'):
269                # Close off the current harness segment, if any.
270                if harness:
271                    segment.lines -= final_using
272                    harness.add_segment (first_key, segment)
273                name = line[len ('Running '):-len(' ...\n')]
274                harness = variation.get_harness (name)
275                segment = Segment (filename, file.tell())
276                first_key = None
277                final_using = 0
278                continue
279
280            # Record test results.  Associate the first test result with
281            # the harness segment, so that if a run for a particular harness
282            # has been split up, we can reassemble the individual segments
283            # in a sensible order.
284            #
285            # dejagnu sometimes issues warnings about the testing environment
286            # before running any tests.  Treat them as part of the header
287            # rather than as a test result.
288            match = self.result_re.match (line)
289            if match and (harness or not line.startswith ('WARNING:')):
290                if not harness:
291                    self.fatal (filename, 'saw test result before harness name')
292                name = match.group (2)
293                # Ugly hack to get the right order for gfortran.
294                if name.startswith ('gfortran.dg/g77/'):
295                    name = 'h' + name
296                # If we have a time out warning, make sure it appears
297                # before the following testcase diagnostic: we insert
298                # the testname before 'program' so that sort faces a
299                # list of testnames.
300                if line.startswith ('WARNING: program timed out'):
301                  has_warning = 1
302                else:
303                  if has_warning == 1:
304                      key = (name, len (harness.results))
305                      myline = 'WARNING: %s program timed out.\n' % name
306                      harness.results.append ((key, myline))
307                      has_warning = 0
308                  key = (name, len (harness.results))
309                  harness.results.append ((key, line))
310                  if not first_key and sort_logs:
311                      first_key = key
312                if line.startswith ('ERROR: (DejaGnu)'):
313                    for i in range (len (self.count_names)):
314                        if 'DejaGnu errors' in self.count_names[i]:
315                            variation.counts[i] += 1
316                            break
317
318            # 'Using ...' lines are only interesting in a header.  Splitting
319            # the test up into parallel runs leads to more 'Using ...' lines
320            # than there would be in a single log.
321            if line.startswith ('Using '):
322                final_using += 1
323            else:
324                final_using = 0
325
326            # Add other text to the current segment, if any.
327            if segment:
328                segment.lines += 1
329
330        # Close off the final harness segment, if any.
331        if harness:
332            segment.lines -= final_using
333            harness.add_segment (first_key, segment)
334
335        while True:
336            before = file.tell()
337            line = file.readline()
338            if line == '':
339                break
340            if line == '\n':
341                continue
342            if not line.startswith ('# '):
343                file.seek (before)
344                break
345            found = False
346            for i in range (len (self.count_names)):
347                if line.startswith (self.count_names[i]):
348                    count = line[len (self.count_names[i]):-1].strip()
349                    variation.counts[i] += self.parse_int (filename, count)
350                    found = True
351                    break
352            if not found:
353                self.fatal (filename, 'unknown test result: ' + line[:-1])
354
355    # Parse an acats run, which uses a different format from dejagnu.
356    # We have just skipped over '=== acats configuration ==='.
357    def parse_acats_run (self, filename, file):
358        # Parse the preamble, which describes the configuration and logs
359        # the creation of support files.
360        record = (self.acats_premable == '')
361        if record:
362            self.acats_premable = '\t\t=== acats configuration ===\n'
363        while True:
364            line = file.readline()
365            if line == '':
366                self.fatal (filename, 'could not parse acats preamble')
367            if line == '\t\t=== acats tests ===\n':
368                break
369            if record:
370                self.acats_premable += line
371
372        # Parse the test results themselves, using a dummy variation name.
373        tool = self.get_tool ('acats')
374        variation = tool.get_variation ('none')
375        self.parse_run (filename, file, tool, variation, 1)
376
377        # Parse the failure list.
378        while True:
379            before = file.tell()
380            line = file.readline()
381            if line.startswith ('*** FAILURES: '):
382                self.acats_failures.append (line[len ('*** FAILURES: '):-1])
383                continue
384            file.seek (before)
385            break
386
387    # Parse the final summary at the end of a log in order to capture
388    # the version output that follows it.
389    def parse_final_summary (self, filename, file):
390        record = (self.version_output == '')
391        while True:
392            line = file.readline()
393            if line == '':
394                break
395            if line.startswith ('# of '):
396                continue
397            if record:
398                self.version_output += line
399            if line == '\n':
400                break
401
402    # Parse a .log or .sum file.
403    def parse_file (self, filename, file):
404        tool = None
405        target = None
406        num_variations = 1
407        while True:
408            line = file.readline()
409            if line == '':
410                return
411
412            # Parse the list of variations, which comes before the test
413            # runs themselves.
414            if line.startswith ('Schedule of variations:'):
415                num_variations = self.parse_variations (filename, file)
416                continue
417
418            # Parse a testsuite run for one tool/variation combination.
419            if line.startswith ('Running target '):
420                name = line[len ('Running target '):-1]
421                if not tool:
422                    self.fatal (filename, 'could not parse tool name')
423                if name not in self.known_variations:
424                    self.fatal (filename, 'unknown target: ' + name)
425                self.parse_run (filename, file, tool,
426                                tool.get_variation (name),
427                                num_variations)
428                # If there is only one variation then there is no separate
429                # summary for it.  Record any following version output.
430                if num_variations == 1:
431                    self.parse_final_summary (filename, file)
432                continue
433
434            # Parse the start line.  In the case where several files are being
435            # parsed, pick the one with the earliest time.
436            match = self.test_run_re.match (line)
437            if match:
438                time = self.parse_time (match.group (2))
439                if not self.start_line or self.start_line[0] > time:
440                    self.start_line = (time, line)
441                continue
442
443            # Parse the form used for native testing.
444            if line.startswith ('Native configuration is '):
445                self.native_line = line
446                continue
447
448            # Parse the target triplet.
449            if line.startswith ('Target is '):
450                self.target_line = line
451                continue
452
453            # Parse the host triplet.
454            if line.startswith ('Host   is '):
455                self.host_line = line
456                continue
457
458            # Parse the acats premable.
459            if line == '\t\t=== acats configuration ===\n':
460                self.parse_acats_run (filename, file)
461                continue
462
463            # Parse the tool name.
464            match = self.tool_re.match (line)
465            if match:
466                tool = self.get_tool (match.group (1))
467                continue
468
469            # Skip over the final summary (which we instead create from
470            # individual runs) and parse the version output.
471            if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n':
472                if file.readline() != '\n':
473                    self.fatal (filename, 'expected blank line after summary')
474                self.parse_final_summary (filename, file)
475                continue
476
477            # Parse the completion line.  In the case where several files
478            # are being parsed, pick the one with the latest time.
479            match = self.completed_re.match (line)
480            if match:
481                time = self.parse_time (match.group (1))
482                if not self.end_line or self.end_line[0] < time:
483                    self.end_line = (time, line)
484                continue
485
486            # Sanity check to make sure that important text doesn't get
487            # dropped accidentally.
488            if strict and line.strip() != '':
489                self.fatal (filename, 'unrecognised line: ' + line[:-1])
490
491    # Output a segment of text.
492    def output_segment (self, segment):
493        with safe_open (segment.filename) as file:
494            file.seek (segment.start)
495            for i in range (segment.lines):
496                sys.stdout.write (file.readline())
497
498    # Output a summary giving the number of times each type of result has
499    # been seen.
500    def output_summary (self, tool, counts):
501        for i in range (len (self.count_names)):
502            name = self.count_names[i]
503            # dejagnu only prints result types that were seen at least once,
504            # but acats always prints a number of unexpected failures.
505            if (counts[i] > 0
506                or (tool.name == 'acats'
507                    and name.startswith ('# of unexpected failures'))):
508                sys.stdout.write ('%s%d\n' % (name, counts[i]))
509
510    # Output unified .log or .sum information for a particular variation,
511    # with a summary at the end.
512    def output_variation (self, tool, variation):
513        self.output_segment (variation.header)
514        for harness in sorted (variation.harnesses.values(),
515                               key = attrgetter ('name')):
516            sys.stdout.write ('Running ' + harness.name + ' ...\n')
517            if self.do_sum:
518                harness.results.sort()
519                for (key, line) in harness.results:
520                    sys.stdout.write (line)
521            else:
522                # Rearrange the log segments into test order (but without
523                # rearranging text within those segments).
524                for key in sorted (harness.segments.keys()):
525                    self.output_segment (harness.segments[key])
526                for segment in harness.empty:
527                    self.output_segment (segment)
528        if len (self.variations) > 1:
529            sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for '
530                              + variation.name + ' ===\n\n')
531            self.output_summary (tool, variation.counts)
532
533    # Output unified .log or .sum information for a particular tool,
534    # with a summary at the end.
535    def output_tool (self, tool):
536        counts = self.zero_counts()
537        if tool.name == 'acats':
538            # acats doesn't use variations, so just output everything.
539            # It also has a different approach to whitespace.
540            sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n')
541            for variation in tool.variations.values():
542                self.output_variation (tool, variation)
543                self.accumulate_counts (counts, variation.counts)
544            sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n')
545        else:
546            # Output the results in the usual dejagnu runtest format.
547            sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n'
548                              'Schedule of variations:\n')
549            for name in self.variations:
550                if name in tool.variations:
551                    sys.stdout.write ('    ' + name + '\n')
552            sys.stdout.write ('\n')
553            for name in self.variations:
554                if name in tool.variations:
555                    variation = tool.variations[name]
556                    sys.stdout.write ('Running target '
557                                      + variation.name + '\n')
558                    self.output_variation (tool, variation)
559                    self.accumulate_counts (counts, variation.counts)
560            sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n')
561        self.output_summary (tool, counts)
562
563    def main (self):
564        self.parse_cmdline()
565        try:
566            # Parse the input files.
567            for filename in self.files:
568                with safe_open (filename) as file:
569                    self.parse_file (filename, file)
570
571            # Decide what to output.
572            if len (self.variations) == 0:
573                self.variations = sorted (self.known_variations)
574            else:
575                for name in self.variations:
576                    if name not in self.known_variations:
577                        self.fatal (None, 'no results for ' + name)
578            if len (self.tools) == 0:
579                self.tools = sorted (self.runs.keys())
580
581            # Output the header.
582            if self.start_line:
583                sys.stdout.write (self.start_line[1])
584            sys.stdout.write (self.native_line)
585            sys.stdout.write (self.target_line)
586            sys.stdout.write (self.host_line)
587            sys.stdout.write (self.acats_premable)
588
589            # Output the main body.
590            for name in self.tools:
591                if name not in self.runs:
592                    self.fatal (None, 'no results for ' + name)
593                self.output_tool (self.runs[name])
594
595            # Output the footer.
596            if len (self.acats_failures) > 0:
597                sys.stdout.write ('*** FAILURES: '
598                                  + ' '.join (self.acats_failures) + '\n')
599            sys.stdout.write (self.version_output)
600            if self.end_line:
601                sys.stdout.write (self.end_line[1])
602        except IOError as e:
603            self.fatal (e.filename, e.strerror)
604
605Prog().main()
606