1#!/usr/bin/python
2#
3# Copyright (C) 2014 Free Software Foundation, Inc.
4#
5# This script is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation; either version 3, or (at your option)
8# any later version.
9
10import sys
11import getopt
12import re
13import io
14from datetime import datetime
15from operator import attrgetter
16
17# True if unrecognised lines should cause a fatal error.  Might want to turn
18# this on by default later.
19strict = False
20
21# True if the order of .log segments should match the .sum file, false if
22# they should keep the original order.
23sort_logs = True
24
25# A version of open() that is safe against whatever binary output
26# might be added to the log.
27def safe_open (filename):
28    if sys.version_info >= (3, 0):
29        return open (filename, 'r', errors = 'surrogateescape')
30    return open (filename, 'r')
31
32# Force stdout to handle escape sequences from a safe_open file.
33if sys.version_info >= (3, 0):
34    sys.stdout = io.TextIOWrapper (sys.stdout.buffer,
35                                   errors = 'surrogateescape')
36
37class Named:
38    def __init__ (self, name):
39        self.name = name
40
41class ToolRun (Named):
42    def __init__ (self, name):
43        Named.__init__ (self, name)
44        # The variations run for this tool, mapped by --target_board name.
45        self.variations = dict()
46
47    # Return the VariationRun for variation NAME.
48    def get_variation (self, name):
49        if name not in self.variations:
50            self.variations[name] = VariationRun (name)
51        return self.variations[name]
52
53class VariationRun (Named):
54    def __init__ (self, name):
55        Named.__init__ (self, name)
56        # A segment of text before the harness runs start, describing which
57        # baseboard files were loaded for the target.
58        self.header = None
59        # The harnesses run for this variation, mapped by filename.
60        self.harnesses = dict()
61        # A list giving the number of times each type of result has
62        # been seen.
63        self.counts = []
64
65    # Return the HarnessRun for harness NAME.
66    def get_harness (self, name):
67        if name not in self.harnesses:
68            self.harnesses[name] = HarnessRun (name)
69        return self.harnesses[name]
70
71class HarnessRun (Named):
72    def __init__ (self, name):
73        Named.__init__ (self, name)
74        # Segments of text that make up the harness run, mapped by a test-based
75        # key that can be used to order them.
76        self.segments = dict()
77        # Segments of text that make up the harness run but which have
78        # no recognized test results.  These are typically harnesses that
79        # are completely skipped for the target.
80        self.empty = []
81        # A list of results.  Each entry is a pair in which the first element
82        # is a unique sorting key and in which the second is the full
83        # PASS/FAIL line.
84        self.results = []
85
86    # Add a segment of text to the harness run.  If the segment includes
87    # test results, KEY is an example of one of them, and can be used to
88    # combine the individual segments in order.  If the segment has no
89    # test results (e.g. because the harness doesn't do anything for the
90    # current configuration) then KEY is None instead.  In that case
91    # just collect the segments in the order that we see them.
92    def add_segment (self, key, segment):
93        if key:
94            assert key not in self.segments
95            self.segments[key] = segment
96        else:
97            self.empty.append (segment)
98
99class Segment:
100    def __init__ (self, filename, start):
101        self.filename = filename
102        self.start = start
103        self.lines = 0
104
105class Prog:
106    def __init__ (self):
107        # The variations specified on the command line.
108        self.variations = []
109        # The variations seen in the input files.
110        self.known_variations = set()
111        # The tools specified on the command line.
112        self.tools = []
113        # Whether to create .sum rather than .log output.
114        self.do_sum = True
115        # Regexps used while parsing.
116        self.test_run_re = re.compile (r'^Test Run By (\S+) on (.*)$')
117        self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$')
118        self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED'
119                                     r'|WARNING|ERROR|UNSUPPORTED|UNTESTED'
120                                     r'|KFAIL|KPASS|PATH|DUPLICATE):\s*(.+)')
121        self.completed_re = re.compile (r'.* completed at (.*)')
122        # Pieces of text to write at the head of the output.
123        # start_line is a pair in which the first element is a datetime
124        # and in which the second is the associated 'Test Run By' line.
125        self.start_line = None
126        self.native_line = ''
127        self.target_line = ''
128        self.host_line = ''
129        self.acats_premable = ''
130        # Pieces of text to write at the end of the output.
131        # end_line is like start_line but for the 'runtest completed' line.
132        self.acats_failures = []
133        self.version_output = ''
134        self.end_line = None
135        # Known summary types.
136        self.count_names = [
137            '# of DejaGnu errors\t\t',
138            '# of expected passes\t\t',
139            '# of unexpected failures\t',
140            '# of unexpected successes\t',
141            '# of expected failures\t\t',
142            '# of unknown successes\t\t',
143            '# of known failures\t\t',
144            '# of untested testcases\t\t',
145            '# of unresolved testcases\t',
146            '# of unsupported tests\t\t',
147            '# of paths in test names\t',
148            '# of duplicate test names\t'
149        ]
150        self.runs = dict()
151
152    def usage (self):
153        name = sys.argv[0]
154        sys.stderr.write ('Usage: ' + name
155                          + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ...
156
157    tool           The tool (e.g. g++, libffi) for which to create a
158                   new test summary file.  If not specified then output
159                   is created for all tools.
160    variant-list   One or more test variant names.  If the list is
161                   not specified then one is constructed from all
162                   variants in the files for <tool>.
163    sum-file       A test summary file with the format of those
164                   created by runtest from DejaGnu.
165    If -L is used, merge *.log files instead of *.sum.  In this
166    mode the exact order of lines may not be preserved, just different
167    Running *.exp chunks should be in correct order.
168''')
169        sys.exit (1)
170
171    def fatal (self, what, string):
172        if not what:
173            what = sys.argv[0]
174        sys.stderr.write (what + ': ' + string + '\n')
175        sys.exit (1)
176
177    # Parse the command-line arguments.
178    def parse_cmdline (self):
179        try:
180            (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L')
181            if len (self.files) == 0:
182                self.usage()
183            for (option, value) in options:
184                if option == '-l':
185                    self.variations.append (value)
186                elif option == '-t':
187                    self.tools.append (value)
188                else:
189                    self.do_sum = False
190        except getopt.GetoptError as e:
191            self.fatal (None, e.msg)
192
193    # Try to parse time string TIME, returning an arbitrary time on failure.
194    # Getting this right is just a nice-to-have so failures should be silent.
195    def parse_time (self, time):
196        try:
197            return datetime.strptime (time, '%c')
198        except ValueError:
199            return datetime.now()
200
201    # Parse an integer and abort on failure.
202    def parse_int (self, filename, value):
203        try:
204            return int (value)
205        except ValueError:
206            self.fatal (filename, 'expected an integer, got: ' + value)
207
208    # Return a list that represents no test results.
209    def zero_counts (self):
210        return [0 for x in self.count_names]
211
212    # Return the ToolRun for tool NAME.
213    def get_tool (self, name):
214        if name not in self.runs:
215            self.runs[name] = ToolRun (name)
216        return self.runs[name]
217
218    # Add the result counts in list FROMC to TOC.
219    def accumulate_counts (self, toc, fromc):
220        for i in range (len (self.count_names)):
221            toc[i] += fromc[i]
222
223    # Parse the list of variations after 'Schedule of variations:'.
224    # Return the number seen.
225    def parse_variations (self, filename, file):
226        num_variations = 0
227        while True:
228            line = file.readline()
229            if line == '':
230                self.fatal (filename, 'could not parse variation list')
231            if line == '\n':
232                break
233            self.known_variations.add (line.strip())
234            num_variations += 1
235        return num_variations
236
237    # Parse from the first line after 'Running target ...' to the end
238    # of the run's summary.
239    def parse_run (self, filename, file, tool, variation, num_variations):
240        header = None
241        harness = None
242        segment = None
243        final_using = 0
244        has_warning = 0
245
246        # If this is the first run for this variation, add any text before
247        # the first harness to the header.
248        if not variation.header:
249            segment = Segment (filename, file.tell())
250            variation.header = segment
251
252        # Parse the rest of the summary (the '# of ' lines).
253        if len (variation.counts) == 0:
254            variation.counts = self.zero_counts()
255
256        # Parse up until the first line of the summary.
257        if num_variations == 1:
258            end = '\t\t=== ' + tool.name + ' Summary ===\n'
259        else:
260            end = ('\t\t=== ' + tool.name + ' Summary for '
261                   + variation.name + ' ===\n')
262        while True:
263            line = file.readline()
264            if line == '':
265                self.fatal (filename, 'no recognised summary line')
266            if line == end:
267                break
268
269            # Look for the start of a new harness.
270            if line.startswith ('Running ') and line.endswith (' ...\n'):
271                # Close off the current harness segment, if any.
272                if harness:
273                    segment.lines -= final_using
274                    harness.add_segment (first_key, segment)
275                name = line[len ('Running '):-len(' ...\n')]
276                harness = variation.get_harness (name)
277                segment = Segment (filename, file.tell())
278                first_key = None
279                final_using = 0
280                continue
281
282            # Record test results.  Associate the first test result with
283            # the harness segment, so that if a run for a particular harness
284            # has been split up, we can reassemble the individual segments
285            # in a sensible order.
286            #
287            # dejagnu sometimes issues warnings about the testing environment
288            # before running any tests.  Treat them as part of the header
289            # rather than as a test result.
290            match = self.result_re.match (line)
291            if match and (harness or not line.startswith ('WARNING:')):
292                if not harness:
293                    self.fatal (filename, 'saw test result before harness name')
294                name = match.group (2)
295                # Ugly hack to get the right order for gfortran.
296                if name.startswith ('gfortran.dg/g77/'):
297                    name = 'h' + name
298                # If we have a time out warning, make sure it appears
299                # before the following testcase diagnostic: we insert
300                # the testname before 'program' so that sort faces a
301                # list of testnames.
302                if line.startswith ('WARNING: program timed out'):
303                  has_warning = 1
304                else:
305                  if has_warning == 1:
306                      key = (name, len (harness.results))
307                      myline = 'WARNING: %s program timed out.\n' % name
308                      harness.results.append ((key, myline))
309                      has_warning = 0
310                  key = (name, len (harness.results))
311                  harness.results.append ((key, line))
312                  if not first_key and sort_logs:
313                      first_key = key
314                if line.startswith ('ERROR: (DejaGnu)'):
315                    for i in range (len (self.count_names)):
316                        if 'DejaGnu errors' in self.count_names[i]:
317                            variation.counts[i] += 1
318                            break
319
320            # 'Using ...' lines are only interesting in a header.  Splitting
321            # the test up into parallel runs leads to more 'Using ...' lines
322            # than there would be in a single log.
323            if line.startswith ('Using '):
324                final_using += 1
325            else:
326                final_using = 0
327
328            # Add other text to the current segment, if any.
329            if segment:
330                segment.lines += 1
331
332        # Close off the final harness segment, if any.
333        if harness:
334            segment.lines -= final_using
335            harness.add_segment (first_key, segment)
336
337        while True:
338            before = file.tell()
339            line = file.readline()
340            if line == '':
341                break
342            if line == '\n':
343                continue
344            if not line.startswith ('# '):
345                file.seek (before)
346                break
347            found = False
348            for i in range (len (self.count_names)):
349                if line.startswith (self.count_names[i]):
350                    count = line[len (self.count_names[i]):-1].strip()
351                    variation.counts[i] += self.parse_int (filename, count)
352                    found = True
353                    break
354            if not found:
355                self.fatal (filename, 'unknown test result: ' + line[:-1])
356
357    # Parse an acats run, which uses a different format from dejagnu.
358    # We have just skipped over '=== acats configuration ==='.
359    def parse_acats_run (self, filename, file):
360        # Parse the preamble, which describes the configuration and logs
361        # the creation of support files.
362        record = (self.acats_premable == '')
363        if record:
364            self.acats_premable = '\t\t=== acats configuration ===\n'
365        while True:
366            line = file.readline()
367            if line == '':
368                self.fatal (filename, 'could not parse acats preamble')
369            if line == '\t\t=== acats tests ===\n':
370                break
371            if record:
372                self.acats_premable += line
373
374        # Parse the test results themselves, using a dummy variation name.
375        tool = self.get_tool ('acats')
376        variation = tool.get_variation ('none')
377        self.parse_run (filename, file, tool, variation, 1)
378
379        # Parse the failure list.
380        while True:
381            before = file.tell()
382            line = file.readline()
383            if line.startswith ('*** FAILURES: '):
384                self.acats_failures.append (line[len ('*** FAILURES: '):-1])
385                continue
386            file.seek (before)
387            break
388
389    # Parse the final summary at the end of a log in order to capture
390    # the version output that follows it.
391    def parse_final_summary (self, filename, file):
392        record = (self.version_output == '')
393        while True:
394            line = file.readline()
395            if line == '':
396                break
397            if line.startswith ('# of '):
398                continue
399            if record:
400                self.version_output += line
401            if line == '\n':
402                break
403
404    # Parse a .log or .sum file.
405    def parse_file (self, filename, file):
406        tool = None
407        target = None
408        num_variations = 1
409        while True:
410            line = file.readline()
411            if line == '':
412                return
413
414            # Parse the list of variations, which comes before the test
415            # runs themselves.
416            if line.startswith ('Schedule of variations:'):
417                num_variations = self.parse_variations (filename, file)
418                continue
419
420            # Parse a testsuite run for one tool/variation combination.
421            if line.startswith ('Running target '):
422                name = line[len ('Running target '):-1]
423                if not tool:
424                    self.fatal (filename, 'could not parse tool name')
425                if name not in self.known_variations:
426                    self.fatal (filename, 'unknown target: ' + name)
427                self.parse_run (filename, file, tool,
428                                tool.get_variation (name),
429                                num_variations)
430                # If there is only one variation then there is no separate
431                # summary for it.  Record any following version output.
432                if num_variations == 1:
433                    self.parse_final_summary (filename, file)
434                continue
435
436            # Parse the start line.  In the case where several files are being
437            # parsed, pick the one with the earliest time.
438            match = self.test_run_re.match (line)
439            if match:
440                time = self.parse_time (match.group (2))
441                if not self.start_line or self.start_line[0] > time:
442                    self.start_line = (time, line)
443                continue
444
445            # Parse the form used for native testing.
446            if line.startswith ('Native configuration is '):
447                self.native_line = line
448                continue
449
450            # Parse the target triplet.
451            if line.startswith ('Target is '):
452                self.target_line = line
453                continue
454
455            # Parse the host triplet.
456            if line.startswith ('Host   is '):
457                self.host_line = line
458                continue
459
460            # Parse the acats premable.
461            if line == '\t\t=== acats configuration ===\n':
462                self.parse_acats_run (filename, file)
463                continue
464
465            # Parse the tool name.
466            match = self.tool_re.match (line)
467            if match:
468                tool = self.get_tool (match.group (1))
469                continue
470
471            # Skip over the final summary (which we instead create from
472            # individual runs) and parse the version output.
473            if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n':
474                if file.readline() != '\n':
475                    self.fatal (filename, 'expected blank line after summary')
476                self.parse_final_summary (filename, file)
477                continue
478
479            # Parse the completion line.  In the case where several files
480            # are being parsed, pick the one with the latest time.
481            match = self.completed_re.match (line)
482            if match:
483                time = self.parse_time (match.group (1))
484                if not self.end_line or self.end_line[0] < time:
485                    self.end_line = (time, line)
486                continue
487
488            # Sanity check to make sure that important text doesn't get
489            # dropped accidentally.
490            if strict and line.strip() != '':
491                self.fatal (filename, 'unrecognised line: ' + line[:-1])
492
493    # Output a segment of text.
494    def output_segment (self, segment):
495        with safe_open (segment.filename) as file:
496            file.seek (segment.start)
497            for i in range (segment.lines):
498                sys.stdout.write (file.readline())
499
500    # Output a summary giving the number of times each type of result has
501    # been seen.
502    def output_summary (self, tool, counts):
503        for i in range (len (self.count_names)):
504            name = self.count_names[i]
505            # dejagnu only prints result types that were seen at least once,
506            # but acats always prints a number of unexpected failures.
507            if (counts[i] > 0
508                or (tool.name == 'acats'
509                    and name.startswith ('# of unexpected failures'))):
510                sys.stdout.write ('%s%d\n' % (name, counts[i]))
511
512    # Output unified .log or .sum information for a particular variation,
513    # with a summary at the end.
514    def output_variation (self, tool, variation):
515        self.output_segment (variation.header)
516        for harness in sorted (variation.harnesses.values(),
517                               key = attrgetter ('name')):
518            sys.stdout.write ('Running ' + harness.name + ' ...\n')
519            if self.do_sum:
520                harness.results.sort()
521                for (key, line) in harness.results:
522                    sys.stdout.write (line)
523            else:
524                # Rearrange the log segments into test order (but without
525                # rearranging text within those segments).
526                for key in sorted (harness.segments.keys()):
527                    self.output_segment (harness.segments[key])
528                for segment in harness.empty:
529                    self.output_segment (segment)
530        if len (self.variations) > 1:
531            sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for '
532                              + variation.name + ' ===\n\n')
533            self.output_summary (tool, variation.counts)
534
535    # Output unified .log or .sum information for a particular tool,
536    # with a summary at the end.
537    def output_tool (self, tool):
538        counts = self.zero_counts()
539        if tool.name == 'acats':
540            # acats doesn't use variations, so just output everything.
541            # It also has a different approach to whitespace.
542            sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n')
543            for variation in tool.variations.values():
544                self.output_variation (tool, variation)
545                self.accumulate_counts (counts, variation.counts)
546            sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n')
547        else:
548            # Output the results in the usual dejagnu runtest format.
549            sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n'
550                              'Schedule of variations:\n')
551            for name in self.variations:
552                if name in tool.variations:
553                    sys.stdout.write ('    ' + name + '\n')
554            sys.stdout.write ('\n')
555            for name in self.variations:
556                if name in tool.variations:
557                    variation = tool.variations[name]
558                    sys.stdout.write ('Running target '
559                                      + variation.name + '\n')
560                    self.output_variation (tool, variation)
561                    self.accumulate_counts (counts, variation.counts)
562            sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n')
563        self.output_summary (tool, counts)
564
565    def main (self):
566        self.parse_cmdline()
567        try:
568            # Parse the input files.
569            for filename in self.files:
570                with safe_open (filename) as file:
571                    self.parse_file (filename, file)
572
573            # Decide what to output.
574            if len (self.variations) == 0:
575                self.variations = sorted (self.known_variations)
576            else:
577                for name in self.variations:
578                    if name not in self.known_variations:
579                        self.fatal (None, 'no results for ' + name)
580            if len (self.tools) == 0:
581                self.tools = sorted (self.runs.keys())
582
583            # Output the header.
584            if self.start_line:
585                sys.stdout.write (self.start_line[1])
586            sys.stdout.write (self.native_line)
587            sys.stdout.write (self.target_line)
588            sys.stdout.write (self.host_line)
589            sys.stdout.write (self.acats_premable)
590
591            # Output the main body.
592            for name in self.tools:
593                if name not in self.runs:
594                    self.fatal (None, 'no results for ' + name)
595                self.output_tool (self.runs[name])
596
597            # Output the footer.
598            if len (self.acats_failures) > 0:
599                sys.stdout.write ('*** FAILURES: '
600                                  + ' '.join (self.acats_failures) + '\n')
601            sys.stdout.write (self.version_output)
602            if self.end_line:
603                sys.stdout.write (self.end_line[1])
604        except IOError as e:
605            self.fatal (e.filename, e.strerror)
606
607Prog().main()
608