1#!/usr/bin/env python3
2
3# Copyright (C) 2016-2023 Free Software Foundation, Inc.
4#
5# This file is part of GDB.
6#
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation; either version 3 of the License, or
10# (at your option) any later version.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License
18# along with this program.  If not, see <http://www.gnu.org/licenses/>.
19
20
21# This program is used to analyze the test results (i.e., *.sum files)
22# generated by GDB's testsuite, and print the testcases that are found
23# to be racy.
24#
25# Racy testcases are considered as being testcases which can
26# intermittently FAIL (or PASS) when run two or more times
27# consecutively, i.e., tests whose results are not deterministic.
28#
29# This program is invoked when the user runs "make check" and
30# specifies the RACY_ITER environment variable.
31
32import sys
33import os
34import re
35
36# The (global) dictionary that stores the associations between a *.sum
37# file and its results.  The data inside it will be stored as:
38#
39# files_and_tests = { 'file1.sum' : { 'PASS' : { 'test1', 'test2' ... },
40#                                     'FAIL' : { 'test5', 'test6' ... },
41#                                     ...
42#                                   },
43#                   { 'file2.sum' : { 'PASS' : { 'test1', 'test3' ... },
44#                                   ...
45#                                   }
46#                   }
47
48files_and_tests = dict()
49
50# The relatioships between various states of the same tests that
51# should be ignored.  For example, if the same test PASSes on a
52# testcase run but KFAILs on another, this test should be considered
53# racy because a known-failure is...  known.
54
55ignore_relations = {"PASS": "KFAIL"}
56
57# We are interested in lines that start with '.?(PASS|FAIL)'.  In
58# other words, we don't process errors (maybe we should).
59
60sum_matcher = re.compile("^(.?(PASS|FAIL)): (.*)$")
61
62
63def parse_sum_line(line, dic):
64    """Parse a single LINE from a sumfile, and store the results in the
65    dictionary referenced by DIC."""
66    global sum_matcher
67
68    line = line.rstrip()
69    m = re.match(sum_matcher, line)
70    if m:
71        result = m.group(1)
72        test_name = m.group(3)
73        # Remove tail parentheses.  These are likely to be '(timeout)'
74        # and other extra information that will only confuse us.
75        test_name = re.sub("(\s+)?\(.*$", "", test_name)
76        if result not in dic.keys():
77            dic[result] = set()
78        if test_name in dic[result]:
79            # If the line is already present in the dictionary, then
80            # we include a unique identifier in the end of it, in the
81            # form or '<<N>>' (where N is a number >= 2).  This is
82            # useful because the GDB testsuite is full of non-unique
83            # test messages; however, if you process the racy summary
84            # file you will also need to perform this same operation
85            # in order to identify the racy test.
86            i = 2
87            while True:
88                nname = test_name + " <<" + str(i) + ">>"
89                if nname not in dic[result]:
90                    break
91                i += 1
92            test_name = nname
93        dic[result].add(test_name)
94
95
96def read_sum_files(files):
97    """Read the sumfiles (passed as a list in the FILES variable), and
98    process each one, filling the FILES_AND_TESTS global dictionary with
99    information about them."""
100    global files_and_tests
101
102    for x in files:
103        with open(x, "r") as f:
104            files_and_tests[x] = dict()
105            for line in f.readlines():
106                parse_sum_line(line, files_and_tests[x])
107
108
109def identify_racy_tests():
110    """Identify and print the racy tests.  This function basically works
111    on sets, and the idea behind it is simple.  It takes all the sets that
112    refer to the same result (for example, all the sets that contain PASS
113    tests), and compare them.  If a test is present in all PASS sets, then
114    it is not racy.  Otherwise, it is.
115
116    This function does that for all sets (PASS, FAIL, KPASS, KFAIL, etc.),
117    and then print a sorted list (without duplicates) of all the tests
118    that were found to be racy."""
119    global files_and_tests
120
121    # First, construct two dictionaries that will hold one set of
122    # testcases for each state (PASS, FAIL, etc.).
123    #
124    # Each set in NONRACY_TESTS will contain only the non-racy
125    # testcases for that state.  A non-racy testcase is a testcase
126    # that has the same state in all test runs.
127    #
128    # Each set in ALL_TESTS will contain all tests, racy or not, for
129    # that state.
130    nonracy_tests = dict()
131    all_tests = dict()
132    for f in files_and_tests:
133        for state in files_and_tests[f]:
134            try:
135                nonracy_tests[state] &= files_and_tests[f][state].copy()
136            except KeyError:
137                nonracy_tests[state] = files_and_tests[f][state].copy()
138
139            try:
140                all_tests[state] |= files_and_tests[f][state].copy()
141            except KeyError:
142                all_tests[state] = files_and_tests[f][state].copy()
143
144    # Now, we eliminate the tests that are present in states that need
145    # to be ignored.  For example, tests both in the PASS and KFAIL
146    # states should not be considered racy.
147    ignored_tests = set()
148    for s1, s2 in ignore_relations.items():
149        try:
150            ignored_tests |= all_tests[s1] & all_tests[s2]
151        except:
152            continue
153
154    racy_tests = set()
155    for f in files_and_tests:
156        for state in files_and_tests[f]:
157            racy_tests |= files_and_tests[f][state] - nonracy_tests[state]
158
159    racy_tests = racy_tests - ignored_tests
160
161    # Print the header.
162    print("\t\t=== gdb racy tests ===\n")
163
164    # Print each test.
165    for line in sorted(racy_tests):
166        print(line)
167
168    # Print the summary.
169    print("\n")
170    print("\t\t=== gdb Summary ===\n")
171    print("# of racy tests:\t\t%d" % len(racy_tests))
172
173
174if __name__ == "__main__":
175    if len(sys.argv) < 3:
176        # It only makes sense to invoke this program if you pass two
177        # or more files to be analyzed.
178        sys.exit("Usage: %s [FILE] [FILE] ..." % sys.argv[0])
179    read_sum_files(sys.argv[1:])
180    identify_racy_tests()
181    exit(0)
182