1#!/usr/bin/env python3 2 3# Copyright (C) 2016-2023 Free Software Foundation, Inc. 4# 5# This file is part of GDB. 6# 7# This program is free software; you can redistribute it and/or modify 8# it under the terms of the GNU General Public License as published by 9# the Free Software Foundation; either version 3 of the License, or 10# (at your option) any later version. 11# 12# This program is distributed in the hope that it will be useful, 13# but WITHOUT ANY WARRANTY; without even the implied warranty of 14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15# GNU General Public License for more details. 16# 17# You should have received a copy of the GNU General Public License 18# along with this program. If not, see <http://www.gnu.org/licenses/>. 19 20 21# This program is used to analyze the test results (i.e., *.sum files) 22# generated by GDB's testsuite, and print the testcases that are found 23# to be racy. 24# 25# Racy testcases are considered as being testcases which can 26# intermittently FAIL (or PASS) when run two or more times 27# consecutively, i.e., tests whose results are not deterministic. 28# 29# This program is invoked when the user runs "make check" and 30# specifies the RACY_ITER environment variable. 31 32import sys 33import os 34import re 35 36# The (global) dictionary that stores the associations between a *.sum 37# file and its results. The data inside it will be stored as: 38# 39# files_and_tests = { 'file1.sum' : { 'PASS' : { 'test1', 'test2' ... }, 40# 'FAIL' : { 'test5', 'test6' ... }, 41# ... 42# }, 43# { 'file2.sum' : { 'PASS' : { 'test1', 'test3' ... }, 44# ... 45# } 46# } 47 48files_and_tests = dict() 49 50# The relatioships between various states of the same tests that 51# should be ignored. For example, if the same test PASSes on a 52# testcase run but KFAILs on another, this test should be considered 53# racy because a known-failure is... known. 54 55ignore_relations = {"PASS": "KFAIL"} 56 57# We are interested in lines that start with '.?(PASS|FAIL)'. In 58# other words, we don't process errors (maybe we should). 59 60sum_matcher = re.compile("^(.?(PASS|FAIL)): (.*)$") 61 62 63def parse_sum_line(line, dic): 64 """Parse a single LINE from a sumfile, and store the results in the 65 dictionary referenced by DIC.""" 66 global sum_matcher 67 68 line = line.rstrip() 69 m = re.match(sum_matcher, line) 70 if m: 71 result = m.group(1) 72 test_name = m.group(3) 73 # Remove tail parentheses. These are likely to be '(timeout)' 74 # and other extra information that will only confuse us. 75 test_name = re.sub("(\s+)?\(.*$", "", test_name) 76 if result not in dic.keys(): 77 dic[result] = set() 78 if test_name in dic[result]: 79 # If the line is already present in the dictionary, then 80 # we include a unique identifier in the end of it, in the 81 # form or '<<N>>' (where N is a number >= 2). This is 82 # useful because the GDB testsuite is full of non-unique 83 # test messages; however, if you process the racy summary 84 # file you will also need to perform this same operation 85 # in order to identify the racy test. 86 i = 2 87 while True: 88 nname = test_name + " <<" + str(i) + ">>" 89 if nname not in dic[result]: 90 break 91 i += 1 92 test_name = nname 93 dic[result].add(test_name) 94 95 96def read_sum_files(files): 97 """Read the sumfiles (passed as a list in the FILES variable), and 98 process each one, filling the FILES_AND_TESTS global dictionary with 99 information about them.""" 100 global files_and_tests 101 102 for x in files: 103 with open(x, "r") as f: 104 files_and_tests[x] = dict() 105 for line in f.readlines(): 106 parse_sum_line(line, files_and_tests[x]) 107 108 109def identify_racy_tests(): 110 """Identify and print the racy tests. This function basically works 111 on sets, and the idea behind it is simple. It takes all the sets that 112 refer to the same result (for example, all the sets that contain PASS 113 tests), and compare them. If a test is present in all PASS sets, then 114 it is not racy. Otherwise, it is. 115 116 This function does that for all sets (PASS, FAIL, KPASS, KFAIL, etc.), 117 and then print a sorted list (without duplicates) of all the tests 118 that were found to be racy.""" 119 global files_and_tests 120 121 # First, construct two dictionaries that will hold one set of 122 # testcases for each state (PASS, FAIL, etc.). 123 # 124 # Each set in NONRACY_TESTS will contain only the non-racy 125 # testcases for that state. A non-racy testcase is a testcase 126 # that has the same state in all test runs. 127 # 128 # Each set in ALL_TESTS will contain all tests, racy or not, for 129 # that state. 130 nonracy_tests = dict() 131 all_tests = dict() 132 for f in files_and_tests: 133 for state in files_and_tests[f]: 134 try: 135 nonracy_tests[state] &= files_and_tests[f][state].copy() 136 except KeyError: 137 nonracy_tests[state] = files_and_tests[f][state].copy() 138 139 try: 140 all_tests[state] |= files_and_tests[f][state].copy() 141 except KeyError: 142 all_tests[state] = files_and_tests[f][state].copy() 143 144 # Now, we eliminate the tests that are present in states that need 145 # to be ignored. For example, tests both in the PASS and KFAIL 146 # states should not be considered racy. 147 ignored_tests = set() 148 for s1, s2 in ignore_relations.items(): 149 try: 150 ignored_tests |= all_tests[s1] & all_tests[s2] 151 except: 152 continue 153 154 racy_tests = set() 155 for f in files_and_tests: 156 for state in files_and_tests[f]: 157 racy_tests |= files_and_tests[f][state] - nonracy_tests[state] 158 159 racy_tests = racy_tests - ignored_tests 160 161 # Print the header. 162 print("\t\t=== gdb racy tests ===\n") 163 164 # Print each test. 165 for line in sorted(racy_tests): 166 print(line) 167 168 # Print the summary. 169 print("\n") 170 print("\t\t=== gdb Summary ===\n") 171 print("# of racy tests:\t\t%d" % len(racy_tests)) 172 173 174if __name__ == "__main__": 175 if len(sys.argv) < 3: 176 # It only makes sense to invoke this program if you pass two 177 # or more files to be analyzed. 178 sys.exit("Usage: %s [FILE] [FILE] ..." % sys.argv[0]) 179 read_sum_files(sys.argv[1:]) 180 identify_racy_tests() 181 exit(0) 182