1#!/usr/bin/python 2 3# Script to compare testsuite failures against a list of known-to-fail 4# tests. 5# 6# NOTE: This script is used in installations that are running Python 2.4. 7# Please stick to syntax features available in 2.4 and earlier 8# versions. 9 10# Contributed by Diego Novillo <dnovillo@google.com> 11# 12# Copyright (C) 2011-2013 Free Software Foundation, Inc. 13# 14# This file is part of GCC. 15# 16# GCC is free software; you can redistribute it and/or modify 17# it under the terms of the GNU General Public License as published by 18# the Free Software Foundation; either version 3, or (at your option) 19# any later version. 20# 21# GCC is distributed in the hope that it will be useful, 22# but WITHOUT ANY WARRANTY; without even the implied warranty of 23# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 24# GNU General Public License for more details. 25# 26# You should have received a copy of the GNU General Public License 27# along with GCC; see the file COPYING. If not, write to 28# the Free Software Foundation, 51 Franklin Street, Fifth Floor, 29# Boston, MA 02110-1301, USA. 30 31"""This script provides a coarser XFAILing mechanism that requires no 32detailed DejaGNU markings. This is useful in a variety of scenarios: 33 34- Development branches with many known failures waiting to be fixed. 35- Release branches with known failures that are not considered 36 important for the particular release criteria used in that branch. 37 38The script must be executed from the toplevel build directory. When 39executed it will: 40 411- Determine the target built: TARGET 422- Determine the source directory: SRCDIR 433- Look for a failure manifest file in 44 <SRCDIR>/<MANIFEST_SUBDIR>/<MANIFEST_NAME>.xfail 454- Collect all the <tool>.sum files from the build tree. 465- Produce a report stating: 47 a- Failures expected in the manifest but not present in the build. 48 b- Failures in the build not expected in the manifest. 496- If all the build failures are expected in the manifest, it exits 50 with exit code 0. Otherwise, it exits with error code 1. 51 52Manifest files contain expected DejaGNU results that are otherwise 53treated as failures. 54They may also contain additional text: 55 56# This is a comment. - self explanatory 57@include file - the file is a path relative to the includer 58@remove result text - result text is removed from the expected set 59""" 60 61import datetime 62import optparse 63import os 64import re 65import sys 66 67# Handled test results. 68_VALID_TEST_RESULTS = [ 'FAIL', 'UNRESOLVED', 'XPASS', 'ERROR' ] 69_VALID_TEST_RESULTS_REX = re.compile("%s" % "|".join(_VALID_TEST_RESULTS)) 70 71# Subdirectory of srcdir in which to find the manifest file. 72_MANIFEST_SUBDIR = 'contrib/testsuite-management' 73 74# Pattern for naming manifest files. 75# The first argument should be the toplevel GCC(/GNU tool) source directory. 76# The second argument is the manifest subdir. 77# The third argument is the manifest target, which defaults to the target 78# triplet used during the build. 79_MANIFEST_PATH_PATTERN = '%s/%s/%s.xfail' 80 81# The options passed to the program. 82_OPTIONS = None 83 84def Error(msg): 85 print >>sys.stderr, 'error: %s' % msg 86 sys.exit(1) 87 88 89class TestResult(object): 90 """Describes a single DejaGNU test result as emitted in .sum files. 91 92 We are only interested in representing unsuccessful tests. So, only 93 a subset of all the tests are loaded. 94 95 The summary line used to build the test result should have this format: 96 97 attrlist | XPASS: gcc.dg/unroll_1.c (test for excess errors) 98 ^^^^^^^^ ^^^^^ ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^ 99 optional state name description 100 attributes 101 102 Attributes: 103 attrlist: A comma separated list of attributes. 104 Valid values: 105 flaky Indicates that this test may not always fail. These 106 tests are reported, but their presence does not affect 107 the results. 108 109 expire=YYYYMMDD After this date, this test will produce an error 110 whether it is in the manifest or not. 111 112 state: One of UNRESOLVED, XPASS or FAIL. 113 name: File name for the test. 114 description: String describing the test (flags used, dejagnu message, etc) 115 ordinal: Monotonically increasing integer. 116 It is used to keep results for one .exp file sorted 117 by the order the tests were run. 118 """ 119 120 def __init__(self, summary_line, ordinal=-1): 121 try: 122 (self.attrs, summary_line) = SplitAttributesFromSummaryLine(summary_line) 123 try: 124 (self.state, 125 self.name, 126 self.description) = re.match(r'([A-Z]+):\s*(\S+)\s*(.*)', 127 summary_line).groups() 128 except: 129 print 'Failed to parse summary line: "%s"' % summary_line 130 raise 131 self.ordinal = ordinal 132 except ValueError: 133 Error('Cannot parse summary line "%s"' % summary_line) 134 135 if self.state not in _VALID_TEST_RESULTS: 136 Error('Invalid test result %s in "%s" (parsed as "%s")' % ( 137 self.state, summary_line, self)) 138 139 def __lt__(self, other): 140 return (self.name < other.name or 141 (self.name == other.name and self.ordinal < other.ordinal)) 142 143 def __hash__(self): 144 return hash(self.state) ^ hash(self.name) ^ hash(self.description) 145 146 def __eq__(self, other): 147 return (self.state == other.state and 148 self.name == other.name and 149 self.description == other.description) 150 151 def __ne__(self, other): 152 return not (self == other) 153 154 def __str__(self): 155 attrs = '' 156 if self.attrs: 157 attrs = '%s | ' % self.attrs 158 return '%s%s: %s %s' % (attrs, self.state, self.name, self.description) 159 160 def ExpirationDate(self): 161 # Return a datetime.date object with the expiration date for this 162 # test result. Return None, if no expiration has been set. 163 if re.search(r'expire=', self.attrs): 164 expiration = re.search(r'expire=(\d\d\d\d)(\d\d)(\d\d)', self.attrs) 165 if not expiration: 166 Error('Invalid expire= format in "%s". Must be of the form ' 167 '"expire=YYYYMMDD"' % self) 168 return datetime.date(int(expiration.group(1)), 169 int(expiration.group(2)), 170 int(expiration.group(3))) 171 return None 172 173 def HasExpired(self): 174 # Return True if the expiration date of this result has passed. 175 expiration_date = self.ExpirationDate() 176 if expiration_date: 177 now = datetime.date.today() 178 return now > expiration_date 179 180 181def GetMakefileValue(makefile_name, value_name): 182 if os.path.exists(makefile_name): 183 makefile = open(makefile_name) 184 for line in makefile: 185 if line.startswith(value_name): 186 (_, value) = line.split('=', 1) 187 value = value.strip() 188 makefile.close() 189 return value 190 makefile.close() 191 return None 192 193 194def ValidBuildDirectory(builddir): 195 if (not os.path.exists(builddir) or 196 not os.path.exists('%s/Makefile' % builddir)): 197 return False 198 return True 199 200 201def IsComment(line): 202 """Return True if line is a comment.""" 203 return line.startswith('#') 204 205 206def SplitAttributesFromSummaryLine(line): 207 """Splits off attributes from a summary line, if present.""" 208 if '|' in line and not _VALID_TEST_RESULTS_REX.match(line): 209 (attrs, line) = line.split('|', 1) 210 attrs = attrs.strip() 211 else: 212 attrs = '' 213 line = line.strip() 214 return (attrs, line) 215 216 217def IsInterestingResult(line): 218 """Return True if line is one of the summary lines we care about.""" 219 (_, line) = SplitAttributesFromSummaryLine(line) 220 return bool(_VALID_TEST_RESULTS_REX.match(line)) 221 222 223def IsInclude(line): 224 """Return True if line is an include of another file.""" 225 return line.startswith("@include ") 226 227 228def GetIncludeFile(line, includer): 229 """Extract the name of the include file from line.""" 230 includer_dir = os.path.dirname(includer) 231 include_file = line[len("@include "):] 232 return os.path.join(includer_dir, include_file.strip()) 233 234 235def IsNegativeResult(line): 236 """Return True if line should be removed from the expected results.""" 237 return line.startswith("@remove ") 238 239 240def GetNegativeResult(line): 241 """Extract the name of the negative result from line.""" 242 line = line[len("@remove "):] 243 return line.strip() 244 245 246def ParseManifestWorker(result_set, manifest_path): 247 """Read manifest_path, adding the contents to result_set.""" 248 if _OPTIONS.verbosity >= 1: 249 print 'Parsing manifest file %s.' % manifest_path 250 manifest_file = open(manifest_path) 251 for line in manifest_file: 252 line = line.strip() 253 if line == "": 254 pass 255 elif IsComment(line): 256 pass 257 elif IsNegativeResult(line): 258 result_set.remove(TestResult(GetNegativeResult(line))) 259 elif IsInclude(line): 260 ParseManifestWorker(result_set, GetIncludeFile(line, manifest_path)) 261 elif IsInterestingResult(line): 262 result_set.add(TestResult(line)) 263 else: 264 Error('Unrecognized line in manifest file: %s' % line) 265 manifest_file.close() 266 267 268def ParseManifest(manifest_path): 269 """Create a set of TestResult instances from the given manifest file.""" 270 result_set = set() 271 ParseManifestWorker(result_set, manifest_path) 272 return result_set 273 274 275def ParseSummary(sum_fname): 276 """Create a set of TestResult instances from the given summary file.""" 277 result_set = set() 278 # ordinal is used when sorting the results so that tests within each 279 # .exp file are kept sorted. 280 ordinal=0 281 sum_file = open(sum_fname) 282 for line in sum_file: 283 if IsInterestingResult(line): 284 result = TestResult(line, ordinal) 285 ordinal += 1 286 if result.HasExpired(): 287 # Tests that have expired are not added to the set of expected 288 # results. If they are still present in the set of actual results, 289 # they will cause an error to be reported. 290 print 'WARNING: Expected failure "%s" has expired.' % line.strip() 291 continue 292 result_set.add(result) 293 sum_file.close() 294 return result_set 295 296 297def GetManifest(manifest_path): 298 """Build a set of expected failures from the manifest file. 299 300 Each entry in the manifest file should have the format understood 301 by the TestResult constructor. 302 303 If no manifest file exists for this target, it returns an empty set. 304 """ 305 if os.path.exists(manifest_path): 306 return ParseManifest(manifest_path) 307 else: 308 return set() 309 310 311def CollectSumFiles(builddir): 312 sum_files = [] 313 for root, dirs, files in os.walk(builddir): 314 for ignored in ('.svn', '.git'): 315 if ignored in dirs: 316 dirs.remove(ignored) 317 for fname in files: 318 if fname.endswith('.sum'): 319 sum_files.append(os.path.join(root, fname)) 320 return sum_files 321 322 323def GetResults(sum_files): 324 """Collect all the test results from the given .sum files.""" 325 build_results = set() 326 for sum_fname in sum_files: 327 print '\t%s' % sum_fname 328 build_results |= ParseSummary(sum_fname) 329 return build_results 330 331 332def CompareResults(manifest, actual): 333 """Compare sets of results and return two lists: 334 - List of results present in ACTUAL but missing from MANIFEST. 335 - List of results present in MANIFEST but missing from ACTUAL. 336 """ 337 # Collect all the actual results not present in the manifest. 338 # Results in this set will be reported as errors. 339 actual_vs_manifest = set() 340 for actual_result in actual: 341 if actual_result not in manifest: 342 actual_vs_manifest.add(actual_result) 343 344 # Collect all the tests in the manifest that were not found 345 # in the actual results. 346 # Results in this set will be reported as warnings (since 347 # they are expected failures that are not failing anymore). 348 manifest_vs_actual = set() 349 for expected_result in manifest: 350 # Ignore tests marked flaky. 351 if 'flaky' in expected_result.attrs: 352 continue 353 if expected_result not in actual: 354 manifest_vs_actual.add(expected_result) 355 356 return actual_vs_manifest, manifest_vs_actual 357 358 359def GetManifestPath(srcdir, target, user_provided_must_exist): 360 """Return the full path to the manifest file.""" 361 manifest_path = _OPTIONS.manifest 362 if manifest_path: 363 if user_provided_must_exist and not os.path.exists(manifest_path): 364 Error('Manifest does not exist: %s' % manifest_path) 365 return manifest_path 366 else: 367 if not srcdir: 368 Error('Could not determine the location of GCC\'s source tree. ' 369 'The Makefile does not contain a definition for "srcdir".') 370 if not target: 371 Error('Could not determine the target triplet for this build. ' 372 'The Makefile does not contain a definition for "target_alias".') 373 return _MANIFEST_PATH_PATTERN % (srcdir, _MANIFEST_SUBDIR, target) 374 375 376def GetBuildData(): 377 if not ValidBuildDirectory(_OPTIONS.build_dir): 378 # If we have been given a set of results to use, we may 379 # not be inside a valid GCC build directory. In that case, 380 # the user must provide both a manifest file and a set 381 # of results to check against it. 382 if not _OPTIONS.results or not _OPTIONS.manifest: 383 Error('%s is not a valid GCC top level build directory. ' 384 'You must use --manifest and --results to do the validation.' % 385 _OPTIONS.build_dir) 386 else: 387 return None, None 388 srcdir = GetMakefileValue('%s/Makefile' % _OPTIONS.build_dir, 'srcdir =') 389 target = GetMakefileValue('%s/Makefile' % _OPTIONS.build_dir, 'target_alias=') 390 print 'Source directory: %s' % srcdir 391 print 'Build target: %s' % target 392 return srcdir, target 393 394 395def PrintSummary(msg, summary): 396 print '\n\n%s' % msg 397 for result in sorted(summary): 398 print result 399 400 401def GetSumFiles(results, build_dir): 402 if not results: 403 print 'Getting actual results from build directory %s' % build_dir 404 sum_files = CollectSumFiles(build_dir) 405 else: 406 print 'Getting actual results from user-provided results' 407 sum_files = results.split() 408 return sum_files 409 410 411def PerformComparison(expected, actual, ignore_missing_failures): 412 actual_vs_expected, expected_vs_actual = CompareResults(expected, actual) 413 414 tests_ok = True 415 if len(actual_vs_expected) > 0: 416 PrintSummary('Unexpected results in this build (new failures)', 417 actual_vs_expected) 418 tests_ok = False 419 420 if not ignore_missing_failures and len(expected_vs_actual) > 0: 421 PrintSummary('Expected results not present in this build (fixed tests)' 422 '\n\nNOTE: This is not a failure. It just means that these ' 423 'tests were expected\nto fail, but either they worked in ' 424 'this configuration or they were not\npresent at all.\n', 425 expected_vs_actual) 426 427 if tests_ok: 428 print '\nSUCCESS: No unexpected failures.' 429 430 return tests_ok 431 432 433def CheckExpectedResults(): 434 srcdir, target = GetBuildData() 435 manifest_path = GetManifestPath(srcdir, target, True) 436 print 'Manifest: %s' % manifest_path 437 manifest = GetManifest(manifest_path) 438 sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir) 439 actual = GetResults(sum_files) 440 441 if _OPTIONS.verbosity >= 1: 442 PrintSummary('Tests expected to fail', manifest) 443 PrintSummary('\nActual test results', actual) 444 445 return PerformComparison(manifest, actual, _OPTIONS.ignore_missing_failures) 446 447 448def ProduceManifest(): 449 (srcdir, target) = GetBuildData() 450 manifest_path = GetManifestPath(srcdir, target, False) 451 print 'Manifest: %s' % manifest_path 452 if os.path.exists(manifest_path) and not _OPTIONS.force: 453 Error('Manifest file %s already exists.\nUse --force to overwrite.' % 454 manifest_path) 455 456 sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir) 457 actual = GetResults(sum_files) 458 manifest_file = open(manifest_path, 'w') 459 for result in sorted(actual): 460 print result 461 manifest_file.write('%s\n' % result) 462 manifest_file.close() 463 464 return True 465 466 467def CompareBuilds(): 468 (srcdir, target) = GetBuildData() 469 470 sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.build_dir) 471 actual = GetResults(sum_files) 472 473 clean_sum_files = GetSumFiles(_OPTIONS.results, _OPTIONS.clean_build) 474 clean = GetResults(clean_sum_files) 475 476 return PerformComparison(clean, actual, _OPTIONS.ignore_missing_failures) 477 478 479def Main(argv): 480 parser = optparse.OptionParser(usage=__doc__) 481 482 # Keep the following list sorted by option name. 483 parser.add_option('--build_dir', action='store', type='string', 484 dest='build_dir', default='.', 485 help='Build directory to check (default = .)') 486 parser.add_option('--clean_build', action='store', type='string', 487 dest='clean_build', default=None, 488 help='Compare test results from this build against ' 489 'those of another (clean) build. Use this option ' 490 'when comparing the test results of your patch versus ' 491 'the test results of a clean build without your patch. ' 492 'You must provide the path to the top directory of your ' 493 'clean build.') 494 parser.add_option('--force', action='store_true', dest='force', 495 default=False, help='When used with --produce_manifest, ' 496 'it will overwrite an existing manifest file ' 497 '(default = False)') 498 parser.add_option('--ignore_missing_failures', action='store_true', 499 dest='ignore_missing_failures', default=False, 500 help='When a failure is expected in the manifest but ' 501 'it is not found in the actual results, the script ' 502 'produces a note alerting to this fact. This means ' 503 'that the expected failure has been fixed, or ' 504 'it did not run, or it may simply be flaky ' 505 '(default = False)') 506 parser.add_option('--manifest', action='store', type='string', 507 dest='manifest', default=None, 508 help='Name of the manifest file to use (default = ' 509 'taken from ' 510 'contrib/testsuite-managment/<target_alias>.xfail)') 511 parser.add_option('--produce_manifest', action='store_true', 512 dest='produce_manifest', default=False, 513 help='Produce the manifest for the current ' 514 'build (default = False)') 515 parser.add_option('--results', action='store', type='string', 516 dest='results', default=None, help='Space-separated list ' 517 'of .sum files with the testing results to check. The ' 518 'only content needed from these files are the lines ' 519 'starting with FAIL, XPASS or UNRESOLVED (default = ' 520 '.sum files collected from the build directory).') 521 parser.add_option('--verbosity', action='store', dest='verbosity', 522 type='int', default=0, help='Verbosity level (default = 0)') 523 global _OPTIONS 524 (_OPTIONS, _) = parser.parse_args(argv[1:]) 525 526 if _OPTIONS.produce_manifest: 527 retval = ProduceManifest() 528 elif _OPTIONS.clean_build: 529 retval = CompareBuilds() 530 else: 531 retval = CheckExpectedResults() 532 533 if retval: 534 return 0 535 else: 536 return 1 537 538 539if __name__ == '__main__': 540 retval = Main(sys.argv) 541 sys.exit(retval) 542