1#!/usr/bin/env python 2# 3# Copyright 2020 Google Inc. All Rights Reserved. 4# 5# Redistribution and use in source and binary forms, with or without 6# modification, are permitted provided that the following conditions are 7# met: 8# 9# * Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# * Redistributions in binary form must reproduce the above 12# copyright notice, this list of conditions and the following disclaimer 13# in the documentation and/or other materials provided with the 14# distribution. 15# * Neither the name of Google Inc. nor the names of its 16# contributors may be used to endorse or promote products derived from 17# this software without specific prior written permission. 18# 19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31"""Unit test for Google Test fail_fast. 32 33A user can specify if a Google Test program should continue test execution 34after a test failure via the GTEST_FAIL_FAST environment variable or the 35--gtest_fail_fast flag. The default value of the flag can also be changed 36by Bazel fail fast environment variable TESTBRIDGE_TEST_RUNNER_FAIL_FAST. 37 38This script tests such functionality by invoking googletest-failfast-unittest_ 39(a program written with Google Test) with different environments and command 40line flags. 41""" 42 43import os 44from googletest.test import gtest_test_utils 45 46# Constants. 47 48# Bazel testbridge environment variable for fail fast 49BAZEL_FAIL_FAST_ENV_VAR = 'TESTBRIDGE_TEST_RUNNER_FAIL_FAST' 50 51# The environment variable for specifying fail fast. 52FAIL_FAST_ENV_VAR = 'GTEST_FAIL_FAST' 53 54# The command line flag for specifying fail fast. 55FAIL_FAST_FLAG = 'gtest_fail_fast' 56 57# The command line flag to run disabled tests. 58RUN_DISABLED_FLAG = 'gtest_also_run_disabled_tests' 59 60# The command line flag for specifying a filter. 61FILTER_FLAG = 'gtest_filter' 62 63# Command to run the googletest-failfast-unittest_ program. 64COMMAND = gtest_test_utils.GetTestExecutablePath( 65 'googletest-failfast-unittest_' 66) 67 68# The command line flag to tell Google Test to output the list of tests it 69# will run. 70LIST_TESTS_FLAG = '--gtest_list_tests' 71 72# Indicates whether Google Test supports death tests. 73SUPPORTS_DEATH_TESTS = ( 74 'HasDeathTest' 75 in gtest_test_utils.Subprocess([COMMAND, LIST_TESTS_FLAG]).output 76) 77 78# Utilities. 79 80environ = os.environ.copy() 81 82 83def SetEnvVar(env_var, value): 84 """Sets the env variable to 'value'; unsets it when 'value' is None.""" 85 86 if value is not None: 87 environ[env_var] = value 88 elif env_var in environ: 89 del environ[env_var] 90 91 92def RunAndReturnOutput(test_suite=None, fail_fast=None, run_disabled=False): 93 """Runs the test program and returns its output.""" 94 95 args = [] 96 xml_path = os.path.join( 97 gtest_test_utils.GetTempDir(), '.GTestFailFastUnitTest.xml' 98 ) 99 args += ['--gtest_output=xml:' + xml_path] 100 if fail_fast is not None: 101 if isinstance(fail_fast, str): 102 args += ['--%s=%s' % (FAIL_FAST_FLAG, fail_fast)] 103 elif fail_fast: 104 args += ['--%s' % FAIL_FAST_FLAG] 105 else: 106 args += ['--no%s' % FAIL_FAST_FLAG] 107 if test_suite: 108 args += ['--%s=%s.*' % (FILTER_FLAG, test_suite)] 109 if run_disabled: 110 args += ['--%s' % RUN_DISABLED_FLAG] 111 txt_out = gtest_test_utils.Subprocess([COMMAND] + args, env=environ).output 112 with open(xml_path) as xml_file: 113 return txt_out, xml_file.read() 114 115 116# The unit test. 117class GTestFailFastUnitTest(gtest_test_utils.TestCase): 118 """Tests the env variable or the command line flag for fail_fast.""" 119 120 def testDefaultBehavior(self): 121 """Tests the behavior of not specifying the fail_fast.""" 122 123 txt, _ = RunAndReturnOutput() 124 self.assertIn('22 FAILED TEST', txt) 125 126 def testGoogletestFlag(self): 127 txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=True) 128 self.assertIn('1 FAILED TEST', txt) 129 self.assertIn('[ SKIPPED ] 3 tests', txt) 130 131 txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=False) 132 self.assertIn('4 FAILED TEST', txt) 133 self.assertNotIn('[ SKIPPED ]', txt) 134 135 def testGoogletestEnvVar(self): 136 """Tests the behavior of specifying fail_fast via Googletest env var.""" 137 138 try: 139 SetEnvVar(FAIL_FAST_ENV_VAR, '1') 140 txt, _ = RunAndReturnOutput('HasSimpleTest') 141 self.assertIn('1 FAILED TEST', txt) 142 self.assertIn('[ SKIPPED ] 3 tests', txt) 143 144 SetEnvVar(FAIL_FAST_ENV_VAR, '0') 145 txt, _ = RunAndReturnOutput('HasSimpleTest') 146 self.assertIn('4 FAILED TEST', txt) 147 self.assertNotIn('[ SKIPPED ]', txt) 148 finally: 149 SetEnvVar(FAIL_FAST_ENV_VAR, None) 150 151 def testBazelEnvVar(self): 152 """Tests the behavior of specifying fail_fast via Bazel testbridge.""" 153 154 try: 155 SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '1') 156 txt, _ = RunAndReturnOutput('HasSimpleTest') 157 self.assertIn('1 FAILED TEST', txt) 158 self.assertIn('[ SKIPPED ] 3 tests', txt) 159 160 SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0') 161 txt, _ = RunAndReturnOutput('HasSimpleTest') 162 self.assertIn('4 FAILED TEST', txt) 163 self.assertNotIn('[ SKIPPED ]', txt) 164 finally: 165 SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None) 166 167 def testFlagOverridesEnvVar(self): 168 """Tests precedence of flag over env var.""" 169 170 try: 171 SetEnvVar(FAIL_FAST_ENV_VAR, '0') 172 txt, _ = RunAndReturnOutput('HasSimpleTest', True) 173 self.assertIn('1 FAILED TEST', txt) 174 self.assertIn('[ SKIPPED ] 3 tests', txt) 175 finally: 176 SetEnvVar(FAIL_FAST_ENV_VAR, None) 177 178 def testGoogletestEnvVarOverridesBazelEnvVar(self): 179 """Tests that the Googletest native env var over Bazel testbridge.""" 180 181 try: 182 SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0') 183 SetEnvVar(FAIL_FAST_ENV_VAR, '1') 184 txt, _ = RunAndReturnOutput('HasSimpleTest') 185 self.assertIn('1 FAILED TEST', txt) 186 self.assertIn('[ SKIPPED ] 3 tests', txt) 187 finally: 188 SetEnvVar(FAIL_FAST_ENV_VAR, None) 189 SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None) 190 191 def testEventListener(self): 192 txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=True) 193 self.assertIn('1 FAILED TEST', txt) 194 self.assertIn('[ SKIPPED ] 3 tests', txt) 195 for expected_count, callback in [ 196 (1, 'OnTestSuiteStart'), 197 (5, 'OnTestStart'), 198 (5, 'OnTestEnd'), 199 (5, 'OnTestPartResult'), 200 (1, 'OnTestSuiteEnd'), 201 ]: 202 self.assertEqual( 203 expected_count, 204 txt.count(callback), 205 'Expected %d calls to callback %s match count on output: %s ' 206 % (expected_count, callback, txt), 207 ) 208 209 txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=False) 210 self.assertIn('3 FAILED TEST', txt) 211 self.assertIn('[ SKIPPED ] 1 test', txt) 212 for expected_count, callback in [ 213 (1, 'OnTestSuiteStart'), 214 (5, 'OnTestStart'), 215 (5, 'OnTestEnd'), 216 (5, 'OnTestPartResult'), 217 (1, 'OnTestSuiteEnd'), 218 ]: 219 self.assertEqual( 220 expected_count, 221 txt.count(callback), 222 'Expected %d calls to callback %s match count on output: %s ' 223 % (expected_count, callback, txt), 224 ) 225 226 def assertXmlResultCount(self, result, count, xml): 227 self.assertEqual( 228 count, 229 xml.count('result="%s"' % result), 230 'Expected \'result="%s"\' match count of %s: %s ' 231 % (result, count, xml), 232 ) 233 234 def assertXmlStatusCount(self, status, count, xml): 235 self.assertEqual( 236 count, 237 xml.count('status="%s"' % status), 238 'Expected \'status="%s"\' match count of %s: %s ' 239 % (status, count, xml), 240 ) 241 242 def assertFailFastXmlAndTxtOutput( 243 self, 244 fail_fast, 245 test_suite, 246 passed_count, 247 failure_count, 248 skipped_count, 249 suppressed_count, 250 run_disabled=False, 251 ): 252 """Assert XML and text output of a test execution.""" 253 254 txt, xml = RunAndReturnOutput(test_suite, fail_fast, run_disabled) 255 if failure_count > 0: 256 self.assertIn('%s FAILED TEST' % failure_count, txt) 257 if suppressed_count > 0: 258 self.assertIn('%s DISABLED TEST' % suppressed_count, txt) 259 if skipped_count > 0: 260 self.assertIn('[ SKIPPED ] %s tests' % skipped_count, txt) 261 self.assertXmlStatusCount( 262 'run', passed_count + failure_count + skipped_count, xml 263 ) 264 self.assertXmlStatusCount('notrun', suppressed_count, xml) 265 self.assertXmlResultCount('completed', passed_count + failure_count, xml) 266 self.assertXmlResultCount('skipped', skipped_count, xml) 267 self.assertXmlResultCount('suppressed', suppressed_count, xml) 268 269 def assertFailFastBehavior( 270 self, 271 test_suite, 272 passed_count, 273 failure_count, 274 skipped_count, 275 suppressed_count, 276 run_disabled=False, 277 ): 278 """Assert --fail_fast via flag.""" 279 280 for fail_fast in ('true', '1', 't', True): 281 self.assertFailFastXmlAndTxtOutput( 282 fail_fast, 283 test_suite, 284 passed_count, 285 failure_count, 286 skipped_count, 287 suppressed_count, 288 run_disabled, 289 ) 290 291 def assertNotFailFastBehavior( 292 self, 293 test_suite, 294 passed_count, 295 failure_count, 296 skipped_count, 297 suppressed_count, 298 run_disabled=False, 299 ): 300 """Assert --nofail_fast via flag.""" 301 302 for fail_fast in ('false', '0', 'f', False): 303 self.assertFailFastXmlAndTxtOutput( 304 fail_fast, 305 test_suite, 306 passed_count, 307 failure_count, 308 skipped_count, 309 suppressed_count, 310 run_disabled, 311 ) 312 313 def testFlag_HasFixtureTest(self): 314 """Tests the behavior of fail_fast and TEST_F.""" 315 self.assertFailFastBehavior( 316 test_suite='HasFixtureTest', 317 passed_count=1, 318 failure_count=1, 319 skipped_count=3, 320 suppressed_count=0, 321 ) 322 self.assertNotFailFastBehavior( 323 test_suite='HasFixtureTest', 324 passed_count=1, 325 failure_count=4, 326 skipped_count=0, 327 suppressed_count=0, 328 ) 329 330 def testFlag_HasSimpleTest(self): 331 """Tests the behavior of fail_fast and TEST.""" 332 self.assertFailFastBehavior( 333 test_suite='HasSimpleTest', 334 passed_count=1, 335 failure_count=1, 336 skipped_count=3, 337 suppressed_count=0, 338 ) 339 self.assertNotFailFastBehavior( 340 test_suite='HasSimpleTest', 341 passed_count=1, 342 failure_count=4, 343 skipped_count=0, 344 suppressed_count=0, 345 ) 346 347 def testFlag_HasParametersTest(self): 348 """Tests the behavior of fail_fast and TEST_P.""" 349 self.assertFailFastBehavior( 350 test_suite='HasParametersSuite/HasParametersTest', 351 passed_count=0, 352 failure_count=1, 353 skipped_count=3, 354 suppressed_count=0, 355 ) 356 self.assertNotFailFastBehavior( 357 test_suite='HasParametersSuite/HasParametersTest', 358 passed_count=0, 359 failure_count=4, 360 skipped_count=0, 361 suppressed_count=0, 362 ) 363 364 def testFlag_HasDisabledTest(self): 365 """Tests the behavior of fail_fast and Disabled test cases.""" 366 self.assertFailFastBehavior( 367 test_suite='HasDisabledTest', 368 passed_count=1, 369 failure_count=1, 370 skipped_count=2, 371 suppressed_count=1, 372 run_disabled=False, 373 ) 374 self.assertNotFailFastBehavior( 375 test_suite='HasDisabledTest', 376 passed_count=1, 377 failure_count=3, 378 skipped_count=0, 379 suppressed_count=1, 380 run_disabled=False, 381 ) 382 383 def testFlag_HasDisabledRunDisabledTest(self): 384 """Tests the behavior of fail_fast and Disabled test cases enabled.""" 385 self.assertFailFastBehavior( 386 test_suite='HasDisabledTest', 387 passed_count=1, 388 failure_count=1, 389 skipped_count=3, 390 suppressed_count=0, 391 run_disabled=True, 392 ) 393 self.assertNotFailFastBehavior( 394 test_suite='HasDisabledTest', 395 passed_count=1, 396 failure_count=4, 397 skipped_count=0, 398 suppressed_count=0, 399 run_disabled=True, 400 ) 401 402 def testFlag_HasDisabledSuiteTest(self): 403 """Tests the behavior of fail_fast and Disabled test suites.""" 404 self.assertFailFastBehavior( 405 test_suite='DISABLED_HasDisabledSuite', 406 passed_count=0, 407 failure_count=0, 408 skipped_count=0, 409 suppressed_count=5, 410 run_disabled=False, 411 ) 412 self.assertNotFailFastBehavior( 413 test_suite='DISABLED_HasDisabledSuite', 414 passed_count=0, 415 failure_count=0, 416 skipped_count=0, 417 suppressed_count=5, 418 run_disabled=False, 419 ) 420 421 def testFlag_HasDisabledSuiteRunDisabledTest(self): 422 """Tests the behavior of fail_fast and Disabled test suites enabled.""" 423 self.assertFailFastBehavior( 424 test_suite='DISABLED_HasDisabledSuite', 425 passed_count=1, 426 failure_count=1, 427 skipped_count=3, 428 suppressed_count=0, 429 run_disabled=True, 430 ) 431 self.assertNotFailFastBehavior( 432 test_suite='DISABLED_HasDisabledSuite', 433 passed_count=1, 434 failure_count=4, 435 skipped_count=0, 436 suppressed_count=0, 437 run_disabled=True, 438 ) 439 440 if SUPPORTS_DEATH_TESTS: 441 442 def testFlag_HasDeathTest(self): 443 """Tests the behavior of fail_fast and death tests.""" 444 self.assertFailFastBehavior( 445 test_suite='HasDeathTest', 446 passed_count=1, 447 failure_count=1, 448 skipped_count=3, 449 suppressed_count=0, 450 ) 451 self.assertNotFailFastBehavior( 452 test_suite='HasDeathTest', 453 passed_count=1, 454 failure_count=4, 455 skipped_count=0, 456 suppressed_count=0, 457 ) 458 459 460if __name__ == '__main__': 461 gtest_test_utils.Main() 462