1#!/usr/bin/env python
2#
3# Copyright 2020 Google Inc. All Rights Reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met:
8#
9#     * Redistributions of source code must retain the above copyright
10# notice, this list of conditions and the following disclaimer.
11#     * Redistributions in binary form must reproduce the above
12# copyright notice, this list of conditions and the following disclaimer
13# in the documentation and/or other materials provided with the
14# distribution.
15#     * Neither the name of Google Inc. nor the names of its
16# contributors may be used to endorse or promote products derived from
17# this software without specific prior written permission.
18#
19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31"""Unit test for Google Test fail_fast.
32
33A user can specify if a Google Test program should continue test execution
34after a test failure via the GTEST_FAIL_FAST environment variable or the
35--gtest_fail_fast flag. The default value of the flag can also be changed
36by Bazel fail fast environment variable TESTBRIDGE_TEST_RUNNER_FAIL_FAST.
37
38This script tests such functionality by invoking googletest-failfast-unittest_
39(a program written with Google Test) with different environments and command
40line flags.
41"""
42
43import os
44from googletest.test import gtest_test_utils
45
46# Constants.
47
48# Bazel testbridge environment variable for fail fast
49BAZEL_FAIL_FAST_ENV_VAR = 'TESTBRIDGE_TEST_RUNNER_FAIL_FAST'
50
51# The environment variable for specifying fail fast.
52FAIL_FAST_ENV_VAR = 'GTEST_FAIL_FAST'
53
54# The command line flag for specifying fail fast.
55FAIL_FAST_FLAG = 'gtest_fail_fast'
56
57# The command line flag to run disabled tests.
58RUN_DISABLED_FLAG = 'gtest_also_run_disabled_tests'
59
60# The command line flag for specifying a filter.
61FILTER_FLAG = 'gtest_filter'
62
63# Command to run the googletest-failfast-unittest_ program.
64COMMAND = gtest_test_utils.GetTestExecutablePath(
65    'googletest-failfast-unittest_'
66)
67
68# The command line flag to tell Google Test to output the list of tests it
69# will run.
70LIST_TESTS_FLAG = '--gtest_list_tests'
71
72# Indicates whether Google Test supports death tests.
73SUPPORTS_DEATH_TESTS = (
74    'HasDeathTest'
75    in gtest_test_utils.Subprocess([COMMAND, LIST_TESTS_FLAG]).output
76)
77
78# Utilities.
79
80environ = os.environ.copy()
81
82
83def SetEnvVar(env_var, value):
84  """Sets the env variable to 'value'; unsets it when 'value' is None."""
85
86  if value is not None:
87    environ[env_var] = value
88  elif env_var in environ:
89    del environ[env_var]
90
91
92def RunAndReturnOutput(test_suite=None, fail_fast=None, run_disabled=False):
93  """Runs the test program and returns its output."""
94
95  args = []
96  xml_path = os.path.join(
97      gtest_test_utils.GetTempDir(), '.GTestFailFastUnitTest.xml'
98  )
99  args += ['--gtest_output=xml:' + xml_path]
100  if fail_fast is not None:
101    if isinstance(fail_fast, str):
102      args += ['--%s=%s' % (FAIL_FAST_FLAG, fail_fast)]
103    elif fail_fast:
104      args += ['--%s' % FAIL_FAST_FLAG]
105    else:
106      args += ['--no%s' % FAIL_FAST_FLAG]
107  if test_suite:
108    args += ['--%s=%s.*' % (FILTER_FLAG, test_suite)]
109  if run_disabled:
110    args += ['--%s' % RUN_DISABLED_FLAG]
111  txt_out = gtest_test_utils.Subprocess([COMMAND] + args, env=environ).output
112  with open(xml_path) as xml_file:
113    return txt_out, xml_file.read()
114
115
116# The unit test.
117class GTestFailFastUnitTest(gtest_test_utils.TestCase):
118  """Tests the env variable or the command line flag for fail_fast."""
119
120  def testDefaultBehavior(self):
121    """Tests the behavior of not specifying the fail_fast."""
122
123    txt, _ = RunAndReturnOutput()
124    self.assertIn('22 FAILED TEST', txt)
125
126  def testGoogletestFlag(self):
127    txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=True)
128    self.assertIn('1 FAILED TEST', txt)
129    self.assertIn('[  SKIPPED ] 3 tests', txt)
130
131    txt, _ = RunAndReturnOutput(test_suite='HasSimpleTest', fail_fast=False)
132    self.assertIn('4 FAILED TEST', txt)
133    self.assertNotIn('[  SKIPPED ]', txt)
134
135  def testGoogletestEnvVar(self):
136    """Tests the behavior of specifying fail_fast via Googletest env var."""
137
138    try:
139      SetEnvVar(FAIL_FAST_ENV_VAR, '1')
140      txt, _ = RunAndReturnOutput('HasSimpleTest')
141      self.assertIn('1 FAILED TEST', txt)
142      self.assertIn('[  SKIPPED ] 3 tests', txt)
143
144      SetEnvVar(FAIL_FAST_ENV_VAR, '0')
145      txt, _ = RunAndReturnOutput('HasSimpleTest')
146      self.assertIn('4 FAILED TEST', txt)
147      self.assertNotIn('[  SKIPPED ]', txt)
148    finally:
149      SetEnvVar(FAIL_FAST_ENV_VAR, None)
150
151  def testBazelEnvVar(self):
152    """Tests the behavior of specifying fail_fast via Bazel testbridge."""
153
154    try:
155      SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '1')
156      txt, _ = RunAndReturnOutput('HasSimpleTest')
157      self.assertIn('1 FAILED TEST', txt)
158      self.assertIn('[  SKIPPED ] 3 tests', txt)
159
160      SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0')
161      txt, _ = RunAndReturnOutput('HasSimpleTest')
162      self.assertIn('4 FAILED TEST', txt)
163      self.assertNotIn('[  SKIPPED ]', txt)
164    finally:
165      SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None)
166
167  def testFlagOverridesEnvVar(self):
168    """Tests precedence of flag over env var."""
169
170    try:
171      SetEnvVar(FAIL_FAST_ENV_VAR, '0')
172      txt, _ = RunAndReturnOutput('HasSimpleTest', True)
173      self.assertIn('1 FAILED TEST', txt)
174      self.assertIn('[  SKIPPED ] 3 tests', txt)
175    finally:
176      SetEnvVar(FAIL_FAST_ENV_VAR, None)
177
178  def testGoogletestEnvVarOverridesBazelEnvVar(self):
179    """Tests that the Googletest native env var over Bazel testbridge."""
180
181    try:
182      SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, '0')
183      SetEnvVar(FAIL_FAST_ENV_VAR, '1')
184      txt, _ = RunAndReturnOutput('HasSimpleTest')
185      self.assertIn('1 FAILED TEST', txt)
186      self.assertIn('[  SKIPPED ] 3 tests', txt)
187    finally:
188      SetEnvVar(FAIL_FAST_ENV_VAR, None)
189      SetEnvVar(BAZEL_FAIL_FAST_ENV_VAR, None)
190
191  def testEventListener(self):
192    txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=True)
193    self.assertIn('1 FAILED TEST', txt)
194    self.assertIn('[  SKIPPED ] 3 tests', txt)
195    for expected_count, callback in [
196        (1, 'OnTestSuiteStart'),
197        (5, 'OnTestStart'),
198        (5, 'OnTestEnd'),
199        (5, 'OnTestPartResult'),
200        (1, 'OnTestSuiteEnd'),
201    ]:
202      self.assertEqual(
203          expected_count,
204          txt.count(callback),
205          'Expected %d calls to callback %s match count on output: %s '
206          % (expected_count, callback, txt),
207      )
208
209    txt, _ = RunAndReturnOutput(test_suite='HasSkipTest', fail_fast=False)
210    self.assertIn('3 FAILED TEST', txt)
211    self.assertIn('[  SKIPPED ] 1 test', txt)
212    for expected_count, callback in [
213        (1, 'OnTestSuiteStart'),
214        (5, 'OnTestStart'),
215        (5, 'OnTestEnd'),
216        (5, 'OnTestPartResult'),
217        (1, 'OnTestSuiteEnd'),
218    ]:
219      self.assertEqual(
220          expected_count,
221          txt.count(callback),
222          'Expected %d calls to callback %s match count on output: %s '
223          % (expected_count, callback, txt),
224      )
225
226  def assertXmlResultCount(self, result, count, xml):
227    self.assertEqual(
228        count,
229        xml.count('result="%s"' % result),
230        'Expected \'result="%s"\' match count of %s: %s '
231        % (result, count, xml),
232    )
233
234  def assertXmlStatusCount(self, status, count, xml):
235    self.assertEqual(
236        count,
237        xml.count('status="%s"' % status),
238        'Expected \'status="%s"\' match count of %s: %s '
239        % (status, count, xml),
240    )
241
242  def assertFailFastXmlAndTxtOutput(
243      self,
244      fail_fast,
245      test_suite,
246      passed_count,
247      failure_count,
248      skipped_count,
249      suppressed_count,
250      run_disabled=False,
251  ):
252    """Assert XML and text output of a test execution."""
253
254    txt, xml = RunAndReturnOutput(test_suite, fail_fast, run_disabled)
255    if failure_count > 0:
256      self.assertIn('%s FAILED TEST' % failure_count, txt)
257    if suppressed_count > 0:
258      self.assertIn('%s DISABLED TEST' % suppressed_count, txt)
259    if skipped_count > 0:
260      self.assertIn('[  SKIPPED ] %s tests' % skipped_count, txt)
261    self.assertXmlStatusCount(
262        'run', passed_count + failure_count + skipped_count, xml
263    )
264    self.assertXmlStatusCount('notrun', suppressed_count, xml)
265    self.assertXmlResultCount('completed', passed_count + failure_count, xml)
266    self.assertXmlResultCount('skipped', skipped_count, xml)
267    self.assertXmlResultCount('suppressed', suppressed_count, xml)
268
269  def assertFailFastBehavior(
270      self,
271      test_suite,
272      passed_count,
273      failure_count,
274      skipped_count,
275      suppressed_count,
276      run_disabled=False,
277  ):
278    """Assert --fail_fast via flag."""
279
280    for fail_fast in ('true', '1', 't', True):
281      self.assertFailFastXmlAndTxtOutput(
282          fail_fast,
283          test_suite,
284          passed_count,
285          failure_count,
286          skipped_count,
287          suppressed_count,
288          run_disabled,
289      )
290
291  def assertNotFailFastBehavior(
292      self,
293      test_suite,
294      passed_count,
295      failure_count,
296      skipped_count,
297      suppressed_count,
298      run_disabled=False,
299  ):
300    """Assert --nofail_fast via flag."""
301
302    for fail_fast in ('false', '0', 'f', False):
303      self.assertFailFastXmlAndTxtOutput(
304          fail_fast,
305          test_suite,
306          passed_count,
307          failure_count,
308          skipped_count,
309          suppressed_count,
310          run_disabled,
311      )
312
313  def testFlag_HasFixtureTest(self):
314    """Tests the behavior of fail_fast and TEST_F."""
315    self.assertFailFastBehavior(
316        test_suite='HasFixtureTest',
317        passed_count=1,
318        failure_count=1,
319        skipped_count=3,
320        suppressed_count=0,
321    )
322    self.assertNotFailFastBehavior(
323        test_suite='HasFixtureTest',
324        passed_count=1,
325        failure_count=4,
326        skipped_count=0,
327        suppressed_count=0,
328    )
329
330  def testFlag_HasSimpleTest(self):
331    """Tests the behavior of fail_fast and TEST."""
332    self.assertFailFastBehavior(
333        test_suite='HasSimpleTest',
334        passed_count=1,
335        failure_count=1,
336        skipped_count=3,
337        suppressed_count=0,
338    )
339    self.assertNotFailFastBehavior(
340        test_suite='HasSimpleTest',
341        passed_count=1,
342        failure_count=4,
343        skipped_count=0,
344        suppressed_count=0,
345    )
346
347  def testFlag_HasParametersTest(self):
348    """Tests the behavior of fail_fast and TEST_P."""
349    self.assertFailFastBehavior(
350        test_suite='HasParametersSuite/HasParametersTest',
351        passed_count=0,
352        failure_count=1,
353        skipped_count=3,
354        suppressed_count=0,
355    )
356    self.assertNotFailFastBehavior(
357        test_suite='HasParametersSuite/HasParametersTest',
358        passed_count=0,
359        failure_count=4,
360        skipped_count=0,
361        suppressed_count=0,
362    )
363
364  def testFlag_HasDisabledTest(self):
365    """Tests the behavior of fail_fast and Disabled test cases."""
366    self.assertFailFastBehavior(
367        test_suite='HasDisabledTest',
368        passed_count=1,
369        failure_count=1,
370        skipped_count=2,
371        suppressed_count=1,
372        run_disabled=False,
373    )
374    self.assertNotFailFastBehavior(
375        test_suite='HasDisabledTest',
376        passed_count=1,
377        failure_count=3,
378        skipped_count=0,
379        suppressed_count=1,
380        run_disabled=False,
381    )
382
383  def testFlag_HasDisabledRunDisabledTest(self):
384    """Tests the behavior of fail_fast and Disabled test cases enabled."""
385    self.assertFailFastBehavior(
386        test_suite='HasDisabledTest',
387        passed_count=1,
388        failure_count=1,
389        skipped_count=3,
390        suppressed_count=0,
391        run_disabled=True,
392    )
393    self.assertNotFailFastBehavior(
394        test_suite='HasDisabledTest',
395        passed_count=1,
396        failure_count=4,
397        skipped_count=0,
398        suppressed_count=0,
399        run_disabled=True,
400    )
401
402  def testFlag_HasDisabledSuiteTest(self):
403    """Tests the behavior of fail_fast and Disabled test suites."""
404    self.assertFailFastBehavior(
405        test_suite='DISABLED_HasDisabledSuite',
406        passed_count=0,
407        failure_count=0,
408        skipped_count=0,
409        suppressed_count=5,
410        run_disabled=False,
411    )
412    self.assertNotFailFastBehavior(
413        test_suite='DISABLED_HasDisabledSuite',
414        passed_count=0,
415        failure_count=0,
416        skipped_count=0,
417        suppressed_count=5,
418        run_disabled=False,
419    )
420
421  def testFlag_HasDisabledSuiteRunDisabledTest(self):
422    """Tests the behavior of fail_fast and Disabled test suites enabled."""
423    self.assertFailFastBehavior(
424        test_suite='DISABLED_HasDisabledSuite',
425        passed_count=1,
426        failure_count=1,
427        skipped_count=3,
428        suppressed_count=0,
429        run_disabled=True,
430    )
431    self.assertNotFailFastBehavior(
432        test_suite='DISABLED_HasDisabledSuite',
433        passed_count=1,
434        failure_count=4,
435        skipped_count=0,
436        suppressed_count=0,
437        run_disabled=True,
438    )
439
440  if SUPPORTS_DEATH_TESTS:
441
442    def testFlag_HasDeathTest(self):
443      """Tests the behavior of fail_fast and death tests."""
444      self.assertFailFastBehavior(
445          test_suite='HasDeathTest',
446          passed_count=1,
447          failure_count=1,
448          skipped_count=3,
449          suppressed_count=0,
450      )
451      self.assertNotFailFastBehavior(
452          test_suite='HasDeathTest',
453          passed_count=1,
454          failure_count=4,
455          skipped_count=0,
456          suppressed_count=0,
457      )
458
459
460if __name__ == '__main__':
461  gtest_test_utils.Main()
462