?login_element?

Subversion Repositories NedoOS

Rev

Blame | Last modification | View Log | Download

  1. #!/usr/bin/env python
  2. #
  3. # Copyright 2009 Google Inc. All Rights Reserved.
  4. #
  5. # Redistribution and use in source and binary forms, with or without
  6. # modification, are permitted provided that the following conditions are
  7. # met:
  8. #
  9. #     * Redistributions of source code must retain the above copyright
  10. # notice, this list of conditions and the following disclaimer.
  11. #     * Redistributions in binary form must reproduce the above
  12. # copyright notice, this list of conditions and the following disclaimer
  13. # in the documentation and/or other materials provided with the
  14. # distribution.
  15. #     * Neither the name of Google Inc. nor the names of its
  16. # contributors may be used to endorse or promote products derived from
  17. # this software without specific prior written permission.
  18. #
  19. # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20. # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21. # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22. # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23. # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24. # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25. # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26. # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27. # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28. # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29. # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30.  
  31. """Verifies that test shuffling works."""
  32.  
  33. import os
  34. import gtest_test_utils
  35.  
  36. # Command to run the googletest-shuffle-test_ program.
  37. COMMAND = gtest_test_utils.GetTestExecutablePath('googletest-shuffle-test_')
  38.  
  39. # The environment variables for test sharding.
  40. TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS'
  41. SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX'
  42.  
  43. TEST_FILTER = 'A*.A:A*.B:C*'
  44.  
  45. ALL_TESTS = []
  46. ACTIVE_TESTS = []
  47. FILTERED_TESTS = []
  48. SHARDED_TESTS = []
  49.  
  50. SHUFFLED_ALL_TESTS = []
  51. SHUFFLED_ACTIVE_TESTS = []
  52. SHUFFLED_FILTERED_TESTS = []
  53. SHUFFLED_SHARDED_TESTS = []
  54.  
  55.  
  56. def AlsoRunDisabledTestsFlag():
  57.   return '--gtest_also_run_disabled_tests'
  58.  
  59.  
  60. def FilterFlag(test_filter):
  61.   return '--gtest_filter=%s' % (test_filter,)
  62.  
  63.  
  64. def RepeatFlag(n):
  65.   return '--gtest_repeat=%s' % (n,)
  66.  
  67.  
  68. def ShuffleFlag():
  69.   return '--gtest_shuffle'
  70.  
  71.  
  72. def RandomSeedFlag(n):
  73.   return '--gtest_random_seed=%s' % (n,)
  74.  
  75.  
  76. def RunAndReturnOutput(extra_env, args):
  77.   """Runs the test program and returns its output."""
  78.  
  79.   environ_copy = os.environ.copy()
  80.   environ_copy.update(extra_env)
  81.  
  82.   return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy).output
  83.  
  84.  
  85. def GetTestsForAllIterations(extra_env, args):
  86.   """Runs the test program and returns a list of test lists.
  87.  
  88.  Args:
  89.    extra_env: a map from environment variables to their values
  90.    args: command line flags to pass to googletest-shuffle-test_
  91.  
  92.  Returns:
  93.    A list where the i-th element is the list of tests run in the i-th
  94.    test iteration.
  95.  """
  96.  
  97.   test_iterations = []
  98.   for line in RunAndReturnOutput(extra_env, args).split('\n'):
  99.     if line.startswith('----'):
  100.       tests = []
  101.       test_iterations.append(tests)
  102.     elif line.strip():
  103.       tests.append(line.strip())  # 'TestCaseName.TestName'
  104.  
  105.   return test_iterations
  106.  
  107.  
  108. def GetTestCases(tests):
  109.   """Returns a list of test cases in the given full test names.
  110.  
  111.  Args:
  112.    tests: a list of full test names
  113.  
  114.  Returns:
  115.    A list of test cases from 'tests', in their original order.
  116.    Consecutive duplicates are removed.
  117.  """
  118.  
  119.   test_cases = []
  120.   for test in tests:
  121.     test_case = test.split('.')[0]
  122.     if not test_case in test_cases:
  123.       test_cases.append(test_case)
  124.  
  125.   return test_cases
  126.  
  127.  
  128. def CalculateTestLists():
  129.   """Calculates the list of tests run under different flags."""
  130.  
  131.   if not ALL_TESTS:
  132.     ALL_TESTS.extend(
  133.         GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0])
  134.  
  135.   if not ACTIVE_TESTS:
  136.     ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0])
  137.  
  138.   if not FILTERED_TESTS:
  139.     FILTERED_TESTS.extend(
  140.         GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0])
  141.  
  142.   if not SHARDED_TESTS:
  143.     SHARDED_TESTS.extend(
  144.         GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
  145.                                   SHARD_INDEX_ENV_VAR: '1'},
  146.                                  [])[0])
  147.  
  148.   if not SHUFFLED_ALL_TESTS:
  149.     SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations(
  150.         {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0])
  151.  
  152.   if not SHUFFLED_ACTIVE_TESTS:
  153.     SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations(
  154.         {}, [ShuffleFlag(), RandomSeedFlag(1)])[0])
  155.  
  156.   if not SHUFFLED_FILTERED_TESTS:
  157.     SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations(
  158.         {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0])
  159.  
  160.   if not SHUFFLED_SHARDED_TESTS:
  161.     SHUFFLED_SHARDED_TESTS.extend(
  162.         GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
  163.                                   SHARD_INDEX_ENV_VAR: '1'},
  164.                                  [ShuffleFlag(), RandomSeedFlag(1)])[0])
  165.  
  166.  
  167. class GTestShuffleUnitTest(gtest_test_utils.TestCase):
  168.   """Tests test shuffling."""
  169.  
  170.   def setUp(self):
  171.     CalculateTestLists()
  172.  
  173.   def testShufflePreservesNumberOfTests(self):
  174.     self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS))
  175.     self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS))
  176.     self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS))
  177.     self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS))
  178.  
  179.   def testShuffleChangesTestOrder(self):
  180.     self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS)
  181.     self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS)
  182.     self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS,
  183.                  SHUFFLED_FILTERED_TESTS)
  184.     self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS,
  185.                  SHUFFLED_SHARDED_TESTS)
  186.  
  187.   def testShuffleChangesTestCaseOrder(self):
  188.     self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS),
  189.                  GetTestCases(SHUFFLED_ALL_TESTS))
  190.     self.assert_(
  191.         GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS),
  192.         GetTestCases(SHUFFLED_ACTIVE_TESTS))
  193.     self.assert_(
  194.         GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS),
  195.         GetTestCases(SHUFFLED_FILTERED_TESTS))
  196.     self.assert_(
  197.         GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS),
  198.         GetTestCases(SHUFFLED_SHARDED_TESTS))
  199.  
  200.   def testShuffleDoesNotRepeatTest(self):
  201.     for test in SHUFFLED_ALL_TESTS:
  202.       self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test),
  203.                        '%s appears more than once' % (test,))
  204.     for test in SHUFFLED_ACTIVE_TESTS:
  205.       self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test),
  206.                        '%s appears more than once' % (test,))
  207.     for test in SHUFFLED_FILTERED_TESTS:
  208.       self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test),
  209.                        '%s appears more than once' % (test,))
  210.     for test in SHUFFLED_SHARDED_TESTS:
  211.       self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test),
  212.                        '%s appears more than once' % (test,))
  213.  
  214.   def testShuffleDoesNotCreateNewTest(self):
  215.     for test in SHUFFLED_ALL_TESTS:
  216.       self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,))
  217.     for test in SHUFFLED_ACTIVE_TESTS:
  218.       self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,))
  219.     for test in SHUFFLED_FILTERED_TESTS:
  220.       self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,))
  221.     for test in SHUFFLED_SHARDED_TESTS:
  222.       self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,))
  223.  
  224.   def testShuffleIncludesAllTests(self):
  225.     for test in ALL_TESTS:
  226.       self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,))
  227.     for test in ACTIVE_TESTS:
  228.       self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,))
  229.     for test in FILTERED_TESTS:
  230.       self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,))
  231.     for test in SHARDED_TESTS:
  232.       self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,))
  233.  
  234.   def testShuffleLeavesDeathTestsAtFront(self):
  235.     non_death_test_found = False
  236.     for test in SHUFFLED_ACTIVE_TESTS:
  237.       if 'DeathTest.' in test:
  238.         self.assert_(not non_death_test_found,
  239.                      '%s appears after a non-death test' % (test,))
  240.       else:
  241.         non_death_test_found = True
  242.  
  243.   def _VerifyTestCasesDoNotInterleave(self, tests):
  244.     test_cases = []
  245.     for test in tests:
  246.       [test_case, _] = test.split('.')
  247.       if test_cases and test_cases[-1] != test_case:
  248.         test_cases.append(test_case)
  249.         self.assertEqual(1, test_cases.count(test_case),
  250.                          'Test case %s is not grouped together in %s' %
  251.                          (test_case, tests))
  252.  
  253.   def testShuffleDoesNotInterleaveTestCases(self):
  254.     self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS)
  255.     self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS)
  256.     self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS)
  257.     self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS)
  258.  
  259.   def testShuffleRestoresOrderAfterEachIteration(self):
  260.     # Get the test lists in all 3 iterations, using random seed 1, 2,
  261.     # and 3 respectively.  Google Test picks a different seed in each
  262.     # iteration, and this test depends on the current implementation
  263.     # picking successive numbers.  This dependency is not ideal, but
  264.     # makes the test much easier to write.
  265.     [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
  266.         GetTestsForAllIterations(
  267.             {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))
  268.  
  269.     # Make sure running the tests with random seed 1 gets the same
  270.     # order as in iteration 1 above.
  271.     [tests_with_seed1] = GetTestsForAllIterations(
  272.         {}, [ShuffleFlag(), RandomSeedFlag(1)])
  273.     self.assertEqual(tests_in_iteration1, tests_with_seed1)
  274.  
  275.     # Make sure running the tests with random seed 2 gets the same
  276.     # order as in iteration 2 above.  Success means that Google Test
  277.     # correctly restores the test order before re-shuffling at the
  278.     # beginning of iteration 2.
  279.     [tests_with_seed2] = GetTestsForAllIterations(
  280.         {}, [ShuffleFlag(), RandomSeedFlag(2)])
  281.     self.assertEqual(tests_in_iteration2, tests_with_seed2)
  282.  
  283.     # Make sure running the tests with random seed 3 gets the same
  284.     # order as in iteration 3 above.  Success means that Google Test
  285.     # correctly restores the test order before re-shuffling at the
  286.     # beginning of iteration 3.
  287.     [tests_with_seed3] = GetTestsForAllIterations(
  288.         {}, [ShuffleFlag(), RandomSeedFlag(3)])
  289.     self.assertEqual(tests_in_iteration3, tests_with_seed3)
  290.  
  291.   def testShuffleGeneratesNewOrderInEachIteration(self):
  292.     [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
  293.         GetTestsForAllIterations(
  294.             {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))
  295.  
  296.     self.assert_(tests_in_iteration1 != tests_in_iteration2,
  297.                  tests_in_iteration1)
  298.     self.assert_(tests_in_iteration1 != tests_in_iteration3,
  299.                  tests_in_iteration1)
  300.     self.assert_(tests_in_iteration2 != tests_in_iteration3,
  301.                  tests_in_iteration2)
  302.  
  303.   def testShuffleShardedTestsPreservesPartition(self):
  304.     # If we run M tests on N shards, the same M tests should be run in
  305.     # total, regardless of the random seeds used by the shards.
  306.     [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
  307.                                          SHARD_INDEX_ENV_VAR: '0'},
  308.                                         [ShuffleFlag(), RandomSeedFlag(1)])
  309.     [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
  310.                                          SHARD_INDEX_ENV_VAR: '1'},
  311.                                         [ShuffleFlag(), RandomSeedFlag(20)])
  312.     [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
  313.                                          SHARD_INDEX_ENV_VAR: '2'},
  314.                                         [ShuffleFlag(), RandomSeedFlag(25)])
  315.     sorted_sharded_tests = tests1 + tests2 + tests3
  316.     sorted_sharded_tests.sort()
  317.     sorted_active_tests = []
  318.     sorted_active_tests.extend(ACTIVE_TESTS)
  319.     sorted_active_tests.sort()
  320.     self.assertEqual(sorted_active_tests, sorted_sharded_tests)
  321.  
  322. if __name__ == '__main__':
  323.   gtest_test_utils.Main()
  324.