diff options
Diffstat (limited to 'WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py')
| -rw-r--r-- | WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py | 93 |
1 files changed, 74 insertions, 19 deletions
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py index 0a60cc7..785cc1c 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py @@ -45,6 +45,13 @@ class JSONGeneratorTest(unittest.TestCase): self.builder_name = 'DUMMY_BUILDER_NAME' self.build_name = 'DUMMY_BUILD_NAME' self.build_number = 'DUMMY_BUILDER_NUMBER' + self._json = None + self._num_runs = 0 + self._tests_list = set([]) + self._test_timings = {} + self._failed_tests = {} + self._passed_tests = set([]) + self._skipped_tests = set([]) def _test_json_generation(self, passed_tests, failed_tests, skipped_tests): # Make sure we have sets (rather than lists). @@ -52,12 +59,13 @@ class JSONGeneratorTest(unittest.TestCase): skipped_tests = set(skipped_tests) tests_list = passed_tests | set(failed_tests.keys()) test_timings = {} + i = 0 for test in tests_list: - test_timings[test] = float(random.randint(1, 10)) + test_timings[test] = float(self._num_runs * 100 + i) + i += 1 port_obj = port.get(None) - # Generate a JSON file. generator = json_results_generator.JSONResultsGenerator(port_obj, self.builder_name, self.build_name, self.build_number, '', @@ -68,8 +76,39 @@ class JSONGeneratorTest(unittest.TestCase): skipped_tests, tests_list) - json = generator.get_json() - + # Test incremental json results + incremental_json = generator.get_json(incremental=True) + self._verify_json_results( + tests_list, + test_timings, + passed_tests, + failed_tests, + skipped_tests, + incremental_json, + 1) + + # Test aggregated json results + generator.set_archived_results(self._json) + json = generator.get_json(incremental=False) + self._json = json + self._num_runs += 1 + self._tests_list |= tests_list + self._test_timings.update(test_timings) + self._failed_tests.update(failed_tests) + self._passed_tests |= passed_tests + self._skipped_tests |= skipped_tests + self._verify_json_results( + self._tests_list, + self._test_timings, + self._passed_tests, + self._failed_tests, + self._skipped_tests, + self._json, + self._num_runs) + + def _verify_json_results(self, tests_list, test_timings, + passed_tests, failed_tests, + skipped_tests, json, num_runs): # Aliasing to a short name for better access to its constants. JRG = json_results_generator.JSONResultsGenerator @@ -79,11 +118,18 @@ class JSONGeneratorTest(unittest.TestCase): buildinfo = json[self.builder_name] self.assertTrue(JRG.FIXABLE in buildinfo) self.assertTrue(JRG.TESTS in buildinfo) - self.assertTrue(len(buildinfo[JRG.BUILD_NUMBERS]) == 1) + self.assertTrue(len(buildinfo[JRG.BUILD_NUMBERS]) == num_runs) self.assertTrue(buildinfo[JRG.BUILD_NUMBERS][0] == self.build_number) - if tests_list or skipped_tests: - fixable = buildinfo[JRG.FIXABLE][0] + if tests_list or skipped_tests: + fixable = {} + for fixable_items in buildinfo[JRG.FIXABLE]: + for (type, count) in fixable_items.iteritems(): + if type in fixable: + fixable[type] = fixable[type] + count + else: + fixable[type] = count + if passed_tests: self.assertTrue(fixable[JRG.PASS_RESULT] == len(passed_tests)) else: @@ -100,26 +146,35 @@ class JSONGeneratorTest(unittest.TestCase): for test_name, failure in failed_tests.iteritems(): self.assertTrue(test_name in tests) test = tests[test_name] - self.assertTrue(test[JRG.RESULTS][0][0] == 1) - self.assertTrue(test[JRG.RESULTS][0][1] == JRG.FAIL_RESULT) - self.assertTrue(test[JRG.TIMES][0][0] == 1) - self.assertTrue(test[JRG.TIMES][0][1] == - int(test_timings[test_name])) + + failed = 0 + for result in test[JRG.RESULTS]: + if result[1] == JRG.FAIL_RESULT: + failed = result[0] + self.assertTrue(failed == 1) + + timing_count = 0 + for timings in test[JRG.TIMES]: + if timings[1] == test_timings[test_name]: + timing_count = timings[0] + self.assertTrue(timing_count == 1) fixable_count = len(skipped_tests) + len(failed_tests.keys()) if skipped_tests or failed_tests: - self.assertTrue(buildinfo[JRG.FIXABLE_COUNT][0] == fixable_count) + self.assertTrue(sum(buildinfo[JRG.FIXABLE_COUNT]) == fixable_count) def test_json_generation(self): reason = test_expectations.TEXT self._test_json_generation([], {}, []) - self._test_json_generation(['A', 'B'], {}, []) - self._test_json_generation([], {'A': reason, 'B': reason}, []) - self._test_json_generation([], {}, ['A', 'B']) - self._test_json_generation(['A'], {'B': reason, 'C': reason}, []) - self._test_json_generation([], {'A': reason, 'B': reason}, ['C', 'D']) - self._test_json_generation(['A', 'B', 'C'], {'D': reason}, ['E', 'F']) + self._test_json_generation(['A1', 'B1'], {}, []) + self._test_json_generation([], {'A2': reason, 'B2': reason}, []) + self._test_json_generation([], {}, ['A3', 'B3']) + self._test_json_generation(['A4'], {'B4': reason, 'C4': reason}, []) + self._test_json_generation( + [], {'A5': reason, 'B5': reason}, ['C5', 'D5']) + self._test_json_generation( + ['A6', 'B6', 'C6'], {'D6': reason}, ['E6', 'F6']) if __name__ == '__main__': |