diff options
Diffstat (limited to 'Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py')
| -rw-r--r-- | Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py | 158 |
1 files changed, 124 insertions, 34 deletions
diff --git a/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py b/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py index 7179bb7..73bc1a7 100644 --- a/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py @@ -65,13 +65,9 @@ def test_options(): html_directory='/tmp', archive_url=ARCHIVE_URL, force_archive_url=None, - webkit_canary=True, - use_drt=False, - target_platform='chromium', verbose=False, quiet=False, - platforms='mac,win,win-xp', - gpu=False) + platforms=None) def test_host_port_and_filesystem(options, expectations): @@ -86,8 +82,12 @@ def test_host_port_and_filesystem(options, expectations): def test_url_fetcher(filesystem): urls = { + ARCHIVE_URL + '/Webkit_Mac10_6/': '<a href="4/">', ARCHIVE_URL + '/Webkit_Mac10_5/': '<a href="1/"><a href="2/">', + ARCHIVE_URL + '/Webkit_Win7/': '<a href="1/">', + ARCHIVE_URL + '/Webkit_Vista/': '<a href="1/">', ARCHIVE_URL + '/Webkit_Win/': '<a href="1/">', + ARCHIVE_URL + '/Webkit_Linux/': '<a href="1/">', } return urlfetcher_mock.make_fetcher_cls(urls)(filesystem) @@ -98,8 +98,31 @@ def test_zip_factory(): 'layout-test-results/failures/expected/image-actual.txt': 'new-image-txt', 'layout-test-results/failures/expected/image-actual.checksum': 'new-image-checksum', 'layout-test-results/failures/expected/image-actual.png': 'new-image-png', + 'layout-test-results/failures/expected/image_checksum-actual.txt': 'png-comment-txt', + 'layout-test-results/failures/expected/image_checksum-actual.checksum': '0123456789', + 'layout-test-results/failures/expected/image_checksum-actual.png': 'tEXtchecksum\x000123456789', }, - ARCHIVE_URL + '/Webkit_Win/1/layout-test-results.zip': { + ARCHIVE_URL + '/Webkit_Mac10_6/4/layout-test-results.zip': { + 'layout-test-results/failures/expected/image-actual.txt': 'new-image-txt', + 'layout-test-results/failures/expected/image-actual.checksum': 'new-image-checksum', + 'layout-test-results/failures/expected/image-actual.png': 'new-image-png', + }, + ARCHIVE_URL + '/Webkit_Vista/1/layout-test-results.zip': { + 'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt', + 'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum', + 'layout-test-results/failures/expected/image-actual.png': 'win-image-png', + }, + ARCHIVE_URL + '/Webkit_Win7/1/layout-test-results.zip': { + 'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt', + 'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum', + 'layout-test-results/failures/expected/image-actual.png': 'win-image-png', + }, + ARCHIVE_URL + '/Webkit_Win/1/layout-test-results.zip': { + 'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt', + 'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum', + 'layout-test-results/failures/expected/image-actual.png': 'win-image-png', + }, + ARCHIVE_URL + '/Webkit_Linux/1/layout-test-results.zip': { 'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt', 'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum', 'layout-test-results/failures/expected/image-actual.png': 'win-image-png', @@ -108,6 +131,14 @@ def test_zip_factory(): return zipfileset_mock.make_factory(ziphashes) +def test_archive(orig_archive_dict): + new_archive_dict = {} + for platform, dirname in orig_archive_dict.iteritems(): + platform = platform.replace('chromium', 'test') + new_archive_dict[platform] = dirname + return new_archive_dict + + class TestGetHostPortObject(unittest.TestCase): def assert_result(self, release_present, debug_present, valid_port_obj): # Tests whether we get a valid port object returned when we claim @@ -150,6 +181,14 @@ class TestOptions(unittest.TestCase): class TestRebaseliner(unittest.TestCase): + def setUp(self): + if not hasattr(self, '_orig_archive'): + self._orig_archive = rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT + rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = test_archive(self._orig_archive) + + def tearDown(self): + rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = self._orig_archive + def make_rebaseliner(self, expectations): options = test_options() host_port_obj, filesystem = test_host_port_and_filesystem(options, expectations) @@ -158,11 +197,11 @@ class TestRebaseliner(unittest.TestCase): target_port_obj = port.get('test', target_options, filesystem=filesystem) target_port_obj._expectations = expectations - platform = target_port_obj.test_platform_name() + platform = target_port_obj.name() url_fetcher = test_url_fetcher(filesystem) zip_factory = test_zip_factory() - mock_scm = mocktool.MockSCM() + mock_scm = mocktool.MockSCM(filesystem) rebaseliner = rebaseline_chromium_webkit_tests.Rebaseliner(host_port_obj, target_port_obj, platform, options, url_fetcher, zip_factory, mock_scm) return rebaseliner, filesystem @@ -171,7 +210,7 @@ class TestRebaseliner(unittest.TestCase): # this method tests that was can at least instantiate an object, even # if there is nothing to do. rebaseliner, filesystem = self.make_rebaseliner("") - rebaseliner.run(False) + rebaseliner.run() self.assertEqual(len(filesystem.written_files), 1) def test_rebaselining_tests(self): @@ -179,19 +218,19 @@ class TestRebaseliner(unittest.TestCase): "BUGX REBASELINE MAC : failures/expected/image.html = IMAGE") compile_success = rebaseliner._compile_rebaselining_tests() self.assertTrue(compile_success) - self.assertEqual(set(['failures/expected/image.html']), rebaseliner.get_rebaselining_tests()) + self.assertEqual(set(['failures/expected/image.html']), rebaseliner._rebaselining_tests) def test_rebaselining_tests_should_ignore_reftests(self): rebaseliner, filesystem = self.make_rebaseliner( "BUGX REBASELINE : failures/expected/reftest.html = IMAGE") compile_success = rebaseliner._compile_rebaselining_tests() self.assertFalse(compile_success) - self.assertFalse(rebaseliner.get_rebaselining_tests()) + self.assertFalse(rebaseliner._rebaselining_tests) def test_one_platform(self): rebaseliner, filesystem = self.make_rebaseliner( "BUGX REBASELINE MAC : failures/expected/image.html = IMAGE") - rebaseliner.run(False) + rebaseliner.run() # We expect to have written 12 files over the course of this rebaseline: # *) 3 files in /__im_tmp for the extracted archive members # *) 3 new baselines under '/test.checkout/LayoutTests' @@ -201,25 +240,67 @@ class TestRebaseliner(unittest.TestCase): # create image diffs (FIXME?) and don't display the checksums. # *) 1 updated test_expectations file self.assertEqual(len(filesystem.written_files), 12) - self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test/test_expectations.txt'], '') - self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac/failures/expected/image-expected.checksum'], 'new-image-checksum') - self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac/failures/expected/image-expected.png'], 'new-image-png') - self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac/failures/expected/image-expected.txt'], 'new-image-txt') + self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.checksum'], 'new-image-checksum') + self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.png'], 'new-image-png') + self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.txt'], 'new-image-txt') def test_all_platforms(self): rebaseliner, filesystem = self.make_rebaseliner( "BUGX REBASELINE : failures/expected/image.html = IMAGE") - rebaseliner.run(False) + rebaseliner.run() # See comment in test_one_platform for an explanation of the 12 written tests. # Note that even though the rebaseline is marked for all platforms, each # rebaseliner only ever does one. self.assertEqual(len(filesystem.written_files), 12) - self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test/test_expectations.txt'], - 'BUGX REBASELINE WIN : failures/expected/image.html = IMAGE\n' - 'BUGX REBASELINE WIN-XP : failures/expected/image.html = IMAGE\n') - self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac/failures/expected/image-expected.checksum'], 'new-image-checksum') - self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac/failures/expected/image-expected.png'], 'new-image-png') - self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac/failures/expected/image-expected.txt'], 'new-image-txt') + self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.checksum'], 'new-image-checksum') + self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.png'], 'new-image-png') + self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.txt'], 'new-image-txt') + + def test_png_file_with_comment(self): + rebaseliner, filesystem = self.make_rebaseliner( + "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE") + compile_success = rebaseliner._compile_rebaselining_tests() + self.assertTrue(compile_success) + self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests) + rebaseliner.run() + # There is one less file written than |test_one_platform| because we only + # write 2 expectations (the png and the txt file). + self.assertEqual(len(filesystem.written_files), 11) + self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'], 'tEXtchecksum\x000123456789') + self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'], 'png-comment-txt') + self.assertFalse(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None)) + + def test_png_file_with_comment_remove_old_checksum(self): + rebaseliner, filesystem = self.make_rebaseliner( + "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE") + filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'] = 'old' + filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum'] = 'old' + filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'] = 'old' + + compile_success = rebaseliner._compile_rebaselining_tests() + self.assertTrue(compile_success) + self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests) + rebaseliner.run() + # There is one more file written than |test_png_file_with_comment_remove_old_checksum| + # because we also delete the old checksum. + self.assertEqual(len(filesystem.written_files), 12) + self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'], 'tEXtchecksum\x000123456789') + self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'], 'png-comment-txt') + self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None), None) + + def test_png_file_with_comment_as_duplicate(self): + rebaseliner, filesystem = self.make_rebaseliner( + "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE") + filesystem.files['/test.checkout/LayoutTests/platform/test-mac-snowleopard/failures/expected/image_checksum-expected.png'] = 'tEXtchecksum\x000123456789' + filesystem.files['/test.checkout/LayoutTests/platform/test-mac-snowleopard/failures/expected/image_checksum-expected.txt'] = 'png-comment-txt' + + compile_success = rebaseliner._compile_rebaselining_tests() + self.assertTrue(compile_success) + self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests) + rebaseliner.run() + self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png', None), None) + self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt', None), None) + self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None), None) def test_diff_baselines_txt(self): rebaseliner, filesystem = self.make_rebaseliner("") @@ -239,28 +320,37 @@ class TestRebaseliner(unittest.TestCase): class TestRealMain(unittest.TestCase): + def setUp(self): + if not hasattr(self, '_orig_archive'): + self._orig_archive = rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT + rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = test_archive(self._orig_archive) + + def tearDown(self): + rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = self._orig_archive + def test_all_platforms(self): expectations = "BUGX REBASELINE : failures/expected/image.html = IMAGE" options = test_options() - host_port_obj, filesystem = test_host_port_and_filesystem(options, expectations) url_fetcher = test_url_fetcher(filesystem) zip_factory = test_zip_factory() mock_scm = mocktool.MockSCM() oc = outputcapture.OutputCapture() oc.capture_output() - rebaseline_chromium_webkit_tests.real_main(options, options, host_port_obj, - host_port_obj, url_fetcher, zip_factory, mock_scm) + res = rebaseline_chromium_webkit_tests.real_main(options, options, + host_port_obj, host_port_obj, url_fetcher, zip_factory, mock_scm) oc.restore_output() - # We expect to have written 35 files over the course of this rebaseline: - # *) 11 files * 3 ports for the new baselines and the diffs (see breakdown - # under test_one_platform, above) - # *) the updated test_expectations file - # *) the rebaseline results html file - self.assertEqual(len(filesystem.written_files), 35) - self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test/test_expectations.txt'], '') + # We expect to have written 36 files over the course of this rebaseline: + # *) 6*3 files in /__im_tmp/ for the archived members of the 6 ports + # *) 2*3 files in /test.checkout for actually differing baselines + # *) 1 file in /test.checkout for the updated test_expectations file + # *) 2*4 files in /tmp for the old/new baselines for the two actual ports + # *) 2 files in /tmp for the text diffs for the two ports + # *) 1 file in /tmp for the rebaseline results html file + self.assertEqual(res, 0) + self.assertEqual(len(filesystem.written_files), 36) class TestHtmlGenerator(unittest.TestCase): @@ -268,7 +358,7 @@ class TestHtmlGenerator(unittest.TestCase): options = mocktool.MockOptions(configuration=None, html_directory='/tmp') host_port = port.get('test', options, filesystem=port.unit_test_filesystem(files)) generator = rebaseline_chromium_webkit_tests.HtmlGenerator(host_port, - target_port=None, options=options, platforms=['mac'], rebaselining_tests=tests) + target_port=None, options=options, platforms=['test-mac-leopard'], rebaselining_tests=tests) return generator, host_port def test_generate_baseline_links(self): |