#!/usr/bin/python # Copyright (C) 2010 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Unit tests for rebaseline_chromium_webkit_tests.py.""" import unittest from webkitpy.tool import mocktool from webkitpy.common.system import urlfetcher_mock from webkitpy.common.system import filesystem_mock from webkitpy.common.system import zipfileset_mock from webkitpy.common.system import outputcapture from webkitpy.common.system.executive import Executive, ScriptError from webkitpy.layout_tests import port from webkitpy.layout_tests import rebaseline_chromium_webkit_tests class MockPort(object): def __init__(self, image_diff_exists): self.image_diff_exists = image_diff_exists def check_image_diff(self, override_step, logging): return self.image_diff_exists def get_mock_get(config_expectations): def mock_get(port_name, options): return MockPort(config_expectations[options.configuration]) return mock_get ARCHIVE_URL = 'http://localhost/layout_test_results' def test_options(): return mocktool.MockOptions(configuration=None, backup=False, html_directory='/tmp', archive_url=ARCHIVE_URL, force_archive_url=None, verbose=False, quiet=False, platforms=None) def test_host_port_and_filesystem(options, expectations): filesystem = port.unit_test_filesystem() host_port_obj = port.get('test', options, filesystem=filesystem, user=mocktool.MockUser()) expectations_path = host_port_obj.path_to_test_expectations_file() filesystem.write_text_file(expectations_path, expectations) return (host_port_obj, filesystem) def test_url_fetcher(filesystem): urls = { ARCHIVE_URL + '/Webkit_Mac10_6/': '', ARCHIVE_URL + '/Webkit_Mac10_5/': '', ARCHIVE_URL + '/Webkit_Win7/': '', ARCHIVE_URL + '/Webkit_Vista/': '', ARCHIVE_URL + '/Webkit_Win/': '', ARCHIVE_URL + '/Webkit_Linux/': '', } return urlfetcher_mock.make_fetcher_cls(urls)(filesystem) def test_zip_factory(): ziphashes = { ARCHIVE_URL + '/Webkit_Mac10_5/2/layout-test-results.zip': { 'layout-test-results/failures/expected/image-actual.txt': 'new-image-txt', 'layout-test-results/failures/expected/image-actual.checksum': 'new-image-checksum', 'layout-test-results/failures/expected/image-actual.png': 'new-image-png', 'layout-test-results/failures/expected/image_checksum-actual.txt': 'png-comment-txt', 'layout-test-results/failures/expected/image_checksum-actual.checksum': '0123456789', 'layout-test-results/failures/expected/image_checksum-actual.png': 'tEXtchecksum\x000123456789', }, ARCHIVE_URL + '/Webkit_Mac10_6/4/layout-test-results.zip': { 'layout-test-results/failures/expected/image-actual.txt': 'new-image-txt', 'layout-test-results/failures/expected/image-actual.checksum': 'new-image-checksum', 'layout-test-results/failures/expected/image-actual.png': 'new-image-png', }, ARCHIVE_URL + '/Webkit_Vista/1/layout-test-results.zip': { 'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt', 'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum', 'layout-test-results/failures/expected/image-actual.png': 'win-image-png', }, ARCHIVE_URL + '/Webkit_Win7/1/layout-test-results.zip': { 'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt', 'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum', 'layout-test-results/failures/expected/image-actual.png': 'win-image-png', }, ARCHIVE_URL + '/Webkit_Win/1/layout-test-results.zip': { 'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt', 'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum', 'layout-test-results/failures/expected/image-actual.png': 'win-image-png', }, ARCHIVE_URL + '/Webkit_Linux/1/layout-test-results.zip': { 'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt', 'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum', 'layout-test-results/failures/expected/image-actual.png': 'win-image-png', }, } return zipfileset_mock.make_factory(ziphashes) def test_archive(orig_archive_dict): new_archive_dict = {} for platform, dirname in orig_archive_dict.iteritems(): platform = platform.replace('chromium', 'test') new_archive_dict[platform] = dirname return new_archive_dict class TestGetHostPortObject(unittest.TestCase): def assert_result(self, release_present, debug_present, valid_port_obj): # Tests whether we get a valid port object returned when we claim # that Image diff is (or isn't) present in the two configs. port.get = get_mock_get({'Release': release_present, 'Debug': debug_present}) options = mocktool.MockOptions(configuration=None, html_directory='/tmp') port_obj = rebaseline_chromium_webkit_tests.get_host_port_object(options) if valid_port_obj: self.assertNotEqual(port_obj, None) else: self.assertEqual(port_obj, None) def test_get_host_port_object(self): # Save the normal port.get() function for future testing. old_get = port.get # Test whether we get a valid port object back for the four # possible cases of having ImageDiffs built. It should work when # there is at least one binary present. self.assert_result(False, False, False) self.assert_result(True, False, True) self.assert_result(False, True, True) self.assert_result(True, True, True) # Restore the normal port.get() function. port.get = old_get class TestOptions(unittest.TestCase): def test_parse_options(self): (options, target_options) = rebaseline_chromium_webkit_tests.parse_options([]) self.assertTrue(target_options.chromium) self.assertEqual(options.tolerance, 0) (options, target_options) = rebaseline_chromium_webkit_tests.parse_options(['--target-platform', 'qt']) self.assertFalse(hasattr(target_options, 'chromium')) self.assertEqual(options.tolerance, 0) class TestRebaseliner(unittest.TestCase): def setUp(self): if not hasattr(self, '_orig_archive'): self._orig_archive = rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = test_archive(self._orig_archive) def tearDown(self): rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = self._orig_archive def make_rebaseliner(self, expectations): options = test_options() host_port_obj, filesystem = test_host_port_and_filesystem(options, expectations) target_options = options target_port_obj = port.get('test', target_options, filesystem=filesystem) target_port_obj._expectations = expectations platform = target_port_obj.name() url_fetcher = test_url_fetcher(filesystem) zip_factory = test_zip_factory() mock_scm = mocktool.MockSCM(filesystem) rebaseliner = rebaseline_chromium_webkit_tests.Rebaseliner(host_port_obj, target_port_obj, platform, options, url_fetcher, zip_factory, mock_scm) return rebaseliner, filesystem def test_noop(self): # this method tests that was can at least instantiate an object, even # if there is nothing to do. rebaseliner, filesystem = self.make_rebaseliner("") rebaseliner.run() self.assertEqual(len(filesystem.written_files), 1) def test_rebaselining_tests(self): rebaseliner, filesystem = self.make_rebaseliner( "BUGX REBASELINE MAC : failures/expected/image.html = IMAGE") compile_success = rebaseliner._compile_rebaselining_tests() self.assertTrue(compile_success) self.assertEqual(set(['failures/expected/image.html']), rebaseliner._rebaselining_tests) def test_rebaselining_tests_should_ignore_reftests(self): rebaseliner, filesystem = self.make_rebaseliner( "BUGX REBASELINE : failures/expected/reftest.html = IMAGE") compile_success = rebaseliner._compile_rebaselining_tests() self.assertFalse(compile_success) self.assertFalse(rebaseliner._rebaselining_tests) def test_one_platform(self): rebaseliner, filesystem = self.make_rebaseliner( "BUGX REBASELINE MAC : failures/expected/image.html = IMAGE") rebaseliner.run() # We expect to have written 12 files over the course of this rebaseline: # *) 3 files in /__im_tmp for the extracted archive members # *) 3 new baselines under '/test.checkout/LayoutTests' # *) 4 files in /tmp for the new and old baselines in the result file # (-{old,new}.{txt,png} # *) 1 text diff in /tmp for the result file (-diff.txt). We don't # create image diffs (FIXME?) and don't display the checksums. # *) 1 updated test_expectations file self.assertEqual(len(filesystem.written_files), 12) self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.checksum'], 'new-image-checksum') self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.png'], 'new-image-png') self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.txt'], 'new-image-txt') def test_all_platforms(self): rebaseliner, filesystem = self.make_rebaseliner( "BUGX REBASELINE : failures/expected/image.html = IMAGE") rebaseliner.run() # See comment in test_one_platform for an explanation of the 12 written tests. # Note that even though the rebaseline is marked for all platforms, each # rebaseliner only ever does one. self.assertEqual(len(filesystem.written_files), 12) self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.checksum'], 'new-image-checksum') self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.png'], 'new-image-png') self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.txt'], 'new-image-txt') def test_png_file_with_comment(self): rebaseliner, filesystem = self.make_rebaseliner( "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE") compile_success = rebaseliner._compile_rebaselining_tests() self.assertTrue(compile_success) self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests) rebaseliner.run() # There is one less file written than |test_one_platform| because we only # write 2 expectations (the png and the txt file). self.assertEqual(len(filesystem.written_files), 11) self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'], 'tEXtchecksum\x000123456789') self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'], 'png-comment-txt') self.assertFalse(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None)) def test_png_file_with_comment_remove_old_checksum(self): rebaseliner, filesystem = self.make_rebaseliner( "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE") filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'] = 'old' filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum'] = 'old' filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'] = 'old' compile_success = rebaseliner._compile_rebaselining_tests() self.assertTrue(compile_success) self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests) rebaseliner.run() # There is one more file written than |test_png_file_with_comment_remove_old_checksum| # because we also delete the old checksum. self.assertEqual(len(filesystem.written_files), 12) self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'], 'tEXtchecksum\x000123456789') self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'], 'png-comment-txt') self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None), None) def test_png_file_with_comment_as_duplicate(self): rebaseliner, filesystem = self.make_rebaseliner( "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE") filesystem.files['/test.checkout/LayoutTests/platform/test-mac-snowleopard/failures/expected/image_checksum-expected.png'] = 'tEXtchecksum\x000123456789' filesystem.files['/test.checkout/LayoutTests/platform/test-mac-snowleopard/failures/expected/image_checksum-expected.txt'] = 'png-comment-txt' compile_success = rebaseliner._compile_rebaselining_tests() self.assertTrue(compile_success) self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests) rebaseliner.run() self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png', None), None) self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt', None), None) self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None), None) def test_diff_baselines_txt(self): rebaseliner, filesystem = self.make_rebaseliner("") port = rebaseliner._port output = port.expected_text( port._filesystem.join(port.layout_tests_dir(), 'passes/text.html')) self.assertFalse(rebaseliner._diff_baselines(output, output, is_image=False)) def test_diff_baselines_png(self): rebaseliner, filesystem = self.make_rebaseliner('') port = rebaseliner._port image = port.expected_image( port._filesystem.join(port.layout_tests_dir(), 'passes/image.html')) self.assertFalse(rebaseliner._diff_baselines(image, image, is_image=True)) class TestRealMain(unittest.TestCase): def setUp(self): if not hasattr(self, '_orig_archive'): self._orig_archive = rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = test_archive(self._orig_archive) def tearDown(self): rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = self._orig_archive def test_all_platforms(self): expectations = "BUGX REBASELINE : failures/expected/image.html = IMAGE" options = test_options() host_port_obj, filesystem = test_host_port_and_filesystem(options, expectations) url_fetcher = test_url_fetcher(filesystem) zip_factory = test_zip_factory() mock_scm = mocktool.MockSCM() oc = outputcapture.OutputCapture() oc.capture_output() res = rebaseline_chromium_webkit_tests.real_main(options, options, host_port_obj, host_port_obj, url_fetcher, zip_factory, mock_scm) oc.restore_output() # We expect to have written 36 files over the course of this rebaseline: # *) 6*3 files in /__im_tmp/ for the archived members of the 6 ports # *) 2*3 files in /test.checkout for actually differing baselines # *) 1 file in /test.checkout for the updated test_expectations file # *) 2*4 files in /tmp for the old/new baselines for the two actual ports # *) 2 files in /tmp for the text diffs for the two ports # *) 1 file in /tmp for the rebaseline results html file self.assertEqual(res, 0) self.assertEqual(len(filesystem.written_files), 36) class TestHtmlGenerator(unittest.TestCase): def make_generator(self, files, tests): options = mocktool.MockOptions(configuration=None, html_directory='/tmp') host_port = port.get('test', options, filesystem=port.unit_test_filesystem(files)) generator = rebaseline_chromium_webkit_tests.HtmlGenerator(host_port, target_port=None, options=options, platforms=['test-mac-leopard'], rebaselining_tests=tests) return generator, host_port def test_generate_baseline_links(self): files = { "/tmp/foo-expected-mac-old.txt": "", "/tmp/foo-expected-mac-new.txt": "", "/tmp/foo-expected-mac-diff.txt": "", } tests = ["foo.txt"] generator, host_port = self.make_generator(files, tests) links = generator._generate_baseline_links("foo", ".txt", "mac") expected_links = 'foo-expected.txtfoo-expected.txtDiff' self.assertEqual(links, expected_links) if __name__ == '__main__': unittest.main()