diff options
author | Kristian Monsen <kristianm@google.com> | 2010-09-30 15:42:16 +0100 |
---|---|---|
committer | Steve Block <steveblock@google.com> | 2010-10-07 10:59:29 +0100 |
commit | bec39347bb3bb5bf1187ccaf471d26247f28b585 (patch) | |
tree | 56bdc4c2978fbfd3d79d0d36d5d6c640ecc09cc8 /WebKitTools/Scripts/webkitpy/layout_tests/port | |
parent | 90b7966e7815b262cd19ac25f03aaad9b21fdc06 (diff) | |
download | external_webkit-bec39347bb3bb5bf1187ccaf471d26247f28b585.zip external_webkit-bec39347bb3bb5bf1187ccaf471d26247f28b585.tar.gz external_webkit-bec39347bb3bb5bf1187ccaf471d26247f28b585.tar.bz2 |
Merge WebKit at r68651 : Initial merge by git.
Change-Id: I3d6bff59f17eedd6722723354f386fec9be8ad12
Diffstat (limited to 'WebKitTools/Scripts/webkitpy/layout_tests/port')
25 files changed, 872 insertions, 291 deletions
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py index 70beac3..6a5d43b 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py @@ -42,11 +42,13 @@ import sys import time import apache_http_server +import test_files import http_server import websocket_server from webkitpy.common.system import logutils from webkitpy.common.system.executive import Executive, ScriptError +from webkitpy.common.system.user import User _log = logutils.get_logger(__file__) @@ -81,14 +83,15 @@ class Port(object): } return flags_by_configuration[configuration] - def __init__(self, port_name=None, options=None, executive=Executive()): - self._name = port_name - self._options = options + def __init__(self, **kwargs): + self._name = kwargs.get('port_name', None) + self._options = kwargs.get('options', None) + self._executive = kwargs.get('executive', Executive()) + self._user = kwargs.get('user', User()) self._helper = None self._http_server = None self._webkit_base_dir = None self._websocket_server = None - self._executive = executive def default_child_processes(self): """Return the number of DumpRenderTree instances to use for this @@ -130,11 +133,11 @@ class Port(object): interface so that it can be overriden for testing purposes.""" return expected_text != actual_text - def diff_image(self, expected_filename, actual_filename, + def diff_image(self, expected_contents, actual_contents, diff_filename=None, tolerance=0): - """Compare two image files and produce a delta image file. + """Compare two images and produce a delta image file. - Return True if the two files are different, False if they are the same. + Return True if the two images are different, False if they are the same. Also produce a delta image of the two images and write that into |diff_filename| if it is not None. @@ -252,6 +255,31 @@ class Port(object): return os.path.join(platform_dir, baseline_filename) return os.path.join(self.layout_tests_dir(), baseline_filename) + def _expected_file_contents(self, test, extension, encoding): + path = self.expected_filename(test, extension) + if not os.path.exists(path): + return None + with codecs.open(path, 'r', encoding) as file: + return file.read() + + def expected_checksum(self, test): + """Returns the checksum of the image we expect the test to produce, or None if it is a text-only test.""" + return self._expected_file_contents(test, '.checksum', 'ascii') + + def expected_image(self, test): + """Returns the image we expect the test to produce.""" + return self._expected_file_contents(test, '.png', None) + + def expected_text(self, test): + """Returns the text output we expect the test to produce.""" + # NOTE: -expected.txt files are ALWAYS utf-8. However, + # we do not decode the output from DRT, so we should not + # decode the -expected.txt values either to allow comparisons. + text = self._expected_file_contents(test, '.txt', None) + if not text: + return '' + return text.strip("\r\n").replace("\r\n", "\n") + "\n" + def filename_to_uri(self, filename): """Convert a test file to a URI.""" LAYOUTTEST_HTTP_DIR = "http/tests/" @@ -287,6 +315,73 @@ class Port(object): return "file:///" + self.get_absolute_path(filename) return "file://" + self.get_absolute_path(filename) + def tests(self, paths): + """Return the list of tests found (relative to layout_tests_dir().""" + return test_files.find(self, paths) + + def test_dirs(self): + """Returns the list of top-level test directories. + + Used by --clobber-old-results.""" + layout_tests_dir = self.layout_tests_dir() + return filter(lambda x: os.path.isdir(os.path.join(layout_tests_dir, x)), + os.listdir(layout_tests_dir)) + + def path_isdir(self, path): + """Returns whether the path refers to a directory of tests. + + Used by test_expectations.py to apply rules to whole directories.""" + return os.path.isdir(path) + + def path_exists(self, path): + """Returns whether the path refers to an existing test or baseline.""" + # Used by test_expectations.py to determine if an entry refers to a + # valid test and by printing.py to determine if baselines exist.""" + return os.path.exists(path) + + def update_baseline(self, path, data, encoding): + """Updates the baseline for a test. + + Args: + path: the actual path to use for baseline, not the path to + the test. This function is used to update either generic or + platform-specific baselines, but we can't infer which here. + data: contents of the baseline. + encoding: file encoding to use for the baseline. + """ + with codecs.open(path, "w", encoding=encoding) as file: + file.write(data) + + def uri_to_test_name(self, uri): + """Return the base layout test name for a given URI. + + This returns the test name for a given URI, e.g., if you passed in + "file:///src/LayoutTests/fast/html/keygen.html" it would return + "fast/html/keygen.html". + + """ + test = uri + if uri.startswith("file:///"): + if sys.platform == 'win32': + test = test.replace('file:///', '') + test = test.replace('/', '\\') + else: + test = test.replace('file://', '') + return self.relative_test_filename(test) + + if uri.startswith("http://127.0.0.1:8880/"): + # websocket tests + return test.replace('http://127.0.0.1:8880/', '') + + if uri.startswith("http://"): + # regular HTTP test + return test.replace('http://127.0.0.1:8000/', 'http/tests/') + + if uri.startswith("https://"): + return test.replace('https://127.0.0.1:8443/', 'http/tests/') + + raise NotImplementedError('unknown url type: %s' % uri) + def get_absolute_path(self, filename): """Return the absolute path in unix format for the given filename. @@ -369,10 +464,10 @@ class Port(object): """ return os.environ.copy() - def show_html_results_file(self, results_filename): + def show_results_html_file(self, results_filename): """This routine should display the HTML file pointed at by results_filename in a users' browser.""" - raise NotImplementedError('Port.show_html_results_file') + return self._user.open_url(results_filename) def create_driver(self, image_path, options): """Return a newly created base.Driver subclass for starting/stopping @@ -588,7 +683,7 @@ class Port(object): try: with self._open_configuration_file() as file: return file.readline().rstrip() - except IOError, e: + except: return None # FIXME: This list may be incomplete as Apple has some sekret configs. diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py index 780cd22..71877b3 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py @@ -57,16 +57,17 @@ class MockExecutive(): class UnitTestPort(base.Port): """Subclass of base.Port used for unit testing.""" - def __init__(self, configuration_contents=None, executive_exception=None): + def __init__(self, configuration_contents=None, configuration_exception=IOError, executive_exception=None): base.Port.__init__(self) self._configuration_contents = configuration_contents + self._configuration_exception = configuration_exception if executive_exception: self._executive = MockExecutive(executive_exception) def _open_configuration_file(self): if self._configuration_contents: return NewStringIO(self._configuration_contents) - raise IOError + raise self._configuration_exception class PortTest(unittest.TestCase): @@ -191,9 +192,14 @@ class PortTest(unittest.TestCase): self.assertFalse('nosuchthing' in diff) def test_default_configuration_notfound(self): + # Regular IOError thrown while trying to get the configuration. port = UnitTestPort() self.assertEqual(port.default_configuration(), "Release") + # More exotic OSError thrown. + port = UnitTestPort(configuration_exception=OSError) + self.assertEqual(port.default_configuration(), "Release") + def test_layout_tests_skipping(self): port = base.Port() port.skipped_layout_tests = lambda: ['foo/bar.html', 'media'] @@ -214,6 +220,11 @@ class PortTest(unittest.TestCase): # This routine is a no-op. We just test it for coverage. port.setup_test_run() + def test_test_dirs(self): + port = base.Port() + dirs = port.test_dirs() + self.assertTrue('canvas' in dirs) + self.assertTrue('css2.1' in dirs) class VirtualTest(unittest.TestCase): """Tests that various methods expected to be virtual are.""" @@ -231,7 +242,6 @@ class VirtualTest(unittest.TestCase): self.assertVirtual(port.path_to_test_expectations_file) self.assertVirtual(port.test_platform_name) self.assertVirtual(port.results_directory) - self.assertVirtual(port.show_html_results_file, None) self.assertVirtual(port.test_expectations) self.assertVirtual(port.test_base_platform_names) self.assertVirtual(port.test_platform_name) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py index 3fc4613..a72627a 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py @@ -39,6 +39,7 @@ import shutil import signal import subprocess import sys +import tempfile import time import webbrowser @@ -46,7 +47,6 @@ import base import http_server from webkitpy.common.system.executive import Executive -from webkitpy.layout_tests.layout_package import test_files from webkitpy.layout_tests.layout_package import test_expectations # Chromium DRT on OSX uses WebKitDriver. @@ -82,8 +82,13 @@ def check_file_exists(path_to_file, file_description, override_step=None, class ChromiumPort(base.Port): """Abstract base class for Chromium implementations of the Port class.""" - def __init__(self, port_name=None, options=None, **kwargs): - base.Port.__init__(self, port_name, options, **kwargs) + def __init__(self, **kwargs): + base.Port.__init__(self, **kwargs) + if 'options' in kwargs: + options = kwargs['options'] + if (options and (not hasattr(options, 'configuration') or + options.configuration is None)): + options.configuration = self.default_configuration() self._chromium_base_dir = None def baseline_path(self): @@ -126,14 +131,18 @@ class ChromiumPort(base.Port): return check_file_exists(image_diff_path, 'image diff exe', override_step, logging) - def diff_image(self, expected_filename, actual_filename, + def diff_image(self, expected_contents, actual_contents, diff_filename=None, tolerance=0): executable = self._path_to_image_diff() + expected_tmpfile = tempfile.NamedTemporaryFile() + expected_tmpfile.write(expected_contents) + actual_tmpfile = tempfile.NamedTemporaryFile() + actual_tmpfile.write(actual_contents) if diff_filename: - cmd = [executable, '--diff', expected_filename, actual_filename, - diff_filename] + cmd = [executable, '--diff', expected_tmpfile.name, + actual_tmpfile.name, diff_filename] else: - cmd = [executable, expected_filename, actual_filename] + cmd = [executable, expected_tmpfile.name, actual_tmpfile.name] result = True try: @@ -144,6 +153,9 @@ class ChromiumPort(base.Port): _compare_available = False else: raise e + finally: + expected_tmpfile.close() + actual_tmpfile.close() return result def driver_name(self): @@ -183,15 +195,6 @@ class ChromiumPort(base.Port): if os.path.exists(cachedir): shutil.rmtree(cachedir) - def show_results_html_file(self, results_filename): - uri = self.get_absolute_path(results_filename) - if self._options.use_drt: - # FIXME: This should use User.open_url - webbrowser.open(uri, new=1) - else: - # Note: Not thread safe: http://bugs.python.org/issue2320 - subprocess.Popen([self._path_to_driver(), uri]) - def create_driver(self, image_path, options): """Starts a new Driver and returns a handle to it.""" if options.use_drt and sys.platform == 'darwin': @@ -236,7 +239,7 @@ class ChromiumPort(base.Port): # FIXME: This drt_overrides handling should be removed when we switch # from tes_shell to DRT. drt_overrides = '' - if self._options.use_drt: + if self._options and self._options.use_drt: drt_overrides_path = self.path_from_webkit_base('LayoutTests', 'platform', 'chromium', 'drt_expectations.txt') if os.path.exists(drt_overrides_path): @@ -259,14 +262,13 @@ class ChromiumPort(base.Port): test_platform_name = self.test_platform_name() is_debug_mode = False - all_test_files = test_files.gather_test_files(self, '*') + all_test_files = self.tests([]) if extra_test_files: all_test_files.update(extra_test_files) expectations = test_expectations.TestExpectations( self, all_test_files, expectations_str, test_platform_name, - is_debug_mode, is_lint_mode=True, - tests_are_present=False, overrides=overrides_str) + is_debug_mode, is_lint_mode=True, overrides=overrides_str) tests_dir = self.layout_tests_dir() return [self.relative_test_filename(test) for test in expectations.get_tests_with_result_type(test_expectations.SKIP)] @@ -354,6 +356,12 @@ class ChromiumDriver(base.Driver): if self._options.gp_fault_error_box: driver_args.append('--gp-fault-error-box') + + if self._options.accelerated_compositing: + driver_args.append('--enable-accelerated-compositing') + + if self._options.accelerated_2d_canvas: + driver_args.append('--enable-accelerated-2d-canvas') return driver_args def start(self): diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py new file mode 100644 index 0000000..80602d9 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import with_statement + +import codecs +import os +import sys + +import chromium_linux +import chromium_mac +import chromium_win + + +def get(**kwargs): + """Some tests have slightly different results when run while using + hardware acceleration. In those cases, we prepend an additional directory + to the baseline paths.""" + port_name = kwargs.get('port_name', None) + if port_name == 'chromium-gpu': + if sys.platform in ('cygwin', 'win32'): + port_name = 'chromium-gpu-win' + elif sys.platform == 'linux2': + port_name = 'chromium-gpu-linux' + elif sys.platform == 'darwin': + port_name = 'chromium-gpu-mac' + else: + raise NotImplementedError('unsupported platform: %s' % + sys.platform) + + if port_name == 'chromium-gpu-linux': + return ChromiumGpuLinuxPort(**kwargs) + + if port_name.startswith('chromium-gpu-mac'): + return ChromiumGpuMacPort(**kwargs) + + if port_name.startswith('chromium-gpu-win'): + return ChromiumGpuWinPort(**kwargs) + + raise NotImplementedError('unsupported port: %s' % port_name) + + +def _set_gpu_options(options): + if options: + if options.accelerated_compositing is None: + options.accelerated_composting = True + if options.accelerated_2d_canvas is None: + options.accelerated_2d_canvas = True + + +def _gpu_overrides(port): + try: + overrides_path = port.path_from_chromium_base('webkit', 'tools', + 'layout_tests', 'test_expectations_gpu.txt') + except AssertionError: + return None + if not os.path.exists(overrides_path): + return None + with codecs.open(overrides_path, "r", "utf-8") as file: + return file.read() + + +class ChromiumGpuLinuxPort(chromium_linux.ChromiumLinuxPort): + def __init__(self, **kwargs): + kwargs.setdefault('port_name', 'chromium-gpu-linux') + _set_gpu_options(kwargs.get('options')) + chromium_linux.ChromiumLinuxPort.__init__(self, **kwargs) + + def baseline_search_path(self): + return ([self._webkit_baseline_path('chromium-gpu-linux')] + + chromium_linux.ChromiumLinuxPort.baseline_search_path(self)) + + def path_to_test_expectations_file(self): + return self.path_from_webkit_base('LayoutTests', 'platform', + 'chromium-gpu', 'test_expectations.txt') + + def test_expectations_overrides(self): + return _gpu_overrides(self) + + +class ChromiumGpuMacPort(chromium_mac.ChromiumMacPort): + def __init__(self, **kwargs): + kwargs.setdefault('port_name', 'chromium-gpu-mac') + _set_gpu_options(kwargs.get('options')) + chromium_mac.ChromiumMacPort.__init__(self, **kwargs) + + def baseline_search_path(self): + return ([self._webkit_baseline_path('chromium-gpu-mac')] + + chromium_mac.ChromiumMacPort.baseline_search_path(self)) + + def path_to_test_expectations_file(self): + return self.path_from_webkit_base('LayoutTests', 'platform', + 'chromium-gpu', 'test_expectations.txt') + + def test_expectations_overrides(self): + return _gpu_overrides(self) + + +class ChromiumGpuWinPort(chromium_win.ChromiumWinPort): + def __init__(self, **kwargs): + kwargs.setdefault('port_name', 'chromium-gpu-win' + self.version()) + _set_gpu_options(kwargs.get('options')) + chromium_win.ChromiumWinPort.__init__(self, **kwargs) + + def baseline_search_path(self): + return ([self._webkit_baseline_path('chromium-gpu-win')] + + chromium_win.ChromiumWinPort.baseline_search_path(self)) + + def path_to_test_expectations_file(self): + return self.path_from_webkit_base('LayoutTests', 'platform', + 'chromium-gpu', 'test_expectations.txt') + + def test_expectations_overrides(self): + return _gpu_overrides(self) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py new file mode 100644 index 0000000..5c79a3f --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. + +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import unittest +import chromium_gpu + + +class ChromiumGpuTest(unittest.TestCase): + def test_get_chromium_gpu_linux(self): + self.assertOverridesWorked('chromium-gpu-linux') + + def test_get_chromium_gpu_mac(self): + self.assertOverridesWorked('chromium-gpu-mac') + + def test_get_chromium_gpu_win(self): + self.assertOverridesWorked('chromium-gpu-win') + + def assertOverridesWorked(self, port_name): + # test that we got the right port + port = chromium_gpu.get(port_name=port_name, options=None) + + # we use startswith() instead of Equal to gloss over platform versions. + self.assertTrue(port.name().startswith(port_name)) + + # test that it has the right directory in front of the search path. + path = port.baseline_search_path()[0] + self.assertEqual(port._webkit_baseline_path(port_name), path) + + # test that we have the right expectations file. + self.assertTrue('chromium-gpu' in + port.path_to_test_expectations_file()) + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_linux.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_linux.py index 4df43e0..176991b 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_linux.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_linux.py @@ -41,12 +41,9 @@ _log = logging.getLogger("webkitpy.layout_tests.port.chromium_linux") class ChromiumLinuxPort(chromium.ChromiumPort): """Chromium Linux implementation of the Port class.""" - def __init__(self, port_name=None, options=None): - if port_name is None: - port_name = 'chromium-linux' - if options and not hasattr(options, 'configuration'): - options.configuration = 'Release' - chromium.ChromiumPort.__init__(self, port_name, options) + def __init__(self, **kwargs): + kwargs.setdefault('port_name', 'chromium-linux') + chromium.ChromiumPort.__init__(self, **kwargs) def baseline_search_path(self): port_names = ["chromium-linux", "chromium-win", "chromium", "win", "mac"] diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_mac.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_mac.py index abd84ae..64016ab 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_mac.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_mac.py @@ -44,12 +44,9 @@ _log = logging.getLogger("webkitpy.layout_tests.port.chromium_mac") class ChromiumMacPort(chromium.ChromiumPort): """Chromium Mac implementation of the Port class.""" - def __init__(self, port_name=None, options=None): - if port_name is None: - port_name = 'chromium-mac' - if options and not hasattr(options, 'configuration'): - options.configuration = 'Release' - chromium.ChromiumPort.__init__(self, port_name, options) + def __init__(self, **kwargs): + kwargs.setdefault('port_name', 'chromium-mac') + chromium.ChromiumPort.__init__(self, **kwargs) def baseline_search_path(self): port_names = ["chromium-mac", "chromium", "mac" + self.version(), "mac"] diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py index 7a005b1..a4a9ea6 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py @@ -83,17 +83,39 @@ class ChromiumDriverTest(unittest.TestCase): self.driver._proc.stdout.readline = mock_readline self._assert_write_command_and_read_line(expected_crash=True) + +class ChromiumPortTest(unittest.TestCase): + class TestMacPort(chromium_mac.ChromiumMacPort): + def __init__(self, options): + chromium_mac.ChromiumMacPort.__init__(self, + port_name='test-port', + options=options) + + def default_configuration(self): + self.default_configuration_called = True + return 'default' + + class TestLinuxPort(chromium_linux.ChromiumLinuxPort): + def __init__(self, options): + chromium_linux.ChromiumLinuxPort.__init__(self, + port_name='test-port', + options=options) + + def default_configuration(self): + self.default_configuration_called = True + return 'default' + def test_path_to_image_diff(self): class MockOptions: def __init__(self): self.use_drt = True - port = chromium_linux.ChromiumLinuxPort('test-port', options=MockOptions()) + port = ChromiumPortTest.TestLinuxPort(options=MockOptions()) self.assertTrue(port._path_to_image_diff().endswith( - '/out/Release/ImageDiff')) - port = chromium_mac.ChromiumMacPort('test-port', options=MockOptions()) + '/out/default/ImageDiff'), msg=port._path_to_image_diff()) + port = ChromiumPortTest.TestMacPort(options=MockOptions()) self.assertTrue(port._path_to_image_diff().endswith( - '/xcodebuild/Release/ImageDiff')) + '/xcodebuild/default/ImageDiff')) # FIXME: Figure out how this is going to work on Windows. #port = chromium_win.ChromiumWinPort('test-port', options=MockOptions()) @@ -102,16 +124,37 @@ class ChromiumDriverTest(unittest.TestCase): def __init__(self): self.use_drt = True - port = chromium_linux.ChromiumLinuxPort('test-port', options=MockOptions()) + port = ChromiumPortTest.TestLinuxPort(options=MockOptions()) fake_test = os.path.join(port.layout_tests_dir(), "fast/js/not-good.js") port.test_expectations = lambda: """BUG_TEST SKIP : fast/js/not-good.js = TEXT DEFER LINUX WIN : fast/js/very-good.js = TIMEOUT PASS""" port.test_expectations_overrides = lambda: '' + port.tests = lambda paths: set() + port.path_exists = lambda test: True skipped_tests = port.skipped_layout_tests(extra_test_files=[fake_test, ]) self.assertTrue("fast/js/not-good.js" in skipped_tests) + def test_default_configuration(self): + class EmptyOptions: + def __init__(self): + pass + + options = EmptyOptions() + port = ChromiumPortTest.TestLinuxPort(options) + self.assertEquals(options.configuration, 'default') + self.assertTrue(port.default_configuration_called) + + class OptionsWithUnsetConfiguration: + def __init__(self): + self.configuration = None + + options = OptionsWithUnsetConfiguration() + port = ChromiumPortTest.TestLinuxPort(options) + self.assertEquals(options.configuration, 'default') + self.assertTrue(port.default_configuration_called) + if __name__ == '__main__': unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win.py index e9a81e7..d2b0265 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_win.py @@ -41,12 +41,9 @@ _log = logging.getLogger("webkitpy.layout_tests.port.chromium_win") class ChromiumWinPort(chromium.ChromiumPort): """Chromium Win implementation of the Port class.""" - def __init__(self, port_name=None, options=None): - if port_name is None: - port_name = "chromium-win" + self.version() - if options and not hasattr(options, "configuration"): - options.configuration = "Release" - chromium.ChromiumPort.__init__(self, port_name, options) + def __init__(self, **kwargs): + kwargs.setdefault('port_name', 'chromium-win' + self.version()) + chromium.ChromiumPort.__init__(self, **kwargs) def setup_environ_for_server(self): env = chromium.ChromiumPort.setup_environ_for_server(self) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py index 4940e4c..648ccad 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py @@ -46,48 +46,24 @@ from __future__ import with_statement +import os import sys import base import factory -def _read_file(path, mode='r'): - """Return the contents of a file as a string. - - Returns '' if anything goes wrong, instead of throwing an IOError. - - """ - contents = '' - try: - with open(path, mode) as f: - contents = f.read() - except IOError: - pass - return contents - - -def _write_file(path, contents, mode='w'): - """Write the string to the specified path. - - Writes should never fail, so we may raise IOError. - - """ - with open(path, mode) as f: - f.write(contents) - - class DryRunPort(object): """DryRun implementation of the Port interface.""" - def __init__(self, port_name=None, options=None): + def __init__(self, **kwargs): pfx = 'dryrun-' - if port_name.startswith(pfx): - port_name = port_name[len(pfx):] - else: - port_name = None - self._options = options - self.__delegate = factory.get(port_name, options) + if 'port_name' in kwargs: + if kwargs['port_name'].startswith(pfx): + kwargs['port_name'] = kwargs['port_name'][len(pfx):] + else: + kwargs['port_name'] = None + self.__delegate = factory.get(**kwargs) def __getattr__(self, name): return getattr(self.__delegate, name) @@ -134,19 +110,16 @@ class DryrunDriver(base.Driver): return None def run_test(self, uri, timeoutms, image_hash): - test_name = self._uri_to_test(uri) - - text_filename = self._port.expected_filename(test_name, '.txt') - text_output = _read_file(text_filename) + test_name = self._port.uri_to_test_name(uri) + path = os.path.join(self._port.layout_tests_dir(), test_name) + text_output = self._port.expected_text(path) if image_hash is not None: - image_filename = self._port.expected_filename(test_name, '.png') - image = _read_file(image_filename, 'rb') - if self._image_path: - _write_file(self._image_path, image) - hash_filename = self._port.expected_filename(test_name, - '.checksum') - hash = _read_file(hash_filename) + image = self._port.expected_image(path) + if image and self._image_path: + with open(self._image_path, 'w') as f: + f.write(image) + hash = self._port.expected_checksum(path) else: hash = None return (False, False, hash, text_output, None) @@ -156,39 +129,3 @@ class DryrunDriver(base.Driver): def stop(self): pass - - def _uri_to_test(self, uri): - """Return the base layout test name for a given URI. - - This returns the test name for a given URI, e.g., if you passed in - "file:///src/LayoutTests/fast/html/keygen.html" it would return - "fast/html/keygen.html". - - """ - if not self._layout_tests_dir: - self._layout_tests_dir = self._port.layout_tests_dir() - test = uri - - if uri.startswith("file:///"): - if sys.platform == 'win32': - test = test.replace('file:///', '') - test = test.replace('/', '\\') - else: - test = test.replace('file://', '') - return test - elif uri.startswith("http://127.0.0.1:8880/"): - # websocket tests - test = test.replace('http://127.0.0.1:8880/', - self._layout_tests_dir + '/') - return test - elif uri.startswith("http://"): - # regular HTTP test - test = test.replace('http://127.0.0.1:8000/', - self._layout_tests_dir + '/http/tests/') - return test - elif uri.startswith("https://"): - test = test.replace('https://127.0.0.1:8443/', - self._layout_tests_dir + '/http/tests/') - return test - else: - raise NotImplementedError('unknown url type: %s' % uri) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/factory.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/factory.py index 5704f65..6935744 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/factory.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/factory.py @@ -37,11 +37,21 @@ ALL_PORT_NAMES = ['test', 'dryrun', 'mac', 'win', 'gtk', 'qt', 'chromium-mac', 'google-chrome-mac', 'google-chrome-linux32', 'google-chrome-linux64'] -def get(port_name=None, options=None): +def get(port_name=None, options=None, **kwargs): """Returns an object implementing the Port interface. If port_name is None, this routine attempts to guess at the most appropriate port on this platform.""" - port_to_use = port_name + # Wrapped for backwards-compatibility + if port_name: + kwargs['port_name'] = port_name + if options: + kwargs['options'] = options + return _get_kwargs(**kwargs) + + +def _get_kwargs(**kwargs): + port_to_use = kwargs.get('port_name', None) + options = kwargs.get('options', None) if port_to_use is None: if sys.platform == 'win32' or sys.platform == 'cygwin': if options and hasattr(options, 'chromium') and options.chromium: @@ -62,37 +72,40 @@ def get(port_name=None, options=None): if port_to_use == 'test': import test - return test.TestPort(port_name, options) + maker = test.TestPort elif port_to_use.startswith('dryrun'): import dryrun - return dryrun.DryRunPort(port_name, options) + maker = dryrun.DryRunPort elif port_to_use.startswith('mac'): import mac - return mac.MacPort(port_name, options) + maker = mac.MacPort elif port_to_use.startswith('win'): import win - return win.WinPort(port_name, options) + maker = win.WinPort elif port_to_use.startswith('gtk'): import gtk - return gtk.GtkPort(port_name, options) + maker = gtk.GtkPort elif port_to_use.startswith('qt'): import qt - return qt.QtPort(port_name, options) + maker = qt.QtPort + elif port_to_use.startswith('chromium-gpu'): + import chromium_gpu + maker = chromium_gpu.get elif port_to_use.startswith('chromium-mac'): import chromium_mac - return chromium_mac.ChromiumMacPort(port_name, options) + maker = chromium_mac.ChromiumMacPort elif port_to_use.startswith('chromium-linux'): import chromium_linux - return chromium_linux.ChromiumLinuxPort(port_name, options) + maker = chromium_linux.ChromiumLinuxPort elif port_to_use.startswith('chromium-win'): import chromium_win - return chromium_win.ChromiumWinPort(port_name, options) + maker = chromium_win.ChromiumWinPort elif port_to_use.startswith('google-chrome'): import google_chrome - return google_chrome.GetGoogleChromePort(port_name, options) - - raise NotImplementedError('unsupported port: %s' % port_to_use) - + maker = google_chrome.GetGoogleChromePort + else: + raise NotImplementedError('unsupported port: %s' % port_to_use) + return maker(**kwargs) def get_all(options=None): """Returns all the objects implementing the Port interface.""" diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/factory_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/factory_unittest.py index c0a4c5e..81c3732 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/factory_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/factory_unittest.py @@ -29,6 +29,7 @@ import sys import unittest +import chromium_gpu import chromium_linux import chromium_mac import chromium_win @@ -133,6 +134,15 @@ class FactoryTest(unittest.TestCase): def test_qt(self): self.assert_port("qt", qt.QtPort) + def test_chromium_gpu_linux(self): + self.assert_port("chromium-gpu-linux", chromium_gpu.ChromiumGpuLinuxPort) + + def test_chromium_gpu_mac(self): + self.assert_port("chromium-gpu-mac", chromium_gpu.ChromiumGpuMacPort) + + def test_chromium_gpu_win(self): + self.assert_port("chromium-gpu-win", chromium_gpu.ChromiumGpuWinPort) + def test_chromium_mac(self): self.assert_port("chromium-mac", chromium_mac.ChromiumMacPort) self.assert_platform_port("darwin", self.chromium_options, diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/google_chrome.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/google_chrome.py index 46ab3ed..bffc860 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/google_chrome.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/google_chrome.py @@ -25,10 +25,12 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -def GetGoogleChromePort(port_name, options): +def GetGoogleChromePort(**kwargs): """Some tests have slightly different results when compiled as Google Chrome vs Chromium. In those cases, we prepend an additional directory to to the baseline paths.""" + port_name = kwargs['port_name'] + del kwargs['port_name'] if port_name == 'google-chrome-linux32': import chromium_linux @@ -39,7 +41,7 @@ def GetGoogleChromePort(port_name, options): paths.insert(0, self._webkit_baseline_path( 'google-chrome-linux32')) return paths - return GoogleChromeLinux32Port(None, options) + return GoogleChromeLinux32Port(**kwargs) elif port_name == 'google-chrome-linux64': import chromium_linux @@ -50,7 +52,7 @@ def GetGoogleChromePort(port_name, options): paths.insert(0, self._webkit_baseline_path( 'google-chrome-linux64')) return paths - return GoogleChromeLinux64Port(None, options) + return GoogleChromeLinux64Port(**kwargs) elif port_name.startswith('google-chrome-mac'): import chromium_mac @@ -61,7 +63,7 @@ def GetGoogleChromePort(port_name, options): paths.insert(0, self._webkit_baseline_path( 'google-chrome-mac')) return paths - return GoogleChromeMacPort(None, options) + return GoogleChromeMacPort(**kwargs) elif port_name.startswith('google-chrome-win'): import chromium_win @@ -72,5 +74,5 @@ def GetGoogleChromePort(port_name, options): paths.insert(0, self._webkit_baseline_path( 'google-chrome-win')) return paths - return GoogleChromeWinPort(None, options) + return GoogleChromeWinPort(**kwargs) raise NotImplementedError('unsupported port: %s' % port_name) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/google_chrome_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/google_chrome_unittest.py index a2d7056..85e9338 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/google_chrome_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/google_chrome_unittest.py @@ -41,6 +41,7 @@ class GetGoogleChromePortTest(unittest.TestCase): self._verify_baseline_path('google-chrome-win', 'google-chrome-win-vista') def _verify_baseline_path(self, expected_path, port_name): - port = google_chrome.GetGoogleChromePort(port_name, None) + port = google_chrome.GetGoogleChromePort(port_name=port_name, + options=None) path = port.baseline_search_path()[0] self.assertEqual(expected_path, os.path.split(path)[1]) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/gtk.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/gtk.py index 59dc1d9..c60909e 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/gtk.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/gtk.py @@ -39,10 +39,9 @@ _log = logging.getLogger("webkitpy.layout_tests.port.gtk") class GtkPort(WebKitPort): """WebKit Gtk implementation of the Port class.""" - def __init__(self, port_name=None, options=None): - if port_name is None: - port_name = 'gtk' - WebKitPort.__init__(self, port_name, options) + def __init__(self, **kwargs): + kwargs.setdefault('port_name', 'gtk') + WebKitPort.__init__(self, **kwargs) def _tests_for_other_platforms(self): # FIXME: This list could be dynamic based on platform name and diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/mac.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/mac.py index 413b5f2..696e339 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/mac.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/mac.py @@ -43,10 +43,9 @@ _log = logging.getLogger("webkitpy.layout_tests.port.mac") class MacPort(WebKitPort): """WebKit Mac implementation of the Port class.""" - def __init__(self, port_name=None, options=None): - if port_name is None: - port_name = 'mac' + self.version() - WebKitPort.__init__(self, port_name, options) + def __init__(self, **kwargs): + kwargs.setdefault('port_name', 'mac' + self.version()) + WebKitPort.__init__(self, **kwargs) def default_child_processes(self): # FIXME: new-run-webkit-tests is unstable on Mac running more than diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py index 2d650f5..47597d6 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py @@ -68,14 +68,20 @@ class PortTestCase(unittest.TestCase): dir = port.layout_tests_dir() file1 = os.path.join(dir, 'fast', 'css', 'button_center.png') + fh1 = file(file1) + contents1 = fh1.read() file2 = os.path.join(dir, 'fast', 'css', 'remove-shorthand-expected.png') + fh2 = file(file2) + contents2 = fh2.read() tmpfile = tempfile.mktemp() - self.assertFalse(port.diff_image(file1, file1)) - self.assertTrue(port.diff_image(file1, file2)) + self.assertFalse(port.diff_image(contents1, contents1)) + self.assertTrue(port.diff_image(contents1, contents2)) - self.assertTrue(port.diff_image(file1, file2, tmpfile)) + self.assertTrue(port.diff_image(contents1, contents2, tmpfile)) + fh1.close() + fh2.close() # FIXME: this may not be being written? # self.assertTrue(os.path.exists(tmpfile)) # os.remove(tmpfile) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/qt.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/qt.py index 158c633..4c8fa0a 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/qt.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/qt.py @@ -42,10 +42,9 @@ _log = logging.getLogger("webkitpy.layout_tests.port.qt") class QtPort(WebKitPort): """QtWebKit implementation of the Port class.""" - def __init__(self, port_name=None, options=None): - if port_name is None: - port_name = 'qt' - WebKitPort.__init__(self, port_name, options) + def __init__(self, **kwargs): + kwargs.setdefault('port_name', 'qt') + WebKitPort.__init__(self, **kwargs) def _tests_for_other_platforms(self): # FIXME: This list could be dynamic based on platform name and diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py index 8e0bc11..5a0a40c 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py @@ -179,7 +179,7 @@ class ServerProcess: elif size == 0: index = self._output.find('\n') + 1 - if index or self.crashed or self.timed_out: + if index > 0 or self.crashed or self.timed_out: output = self._output[0:index] self._output = self._output[index:] return output diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py index 2ccddb0..3b81167 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py @@ -31,17 +31,100 @@ from __future__ import with_statement import codecs +import fnmatch import os +import sys import time import base +# This sets basic expectations for a test. Each individual expectation +# can be overridden by a keyword argument in TestList.add(). +class TestInstance: + def __init__(self, name): + self.name = name + self.base = name[(name.rfind("/") + 1):name.rfind(".html")] + self.crash = False + self.exception = False + self.hang = False + self.keyboard = False + self.error = '' + self.timeout = False + self.actual_text = self.base + '-txt\n' + self.actual_checksum = self.base + '-checksum\n' + self.actual_image = self.base + '-png\n' + self.expected_text = self.actual_text + self.expected_checksum = self.actual_checksum + self.expected_image = self.actual_image + + +# This is an in-memory list of tests, what we want them to produce, and +# what we want to claim are the expected results. +class TestList: + def __init__(self, port): + self.port = port + self.tests = {} + + def add(self, name, **kwargs): + test = TestInstance(name) + for key, value in kwargs.items(): + test.__dict__[key] = value + self.tests[name] = test + + def keys(self): + return self.tests.keys() + + def __contains__(self, item): + return item in self.tests + + def __getitem__(self, item): + return self.tests[item] + + class TestPort(base.Port): """Test implementation of the Port interface.""" - def __init__(self, port_name=None, options=None): - base.Port.__init__(self, port_name, options) + def __init__(self, **kwargs): + base.Port.__init__(self, **kwargs) + tests = TestList(self) + tests.add('passes/image.html') + tests.add('passes/text.html') + tests.add('failures/expected/checksum.html', + actual_checksum='checksum_fail-checksum') + tests.add('failures/expected/crash.html', crash=True) + tests.add('failures/expected/exception.html', exception=True) + tests.add('failures/expected/timeout.html', timeout=True) + tests.add('failures/expected/hang.html', hang=True) + tests.add('failures/expected/missing_text.html', + expected_text=None) + tests.add('failures/expected/image.html', + actual_image='image_fail-png', + expected_image='image-png') + tests.add('failures/expected/image_checksum.html', + actual_checksum='image_checksum_fail-checksum', + actual_image='image_checksum_fail-png') + tests.add('failures/expected/keyboard.html', + keyboard=True) + tests.add('failures/expected/missing_check.html', + expected_checksum=None) + tests.add('failures/expected/missing_image.html', + expected_image=None) + tests.add('failures/expected/missing_text.html', + expected_text=None) + tests.add('failures/expected/text.html', + actual_text='text_fail-png') + tests.add('failures/unexpected/text-image-checksum.html', + actual_text='text-image-checksum_fail-txt', + actual_checksum='text-image-checksum_fail-checksum') + tests.add('http/tests/passes/text.html') + tests.add('http/tests/ssl/text.html') + tests.add('passes/error.html', error='stuff going to stderr') + tests.add('passes/image.html') + tests.add('passes/platform_image.html') + tests.add('passes/text.html') + tests.add('websocket/tests/passes/text.html') + self._tests = tests def baseline_path(self): return os.path.join(self.layout_tests_dir(), 'platform', @@ -53,12 +136,8 @@ class TestPort(base.Port): def check_build(self, needs_http): return True - def diff_image(self, expected_filename, actual_filename, + def diff_image(self, expected_contents, actual_contents, diff_filename=None, tolerance=0): - with codecs.open(actual_filename, "r", "utf-8") as actual_fh: - actual_contents = actual_fh.read() - with codecs.open(expected_filename, "r", "utf-8") as expected_fh: - expected_contents = expected_fh.read() diffed = actual_contents != expected_contents if diffed and diff_filename: with codecs.open(diff_filename, "w", "utf-8") as diff_fh: @@ -66,24 +145,79 @@ class TestPort(base.Port): (expected_contents, actual_contents)) return diffed + def expected_checksum(self, test): + test = self.relative_test_filename(test) + return self._tests[test].expected_checksum + + def expected_image(self, test): + test = self.relative_test_filename(test) + return self._tests[test].expected_image + + def expected_text(self, test): + test = self.relative_test_filename(test) + text = self._tests[test].expected_text + if not text: + text = '' + return text + + def tests(self, paths): + # Test the idea of port-specific overrides for test lists. Also + # keep in memory to speed up the test harness. + if not paths: + paths = ['*'] + + matched_tests = [] + for p in paths: + if self.path_isdir(p): + matched_tests.extend(fnmatch.filter(self._tests.keys(), p + '*')) + else: + matched_tests.extend(fnmatch.filter(self._tests.keys(), p)) + layout_tests_dir = self.layout_tests_dir() + return set([os.path.join(layout_tests_dir, p) for p in matched_tests]) + + def path_exists(self, path): + # used by test_expectations.py and printing.py + rpath = self.relative_test_filename(path) + if rpath in self._tests: + return True + if self.path_isdir(rpath): + return True + if rpath.endswith('-expected.txt'): + test = rpath.replace('-expected.txt', '.html') + return (test in self._tests and + self._tests[test].expected_text) + if rpath.endswith('-expected.checksum'): + test = rpath.replace('-expected.checksum', '.html') + return (test in self._tests and + self._tests[test].expected_checksum) + if rpath.endswith('-expected.png'): + test = rpath.replace('-expected.png', '.html') + return (test in self._tests and + self._tests[test].expected_image) + return False + def layout_tests_dir(self): return self.path_from_webkit_base('WebKitTools', 'Scripts', 'webkitpy', 'layout_tests', 'data') + def path_isdir(self, path): + # Used by test_expectations.py + # + # We assume that a path is a directory if we have any tests that + # whose prefix matches the path plus a directory modifier. + if path[-1] != '/': + path += '/' + return any([t.startswith(path) for t in self._tests.keys()]) + + def test_dirs(self): + return ['passes', 'failures'] + def name(self): return self._name def options(self): return self._options - def skipped_layout_tests(self): - return [] - - def path_to_test_expectations_file(self): - return self.path_from_webkit_base('WebKitTools', 'Scripts', - 'webkitpy', 'layout_tests', 'data', 'platform', 'test', - 'test_expectations.txt') - def _path_to_wdiff(self): return None @@ -93,9 +227,6 @@ class TestPort(base.Port): def setup_test_run(self): pass - def show_results_html_file(self, filename): - pass - def create_driver(self, image_path, options): return TestDriver(self, image_path, options, executive=None) @@ -116,9 +247,21 @@ class TestPort(base.Port): Basically this string should contain the equivalent of a test_expectations file. See test_expectations.py for more details.""" - expectations_path = self.path_to_test_expectations_file() - with codecs.open(expectations_path, "r", "utf-8") as file: - return file.read() + return """ +WONTFIX : failures/expected/checksum.html = IMAGE +WONTFIX : failures/expected/crash.html = CRASH +// This one actually passes because the checksums will match. +WONTFIX : failures/expected/image.html = PASS +WONTFIX : failures/expected/image_checksum.html = IMAGE +WONTFIX : failures/expected/missing_check.html = MISSING PASS +WONTFIX : failures/expected/missing_image.html = MISSING PASS +WONTFIX : failures/expected/missing_text.html = MISSING PASS +WONTFIX : failures/expected/text.html = TEXT +WONTFIX : failures/expected/timeout.html = TIMEOUT +WONTFIX SKIP : failures/expected/hang.html = TIMEOUT +WONTFIX SKIP : failures/expected/keyboard.html = CRASH +WONTFIX SKIP : failures/expected/exception.html = CRASH +""" def test_base_platform_names(self): return ('mac', 'win') @@ -150,68 +293,21 @@ class TestDriver(base.Driver): return True def run_test(self, uri, timeoutms, image_hash): - basename = uri[(uri.rfind("/") + 1):uri.rfind(".html")] - - if 'error' in basename: - error = basename + "_error\n" - else: - error = '' - checksum = None - # There are four currently supported types of tests: text, image, - # image hash (checksum), and stderr output. The fake output - # is the basename of the file + "-" plus the type of test output - # (or a blank string for stderr). - # - # If 'image' or 'check' appears in the basename, we assume this is - # simulating a pixel test. - # - # If 'failures' appears in the URI, then we assume this test should - # fail. Which type of failures are determined by which strings appear - # in the basename of the test. For failures that produce outputs, - # we change the fake output to basename + "_failed-". - # - # The fact that each test produces (more or less) unique output data - # will allow us to see if any results get crossed by the rest of the - # program. - if 'failures' in uri: - if 'keyboard' in basename: - raise KeyboardInterrupt - if 'exception' in basename: - raise ValueError('exception from ' + basename) - - crash = 'crash' in basename - timeout = 'timeout' in basename or 'hang' in basename - timeout = 'timeout' in basename - if 'text' in basename: - output = basename + '_failed-txt\n' - else: - output = basename + '-txt\n' - if self._port.options().pixel_tests: - if ('image' in basename or 'check' in basename): - checksum = basename + "-checksum\n" - - if 'image' in basename: - with open(self._image_path, "w") as f: - f.write(basename + "_failed-png\n") - elif 'check' in basename: - with open(self._image_path, "w") as f: - f.write(basename + "-png\n") - if 'checksum' in basename: - checksum = basename + "_failed-checksum\n" - - if 'hang' in basename: - time.sleep((float(timeoutms) * 4) / 1000.0) - else: - crash = False - timeout = False - output = basename + '-txt\n' - if self._options.pixel_tests and ( - 'image' in basename or 'check' in basename): - checksum = basename + '-checksum\n' - with open(self._image_path, "w") as f: - f.write(basename + "-png") - - return (crash, timeout, checksum, output, error) + test_name = self._port.uri_to_test_name(uri) + test = self._port._tests[test_name] + if test.keyboard: + raise KeyboardInterrupt + if test.exception: + raise ValueError('exception from ' + test_name) + if test.hang: + time.sleep((float(timeoutms) * 4) / 1000.0) + + if self._port.options().pixel_tests and test.actual_image: + with open(self._image_path, 'w') as file: + file.write(test.actual_image) + + return (test.crash, test.timeout, test.actual_checksum, + test.actual_text, test.error) def start(self): pass diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files.py new file mode 100644 index 0000000..3fa0fb3 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""This module is used to find all of the layout test files used by +run-webkit-tests. It exposes one public function - find() - +which takes an optional list of paths. If a list is passed in, the returned +list of test files is constrained to those found under the paths passed in, +i.e. calling find(["LayoutTests/fast"]) will only return files +under that directory.""" + +import glob +import os +import time + +from webkitpy.common.system import logutils + + +_log = logutils.get_logger(__file__) + + +# When collecting test cases, we include any file with these extensions. +_supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.xhtmlmp', '.pl', + '.php', '.svg']) +# When collecting test cases, skip these directories +_skipped_directories = set(['.svn', '_svn', 'resources', 'script-tests']) + + +def find(port, paths): + """Finds the set of tests under port.layout_tests_dir(). + + Args: + paths: a list of command line paths relative to the layout_tests_dir() + to limit the search to. glob patterns are ok. + """ + gather_start_time = time.time() + paths_to_walk = set() + # if paths is empty, provide a pre-defined list. + if paths: + _log.debug("Gathering tests from: %s relative to %s" % (paths, port.layout_tests_dir())) + for path in paths: + # If there's an * in the name, assume it's a glob pattern. + path = os.path.join(port.layout_tests_dir(), path) + if path.find('*') > -1: + filenames = glob.glob(path) + paths_to_walk.update(filenames) + else: + paths_to_walk.add(path) + else: + _log.debug("Gathering tests from: %s" % port.layout_tests_dir()) + paths_to_walk.add(port.layout_tests_dir()) + + # Now walk all the paths passed in on the command line and get filenames + test_files = set() + for path in paths_to_walk: + if os.path.isfile(path) and _has_supported_extension(path): + test_files.add(os.path.normpath(path)) + continue + + for root, dirs, files in os.walk(path): + # Don't walk skipped directories or their sub-directories. + if os.path.basename(root) in _skipped_directories: + del dirs[:] + continue + # This copy and for-in is slightly inefficient, but + # the extra walk avoidance consistently shaves .5 seconds + # off of total walk() time on my MacBook Pro. + for directory in dirs[:]: + if directory in _skipped_directories: + dirs.remove(directory) + + for filename in files: + if _has_supported_extension(filename): + filename = os.path.join(root, filename) + filename = os.path.normpath(filename) + test_files.add(filename) + + gather_time = time.time() - gather_start_time + _log.debug("Test gathering took %f seconds" % gather_time) + + return test_files + + +def _has_supported_extension(filename): + """Return true if filename is one of the file extensions we want to run a + test on.""" + extension = os.path.splitext(filename)[1] + return extension in _supported_file_extensions diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py new file mode 100644 index 0000000..c37eb92 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py @@ -0,0 +1,68 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import os +import unittest + +import base +import test_files + + +class TestFilesTest(unittest.TestCase): + def test_find_no_paths_specified(self): + port = base.Port() + layout_tests_dir = port.layout_tests_dir() + port.layout_tests_dir = lambda: os.path.join(layout_tests_dir, + 'fast', 'html') + tests = test_files.find(port, []) + self.assertNotEqual(tests, 0) + + def test_find_one_test(self): + port = base.Port() + # This is just a test picked at random but known to exist. + tests = test_files.find(port, ['fast/html/keygen.html']) + self.assertEqual(len(tests), 1) + + def test_find_glob(self): + port = base.Port() + tests = test_files.find(port, ['fast/html/key*']) + self.assertEqual(len(tests), 1) + + def test_find_with_skipped_directories(self): + port = base.Port() + tests = port.tests('userscripts') + self.assertTrue('userscripts/resources/frame1.html' not in tests) + + def test_find_with_skipped_directories_2(self): + port = base.Port() + tests = test_files.find(port, ['userscripts/resources']) + self.assertEqual(tests, set([])) + + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py index 88c9bdf..ed19c09 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py @@ -58,16 +58,16 @@ _log = logging.getLogger("webkitpy.layout_tests.port.webkit") class WebKitPort(base.Port): """WebKit implementation of the Port class.""" - def __init__(self, port_name=None, options=None, **kwargs): - base.Port.__init__(self, port_name, options, **kwargs) + def __init__(self, **kwargs): + base.Port.__init__(self, **kwargs) self._cached_build_root = None self._cached_apache_path = None # FIXME: disable pixel tests until they are run by default on the # build machines. - if options and (not hasattr(options, "pixel_tests") or - options.pixel_tests is None): - options.pixel_tests = False + if self._options and (not hasattr(self._options, "pixel_tests") or + self._options.pixel_tests is None): + self._options.pixel_tests = False def baseline_path(self): return self._webkit_baseline_path(self._name) @@ -84,10 +84,14 @@ class WebKitPort(base.Port): return '' def _build_driver(self): - return not self._executive.run_command([ + exit_code = self._executive.run_command([ self.script_path("build-dumprendertree"), self.flag_from_configuration(self._options.configuration), ], return_exit_code=True) + if exit_code != 0: + _log.error("Failed to build DumpRenderTree") + return False + return True def _check_driver(self): driver_path = self._path_to_driver() @@ -119,7 +123,7 @@ class WebKitPort(base.Port): return False return True - def diff_image(self, expected_filename, actual_filename, + def diff_image(self, expected_contents, actual_contents, diff_filename=None, tolerance=0.1): """Return True if the two files are different. Also write a delta image of the two images into |diff_filename| if it is not None.""" @@ -128,31 +132,24 @@ class WebKitPort(base.Port): # parameter, or make it go away and always use exact matches. # Handle the case where the test didn't actually generate an image. - actual_length = os.stat(actual_filename).st_size - if actual_length == 0: - if diff_filename: - shutil.copyfile(actual_filename, expected_filename) + if not actual_contents: return True - sp = self._diff_image_request(expected_filename, actual_filename, tolerance) - return self._diff_image_reply(sp, expected_filename, diff_filename) + sp = self._diff_image_request(expected_contents, actual_contents, + tolerance) + return self._diff_image_reply(sp, diff_filename) - def _diff_image_request(self, expected_filename, actual_filename, tolerance): + def _diff_image_request(self, expected_contents, actual_contents, tolerance): command = [self._path_to_image_diff(), '--tolerance', str(tolerance)] sp = server_process.ServerProcess(self, 'ImageDiff', command) - actual_length = os.stat(actual_filename).st_size - with open(actual_filename) as file: - actual_file = file.read() - expected_length = os.stat(expected_filename).st_size - with open(expected_filename) as file: - expected_file = file.read() sp.write('Content-Length: %d\n%sContent-Length: %d\n%s' % - (actual_length, actual_file, expected_length, expected_file)) + (len(actual_contents), actual_contents, + len(expected_contents), expected_contents)) return sp - def _diff_image_reply(self, sp, expected_filename, diff_filename): + def _diff_image_reply(self, sp, diff_filename): timeout = 2.0 deadline = time.time() + timeout output = sp.read_line(timeout) @@ -178,7 +175,7 @@ class WebKitPort(base.Port): with open(diff_filename, 'w') as file: file.write(output) elif sp.timed_out: - _log.error("ImageDiff timed out on %s" % expected_filename) + _log.error("ImageDiff timed out") elif sp.crashed: _log.error("ImageDiff crashed") sp.stop() @@ -193,11 +190,6 @@ class WebKitPort(base.Port): # This port doesn't require any specific configuration. pass - def show_results_html_file(self, results_filename): - uri = self.filename_to_uri(results_filename) - # FIXME: We should open results in the version of WebKit we built. - webbrowser.open(uri, new=1) - def create_driver(self, image_path, options): return WebKitDriver(self, image_path, options, executive=self._executive) @@ -255,7 +247,7 @@ class WebKitPort(base.Port): "MathMLElement": ["mathml"], "GraphicsLayer": ["compositing"], "WebCoreHas3DRendering": ["animations/3d", "transforms/3d"], - "WebGLShader": ["fast/canvas/webgl"], + "WebGLShader": ["fast/canvas/webgl", "compositing/webgl", "http/tests/canvas/webgl"], "WMLElement": ["http/tests/wml", "fast/wml", "wml"], "parseWCSSInputProperty": ["fast/wcss"], "isXHTMLMPDocument": ["fast/xhtmlmp"], @@ -418,12 +410,17 @@ class WebKitDriver(base.Driver): def _driver_args(self): driver_args = [] + if self._image_path: driver_args.append('--pixel-tests') - # These are used by the Chromium DRT port if self._options.use_drt: - driver_args.append('--test-shell') + if self._options.accelerated_compositing: + driver_args.append('--enable-accelerated-compositing') + + if self._options.accelerated_2d_canvas: + driver_args.append('--enable-accelerated-2d-canvas') + return driver_args def start(self): diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py index fbfadc3..7b68310 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py @@ -53,7 +53,7 @@ class WebKitPortTest(unittest.TestCase): def test_skipped_directories_for_symbols(self): supported_symbols = ["GraphicsLayer", "WebCoreHas3DRendering", "isXHTMLMPDocument", "fooSymbol"] - expected_directories = set(["mathml", "fast/canvas/webgl", "http/tests/wml", "fast/wml", "wml", "fast/wcss"]) + expected_directories = set(["mathml", "fast/canvas/webgl", "compositing/webgl", "http/tests/canvas/webgl", "http/tests/wml", "fast/wml", "wml", "fast/wcss"]) result_directories = set(TestWebKitPort(supported_symbols, None)._skipped_tests_for_unsupported_features()) self.assertEqual(result_directories, expected_directories) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/win.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/win.py index e05a69d..9e30155 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/win.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/win.py @@ -39,10 +39,9 @@ _log = logging.getLogger("webkitpy.layout_tests.port.win") class WinPort(WebKitPort): """WebKit Win implementation of the Port class.""" - def __init__(self, port_name=None, options=None): - if port_name is None: - port_name = 'win' - WebKitPort.__init__(self, port_name, options) + def __init__(self, **kwargs): + kwargs.setdefault('port_name', 'win') + WebKitPort.__init__(self, **kwargs) def baseline_search_path(self): # Based on code from old-run-webkit-tests expectedDirectoryForTest() |