diff options
Diffstat (limited to 'Tools/Scripts/webkitpy/layout_tests/port')
25 files changed, 1241 insertions, 350 deletions
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/base.py b/Tools/Scripts/webkitpy/layout_tests/port/base.py index 6e5fabc..5ff4bff 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/base.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/base.py @@ -121,15 +121,18 @@ class Port(object): # certainly won't be available, so it's a good test to keep us # from erroring out later. self._pretty_patch_available = self._filesystem.exists(self._pretty_patch_path) - self.set_option_default('configuration', None) - if self._options.configuration is None: + if not hasattr(self._options, 'configuration') or self._options.configuration is None: self._options.configuration = self.default_configuration() + self._test_configuration = None def default_child_processes(self): """Return the number of DumpRenderTree instances to use for this port.""" return self._executive.cpu_count() + def default_worker_model(self): + return 'old-threads' + def baseline_path(self): """Return the absolute path to the directory to store new baselines in for this port.""" @@ -315,7 +318,7 @@ class Port(object): path = self.expected_filename(test, '.checksum') if not self.path_exists(path): return None - return self._filesystem.read_text_file(path) + return self._filesystem.read_binary_file(path) def expected_image(self, test): """Returns the image we expect the test to produce.""" @@ -393,7 +396,7 @@ class Port(object): driver = self.create_driver(0) return driver.cmd_line() - def update_baseline(self, path, data, encoding): + def update_baseline(self, path, data): """Updates the baseline for a test. Args: @@ -401,14 +404,8 @@ class Port(object): the test. This function is used to update either generic or platform-specific baselines, but we can't infer which here. data: contents of the baseline. - encoding: file encoding to use for the baseline. """ - # FIXME: remove the encoding parameter in favor of text/binary - # functions. - if encoding is None: - self._filesystem.write_binary_file(path, data) - else: - self._filesystem.write_text_file(path, data) + self._filesystem.write_binary_file(path, data) def uri_to_test_name(self, uri): """Return the base layout test name for a given URI. @@ -465,6 +462,15 @@ class Port(object): may be different (e.g., 'win-xp' instead of 'chromium-win-xp'.""" return self._name + def graphics_type(self): + """Returns whether the port uses accelerated graphics ('gpu') or not + ('cpu').""" + return 'cpu' + + def real_name(self): + """Returns the actual name of the port, not the delegate's.""" + return self.name() + def get_option(self, name, default_value=None): # FIXME: Eventually we should not have to do a test for # hasattr(), and we should be able to just do @@ -496,9 +502,16 @@ class Port(object): """Relative unix-style path for a filename under the LayoutTests directory. Filenames outside the LayoutTests directory should raise an error.""" + # FIXME: On Windows, does this return test_names with forward slashes, + # or windows-style relative paths? assert filename.startswith(self.layout_tests_dir()), "%s did not start with %s" % (filename, self.layout_tests_dir()) return filename[len(self.layout_tests_dir()) + 1:] + def abspath_for_test(self, test_name): + """Returns the full path to the file for a given test name. This is the + inverse of relative_test_filename().""" + return self._filesystem.normpath(self._filesystem.join(self.layout_tests_dir(), test_name)) + def results_directory(self): """Absolute path to the place to store the test results.""" raise NotImplementedError('Port.results_directory') @@ -577,12 +590,25 @@ class Port(object): if self._http_lock: self._http_lock.cleanup_http_lock() + # + # TEST EXPECTATION-RELATED METHODS + # + + def test_configuration(self): + """Returns the current TestConfiguration for the port.""" + if not self._test_configuration: + self._test_configuration = TestConfiguration(self) + return self._test_configuration + + def all_test_configurations(self): + return self.test_configuration().all_test_configurations() + def test_expectations(self): """Returns the test expectations for this port. Basically this string should contain the equivalent of a test_expectations file. See test_expectations.py for more details.""" - raise NotImplementedError('Port.test_expectations') + return self._filesystem.read_text_file(self.path_to_test_expectations_file()) def test_expectations_overrides(self): """Returns an optional set of overrides for the test_expectations. @@ -593,18 +619,6 @@ class Port(object): sync up the two repos.""" return None - def test_base_platform_names(self): - """Return a list of the 'base' platforms on your port. The base - platforms represent different architectures, operating systems, - or implementations (as opposed to different versions of a single - platform). For example, 'mac' and 'win' might be different base - platforms, wherease 'mac-tiger' and 'mac-leopard' might be - different platforms. This routine is used by the rebaselining tool - and the dashboards, and the strings correspond to the identifiers - in your test expectations (*not* necessarily the platform names - themselves).""" - raise NotImplementedError('Port.base_test_platforms') - def test_platform_name(self): """Returns the string that corresponds to the given platform name in the test expectations. This may be the same as name(), or it @@ -810,6 +824,48 @@ class Port(object): platform) +class DriverInput(object): + """Holds the input parameters for a driver.""" + + def __init__(self, filename, timeout, image_hash): + """Initializes a DriverInput object. + + Args: + filename: Full path to the test. + timeout: Timeout in msecs the driver should use while running the test + image_hash: A image checksum which is used to avoid doing an image dump if + the checksums match. + """ + self.filename = filename + self.timeout = timeout + self.image_hash = image_hash + + +class DriverOutput(object): + """Groups information about a output from driver for easy passing of data.""" + + def __init__(self, text, image, image_hash, + crash=False, test_time=None, timeout=False, error=''): + """Initializes a TestOutput object. + + Args: + text: a text output + image: an image output + image_hash: a string containing the checksum of the image + crash: a boolean indicating whether the driver crashed on the test + test_time: a time which the test has taken + timeout: a boolean indicating whehter the test timed out + error: any unexpected or additional (or error) text output + """ + self.text = text + self.image = image + self.image_hash = image_hash + self.crash = crash + self.test_time = test_time + self.timeout = timeout + self.error = error + + class Driver: """Abstract interface for the DumpRenderTree interface.""" @@ -824,7 +880,7 @@ class Driver: """ raise NotImplementedError('Driver.__init__') - def run_test(self, test_input): + def run_test(self, driver_input): """Run a single test and return the results. Note that it is okay if a test times out or crashes and leaves @@ -832,9 +888,9 @@ class Driver: are responsible for cleaning up and ensuring things are okay. Args: - test_input: a TestInput object + driver_input: a DriverInput object - Returns a TestOutput object. + Returns a DriverOutput object. """ raise NotImplementedError('Driver.run_test') @@ -863,3 +919,68 @@ class Driver: def stop(self): raise NotImplementedError('Driver.stop') + + +class TestConfiguration(object): + def __init__(self, port=None, os=None, version=None, architecture=None, + build_type=None, graphics_type=None): + + # FIXME: We can get the O/S and version from test_platform_name() + # and version() for now, but those should go away and be cleaned up + # with more generic methods like operation_system() and os_version() + # or something. Note that we need to strip the leading '-' off the + # version string if it is present. + if port: + port_version = port.version() + self.os = os or port.test_platform_name().replace(port_version, '') + self.version = version or port_version[1:] + self.architecture = architecture or 'x86' + self.build_type = build_type or port._options.configuration.lower() + self.graphics_type = graphics_type or port.graphics_type() + + def items(self): + return self.__dict__.items() + + def keys(self): + return self.__dict__.keys() + + def __str__(self): + return ("<%(os)s, %(version)s, %(build_type)s, %(graphics_type)s>" % + self.__dict__) + + def __repr__(self): + return "TestConfig(os='%(os)s', version='%(version)s', architecture='%(architecture)s', build_type='%(build_type)s', graphics_type='%(graphics_type)s')" % self.__dict__ + + def values(self): + """Returns the configuration values of this instance as a tuple.""" + return self.__dict__.values() + + def all_test_configurations(self): + """Returns a sequence of the TestConfigurations the port supports.""" + # By default, we assume we want to test every graphics type in + # every configuration on every system. + test_configurations = [] + for system in self.all_systems(): + for build_type in self.all_build_types(): + for graphics_type in self.all_graphics_types(): + test_configurations.append(TestConfiguration( + os=system[0], + version=system[1], + architecture=system[2], + build_type=build_type, + graphics_type=graphics_type)) + return test_configurations + + def all_systems(self): + return (('mac', 'leopard', 'x86'), + ('mac', 'snowleopard', 'x86'), + ('win', 'xp', 'x86'), + ('win', 'vista', 'x86'), + ('win', 'win7', 'x86'), + ('linux', 'hardy', 'x86')) + + def all_build_types(self): + return ('debug', 'release') + + def all_graphics_types(self): + return ('cpu', 'gpu') diff --git a/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py index 72f2d05..ef90484 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py @@ -224,19 +224,6 @@ class PortTest(unittest.TestCase): port = base.Port() self.assertEqual(port.get_option('foo', 'bar'), 'bar') - def test_set_option_default__unset(self): - port = base.Port() - port.set_option_default('foo', 'bar') - self.assertEqual(port.get_option('foo'), 'bar') - - def test_set_option_default__set(self): - options, args = optparse.OptionParser().parse_args([]) - options.foo = 'bar' - port = base.Port(options=options) - # This call should have no effect. - port.set_option_default('foo', 'new_bar') - self.assertEqual(port.get_option('foo'), 'bar') - def test_name__unset(self): port = base.Port() self.assertEqual(port.name(), None) @@ -263,7 +250,6 @@ class VirtualTest(unittest.TestCase): self.assertVirtual(port.test_platform_name) self.assertVirtual(port.results_directory) self.assertVirtual(port.test_expectations) - self.assertVirtual(port.test_base_platform_names) self.assertVirtual(port.test_platform_name) self.assertVirtual(port.test_platforms) self.assertVirtual(port.test_platform_name_to_name, None) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py index ad1bea6..7d56fa2 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py @@ -41,16 +41,9 @@ import webbrowser from webkitpy.common.system import executive from webkitpy.common.system.path import cygpath from webkitpy.layout_tests.layout_package import test_expectations -from webkitpy.layout_tests.layout_package import test_output - -import base -import http_server - -# Chromium DRT on OSX uses WebKitDriver. -if sys.platform == 'darwin': - import webkit - -import websocket_server +from webkitpy.layout_tests.port import base +from webkitpy.layout_tests.port import http_server +from webkitpy.layout_tests.port import websocket_server _log = logging.getLogger("webkitpy.layout_tests.port.chromium") @@ -176,8 +169,6 @@ class ChromiumPort(base.Port): return result def driver_name(self): - if self._options.use_test_shell: - return "test_shell" return "DumpRenderTree" def path_from_chromium_base(self, *comps): @@ -189,7 +180,7 @@ class ChromiumPort(base.Port): if offset == -1: self._chromium_base_dir = self._filesystem.join( abspath[0:abspath.find('Tools')], - 'WebKit', 'chromium') + 'Source', 'WebKit', 'chromium') else: self._chromium_base_dir = abspath[0:offset] return self._filesystem.join(self._chromium_base_dir, *comps) @@ -217,8 +208,6 @@ class ChromiumPort(base.Port): def create_driver(self, worker_number): """Starts a new Driver and returns a handle to it.""" - if not self.get_option('use_test_shell') and sys.platform == 'darwin': - return webkit.WebKitDriver(self, worker_number) return ChromiumDriver(self, worker_number) def start_helper(self): @@ -241,9 +230,6 @@ class ChromiumPort(base.Port): # http://bugs.python.org/issue1731717 self._helper.wait() - def test_base_platform_names(self): - return ('linux', 'mac', 'win') - def test_expectations(self): """Returns the test expectations for this port. @@ -273,15 +259,14 @@ class ChromiumPort(base.Port): all_test_files.update(extra_test_files) expectations = test_expectations.TestExpectations( - self, all_test_files, expectations_str, test_platform_name, - is_debug_mode, is_lint_mode=False, overrides=overrides_str) + self, all_test_files, expectations_str, self.test_configuration(), + is_lint_mode=False, overrides=overrides_str) tests_dir = self.layout_tests_dir() return [self.relative_test_filename(test) for test in expectations.get_tests_with_result_type(test_expectations.SKIP)] def test_platform_names(self): - return self.test_base_platform_names() + ('win-xp', - 'win-vista', 'win-7') + return ('mac', 'win', 'linux', 'win-xp', 'win-vista', 'win-7') def test_platform_name_to_name(self, test_platform_name): if test_platform_name in self.test_platform_names(): @@ -340,13 +325,11 @@ class ChromiumPort(base.Port): def _path_to_image_diff(self): binary_name = 'ImageDiff' - if self.get_option('use_test_shell'): - binary_name = 'image_diff' return self._build_path(self.get_option('configuration'), binary_name) class ChromiumDriver(base.Driver): - """Abstract interface for test_shell.""" + """Abstract interface for DRT.""" def __init__(self, port, worker_number): self._port = port @@ -365,10 +348,7 @@ class ChromiumDriver(base.Driver): cmd.append("--pixel-tests=" + self._port._convert_path(self._image_path)) - if self._port.get_option('use_test_shell'): - cmd.append('--layout-tests') - else: - cmd.append('--test-shell') + cmd.append('--test-shell') if self._port.get_option('startup_dialog'): cmd.append('--testshell-startup-dialog') @@ -385,14 +365,12 @@ class ChromiumDriver(base.Driver): if self._port.get_option('stress_deopt'): cmd.append('--stress-deopt') - # test_shell does not support accelerated compositing. - if not self._port.get_option("use_test_shell"): - if self._port.get_option('accelerated_compositing'): - cmd.append('--enable-accelerated-compositing') - if self._port.get_option('accelerated_2d_canvas'): - cmd.append('--enable-accelerated-2d-canvas') - if self._port.get_option('enable_hardware_gpu'): - cmd.append('--enable-hardware-gpu') + if self._port.get_option('accelerated_compositing'): + cmd.append('--enable-accelerated-compositing') + if self._port.get_option('accelerated_2d_canvas'): + cmd.append('--enable-accelerated-2d-canvas') + if self._port.get_option('enable_hardware_gpu'): + cmd.append('--enable-hardware-gpu') return cmd def start(self): @@ -420,17 +398,17 @@ class ChromiumDriver(base.Driver): try: if input: if isinstance(input, unicode): - # TestShell expects utf-8 + # DRT expects utf-8 input = input.encode("utf-8") self._proc.stdin.write(input) # DumpRenderTree text output is always UTF-8. However some tests # (e.g. webarchive) may spit out binary data instead of text so we - # don't bother to decode the output (for either DRT or test_shell). + # don't bother to decode the output. line = self._proc.stdout.readline() # We could assert() here that line correctly decodes as UTF-8. return (line, False) except IOError, e: - _log.error("IOError communicating w/ test_shell: " + str(e)) + _log.error("IOError communicating w/ DRT: " + str(e)) return (None, True) def _test_shell_command(self, uri, timeoutms, checksum): @@ -465,7 +443,7 @@ class ChromiumDriver(base.Driver): raise e return self._output_image() - def run_test(self, test_input): + def run_test(self, driver_input): output = [] error = [] crash = False @@ -475,9 +453,9 @@ class ChromiumDriver(base.Driver): start_time = time.time() - uri = self._port.filename_to_uri(test_input.filename) - cmd = self._test_shell_command(uri, test_input.timeout, - test_input.image_hash) + uri = self._port.filename_to_uri(driver_input.filename) + cmd = self._test_shell_command(uri, driver_input.timeout, + driver_input.image_hash) (line, crash) = self._write_command_and_read_line(input=cmd) while not crash and line.rstrip() != "#EOF": @@ -485,7 +463,7 @@ class ChromiumDriver(base.Driver): if line == '' and self.poll() is not None: # This is hex code 0xc000001d, which is used for abrupt # termination. This happens if we hit ctrl+c from the prompt - # and we happen to be waiting on test_shell. + # and we happen to be waiting on DRT. # sdoyon: Not sure for which OS and in what circumstances the # above code is valid. What works for me under Linux to detect # ctrl+c is for the subprocess returncode to be negative @@ -519,7 +497,7 @@ class ChromiumDriver(base.Driver): (line, crash) = self._write_command_and_read_line(input=None) run_time = time.time() - start_time - return test_output.TestOutput( + return base.DriverOutput( ''.join(output), self._output_image_with_retry(), actual_checksum, crash, run_time, timeout, ''.join(error)) @@ -532,8 +510,8 @@ class ChromiumDriver(base.Driver): if sys.platform not in ('win32', 'cygwin'): # Closing stdin/stdout/stderr hangs sometimes on OS X, # (see __init__(), above), and anyway we don't want to hang - # the harness if test_shell is buggy, so we wait a couple - # seconds to give test_shell a chance to clean up, but then + # the harness if DRT is buggy, so we wait a couple + # seconds to give DRT a chance to clean up, but then # force-kill the process if necessary. KILL_TIMEOUT = 3.0 timeout = time.time() + KILL_TIMEOUT diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py index b88d8aa..e8c75c4 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py @@ -30,64 +30,64 @@ import chromium_linux import chromium_mac import chromium_win +from webkitpy.layout_tests.port import test_files -def get(**kwargs): + +def get(platform=None, port_name='chromium-gpu', **kwargs): """Some tests have slightly different results when run while using hardware acceleration. In those cases, we prepend an additional directory to the baseline paths.""" - port_name = kwargs.get('port_name', None) + platform = platform or sys.platform if port_name == 'chromium-gpu': - if sys.platform in ('cygwin', 'win32'): + if platform in ('cygwin', 'win32'): port_name = 'chromium-gpu-win' - elif sys.platform == 'linux2': + elif platform == 'linux2': port_name = 'chromium-gpu-linux' - elif sys.platform == 'darwin': + elif platform == 'darwin': port_name = 'chromium-gpu-mac' else: - raise NotImplementedError('unsupported platform: %s' % - sys.platform) + raise NotImplementedError('unsupported platform: %s' % platform) if port_name == 'chromium-gpu-linux': - return ChromiumGpuLinuxPort(**kwargs) - - if port_name.startswith('chromium-gpu-mac'): - return ChromiumGpuMacPort(**kwargs) - - if port_name.startswith('chromium-gpu-win'): - return ChromiumGpuWinPort(**kwargs) - + return ChromiumGpuLinuxPort(port_name=port_name, **kwargs) + if port_name == 'chromium-gpu-mac': + return ChromiumGpuMacPort(port_name=port_name, **kwargs) + if port_name == 'chromium-gpu-win': + return ChromiumGpuWinPort(port_name=port_name, **kwargs) raise NotImplementedError('unsupported port: %s' % port_name) -def _set_gpu_options(options): - if options: - if options.accelerated_compositing is None: - options.accelerated_compositing = True - if options.accelerated_2d_canvas is None: - options.accelerated_2d_canvas = True +# FIXME: These should really be a mixin class. - # FIXME: Remove this after http://codereview.chromium.org/5133001/ is enabled - # on the bots. - if options.builder_name is not None and not ' - GPU' in options.builder_name: - options.builder_name = options.builder_name + ' - GPU' +def _set_gpu_options(port): + if port.get_option('accelerated_compositing') is None: + port._options.accelerated_compositing = True + if port.get_option('accelerated_2d_canvas') is None: + port._options.accelerated_2d_canvas = True + # FIXME: Remove this after http://codereview.chromium.org/5133001/ is enabled + # on the bots. + if port.get_option('builder_name') is not None and not ' - GPU' in port._options.builder_name: + port._options.builder_name += ' - GPU' -def _gpu_overrides(port): - try: - overrides_path = port.path_from_chromium_base('webkit', 'tools', - 'layout_tests', 'test_expectations_gpu.txt') - except AssertionError: - return None - if not port._filesystem.exists(overrides_path): - return None - return port._filesystem.read_text_file(overrides_path) + +def _tests(port, paths): + if not paths: + paths = ['compositing', 'platform/chromium/compositing'] + if not port.name().startswith('chromium-gpu-mac'): + # Canvas is not yet accelerated on the Mac, so there's no point + # in running the tests there. + paths += ['fast/canvas', 'canvas/philip'] + # invalidate_rect.html tests a bug in the compositor. + # See https://bugs.webkit.org/show_bug.cgi?id=53117 + paths += ['plugins/invalidate_rect.html'] + return test_files.find(port, paths) class ChromiumGpuLinuxPort(chromium_linux.ChromiumLinuxPort): - def __init__(self, **kwargs): - kwargs.setdefault('port_name', 'chromium-gpu-linux') - _set_gpu_options(kwargs.get('options')) - chromium_linux.ChromiumLinuxPort.__init__(self, **kwargs) + def __init__(self, port_name='chromium-gpu-linux', **kwargs): + chromium_linux.ChromiumLinuxPort.__init__(self, port_name=port_name, **kwargs) + _set_gpu_options(self) def baseline_search_path(self): # Mimic the Linux -> Win expectations fallback in the ordinary Chromium port. @@ -97,19 +97,18 @@ class ChromiumGpuLinuxPort(chromium_linux.ChromiumLinuxPort): def default_child_processes(self): return 1 - def path_to_test_expectations_file(self): - return self.path_from_webkit_base('LayoutTests', 'platform', - 'chromium-gpu', 'test_expectations.txt') + def graphics_type(self): + return 'gpu' + + def tests(self, paths): + return _tests(self, paths) - def test_expectations_overrides(self): - return _gpu_overrides(self) class ChromiumGpuMacPort(chromium_mac.ChromiumMacPort): - def __init__(self, **kwargs): - kwargs.setdefault('port_name', 'chromium-gpu-mac') - _set_gpu_options(kwargs.get('options')) - chromium_mac.ChromiumMacPort.__init__(self, **kwargs) + def __init__(self, port_name='chromium-gpu-mac', **kwargs): + chromium_mac.ChromiumMacPort.__init__(self, port_name=port_name, **kwargs) + _set_gpu_options(self) def baseline_search_path(self): return (map(self._webkit_baseline_path, ['chromium-gpu-mac', 'chromium-gpu']) + @@ -118,19 +117,18 @@ class ChromiumGpuMacPort(chromium_mac.ChromiumMacPort): def default_child_processes(self): return 1 - def path_to_test_expectations_file(self): - return self.path_from_webkit_base('LayoutTests', 'platform', - 'chromium-gpu', 'test_expectations.txt') + def graphics_type(self): + return 'gpu' + + def tests(self, paths): + return _tests(self, paths) - def test_expectations_overrides(self): - return _gpu_overrides(self) class ChromiumGpuWinPort(chromium_win.ChromiumWinPort): - def __init__(self, **kwargs): - kwargs.setdefault('port_name', 'chromium-gpu-win' + self.version()) - _set_gpu_options(kwargs.get('options')) - chromium_win.ChromiumWinPort.__init__(self, **kwargs) + def __init__(self, port_name='chromium-gpu-win', **kwargs): + chromium_win.ChromiumWinPort.__init__(self, port_name=port_name, **kwargs) + _set_gpu_options(self) def baseline_search_path(self): return (map(self._webkit_baseline_path, ['chromium-gpu-win', 'chromium-gpu']) + @@ -139,9 +137,8 @@ class ChromiumGpuWinPort(chromium_win.ChromiumWinPort): def default_child_processes(self): return 1 - def path_to_test_expectations_file(self): - return self.path_from_webkit_base('LayoutTests', 'platform', - 'chromium-gpu', 'test_expectations.txt') + def graphics_type(self): + return 'gpu' - def test_expectations_overrides(self): - return _gpu_overrides(self) + def tests(self, paths): + return _tests(self, paths) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py index 0bfb127..96962ec 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py @@ -40,20 +40,34 @@ class ChromiumGpuTest(unittest.TestCase): def test_get_chromium_gpu_win(self): self.assertOverridesWorked('chromium-gpu-win') - def assertOverridesWorked(self, port_name): + def test_get_chromium_gpu__on_linux(self): + self.assertOverridesWorked('chromium-gpu-linux', 'chromium-gpu', 'linux2') + + def test_get_chromium_gpu__on_mac(self): + self.assertOverridesWorked('chromium-gpu-mac', 'chromium-gpu', 'darwin') + + def test_get_chromium_gpu__on_win(self): + self.assertOverridesWorked('chromium-gpu-win', 'chromium-gpu', 'win32') + self.assertOverridesWorked('chromium-gpu-win', 'chromium-gpu', 'cygwin') + + def assertOverridesWorked(self, port_name, input_name=None, platform=None): # test that we got the right port mock_options = mocktool.MockOptions(accelerated_compositing=None, accelerated_2d_canvas=None, builder_name='foo', child_processes=None) - port = chromium_gpu.get(port_name=port_name, options=mock_options) + if input_name and platform: + port = chromium_gpu.get(platform=platform, port_name=input_name, + options=mock_options) + else: + port = chromium_gpu.get(port_name=port_name, options=mock_options) self.assertTrue(port._options.accelerated_compositing) self.assertTrue(port._options.accelerated_2d_canvas) self.assertEqual(port.default_child_processes(), 1) self.assertEqual(port._options.builder_name, 'foo - GPU') - # we use startswith() instead of Equal to gloss over platform versions. - self.assertTrue(port.name().startswith(port_name)) + # We don't support platform-specific versions of the GPU port yet. + self.assertEqual(port.name(), port_name) # test that it has the right directories in front of the search path. paths = port.baseline_search_path() @@ -64,9 +78,24 @@ class ChromiumGpuTest(unittest.TestCase): else: self.assertEqual(port._webkit_baseline_path('chromium-gpu'), paths[1]) - # Test that we have the right expectations file. - self.assertTrue('chromium-gpu' in - port.path_to_test_expectations_file()) + + # Test that we're limiting to the correct directories. + # These two tests are picked mostly at random, but we make sure they + # exist separately from being filtered out by the port. + files = port.tests(None) + + path = port.abspath_for_test('compositing/checkerboard.html') + self.assertTrue(port._filesystem.exists(path)) + self.assertTrue(path in files) + + path = port.abspath_for_test('fast/html/keygen.html') + self.assertTrue(port._filesystem.exists(path)) + self.assertFalse(path in files) + if port_name.startswith('chromium-gpu-mac'): + path = port.abspath_for_test('fast/canvas/set-colors.html') + self.assertTrue(port._filesystem.exists(path)) + self.assertFalse(path in files) + if __name__ == '__main__': unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py index c1c85f8..c3c5a21 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py @@ -85,7 +85,7 @@ class ChromiumLinuxPort(chromium.ChromiumPort): base = self.path_from_chromium_base() if self._filesystem.exists(self._filesystem.join(base, 'sconsbuild')): return self._filesystem.join(base, 'sconsbuild', *comps) - if self._filesystem.exists(self._filesystem.join(base, 'out', *comps)) or self.get_option('use_test_shell'): + if self._filesystem.exists(self._filesystem.join(base, 'out', *comps)): return self._filesystem.join(base, 'out', *comps) base = self.path_from_webkit_base() if self._filesystem.exists(self._filesystem.join(base, 'sconsbuild')): @@ -153,8 +153,6 @@ class ChromiumLinuxPort(chromium.ChromiumPort): if not configuration: configuration = self.get_option('configuration') binary_name = 'DumpRenderTree' - if self.get_option('use_test_shell'): - binary_name = 'test_shell' return self._build_path(configuration, binary_name) def _path_to_helper(self): diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py index 5360ab3..17862a2 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py @@ -69,18 +69,18 @@ class ChromiumMacPort(chromium.ChromiumPort): return result def default_child_processes(self): - # FIXME: we need to run single-threaded for now. See - # https://bugs.webkit.org/show_bug.cgi?id=38553. Unfortunately this - # routine is called right before the logger is configured, so if we - # try to _log.warning(), it gets thrown away. - import sys - sys.stderr.write("Defaulting to one child - see https://bugs.webkit.org/show_bug.cgi?id=38553\n") - return 1 + if self.get_option('worker_model') == 'old-threads': + # FIXME: we need to run single-threaded for now. See + # https://bugs.webkit.org/show_bug.cgi?id=38553. Unfortunately this + # routine is called right before the logger is configured, so if we + # try to _log.warning(), it gets thrown away. + import sys + sys.stderr.write("Defaulting to one child - see https://bugs.webkit.org/show_bug.cgi?id=38553\n") + return 1 + + return chromium.ChromiumPort.default_child_processes(self) def driver_name(self): - """name for this port's equivalent of DumpRenderTree.""" - if self.get_option('use_test_shell'): - return "TestShell" return "DumpRenderTree" def test_platform_name(self): @@ -110,7 +110,7 @@ class ChromiumMacPort(chromium.ChromiumPort): *comps) path = self.path_from_chromium_base('xcodebuild', *comps) - if self._filesystem.exists(path) or self.get_option('use_test_shell'): + if self._filesystem.exists(path): return path return self.path_from_webkit_base( 'Source', 'WebKit', 'chromium', 'xcodebuild', *comps) @@ -154,8 +154,6 @@ class ChromiumMacPort(chromium.ChromiumPort): def _path_to_helper(self): binary_name = 'LayoutTestHelper' - if self.get_option('use_test_shell'): - binary_name = 'layout_test_helper' return self._build_path(self.get_option('configuration'), binary_name) def _path_to_wdiff(self): diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py index 6c8987b..b89c8cc 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py @@ -116,13 +116,6 @@ class ChromiumPortTest(unittest.TestCase): port = ChromiumPortTest.TestMacPort(options=mock_options) self.assertTrue(port._path_to_image_diff().endswith( '/xcodebuild/default/ImageDiff')) - mock_options = mocktool.MockOptions(use_test_shell=True) - port = ChromiumPortTest.TestLinuxPort(options=mock_options) - self.assertTrue(port._path_to_image_diff().endswith( - '/out/default/image_diff'), msg=port._path_to_image_diff()) - port = ChromiumPortTest.TestMacPort(options=mock_options) - self.assertTrue(port._path_to_image_diff().endswith( - '/xcodebuild/default/image_diff')) # FIXME: Figure out how this is going to work on Windows. #port = chromium_win.ChromiumWinPort('test-port', options=MockOptions()) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py index 14f2777..f4cbf80 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py @@ -113,9 +113,9 @@ class ChromiumWinPort(chromium.ChromiumPort): if self._filesystem.exists(p): return p p = self.path_from_chromium_base('chrome', *comps) - if self._filesystem.exists(p) or self.get_option('use_test_shell'): + if self._filesystem.exists(p): return p - return self._filesystem.join(self.path_from_webkit_base(), 'WebKit', 'chromium', *comps) + return self._filesystem.join(self.path_from_webkit_base(), 'Source', 'WebKit', 'chromium', *comps) def _lighttpd_path(self, *comps): return self.path_from_chromium_base('third_party', 'lighttpd', 'win', @@ -141,20 +141,14 @@ class ChromiumWinPort(chromium.ChromiumPort): if not configuration: configuration = self.get_option('configuration') binary_name = 'DumpRenderTree.exe' - if self.get_option('use_test_shell'): - binary_name = 'test_shell.exe' return self._build_path(configuration, binary_name) def _path_to_helper(self): binary_name = 'LayoutTestHelper.exe' - if self.get_option('use_test_shell'): - binary_name = 'layout_test_helper.exe' return self._build_path(self.get_option('configuration'), binary_name) def _path_to_image_diff(self): binary_name = 'ImageDiff.exe' - if self.get_option('use_test_shell'): - binary_name = 'image_diff.exe' return self._build_path(self.get_option('configuration'), binary_name) def _path_to_wdiff(self): diff --git a/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py b/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py index 4ed34e6..6b3bd51 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py @@ -50,8 +50,6 @@ import os import sys import time -from webkitpy.layout_tests.layout_package import test_output - import base import factory @@ -71,6 +69,12 @@ class DryRunPort(object): def __getattr__(self, name): return getattr(self.__delegate, name) + def acquire_http_lock(self): + pass + + def release_http_lock(self): + pass + def check_build(self, needs_http): return True @@ -112,18 +116,18 @@ class DryrunDriver(base.Driver): def poll(self): return None - def run_test(self, test_input): + def run_test(self, driver_input): start_time = time.time() - text_output = self._port.expected_text(test_input.filename) + text_output = self._port.expected_text(driver_input.filename) - if test_input.image_hash is not None: - image = self._port.expected_image(test_input.filename) - hash = self._port.expected_checksum(test_input.filename) + if driver_input.image_hash is not None: + image = self._port.expected_image(driver_input.filename) + hash = self._port.expected_checksum(driver_input.filename) else: image = None hash = None - return test_output.TestOutput(text_output, image, hash, False, - time.time() - start_time, False, None) + return base.DriverOutput(text_output, image, hash, False, + time.time() - start_time, False, '') def start(self): pass diff --git a/Tools/Scripts/webkitpy/layout_tests/port/factory.py b/Tools/Scripts/webkitpy/layout_tests/port/factory.py index 6935744..7ae6eb6 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/factory.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/factory.py @@ -70,12 +70,15 @@ def _get_kwargs(**kwargs): raise NotImplementedError('unknown port; sys.platform = "%s"' % sys.platform) - if port_to_use == 'test': + if port_to_use.startswith('test'): import test maker = test.TestPort elif port_to_use.startswith('dryrun'): import dryrun maker = dryrun.DryRunPort + elif port_to_use.startswith('mock-'): + import mock_drt + maker = mock_drt.MockDRTPort elif port_to_use.startswith('mac'): import mac maker = mac.MacPort diff --git a/Tools/Scripts/webkitpy/layout_tests/port/http_server_base.py b/Tools/Scripts/webkitpy/layout_tests/port/http_server_base.py index 52a0403..2a43e81 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/http_server_base.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/http_server_base.py @@ -67,7 +67,7 @@ class HttpServerBase(object): url = 'http%s://127.0.0.1:%d/' % (http_suffix, mapping['port']) try: - response = urllib.urlopen(url) + response = urllib.urlopen(url, proxies={}) _log.debug("Server running at %s" % url) except IOError, e: _log.debug("Server NOT running at %s: %s" % (url, e)) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mac.py b/Tools/Scripts/webkitpy/layout_tests/port/mac.py index 0622196..1398ed3 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/mac.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/mac.py @@ -33,9 +33,7 @@ import os import platform import signal -import webkitpy.common.system.ospath as ospath -import webkitpy.layout_tests.port.server_process as server_process -from webkitpy.layout_tests.port.webkit import WebKitPort, WebKitDriver +from webkitpy.layout_tests.port.webkit import WebKitPort _log = logging.getLogger("webkitpy.layout_tests.port.mac") @@ -52,7 +50,7 @@ class MacPort(WebKitPort): # four threads in parallel. # See https://bugs.webkit.org/show_bug.cgi?id=36622 child_processes = WebKitPort.default_child_processes(self) - if child_processes > 4: + if self.get_option('worker_model') == 'old-threads' and child_processes > 4: return 4 return child_processes diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py index d383a4c..ef04679 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py @@ -35,23 +35,31 @@ import port_testcase class MacTest(port_testcase.PortTestCase): - def make_port(self, options=port_testcase.mock_options): + def make_port(self, port_name=None, options=port_testcase.mock_options): if sys.platform != 'darwin': return None - port_obj = mac.MacPort(options=options) + port_obj = mac.MacPort(port_name=port_name, options=options) port_obj._options.results_directory = port_obj.results_directory() port_obj._options.configuration = 'Release' return port_obj - def test_skipped_file_paths(self): - port = self.make_port() + def assert_skipped_files_for_version(self, port_name, expected_paths): + port = self.make_port(port_name) if not port: return skipped_paths = port._skipped_file_paths() # FIXME: _skipped_file_paths should return WebKit-relative paths. # So to make it unit testable, we strip the WebKit directory from the path. relative_paths = [path[len(port.path_from_webkit_base()):] for path in skipped_paths] - self.assertEqual(relative_paths, ['LayoutTests/platform/mac-leopard/Skipped', 'LayoutTests/platform/mac/Skipped']) + self.assertEqual(relative_paths, expected_paths) + + def test_skipped_file_paths(self): + self.assert_skipped_files_for_version('mac', + ['/LayoutTests/platform/mac/Skipped']) + self.assert_skipped_files_for_version('mac-snowleopard', + ['/LayoutTests/platform/mac-snowleopard/Skipped', '/LayoutTests/platform/mac/Skipped']) + self.assert_skipped_files_for_version('mac-leopard', + ['/LayoutTests/platform/mac-leopard/Skipped', '/LayoutTests/platform/mac/Skipped']) example_skipped_file = u""" # <rdar://problem/5647952> fast/events/mouseout-on-window.html needs mac DRT to issue mouse out events @@ -69,12 +77,11 @@ svg/batik/text/smallFonts.svg "svg/batik/text/smallFonts.svg", ] - def test_skipped_file_paths(self): + def test_tests_from_skipped_file_contents(self): port = self.make_port() if not port: return - skipped_file = StringIO.StringIO(self.example_skipped_file) - self.assertEqual(port._tests_from_skipped_file(skipped_file), self.example_skipped_tests) + self.assertEqual(port._tests_from_skipped_file_contents(self.example_skipped_file), self.example_skipped_tests) if __name__ == '__main__': diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mock_drt.py b/Tools/Scripts/webkitpy/layout_tests/port/mock_drt.py new file mode 100644 index 0000000..1147846 --- /dev/null +++ b/Tools/Scripts/webkitpy/layout_tests/port/mock_drt.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python +# Copyright (C) 2011 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the Google name nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""" +This is an implementation of the Port interface that overrides other +ports and changes the Driver binary to "MockDRT". +""" + +import logging +import optparse +import os +import sys + +from webkitpy.common.system import filesystem + +from webkitpy.layout_tests.port import base +from webkitpy.layout_tests.port import factory + +_log = logging.getLogger(__name__) + + +class MockDRTPort(object): + """MockPort implementation of the Port interface.""" + + def __init__(self, **kwargs): + prefix = 'mock-' + if 'port_name' in kwargs: + kwargs['port_name'] = kwargs['port_name'][len(prefix):] + self.__delegate = factory.get(**kwargs) + self.__real_name = prefix + self.__delegate.name() + + def real_name(self): + return self.__real_name + + def __getattr__(self, name): + return getattr(self.__delegate, name) + + def acquire_http_lock(self): + pass + + def release_http_lock(self): + pass + + def check_build(self, needs_http): + return True + + def check_sys_deps(self, needs_http): + return True + + def driver_cmd_line(self): + driver = self.create_driver(0) + return driver.cmd_line() + + def _path_to_driver(self): + return os.path.abspath(__file__) + + def create_driver(self, worker_number): + # We need to create a driver object as the delegate would, but + # overwrite the path to the driver binary in its command line. We do + # this by actually overwriting its cmd_line() method with a proxy + # method that splices in the mock_drt path and command line arguments + # in place of the actual path to the driver binary. + + def overriding_cmd_line(): + cmd = self.__original_driver_cmd_line() + index = cmd.index(self.__delegate._path_to_driver()) + cmd[index:index + 1] = [sys.executable, self._path_to_driver(), + '--platform', self.name()] + return cmd + + delegated_driver = self.__delegate.create_driver(worker_number) + self.__original_driver_cmd_line = delegated_driver.cmd_line + delegated_driver.cmd_line = overriding_cmd_line + return delegated_driver + + def start_helper(self): + pass + + def start_http_server(self): + pass + + def start_websocket_server(self): + pass + + def stop_helper(self): + pass + + def stop_http_server(self): + pass + + def stop_websocket_server(self): + pass + + +def main(argv, fs, stdin, stdout, stderr): + """Run the tests.""" + + options, args = parse_options(argv) + if options.chromium: + drt = MockChromiumDRT(options, args, fs, stdin, stdout, stderr) + else: + drt = MockDRT(options, args, fs, stdin, stdout, stderr) + return drt.run() + + +def parse_options(argv): + # FIXME: We have to do custom arg parsing instead of using the optparse + # module. First, Chromium and non-Chromium DRTs have a different argument + # syntax. Chromium uses --pixel-tests=<path>, and non-Chromium uses + # --pixel-tests as a boolean flag. Second, we don't want to have to list + # every command line flag DRT accepts, but optparse complains about + # unrecognized flags. At some point it might be good to share a common + # DRT options class between this file and webkit.py and chromium.py + # just to get better type checking. + platform_index = argv.index('--platform') + platform = argv[platform_index + 1] + + pixel_tests = False + pixel_path = None + chromium = False + if platform.startswith('chromium'): + chromium = True + for arg in argv: + if arg.startswith('--pixel-tests'): + pixel_tests = True + pixel_path = arg[len('--pixel-tests='):] + else: + pixel_tests = '--pixel-tests' in argv + options = base.DummyOptions(chromium=chromium, + platform=platform, + pixel_tests=pixel_tests, + pixel_path=pixel_path) + return (options, []) + + +# FIXME: Should probably change this to use DriverInput after +# https://bugs.webkit.org/show_bug.cgi?id=53004 lands. +class _DRTInput(object): + def __init__(self, line): + vals = line.strip().split("'") + if len(vals) == 1: + self.uri = vals[0] + self.checksum = None + else: + self.uri = vals[0] + self.checksum = vals[1] + + +class MockDRT(object): + def __init__(self, options, args, filesystem, stdin, stdout, stderr): + self._options = options + self._args = args + self._filesystem = filesystem + self._stdout = stdout + self._stdin = stdin + self._stderr = stderr + + port_name = None + if options.platform: + port_name = options.platform + self._port = factory.get(port_name, options=options, filesystem=filesystem) + + def run(self): + while True: + line = self._stdin.readline() + if not line: + break + self.run_one_test(self.parse_input(line)) + return 0 + + def parse_input(self, line): + return _DRTInput(line) + + def run_one_test(self, test_input): + port = self._port + if test_input.uri.startswith('http'): + test_name = port.uri_to_test_name(test_input.uri) + test_path = self._filesystem.join(port.layout_tests_dir(), test_name) + else: + test_path = test_input.uri + + actual_text = port.expected_text(test_path) + if self._options.pixel_tests and test_input.checksum: + actual_checksum = port.expected_checksum(test_path) + actual_image = port.expected_image(test_path) + + self._stdout.write('Content-Type: text/plain\n') + + # FIXME: Note that we don't ensure there is a trailing newline! + # This mirrors actual (Mac) DRT behavior but is a bug. + self._stdout.write(actual_text) + self._stdout.write('#EOF\n') + + if self._options.pixel_tests and test_input.checksum: + self._stdout.write('\n') + self._stdout.write('ActualHash: %s\n' % actual_checksum) + self._stdout.write('ExpectedHash: %s\n' % test_input.checksum) + if actual_checksum != test_input.checksum: + self._stdout.write('Content-Type: image/png\n') + self._stdout.write('Content-Length: %s\n\n' % len(actual_image)) + self._stdout.write(actual_image) + self._stdout.write('#EOF\n') + self._stdout.flush() + self._stderr.flush() + + +# FIXME: Should probably change this to use DriverInput after +# https://bugs.webkit.org/show_bug.cgi?id=53004 lands. +class _ChromiumDRTInput(_DRTInput): + def __init__(self, line): + vals = line.strip().split() + if len(vals) == 3: + self.uri, self.timeout, self.checksum = vals + else: + self.uri = vals[0] + self.timeout = vals[1] + self.checksum = None + + +class MockChromiumDRT(MockDRT): + def parse_input(self, line): + return _ChromiumDRTInput(line) + + def run_one_test(self, test_input): + port = self._port + test_name = self._port.uri_to_test_name(test_input.uri) + test_path = self._filesystem.join(port.layout_tests_dir(), test_name) + + actual_text = port.expected_text(test_path) + actual_image = '' + actual_checksum = '' + if self._options.pixel_tests and test_input.checksum: + actual_checksum = port.expected_checksum(test_path) + if actual_checksum != test_input.checksum: + actual_image = port.expected_image(test_path) + + self._stdout.write("#URL:%s\n" % test_input.uri) + if self._options.pixel_tests and test_input.checksum: + self._stdout.write("#MD5:%s\n" % actual_checksum) + self._filesystem.write_binary_file(self._options.pixel_path, + actual_image) + self._stdout.write(actual_text) + + # FIXME: (See above FIXME as well). Chromium DRT appears to always + # ensure the text output has a trailing newline. Mac DRT does not. + if not actual_text.endswith('\n'): + self._stdout.write('\n') + self._stdout.write('#EOF\n') + self._stdout.flush() + + +if __name__ == '__main__': + fs = filesystem.FileSystem() + sys.exit(main(sys.argv[1:], fs, sys.stdin, sys.stdout, sys.stderr)) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py new file mode 100644 index 0000000..1506315 --- /dev/null +++ b/Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python +# Copyright (C) 2011 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit tests for MockDRT.""" + +import unittest + +from webkitpy.common import newstringio + +from webkitpy.layout_tests.port import mock_drt +from webkitpy.layout_tests.port import factory +from webkitpy.layout_tests.port import port_testcase +from webkitpy.layout_tests.port import test + + +class MockDRTPortTest(port_testcase.PortTestCase): + def make_port(self): + return mock_drt.MockDRTPort() + + def test_port_name_in_constructor(self): + self.assertTrue(mock_drt.MockDRTPort(port_name='mock-test')) + + def test_acquire_http_lock(self): + # Only checking that no exception is raised. + self.make_port().acquire_http_lock() + + def test_release_http_lock(self): + # Only checking that no exception is raised. + self.make_port().release_http_lock() + + def test_check_build(self): + port = self.make_port() + self.assertTrue(port.check_build(True)) + + def test_check_sys_deps(self): + port = self.make_port() + self.assertTrue(port.check_sys_deps(True)) + + def test_start_helper(self): + # Only checking that no exception is raised. + self.make_port().start_helper() + + def test_start_http_server(self): + # Only checking that no exception is raised. + self.make_port().start_http_server() + + def test_start_websocket_server(self): + # Only checking that no exception is raised. + self.make_port().start_websocket_server() + + def test_stop_helper(self): + # Only checking that no exception is raised. + self.make_port().stop_helper() + + def test_stop_http_server(self): + # Only checking that no exception is raised. + self.make_port().stop_http_server() + + def test_stop_websocket_server(self): + # Only checking that no exception is raised. + self.make_port().stop_websocket_server() + + +class MockDRTTest(unittest.TestCase): + def to_path(self, port, test_name): + return port._filesystem.join(port.layout_tests_dir(), test_name) + + def input_line(self, port, test_name, checksum=None): + url = port.filename_to_uri(self.to_path(port, test_name)) + # FIXME: we shouldn't have to work around platform-specific issues + # here. + if url.startswith('file:////'): + url = url[len('file:////') - 1:] + if url.startswith('file:///'): + url = url[len('file:///') - 1:] + + if checksum: + return url + "'" + checksum + '\n' + return url + '\n' + + def extra_args(self, pixel_tests): + if pixel_tests: + return ['--pixel-tests', '-'] + return ['-'] + + def make_drt(self, options, args, filesystem, stdin, stdout, stderr): + return mock_drt.MockDRT(options, args, filesystem, stdin, stdout, stderr) + + def make_input_output(self, port, test_name, pixel_tests, + expected_checksum, drt_output, drt_input=None): + path = self.to_path(port, test_name) + if pixel_tests: + if not expected_checksum: + expected_checksum = port.expected_checksum(path) + if not drt_input: + drt_input = self.input_line(port, test_name, expected_checksum) + text_output = port.expected_text(path) + + if not drt_output: + drt_output = self.expected_output(port, test_name, pixel_tests, + text_output, expected_checksum) + return (drt_input, drt_output) + + def expected_output(self, port, test_name, pixel_tests, text_output, expected_checksum): + if pixel_tests and expected_checksum: + return ['Content-Type: text/plain\n', + text_output, + '#EOF\n', + '\n', + 'ActualHash: %s\n' % expected_checksum, + 'ExpectedHash: %s\n' % expected_checksum, + '#EOF\n'] + else: + return ['Content-Type: text/plain\n', + text_output, + '#EOF\n', + '#EOF\n'] + + def assertTest(self, test_name, pixel_tests, expected_checksum=None, + drt_output=None, filesystem=None): + platform = 'test' + filesystem = filesystem or test.unit_test_filesystem() + port = factory.get(platform, filesystem=filesystem) + drt_input, drt_output = self.make_input_output(port, test_name, + pixel_tests, expected_checksum, drt_output) + + args = ['--platform', 'test'] + self.extra_args(pixel_tests) + stdin = newstringio.StringIO(drt_input) + stdout = newstringio.StringIO() + stderr = newstringio.StringIO() + options, args = mock_drt.parse_options(args) + + drt = self.make_drt(options, args, filesystem, stdin, stdout, stderr) + res = drt.run() + + self.assertEqual(res, 0) + + # We use the StringIO.buflist here instead of getvalue() because + # the StringIO might be a mix of unicode/ascii and 8-bit strings. + self.assertEqual(stdout.buflist, drt_output) + self.assertEqual(stderr.getvalue(), '') + + def test_main(self): + filesystem = test.unit_test_filesystem() + stdin = newstringio.StringIO() + stdout = newstringio.StringIO() + stderr = newstringio.StringIO() + res = mock_drt.main(['--platform', 'test'] + self.extra_args(False), + filesystem, stdin, stdout, stderr) + self.assertEqual(res, 0) + self.assertEqual(stdout.getvalue(), '') + self.assertEqual(stderr.getvalue(), '') + self.assertEqual(filesystem.written_files, {}) + + def test_pixeltest_passes(self): + # This also tests that we handle HTTP: test URLs properly. + self.assertTest('http/tests/passes/text.html', True) + + def test_pixeltest__fails(self): + self.assertTest('failures/expected/checksum.html', pixel_tests=True, + expected_checksum='wrong-checksum', + drt_output=['Content-Type: text/plain\n', + 'checksum-txt', + '#EOF\n', + '\n', + 'ActualHash: checksum-checksum\n', + 'ExpectedHash: wrong-checksum\n', + 'Content-Type: image/png\n', + 'Content-Length: 13\n\n', + 'checksum\x8a-png', + '#EOF\n']) + + def test_textonly(self): + self.assertTest('passes/image.html', False) + + +class MockChromiumDRTTest(MockDRTTest): + def extra_args(self, pixel_tests): + if pixel_tests: + return ['--pixel-tests=/tmp/png_result0.png'] + return [] + + def make_drt(self, options, args, filesystem, stdin, stdout, stderr): + options.chromium = True + + # We have to set these by hand because --platform test won't trigger + # the Chromium code paths. + options.pixel_path = '/tmp/png_result0.png' + options.pixel_tests = True + + return mock_drt.MockChromiumDRT(options, args, filesystem, stdin, stdout, stderr) + + def input_line(self, port, test_name, checksum=None): + url = port.filename_to_uri(self.to_path(port, test_name)) + if checksum: + return url + ' 6000 ' + checksum + '\n' + return url + ' 6000\n' + + def expected_output(self, port, test_name, pixel_tests, text_output, expected_checksum): + url = port.filename_to_uri(self.to_path(port, test_name)) + if pixel_tests and expected_checksum: + return ['#URL:%s\n' % url, + '#MD5:%s\n' % expected_checksum, + text_output, + '\n', + '#EOF\n'] + else: + return ['#URL:%s\n' % url, + text_output, + '\n', + '#EOF\n'] + + def test_pixeltest__fails(self): + filesystem = test.unit_test_filesystem() + self.assertTest('failures/expected/checksum.html', pixel_tests=True, + expected_checksum='wrong-checksum', + drt_output=['#URL:file:///test.checkout/LayoutTests/failures/expected/checksum.html\n', + '#MD5:checksum-checksum\n', + 'checksum-txt', + '\n', + '#EOF\n'], + filesystem=filesystem) + self.assertEquals(filesystem.written_files, + {'/tmp/png_result0.png': 'checksum\x8a-png'}) + + def test_chromium_parse_options(self): + options, args = mock_drt.parse_options(['--platform', 'chromium-mac', + '--pixel-tests=/tmp/png_result0.png']) + self.assertTrue(options.chromium) + self.assertTrue(options.pixel_tests) + self.assertEquals(options.pixel_path, '/tmp/png_result0.png') + + +if __name__ == '__main__': + unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py b/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py index 0b03b4c..4146d40 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py @@ -88,3 +88,15 @@ class PortTestCase(unittest.TestCase): return port.start_websocket_server() port.stop_websocket_server() + + def test_test_configuration(self): + port = self.make_port() + if not port: + return + self.assertTrue(port.test_configuration()) + + def test_all_test_configurations(self): + port = self.make_port() + if not port: + return + self.assertTrue(len(port.all_test_configurations()) > 0) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/server_process.py b/Tools/Scripts/webkitpy/layout_tests/port/server_process.py index 5a0a40c..7974f94 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/server_process.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/server_process.py @@ -115,7 +115,11 @@ class ServerProcess: if is not already running.""" if not self._proc: self._start() - self._proc.stdin.write(input) + try: + self._proc.stdin.write(input) + except IOError, e: + self.stop() + self.crashed = True def read_line(self, timeout): """Read a single line from the subprocess, waiting until the deadline. diff --git a/Tools/Scripts/webkitpy/layout_tests/port/server_process_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/server_process_unittest.py new file mode 100644 index 0000000..f3429cb --- /dev/null +++ b/Tools/Scripts/webkitpy/layout_tests/port/server_process_unittest.py @@ -0,0 +1,77 @@ +# Copyright (C) 2011 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.layout_tests.port import server_process + + +class MockFile(object): + def __init__(self, server_process): + self._server_process = server_process + + def fileno(self): + return 1 + + def write(self, line): + self._server_process.broken_pipes.append(self) + raise IOError + + def close(self): + pass + + +class MockProc(object): + def __init__(self, server_process): + self.stdin = MockFile(server_process) + self.stdout = MockFile(server_process) + self.stderr = MockFile(server_process) + self.pid = 1 + + def poll(self): + return 1 + + +class FakeServerProcess(server_process.ServerProcess): + def _start(self): + self._proc = MockProc(self) + self.stdin = self._proc.stdin + self.broken_pipes = [] + + +class TestServerProcess(unittest.TestCase): + def test_broken_pipe(self): + server_process = FakeServerProcess(port_obj=None, name="test", cmd=["test"]) + server_process.write("should break") + self.assertTrue(server_process.crashed) + self.assertEquals(server_process._proc, None) + self.assertEquals(server_process.broken_pipes, [server_process.stdin]) + + +if __name__ == '__main__': + unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/test.py b/Tools/Scripts/webkitpy/layout_tests/port/test.py index 5df5c2d..b94c378 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/test.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/test.py @@ -33,8 +33,7 @@ from __future__ import with_statement import time from webkitpy.common.system import filesystem_mock - -from webkitpy.layout_tests.layout_package import test_output +from webkitpy.tool import mocktool import base @@ -51,9 +50,17 @@ class TestInstance: self.keyboard = False self.error = '' self.timeout = False - self.actual_text = self.base + '-txt\n' - self.actual_checksum = self.base + '-checksum\n' - self.actual_image = self.base + '-png\n' + + # The values of each field are treated as raw byte strings. They + # will be converted to unicode strings where appropriate using + # MockFileSystem.read_text_file(). + self.actual_text = self.base + '-txt' + self.actual_checksum = self.base + '-checksum' + + # We add the '\x8a' for the image file to prevent the value from + # being treated as UTF-8 (the character is invalid) + self.actual_image = self.base + '\x8a' + '-png' + self.expected_text = self.actual_text self.expected_checksum = self.actual_checksum self.expected_image = self.actual_image @@ -84,53 +91,44 @@ class TestList: def unit_test_list(): tests = TestList() tests.add('failures/expected/checksum.html', - actual_checksum='checksum_fail-checksum') + actual_checksum='checksum_fail-checksum') tests.add('failures/expected/crash.html', crash=True) tests.add('failures/expected/exception.html', exception=True) tests.add('failures/expected/timeout.html', timeout=True) tests.add('failures/expected/hang.html', hang=True) - tests.add('failures/expected/missing_text.html', - expected_text=None) + tests.add('failures/expected/missing_text.html', expected_text=None) tests.add('failures/expected/image.html', - actual_image='image_fail-png', - expected_image='image-png') + actual_image='image_fail-png', + expected_image='image-png') tests.add('failures/expected/image_checksum.html', - actual_checksum='image_checksum_fail-checksum', - actual_image='image_checksum_fail-png') - tests.add('failures/expected/keyboard.html', - keyboard=True) - tests.add('failures/expected/missing_check.html', - expected_checksum=None) - tests.add('failures/expected/missing_image.html', - expected_image=None) - tests.add('failures/expected/missing_text.html', - expected_text=None) + actual_checksum='image_checksum_fail-checksum', + actual_image='image_checksum_fail-png') + tests.add('failures/expected/keyboard.html', keyboard=True) + tests.add('failures/expected/missing_check.html', expected_checksum=None) + tests.add('failures/expected/missing_image.html', expected_image=None) + tests.add('failures/expected/missing_text.html', expected_text=None) tests.add('failures/expected/newlines_leading.html', - expected_text="\nfoo\n", - actual_text="foo\n") + expected_text="\nfoo\n", actual_text="foo\n") tests.add('failures/expected/newlines_trailing.html', - expected_text="foo\n\n", - actual_text="foo\n") + expected_text="foo\n\n", actual_text="foo\n") tests.add('failures/expected/newlines_with_excess_CR.html', - expected_text="foo\r\r\r\n", - actual_text="foo\n") - tests.add('failures/expected/text.html', - actual_text='text_fail-png') + expected_text="foo\r\r\r\n", actual_text="foo\n") + tests.add('failures/expected/text.html', actual_text='text_fail-png') tests.add('failures/unexpected/crash.html', crash=True) tests.add('failures/unexpected/text-image-checksum.html', - actual_text='text-image-checksum_fail-txt', - actual_checksum='text-image-checksum_fail-checksum') + actual_text='text-image-checksum_fail-txt', + actual_checksum='text-image-checksum_fail-checksum') tests.add('failures/unexpected/timeout.html', timeout=True) tests.add('http/tests/passes/text.html') tests.add('http/tests/ssl/text.html') tests.add('passes/error.html', error='stuff going to stderr') tests.add('passes/image.html') tests.add('passes/platform_image.html') + # Text output files contain "\r\n" on Windows. This may be # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling. tests.add('passes/text.html', - expected_text='\nfoo\n\n', - actual_text='\nfoo\r\n\r\r\n') + expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n') tests.add('websocket/tests/passes/text.html') return tests @@ -184,6 +182,9 @@ WONTFIX SKIP : failures/expected/keyboard.html = CRASH WONTFIX SKIP : failures/expected/exception.html = CRASH """ + # Add in a file should be ignored by test_files.find(). + files[LAYOUT_TEST_DIR + 'userscripts/resources/iframe.html'] = 'iframe' + fs = filesystem_mock.MockFileSystem(files) fs._tests = test_list return fs @@ -192,30 +193,31 @@ WONTFIX SKIP : failures/expected/exception.html = CRASH class TestPort(base.Port): """Test implementation of the Port interface.""" - def __init__(self, **kwargs): - # FIXME: what happens if we're not passed in the test filesystem - # and the tests don't match what's in the filesystem? - # - # We'll leave as is for now to avoid unnecessary dependencies while - # converting all of the unit tests over to using - # unit_test_filesystem(). If things get out of sync the tests should - # fail in fairly obvious ways. Eventually we want to just do: - # - # assert kwargs['filesystem']._tests - # self._tests = kwargs['filesystem']._tests + def __init__(self, port_name=None, user=None, filesystem=None, **kwargs): + if not filesystem: + filesystem = unit_test_filesystem() + + assert filesystem._tests + self._tests = filesystem._tests + + if not user: + user = mocktool.MockUser() - if 'filesystem' not in kwargs or kwargs['filesystem'] is None: - kwargs['filesystem'] = unit_test_filesystem() - self._tests = kwargs['filesystem']._tests - else: - self._tests = unit_test_list() + if not port_name or port_name == 'test': + port_name = 'test-mac' - kwargs.setdefault('port_name', 'test') - base.Port.__init__(self, **kwargs) + self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt' + base.Port.__init__(self, port_name=port_name, filesystem=filesystem, user=user, + **kwargs) + + def _path_to_driver(self): + # This routine shouldn't normally be called, but it is called by + # the mock_drt Driver. We return something, but make sure it's useless. + return 'junk' def baseline_path(self): - return self._filesystem.join(self.layout_tests_dir(), 'platform', - self.name() + self.version()) + # We don't bother with a fallback path. + return self._filesystem.join(self.layout_tests_dir(), 'platform', self.name()) def baseline_search_path(self): return [self.baseline_path()] @@ -223,11 +225,14 @@ class TestPort(base.Port): def check_build(self, needs_http): return True + def default_configuration(self): + return 'Release' + def diff_image(self, expected_contents, actual_contents, diff_filename=None): diffed = actual_contents != expected_contents if diffed and diff_filename: - self._filesystem.write_text_file(diff_filename, + self._filesystem.write_binary_file(diff_filename, "< %s\n---\n> %s\n" % (expected_contents, actual_contents)) return diffed @@ -261,23 +266,98 @@ class TestPort(base.Port): def stop_websocket_server(self): pass - def test_base_platform_names(self): - return ('mac', 'win') - - def test_expectations(self): - return self._filesystem.read_text_file(LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt') + def path_to_test_expectations_file(self): + return self._expectations_path def test_platform_name(self): - return 'mac' + name_map = { + 'test-mac': 'mac', + 'test-win': 'win', + 'test-win-xp': 'win-xp', + } + return name_map[self._name] def test_platform_names(self): - return self.test_base_platform_names() + return ('mac', 'win', 'win-xp') def test_platform_name_to_name(self, test_platform_name): - return test_platform_name + name_map = { + 'mac': 'test-mac', + 'win': 'test-win', + 'win-xp': 'test-win-xp', + } + return name_map[test_platform_name] + + # FIXME: These next two routines are copied from base.py with + # the calls to path.abspath_to_uri() removed. We shouldn't have + # to do this. + def filename_to_uri(self, filename): + """Convert a test file (which is an absolute path) to a URI.""" + LAYOUTTEST_HTTP_DIR = "http/tests/" + LAYOUTTEST_WEBSOCKET_DIR = "http/tests/websocket/tests/" + + relative_path = self.relative_test_filename(filename) + port = None + use_ssl = False + + if (relative_path.startswith(LAYOUTTEST_WEBSOCKET_DIR) + or relative_path.startswith(LAYOUTTEST_HTTP_DIR)): + relative_path = relative_path[len(LAYOUTTEST_HTTP_DIR):] + port = 8000 + + # Make http/tests/local run as local files. This is to mimic the + # logic in run-webkit-tests. + # + # TODO(dpranke): remove the media reference and the SSL reference? + if (port and not relative_path.startswith("local/") and + not relative_path.startswith("media/")): + if relative_path.startswith("ssl/"): + port += 443 + protocol = "https" + else: + protocol = "http" + return "%s://127.0.0.1:%u/%s" % (protocol, port, relative_path) + + return "file://" + self._filesystem.abspath(filename) + + def uri_to_test_name(self, uri): + """Return the base layout test name for a given URI. + + This returns the test name for a given URI, e.g., if you passed in + "file:///src/LayoutTests/fast/html/keygen.html" it would return + "fast/html/keygen.html". + + """ + test = uri + if uri.startswith("file:///"): + prefix = "file://" + self.layout_tests_dir() + "/" + return test[len(prefix):] + + if uri.startswith("http://127.0.0.1:8880/"): + # websocket tests + return test.replace('http://127.0.0.1:8880/', '') + + if uri.startswith("http://"): + # regular HTTP test + return test.replace('http://127.0.0.1:8000/', 'http/tests/') + + if uri.startswith("https://"): + return test.replace('https://127.0.0.1:8443/', 'http/tests/') + + raise NotImplementedError('unknown url type: %s' % uri) def version(self): - return '' + version_map = { + 'test-win-xp': '-xp', + 'test-win': '-7', + 'test-mac': '-leopard', + } + return version_map[self._name] + + def test_configuration(self): + if not self._test_configuration: + self._test_configuration = TestTestConfiguration(self) + return self._test_configuration class TestDriver(base.Driver): @@ -287,7 +367,7 @@ class TestDriver(base.Driver): self._port = port def cmd_line(self): - return ['None'] + return [self._port._path_to_driver()] def poll(self): return True @@ -302,13 +382,20 @@ class TestDriver(base.Driver): raise ValueError('exception from ' + test_name) if test.hang: time.sleep((float(test_input.timeout) * 4) / 1000.0) - return test_output.TestOutput(test.actual_text, test.actual_image, - test.actual_checksum, test.crash, - time.time() - start_time, test.timeout, - test.error) + return base.DriverOutput(test.actual_text, test.actual_image, + test.actual_checksum, test.crash, + time.time() - start_time, test.timeout, + test.error) def start(self): pass def stop(self): pass + + +class TestTestConfiguration(base.TestConfiguration): + def all_systems(self): + return (('mac', 'leopard', 'x86'), + ('win', 'xp', 'x86'), + ('win', 'win7', 'x86')) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/test_files.py b/Tools/Scripts/webkitpy/layout_tests/port/test_files.py index 41d918f..534462a 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/test_files.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/test_files.py @@ -49,37 +49,47 @@ _supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.xhtmlmp _skipped_directories = set(['.svn', '_svn', 'resources', 'script-tests']) -def find(port, paths): - """Finds the set of tests under port.layout_tests_dir(). +def find(port, paths=None): + """Finds the set of tests under a given list of sub-paths. Args: - paths: a list of command line paths relative to the layout_tests_dir() - to limit the search to. glob patterns are ok. + paths: a list of path expressions relative to port.layout_tests_dir() + to search. Glob patterns are ok, as are path expressions with + forward slashes on Windows. If paths is empty, we look at + everything under the layout_tests_dir(). + """ + paths = paths or ['*'] + filesystem = port._filesystem + return normalized_find(filesystem, normalize(filesystem, port.layout_tests_dir(), paths)) + + +def normalize(filesystem, base_dir, paths): + return [filesystem.normpath(filesystem.join(base_dir, path)) for path in paths] + + +def normalized_find(filesystem, paths): + """Finds the set of tests under the list of paths. + + Args: + paths: a list of absolute path expressions to search. + Glob patterns are ok. """ - fs = port._filesystem gather_start_time = time.time() paths_to_walk = set() - # if paths is empty, provide a pre-defined list. - if paths: - _log.debug("Gathering tests from: %s relative to %s" % (paths, port.layout_tests_dir())) - for path in paths: - # If there's an * in the name, assume it's a glob pattern. - path = fs.join(port.layout_tests_dir(), path) - if path.find('*') > -1: - filenames = fs.glob(path) - paths_to_walk.update(filenames) - else: - paths_to_walk.add(path) - else: - _log.debug("Gathering tests from: %s" % port.layout_tests_dir()) - paths_to_walk.add(port.layout_tests_dir()) + for path in paths: + # If there's an * in the name, assume it's a glob pattern. + if path.find('*') > -1: + filenames = filesystem.glob(path) + paths_to_walk.update(filenames) + else: + paths_to_walk.add(path) # FIXME: I'm not sure there's much point in this being a set. A list would # probably be faster. test_files = set() for path in paths_to_walk: - files = fs.files_under(path, _skipped_directories, _is_test_file) + files = filesystem.files_under(path, _skipped_directories, _is_test_file) test_files.update(set(files)) gather_time = time.time() - gather_start_time @@ -88,10 +98,10 @@ def find(port, paths): return test_files -def _has_supported_extension(fs, filename): +def _has_supported_extension(filesystem, filename): """Return true if filename is one of the file extensions we want to run a test on.""" - extension = fs.splitext(filename)[1] + extension = filesystem.splitext(filename)[1] return extension in _supported_file_extensions @@ -104,7 +114,7 @@ def _is_reference_html_file(filename): return False -def _is_test_file(fs, dirname, filename): +def _is_test_file(filesystem, dirname, filename): """Return true if the filename points to a test file.""" - return (_has_supported_extension(fs, filename) and + return (_has_supported_extension(filesystem, filename) and not _is_reference_html_file(filename)) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py index a68950a..a29ba49 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py @@ -26,44 +26,41 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import sys import unittest -import base +from webkitpy.layout_tests.port import test import test_files - class TestFilesTest(unittest.TestCase): def test_find_no_paths_specified(self): - port = base.Port() + port = test.TestPort() layout_tests_dir = port.layout_tests_dir() - port.layout_tests_dir = lambda: port._filesystem.join(layout_tests_dir, - 'fast', 'html') tests = test_files.find(port, []) - self.assertNotEqual(tests, 0) + self.assertNotEqual(len(tests), 0) def test_find_one_test(self): - port = base.Port() - # This is just a test picked at random but known to exist. - tests = test_files.find(port, ['fast/html/keygen.html']) + port = test.TestPort() + tests = test_files.find(port, ['failures/expected/image.html']) self.assertEqual(len(tests), 1) def test_find_glob(self): - port = base.Port() - tests = test_files.find(port, ['fast/html/key*']) - self.assertEqual(len(tests), 1) + port = test.TestPort() + tests = test_files.find(port, ['failures/expected/im*']) + self.assertEqual(len(tests), 2) def test_find_with_skipped_directories(self): - port = base.Port() + port = test.TestPort() tests = port.tests('userscripts') - self.assertTrue('userscripts/resources/frame1.html' not in tests) + self.assertTrue('userscripts/resources/iframe.html' not in tests) def test_find_with_skipped_directories_2(self): - port = base.Port() + port = test.TestPort() tests = test_files.find(port, ['userscripts/resources']) self.assertEqual(tests, set([])) def test_is_test_file(self): - port = base.Port() + port = test.TestPort() fs = port._filesystem self.assertTrue(test_files._is_test_file(fs, '', 'foo.html')) self.assertTrue(test_files._is_test_file(fs, '', 'foo.shtml')) @@ -72,5 +69,33 @@ class TestFilesTest(unittest.TestCase): self.assertFalse(test_files._is_test_file(fs, '', 'foo-expected-mismatch.html')) +class MockWinFileSystem(object): + def join(self, *paths): + return '\\'.join(paths) + + def normpath(self, path): + return path.replace('/', '\\') + + +class TestWinNormalize(unittest.TestCase): + def assert_filesystem_normalizes(self, filesystem): + self.assertEquals(test_files.normalize(filesystem, "c:\\foo", + ['fast/html', 'fast/canvas/*', 'compositing/foo.html']), + ['c:\\foo\\fast\html', 'c:\\foo\\fast\canvas\*', 'c:\\foo\compositing\\foo.html']) + + def test_mocked_win(self): + # This tests test_files.normalize, using portable behavior emulating + # what we think Windows is supposed to do. This test will run on all + # platforms. + self.assert_filesystem_normalizes(MockWinFileSystem()) + + def test_win(self): + # This tests the actual windows platform, to ensure we get the same + # results that we get in test_mocked_win(). + if sys.platform != 'win': + return + self.assert_filesystem_normalizes(FileSystem()) + + if __name__ == '__main__': unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/webkit.py b/Tools/Scripts/webkitpy/layout_tests/port/webkit.py index 577acd4..65a047d 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/webkit.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/webkit.py @@ -40,10 +40,9 @@ import sys import time import webbrowser -import webkitpy.common.system.ospath as ospath -import webkitpy.layout_tests.layout_package.test_output as test_output -import webkitpy.layout_tests.port.base as base -import webkitpy.layout_tests.port.server_process as server_process +from webkitpy.common.system import ospath +from webkitpy.layout_tests.port import base +from webkitpy.layout_tests.port import server_process _log = logging.getLogger("webkitpy.layout_tests.port.webkit") @@ -57,7 +56,8 @@ class WebKitPort(base.Port): # FIXME: disable pixel tests until they are run by default on the # build machines. - self.set_option_default('pixel_tests', False) + if not hasattr(self._options, "pixel_tests") or self._options.pixel_tests == None: + self._options.pixel_tests = False def baseline_path(self): return self._webkit_baseline_path(self._name) @@ -120,9 +120,9 @@ class WebKitPort(base.Port): return self._diff_image_reply(sp, diff_filename) def _diff_image_request(self, expected_contents, actual_contents): - # FIXME: use self.get_option('tolerance') and - # self.set_option_default('tolerance', 0.1) once that behaves correctly - # with default values. + # FIXME: There needs to be a more sane way of handling default + # values for options so that you can distinguish between a default + # value of None and a default value that wasn't set. if self.get_option('tolerance') is not None: tolerance = self.get_option('tolerance') else: @@ -159,7 +159,7 @@ class WebKitPort(base.Port): if m.group(2) == 'passed': result = False elif output and diff_filename: - self._filesystem.write_text_file(diff_filename, output) + self._filesystem.write_binary_file(diff_filename, output) elif sp.timed_out: _log.error("ImageDiff timed out") elif sp.crashed: @@ -179,11 +179,6 @@ class WebKitPort(base.Port): def create_driver(self, worker_number): return WebKitDriver(self, worker_number) - def test_base_platform_names(self): - # At the moment we don't use test platform names, but we have - # to return something. - return ('mac', 'win') - def _tests_for_other_platforms(self): raise NotImplementedError('WebKitPort._tests_for_other_platforms') # The original run-webkit-tests builds up a "whitelist" of tests to @@ -283,9 +278,9 @@ class WebKitPort(base.Port): unsupported_feature_tests = self._skipped_tests_for_unsupported_features() return disabled_feature_tests + webarchive_tests + unsupported_feature_tests - def _tests_from_skipped_file(self, skipped_file): + def _tests_from_skipped_file_contents(self, skipped_file_contents): tests_to_skip = [] - for line in skipped_file.readlines(): + for line in skipped_file_contents.split('\n'): line = line.strip() if line.startswith('#') or not len(line): continue @@ -301,7 +296,8 @@ class WebKitPort(base.Port): if not self._filesystem.exists(filename): _log.warn("Failed to open Skipped file: %s" % filename) continue - skipped_file = self._filesystem.read_text_file(filename) + skipped_file_contents = self._filesystem.read_text_file(filename) + tests_to_skip.extend(self._tests_from_skipped_file_contents(skipped_file_contents)) return tests_to_skip def test_expectations(self): @@ -335,8 +331,7 @@ class WebKitPort(base.Port): return self._name + self.version() def test_platform_names(self): - return self.test_base_platform_names() + ( - 'mac-tiger', 'mac-leopard', 'mac-snowleopard') + return ('mac', 'win', 'mac-tiger', 'mac-leopard', 'mac-snowleopard') def _build_path(self, *comps): return self._filesystem.join(self._config.build_directory( @@ -409,15 +404,15 @@ class WebKitDriver(base.Driver): return # FIXME: This function is huge. - def run_test(self, test_input): - uri = self._port.filename_to_uri(test_input.filename) + def run_test(self, driver_input): + uri = self._port.filename_to_uri(driver_input.filename) if uri.startswith("file:///"): command = uri[7:] else: command = uri - if test_input.image_hash: - command += "'" + test_input.image_hash + if driver_input.image_hash: + command += "'" + driver_input.image_hash command += "\n" start_time = time.time() @@ -428,7 +423,7 @@ class WebKitDriver(base.Driver): output = str() # Use a byte array for output, even though it should be UTF-8. image = str() - timeout = int(test_input.timeout) / 1000.0 + timeout = int(driver_input.timeout) / 1000.0 deadline = time.time() + timeout line = self._server_process.read_line(timeout) while (not self._server_process.timed_out @@ -475,11 +470,11 @@ class WebKitDriver(base.Driver): # FIXME: This seems like the wrong section of code to be doing # this reset in. self._server_process.error = "" - return test_output.TestOutput(output, image, actual_image_hash, - self._server_process.crashed, - time.time() - start_time, - self._server_process.timed_out, - error) + return base.DriverOutput(output, image, actual_image_hash, + self._server_process.crashed, + time.time() - start_time, + self._server_process.timed_out, + error) def stop(self): if self._server_process: diff --git a/Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py index 7b68310..c72a411 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged +# Copyright (C) 2010 Google Inc. All rights reserved. # # All rights reserved. # @@ -26,13 +27,19 @@ import unittest +from webkitpy.common.system import filesystem_mock + from webkitpy.layout_tests.port.webkit import WebKitPort class TestWebKitPort(WebKitPort): - def __init__(self, symbol_list=None, feature_list=None): + def __init__(self, symbol_list=None, feature_list=None, + expectations_file=None, skips_file=None, **kwargs): self.symbol_list = symbol_list self.feature_list = feature_list + self.expectations_file = expectations_file + self.skips_file = skips_file + WebKitPort.__init__(self, **kwargs) def _runtime_feature_list(self): return self.feature_list @@ -46,7 +53,14 @@ class TestWebKitPort(WebKitPort): def _tests_for_disabled_features(self): return ["accessibility", ] + def path_to_test_expectations_file(self): + if self.expectations_file: + return self.expectations_file + return WebKitPort.path_to_test_expectations_file(self) + def _skipped_file_paths(self): + if self.skips_file: + return [self.skips_file] return [] class WebKitPortTest(unittest.TestCase): @@ -66,3 +80,23 @@ class WebKitPortTest(unittest.TestCase): def test_skipped_layout_tests(self): self.assertEqual(TestWebKitPort(None, None).skipped_layout_tests(), set(["media", "accessibility"])) + + def test_test_expectations(self): + # Check that we read both the expectations file and anything in a + # Skipped file, and that we include the feature and platform checks. + files = { + '/tmp/test_expectations.txt': 'BUG_TESTEXPECTATIONS SKIP : fast/html/article-element.html = FAIL\n', + '/tmp/Skipped': 'fast/html/keygen.html', + } + mock_fs = filesystem_mock.MockFileSystem(files) + port = TestWebKitPort(expectations_file='/tmp/test_expectations.txt', + skips_file='/tmp/Skipped', filesystem=mock_fs) + self.assertEqual(port.test_expectations(), + """BUG_TESTEXPECTATIONS SKIP : fast/html/article-element.html = FAIL +BUG_SKIPPED SKIP : fast/html/keygen.html = FAIL +BUG_SKIPPED SKIP : media = FAIL +BUG_SKIPPED SKIP : accessibility = FAIL""") + + +if __name__ == '__main__': + unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/websocket_server.py b/Tools/Scripts/webkitpy/layout_tests/port/websocket_server.py index 926bc04..713ad21 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/websocket_server.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/websocket_server.py @@ -73,7 +73,7 @@ def url_is_alive(url): wait_time = 10 while wait_time > 0: try: - response = urllib.urlopen(url) + response = urllib.urlopen(url, proxies={}) # Server is up and responding. return True except IOError: |