diff options
Diffstat (limited to 'WebKitTools/Scripts/webkitpy/layout_tests')
12 files changed, 576 insertions, 130 deletions
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py index 3e3ba0b..9f2de7e 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py @@ -341,7 +341,7 @@ class TestShellThread(WatchableThread): def cancel(self): """Clean up http lock and set a flag telling this thread to quit.""" - self._stop_http_lock() + self._stop_servers_with_lock() WatchableThread.cancel(self) def next_timeout(self): @@ -389,23 +389,15 @@ class TestShellThread(WatchableThread): self._current_group, self._filename_list = \ self._filename_list_queue.get_nowait() except Queue.Empty: - self._stop_http_lock() + self._stop_servers_with_lock() self._kill_dump_render_tree() tests_run_file.close() return - if self._options.wait_for_httpd: - if self._current_group == "tests_to_http_lock": - self._http_lock_wait_begin = time.time() - self._port.acquire_http_lock() - - self._port.start_http_server() - self._port.start_websocket_server() - - self._have_http_lock = True - self._http_lock_wait_end = time.time() - elif self._have_http_lock: - self._stop_http_lock() + if self._current_group == "tests_to_http_lock": + self._start_servers_with_lock() + elif self._have_http_lock: + self._stop_servers_with_lock() self._num_tests_in_current_group = len(self._filename_list) self._current_group_start_time = time.time() @@ -440,7 +432,7 @@ class TestShellThread(WatchableThread): self._port.relative_test_filename(filename))) self._result_queue.put(result.dumps()) - if batch_size > 0 and batch_count > batch_size: + if batch_size > 0 and batch_count >= batch_size: # Bounce the shell and reset count. self._kill_dump_render_tree() batch_count = 0 @@ -545,11 +537,26 @@ class TestShellThread(WatchableThread): self._options) self._driver.start() - def _stop_http_lock(self): + def _start_servers_with_lock(self): + """Acquire http lock and start the servers.""" + self._http_lock_wait_begin = time.time() + _log.debug('Acquire http lock ...') + self._port.acquire_http_lock() + _log.debug('Starting HTTP server ...') + self._port.start_http_server() + _log.debug('Starting WebSocket server ...') + self._port.start_websocket_server() + self._http_lock_wait_end = time.time() + self._have_http_lock = True + + def _stop_servers_with_lock(self): """Stop the servers and release http lock.""" if self._have_http_lock: + _log.debug('Stopping HTTP server ...') self._port.stop_http_server() + _log.debug('Stopping WebSocket server ...') self._port.stop_websocket_server() + _log.debug('Release http lock ...') self._port.release_http_lock() self._have_http_lock = False diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py index cd7d663..a98b858 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py @@ -47,6 +47,7 @@ import http_server import test_files import websocket_server +from webkitpy.common.memoized import memoized from webkitpy.common.system import logutils from webkitpy.common.system.executive import Executive, ScriptError from webkitpy.common.system.path import abspath_to_uri @@ -725,13 +726,25 @@ class Port(object): e.message_with_output())) return self._pretty_patch_error_html - def _webkit_build_directory(self, args): - args = ["perl", self.script_path("webkit-build-directory")] + args + def _webkit_build_directory_command(self, args): + return ["perl", self.script_path("webkit-build-directory")] + args + + @memoized + def _webkit_top_level_build_directory(self, top_level=True): + """This directory is above where products are built to and contains things like the Configuration file.""" + args = self._webkit_build_directory_command(["--top-level"]) + return self._executive.run_command(args).rstrip() + + @memoized + def _webkit_configuration_build_directory(self, configuration=None): + """This is where products are normally built to.""" + if not configuration: + configuration = self.flag_from_configuration(self.get_option('configuration')) + args = self._webkit_build_directory_command(["--configuration", configuration]) return self._executive.run_command(args).rstrip() def _configuration_file_path(self): - build_root = self._webkit_build_directory(["--top-level"]) - return os.path.join(build_root, "Configuration") + return os.path.join(self._webkit_top_level_build_directory(), "Configuration") # Easy override for unit tests def _open_configuration_file(self): diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py index 5d28fae..57b6989 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py @@ -71,6 +71,8 @@ def _set_gpu_options(options): options.accelerated_2d_canvas = True if options.builder_name is not None: options.builder_name = options.builder_name + ' - GPU' + if options.use_drt is None: + options.use_drt = True def _gpu_overrides(port): @@ -96,6 +98,9 @@ class ChromiumGpuLinuxPort(chromium_linux.ChromiumLinuxPort): return (map(self._webkit_baseline_path, ['chromium-gpu-linux', 'chromium-gpu-win', 'chromium-gpu']) + chromium_linux.ChromiumLinuxPort.baseline_search_path(self)) + def default_child_processes(self): + return 1 + def path_to_test_expectations_file(self): return self.path_from_webkit_base('LayoutTests', 'platform', 'chromium-gpu', 'test_expectations.txt') @@ -114,6 +119,9 @@ class ChromiumGpuMacPort(chromium_mac.ChromiumMacPort): return (map(self._webkit_baseline_path, ['chromium-gpu-mac', 'chromium-gpu']) + chromium_mac.ChromiumMacPort.baseline_search_path(self)) + def default_child_processes(self): + return 1 + def path_to_test_expectations_file(self): return self.path_from_webkit_base('LayoutTests', 'platform', 'chromium-gpu', 'test_expectations.txt') @@ -132,6 +140,9 @@ class ChromiumGpuWinPort(chromium_win.ChromiumWinPort): return (map(self._webkit_baseline_path, ['chromium-gpu-win', 'chromium-gpu']) + chromium_win.ChromiumWinPort.baseline_search_path(self)) + def default_child_processes(self): + return 1 + def path_to_test_expectations_file(self): return self.path_from_webkit_base('LayoutTests', 'platform', 'chromium-gpu', 'test_expectations.txt') diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py index 88524fc..03bc98a 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py @@ -45,10 +45,14 @@ class ChromiumGpuTest(unittest.TestCase): # test that we got the right port mock_options = mocktool.MockOptions(accelerated_compositing=None, accelerated_2d_canvas=None, - builder_name='foo') + builder_name='foo', + use_drt=None, + child_processes=None) port = chromium_gpu.get(port_name=port_name, options=mock_options) self.assertTrue(port._options.accelerated_compositing) self.assertTrue(port._options.accelerated_2d_canvas) + self.assertTrue(port._options.use_drt) + self.assertEqual(port.default_child_processes(), 1) self.assertEqual(port._options.builder_name, 'foo - GPU') # we use startswith() instead of Equal to gloss over platform versions. diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/config.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/config.py new file mode 100644 index 0000000..2364098 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/config.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Wrapper objects for WebKit-specific utility routines.""" + +# FIXME: This file needs to be unified with common/checkout/scm.py and +# common/config/ports.py . + +from __future__ import with_statement + +import codecs +import os + +from webkitpy.common.checkout import scm +from webkitpy.common.system import logutils + + +_log = logutils.get_logger(__file__) + +# +# This is used to record if we've already hit the filesystem to look +# for a default configuration. We cache this to speed up the unit tests, +# but this can be reset with clear_cached_configuration(). +# +_determined_configuration = None + + +def clear_cached_configuration(): + global _determined_configuration + _determined_configuration = -1 + + +class Config(object): + _FLAGS_FROM_CONFIGURATIONS = { + "Debug": "--debug", + "Release": "--release", + } + + def __init__(self, executive): + self._executive = executive + self._webkit_base_dir = None + self._default_configuration = None + self._build_directories = {} + + def build_directory(self, configuration): + """Returns the path to the build directory for the configuration.""" + if configuration: + flags = ["--configuration", + self._FLAGS_FROM_CONFIGURATIONS[configuration]] + configuration = "" + else: + configuration = "" + flags = ["--top-level"] + + if not self._build_directories.get(configuration): + args = ["perl", self._script_path("webkit-build-directory")] + flags + self._build_directories[configuration] = ( + self._executive.run_command(args).rstrip()) + + return self._build_directories[configuration] + + def build_dumprendertree(self, configuration): + """Builds DRT in the given configuration. + + Returns True if the build was successful and up-to-date.""" + flag = self._FLAGS_FROM_CONFIGURATIONS[configuration] + exit_code = self._executive.run_command([ + self._script_path("build-dumprendertree"), flag], + return_exit_code=True) + if exit_code != 0: + _log.error("Failed to build DumpRenderTree") + return False + return True + + def default_configuration(self): + """Returns the default configuration for the user. + + Returns the value set by 'set-webkit-configuration', or "Release" + if that has not been set. This mirrors the logic in webkitdirs.pm.""" + if not self._default_configuration: + self._default_configuration = self._determine_configuration() + if not self._default_configuration: + self._default_configuration = 'Release' + if self._default_configuration not in self._FLAGS_FROM_CONFIGURATIONS: + _log.warn("Configuration \"%s\" is not a recognized value.\n" % + self._default_configuration) + _log.warn("Scripts may fail. " + "See 'set-webkit-configuration --help'.") + return self._default_configuration + + def path_from_webkit_base(self, *comps): + return os.path.join(self.webkit_base_dir(), *comps) + + def webkit_base_dir(self): + """Returns the absolute path to the top of the WebKit tree. + + Raises an AssertionError if the top dir can't be determined.""" + # FIXME: Consider determining this independently of scm in order + # to be able to run in a bare tree. + if not self._webkit_base_dir: + self._webkit_base_dir = scm.find_checkout_root() + assert self._webkit_base_dir, "Could not determine the top of the WebKit checkout" + return self._webkit_base_dir + + def _script_path(self, script_name): + return os.path.join(self.webkit_base_dir(), "WebKitTools", + "Scripts", script_name) + + def _determine_configuration(self): + # This mirrors the logic in webkitdirs.pm:determineConfiguration(). + global _determined_configuration + if _determined_configuration == -1: + contents = self._read_configuration() + if contents == "Deployment": + contents = "Release" + if contents == "Development": + contents = "Debug" + _determined_configuration = contents + return _determined_configuration + + def _read_configuration(self): + configuration_path = os.path.join(self.build_directory(None), + "Configuration") + if not os.path.exists(configuration_path): + return None + + with codecs.open(configuration_path, "r", "utf-8") as fh: + return fh.read().rstrip() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/config_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/config_unittest.py new file mode 100644 index 0000000..4674cba --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/config_unittest.py @@ -0,0 +1,176 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import codecs +import os +import StringIO +import unittest + +from webkitpy.common.system import outputcapture + +import config + + +# FIXME: This makes StringIO objects work with "with". Remove +# when we upgrade to 2.6. +class NewStringIO(StringIO.StringIO): + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + pass + + +class MockExecutive(object): + def __init__(self, output='', exit_code=0): + self._output = output + self._exit_code = exit_code + + def run_command(self, arg_list, return_exit_code=False, + decode_output=False): + if return_exit_code: + return self._exit_code + return self._output + + +class ConfigTest(unittest.TestCase): + def tearDown(self): + config.clear_cached_configuration() + + def assertConfiguration(self, contents, expected): + # This tests that a configuration file containing + # _contents_ endsd up being interpreted as _expected_. + # + # FIXME: rewrite this when we have a filesystem abstraction we + # can mock out more easily. + config.clear_cached_configuration() + orig_open = codecs.open + + def wrap_open(contents): + def open_wrapper(path, mode, encoding): + return NewStringIO(contents) + return open_wrapper + + try: + orig_exists = os.path.exists + os.path.exists = lambda p: True + codecs.open = wrap_open(contents) + + e = MockExecutive(output='foo') + c = config.Config(e) + self.assertEqual(c.default_configuration(), expected) + finally: + os.path.exists = orig_exists + codecs.open = orig_open + + def test_build_directory_toplevel(self): + e = MockExecutive(output="toplevel") + c = config.Config(e) + self.assertEqual(c.build_directory(None), 'toplevel') + + # Test again to check caching + self.assertEqual(c.build_directory(None), 'toplevel') + + def test_build_directory__release(self): + e = MockExecutive(output="release") + c = config.Config(e) + self.assertEqual(c.build_directory('Release'), 'release') + + def test_build_directory__debug(self): + e = MockExecutive(output="debug") + c = config.Config(e) + self.assertEqual(c.build_directory('Debug'), 'debug') + + def test_build_directory__unknown(self): + e = MockExecutive(output="unknown") + c = config.Config(e) + self.assertRaises(KeyError, c.build_directory, 'Unknown') + + def test_build_dumprendertree__success(self): + e = MockExecutive(exit_code=0) + c = config.Config(e) + self.assertTrue(c.build_dumprendertree("Debug")) + self.assertTrue(c.build_dumprendertree("Release")) + self.assertRaises(KeyError, c.build_dumprendertree, "Unknown") + + def test_build_dumprendertree__failure(self): + e = MockExecutive(exit_code=-1) + c = config.Config(e) + + oc = outputcapture.OutputCapture() + oc.capture_output() + self.assertFalse(c.build_dumprendertree('Debug')) + (out, err) = oc.restore_output() + + oc.capture_output() + self.assertFalse(c.build_dumprendertree('Release')) + oc.restore_output() + + def test_default_configuration__release(self): + self.assertConfiguration('Release', 'Release') + + def test_default_configuration__debug(self): + self.assertConfiguration('Debug', 'Debug') + + def test_default_configuration__deployment(self): + self.assertConfiguration('Deployment', 'Release') + + def test_default_configuration__development(self): + self.assertConfiguration('Development', 'Debug') + + def test_default_configuration__notfound(self): + # This tests what happens if the default configuration file + # doesn't exist. + config.clear_cached_configuration() + try: + orig_exists = os.path.exists + os.path.exists = lambda p: False + e = MockExecutive(output="foo") + c = config.Config(e) + self.assertEqual(c.default_configuration(), "Release") + finally: + os.path.exists = orig_exists + + def test_default_configuration__unknown(self): + # Ignore the warning about an unknown configuration value. + oc = outputcapture.OutputCapture() + oc.capture_output() + self.assertConfiguration('Unknown', 'Unknown') + oc.restore_output() + + def test_path_from_webkit_base(self, *comps): + c = config.Config(None) + self.assertTrue(c.path_from_webkit_base('foo')) + + def test_webkit_base_dir(self): + c = config.Config(None) + self.assertTrue(c.webkit_base_dir()) + + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/http_lock.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/http_lock.py index b2615a3..d65801d 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/http_lock.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/http_lock.py @@ -29,27 +29,38 @@ http and websocket tests in a same time.""" import glob +import logging import os import sys import tempfile import time +from webkitpy.common.system.executive import Executive + + +_log = logging.getLogger("webkitpy.layout_tests.port.http_lock") + class HttpLock(object): def __init__(self, lock_path, lock_file_prefix="WebKitHttpd.lock.", guard_lock="WebKit.lock"): - if not lock_path: + self._lock_path = lock_path + if not self._lock_path: self._lock_path = tempfile.gettempdir() self._lock_file_prefix = lock_file_prefix self._lock_file_path_prefix = os.path.join(self._lock_path, self._lock_file_prefix) self._guard_lock_file = os.path.join(self._lock_path, guard_lock) self._process_lock_file_name = "" + self._executive = Executive() + # maximum wait time for the lock creation + self._guard_lock_max_wait = 1 * 60 def cleanup_http_lock(self): """Delete the lock file if exists.""" if os.path.exists(self._process_lock_file_name): + _log.debug("Removing lock file: %s" % self._process_lock_file_name) os.unlink(self._process_lock_file_name) def _extract_lock_number(self, lock_file_name): @@ -70,17 +81,6 @@ class HttpLock(object): return 0 return self._extract_lock_number(lock_list[-1]) + 1 - def _check_pid(self, current_pid): - """Return True if pid is alive, otherwise return False. - FIXME: os.kill() doesn't work on Windows for checking if - a pid is alive, so always return True""" - if sys.platform in ('darwin', 'linux2'): - try: - os.kill(current_pid, 0) - except OSError: - return False - return True - def _curent_lock_pid(self): """Return with the current lock pid. If the lock is not valid it deletes the lock file.""" @@ -91,7 +91,8 @@ class HttpLock(object): current_lock_file = open(lock_list[0], 'r') current_pid = current_lock_file.readline() current_lock_file.close() - if not (current_pid and self._check_pid(int(current_pid))): + if not (current_pid and self._executive.check_running_pid(int(current_pid))): + _log.debug("Removing stuck lock file: %s" % lock_list[0]) os.unlink(lock_list[0]) return except IOError, OSError: @@ -102,22 +103,34 @@ class HttpLock(object): """The lock files are used to schedule the running test sessions in first come first served order. The sequential guard lock ensures that the lock numbers are sequential.""" + if not os.path.exists(self._lock_path): + _log.debug("Lock directory does not exist: %s" % self._lock_path) + return False + + start_time = time.time() while(True): try: sequential_guard_lock = os.open(self._guard_lock_file, os.O_CREAT | os.O_EXCL) self._process_lock_file_name = (self._lock_file_path_prefix + str(self._next_lock_number())) lock_file = open(self._process_lock_file_name, 'w') + _log.debug("Creating lock file: %s" % self._process_lock_file_name) lock_file.write(str(os.getpid())) lock_file.close() os.close(sequential_guard_lock) os.unlink(self._guard_lock_file) - break + return True except OSError: - pass + if time.time() - start_time > self._guard_lock_max_wait: + _log.debug("Lock does not created: %s" % str(sys.exc_info())) + return False def wait_for_httpd_lock(self): - """Create a lock file and wait until it's turn comes.""" - self._create_lock_file() + """Create a lock file and wait until it's turn comes. If something goes wrong + it wont do any locking.""" + if not self._create_lock_file(): + _log.debug("Warning, http locking failed!") + return + while self._curent_lock_pid() != os.getpid(): time.sleep(1) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py index 0d0d3e0..0b324f5 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py @@ -60,7 +60,6 @@ class WebKitPort(base.Port): def __init__(self, **kwargs): base.Port.__init__(self, **kwargs) - self._cached_build_root = None self._cached_apache_path = None # FIXME: disable pixel tests until they are run by default on the @@ -358,12 +357,8 @@ class WebKitPort(base.Port): 'mac-tiger', 'mac-leopard', 'mac-snowleopard') def _build_path(self, *comps): - if not self._cached_build_root: - self._cached_build_root = self._webkit_build_directory([ - "--configuration", - self.flag_from_configuration(self.get_option('configuration')), - ]) - return os.path.join(self._cached_build_root, *comps) + build_root = self._webkit_configuration_build_directory() + return os.path.join(build_root, *comps) def _path_to_driver(self): return self._build_path('DumpRenderTree') diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py b/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py index 434058e..55c4558 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py @@ -70,11 +70,11 @@ _log = logging.getLogger("webkitpy.layout_tests." BASELINE_SUFFIXES = ['.txt', '.png', '.checksum'] REBASELINE_PLATFORM_ORDER = ['mac', 'win', 'win-xp', 'win-vista', 'linux'] -ARCHIVE_DIR_NAME_DICT = {'win': 'webkit-rel', +ARCHIVE_DIR_NAME_DICT = {'win': 'Webkit_Win', 'win-vista': 'webkit-dbg-vista', - 'win-xp': 'webkit-rel', - 'mac': 'webkit-rel-mac5', - 'linux': 'webkit-rel-linux', + 'win-xp': 'Webkit_Win', + 'mac': 'Webkit_Mac10_5', + 'linux': 'webkit-rel-linux64', 'win-canary': 'webkit-rel-webkit-org', 'win-vista-canary': 'webkit-dbg-vista', 'win-xp-canary': 'webkit-rel-webkit-org', @@ -819,9 +819,8 @@ def get_host_port_object(options): return port_obj -def main(executive=Executive()): - """Main function to produce new baselines.""" - +def parse_options(args): + """Parse options and return a pair of host options and target options.""" option_parser = optparse.OptionParser() option_parser.add_option('-v', '--verbose', action='store_true', @@ -874,7 +873,20 @@ def main(executive=Executive()): help=('The target platform to rebaseline ' '("mac", "chromium", "qt", etc.). Defaults ' 'to "chromium".')) - options = option_parser.parse_args()[0] + options = option_parser.parse_args(args)[0] + + target_options = copy.copy(options) + if options.target_platform == 'chromium': + target_options.chromium = True + options.tolerance = 0 + + return (options, target_options) + + +def main(executive=Executive()): + """Main function to produce new baselines.""" + + (options, target_options) = parse_options(sys.argv[1:]) # We need to create three different Port objects over the life of this # script. |target_port_obj| is used to determine configuration information: @@ -882,9 +894,6 @@ def main(executive=Executive()): # |port_obj| is used for runtime functionality like actually diffing # Then we create a rebaselining port to actual find and manage the # baselines. - target_options = copy.copy(options) - if options.target_platform == 'chromium': - target_options.chromium = True target_port_obj = port.get(None, target_options) # Set up our logging format. diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py index 8db31a6..7c55b94 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py @@ -95,6 +95,15 @@ class TestRebaseliner(unittest.TestCase): return rebaseline_chromium_webkit_tests.Rebaseliner( host_port_obj, target_port_obj, platform, options) + def test_parse_options(self): + (options, target_options) = rebaseline_chromium_webkit_tests.parse_options([]) + self.assertTrue(target_options.chromium) + self.assertEqual(options.tolerance, 0) + + (options, target_options) = rebaseline_chromium_webkit_tests.parse_options(['--target-platform', 'qt']) + self.assertFalse(hasattr(target_options, 'chromium')) + self.assertEqual(options.tolerance, 0) + def test_noop(self): # this method tests that was can at least instantiate an object, even # if there is nothing to do. diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py index 704180c..f360adc 100755 --- a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py @@ -85,6 +85,8 @@ _log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests") # Builder base URL where we have the archived test results. BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/" +LAYOUT_TESTS_DIRECTORY = "LayoutTests" + os.sep + TestExpectationsFile = test_expectations.TestExpectationsFile @@ -283,12 +285,17 @@ class TestRunner: last_unexpected_results: list of unexpected results to retest, if any """ - paths = [arg for arg in args if arg and arg != ''] + paths = [self._strip_test_dir_prefix(arg) for arg in args if arg and arg != ''] paths += last_unexpected_results if self._options.test_list: paths += read_test_files(self._options.test_list) self._test_files = self._port.tests(paths) + def _strip_test_dir_prefix(self, path): + if path.startswith(LAYOUT_TESTS_DIRECTORY): + return path[len(LAYOUT_TESTS_DIRECTORY):] + return path + def lint(self): # Creating the expecations for each platform/configuration pair does # all the test list parsing and ensures it's correct syntax (e.g. no @@ -404,9 +411,8 @@ class TestRunner: # If we reached the end and we don't have enough tests, we run some # from the beginning. - if (self._options.run_chunk and - (slice_end - slice_start < chunk_len)): - extra = 1 + chunk_len - (slice_end - slice_start) + if slice_end - slice_start < chunk_len: + extra = chunk_len - (slice_end - slice_start) extra_msg = (' last chunk is partial, appending [0:%d]' % extra) self._printer.print_expected(extra_msg) @@ -470,9 +476,9 @@ class TestRunner: def _get_dir_for_test_file(self, test_file): """Returns the highest-level directory by which to shard the given test file.""" - index = test_file.rfind(os.sep + 'LayoutTests' + os.sep) + index = test_file.rfind(os.sep + LAYOUT_TESTS_DIRECTORY) - test_file = test_file[index + len('LayoutTests/'):] + test_file = test_file[index + len(LAYOUT_TESTS_DIRECTORY):] test_file_parts = test_file.split(os.sep, 1) directory = test_file_parts[0] test_file = test_file_parts[1] @@ -741,17 +747,6 @@ class TestRunner: if not result_summary: return None - # Do not start when http locking is enabled. - if not self._options.wait_for_httpd: - if self.needs_http(): - self._printer.print_update('Starting HTTP server ...') - self._port.start_http_server() - - if self.needs_websocket(): - self._printer.print_update('Starting WebSocket server ...') - self._port.start_websocket_server() - # self._websocket_secure_server.Start() - return result_summary def run(self, result_summary): @@ -841,11 +836,6 @@ class TestRunner: sys.stdout.flush() _log.debug("flushing stderr") sys.stderr.flush() - if not self._options.wait_for_httpd: - _log.debug("stopping http server") - self._port.stop_http_server() - _log.debug("stopping websocket server") - self._port.stop_websocket_server() _log.debug("stopping helper") self._port.stop_helper() @@ -948,14 +938,15 @@ class TestRunner: if not self._options.test_results_server: return + if not self._options.master_name: + _log.error("--test-results-server was set, but --master-name was not. Not uploading JSON files.") + return + _log.info("Uploading JSON files for builder: %s", self._options.builder_name) - attrs = [("builder", self._options.builder_name), ("testtype", "layout-tests")] - # FIXME: master_name should be required if test_results_server is set. - # Throw an error if master_name isn't set. - if self._options.master_name: - attrs.append(("master", self._options.master_name)) + attrs = [("builder", self._options.builder_name), ("testtype", "layout-tests"), + ("master", self._options.master_name)] json_files = ["expectations.json"] if self._options.upload_full_results: @@ -966,13 +957,6 @@ class TestRunner: files = [(file, os.path.join(self._options.results_directory, file)) for file in json_files] - # FIXME: Remove this. This is temporary debug logging. - if self._options.builder_name.startswith("Webkit Linux"): - for filename in files: - _log.debug(filename[1]) - with codecs.open(filename[1], "r") as results_file: - _log.debug("%s:\n%s" % (filename[0], results_file.read())) - uploader = test_results_uploader.TestResultsUploader( self._options.test_results_server) try: @@ -1530,7 +1514,7 @@ def parse_args(args=None): default=False, help="Don't check the system dependencies (themes)"), optparse.make_option("--use-drt", action="store_true", - default=False, + default=None, help="Use DumpRenderTree instead of test_shell"), optparse.make_option("--accelerated-compositing", action="store_true", @@ -1612,9 +1596,6 @@ def parse_args(args=None): optparse.make_option("--no-record-results", action="store_false", default=True, dest="record_results", help="Don't record the results."), - optparse.make_option("--wait-for-httpd", action="store_true", - default=False, dest="wait_for_httpd", - help="Wait for http locks."), # old-run-webkit-tests also has HTTP toggle options: # --[no-]http Run (or do not run) http tests # (default: run) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py index 0f09ffa..f21e7a5 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py @@ -31,6 +31,7 @@ """Unit tests for run_webkit_tests.""" import codecs +import itertools import logging import os import Queue @@ -48,7 +49,9 @@ from webkitpy.common.system import user from webkitpy.layout_tests import port from webkitpy.layout_tests import run_webkit_tests from webkitpy.layout_tests.layout_package import dump_render_tree_thread -from webkitpy.layout_tests.port.test import TestPort +from webkitpy.layout_tests.port.test import TestPort, TestDriver +from webkitpy.python24.versioning import compare_version +from webkitpy.test.skip import skip_if from webkitpy.thirdparty.mock import Mock @@ -61,41 +64,44 @@ class MockUser(): self.url = url -def passing_run(args=[], port_obj=None, record_results=False, +def passing_run(extra_args=None, port_obj=None, record_results=False, tests_included=False): - new_args = ['--print', 'nothing'] - if not '--platform' in args: - new_args.extend(['--platform', 'test']) + extra_args = extra_args or [] + args = ['--print', 'nothing'] + if not '--platform' in extra_args: + args.extend(['--platform', 'test']) if not record_results: - new_args.append('--no-record-results') - new_args.extend(args) + args.append('--no-record-results') + args.extend(extra_args) if not tests_included: # We use the glob to test that globbing works. - new_args.extend(['passes', - 'http/tests', - 'http/tests/websocket/tests', - 'failures/expected/*']) - options, parsed_args = run_webkit_tests.parse_args(new_args) - if port_obj is None: + args.extend(['passes', + 'http/tests', + 'http/tests/websocket/tests', + 'failures/expected/*']) + options, parsed_args = run_webkit_tests.parse_args(args) + if not port_obj: port_obj = port.get(port_name=options.platform, options=options, user=MockUser()) res = run_webkit_tests.run(port_obj, options, parsed_args) return res == 0 -def logging_run(args=[], tests_included=False): - new_args = ['--no-record-results'] - if not '--platform' in args: - new_args.extend(['--platform', 'test']) - new_args.extend(args) +def logging_run(extra_args=None, port_obj=None, tests_included=False): + extra_args = extra_args or [] + args = ['--no-record-results'] + if not '--platform' in extra_args: + args.extend(['--platform', 'test']) + args.extend(extra_args) if not tests_included: - new_args.extend(['passes', - 'http/tests', - 'http/tests/websocket/tests', - 'failures/expected/*']) - options, parsed_args = run_webkit_tests.parse_args(new_args) + args.extend(['passes', + 'http/tests', + 'http/tests/websocket/tests', + 'failures/expected/*']) + options, parsed_args = run_webkit_tests.parse_args(args) user = MockUser() - port_obj = port.get(port_name=options.platform, options=options, user=user) + if not port_obj: + port_obj = port.get(port_name=options.platform, options=options, user=user) buildbot_output = array_stream.ArrayStream() regular_output = array_stream.ArrayStream() res = run_webkit_tests.run(port_obj, options, parsed_args, @@ -104,6 +110,54 @@ def logging_run(args=[], tests_included=False): return (res, buildbot_output, regular_output, user) +def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False): + extra_args = extra_args or [] + args = [ + '--print', 'nothing', + '--platform', 'test', + '--no-record-results', + '--child-processes', '1'] + args.extend(extra_args) + if not tests_included: + # Not including http tests since they get run out of order (that + # behavior has its own test, see test_get_test_file_queue) + args.extend(['passes', 'failures']) + options, parsed_args = run_webkit_tests.parse_args(args) + user = MockUser() + + test_batches = [] + + class RecordingTestDriver(TestDriver): + def __init__(self, port, image_path, options): + TestDriver.__init__(self, port, image_path, options, executive=None) + self._current_test_batch = None + + def poll(self): + # So that we don't create a new driver for every test + return None + + def stop(self): + self._current_test_batch = None + + def run_test(self, uri, timeoutms, image_hash): + if self._current_test_batch is None: + self._current_test_batch = [] + test_batches.append(self._current_test_batch) + self._current_test_batch.append(self._port.uri_to_test_name(uri)) + return TestDriver.run_test(self, uri, timeoutms, image_hash) + + class RecordingTestPort(TestPort): + def create_driver(self, image_path, options): + return RecordingTestDriver(self, image_path, options) + + recording_port = RecordingTestPort(options=options, user=user) + logging_run(extra_args=args, port_obj=recording_port, tests_included=True) + + if flatten_batches: + return list(itertools.chain(*test_batches)) + + return test_batches + class MainTest(unittest.TestCase): def test_accelerated_compositing(self): # This just tests that we recognize the command line args @@ -119,8 +173,9 @@ class MainTest(unittest.TestCase): self.assertTrue(passing_run()) def test_batch_size(self): - # FIXME: verify # of tests run - self.assertTrue(passing_run(['--batch-size', '2'])) + batch_tests_run = get_tests_run(['--batch-size', '2']) + for batch in batch_tests_run: + self.assertTrue(len(batch) <= 2, '%s had too many tests' % ', '.join(batch)) def test_child_process_1(self): (res, buildbot_output, regular_output, user) = logging_run( @@ -194,8 +249,15 @@ class MainTest(unittest.TestCase): self.assertTrue(passing_run(['--randomize-order'])) def test_run_chunk(self): - # FIXME: verify # of tests run - self.assertTrue(passing_run(['--run-chunk', '1:4'])) + # Test that we actually select the right chunk + all_tests_run = get_tests_run(flatten_batches=True) + chunk_tests_run = get_tests_run(['--run-chunk', '1:4'], flatten_batches=True) + self.assertEquals(all_tests_run[4:8], chunk_tests_run) + + # Test that we wrap around if the number of tests is not evenly divisible by the chunk size + tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html'] + chunk_tests_run = get_tests_run(['--run-chunk', '1:3'] + tests_to_run, tests_included=True, flatten_batches=True) + self.assertEquals(['passes/text.html', 'passes/error.html', 'passes/image.html'], chunk_tests_run) def test_run_force(self): # This raises an exception because we run @@ -203,23 +265,33 @@ class MainTest(unittest.TestCase): self.assertRaises(ValueError, logging_run, ['--force']) def test_run_part(self): - # FIXME: verify # of tests run - self.assertTrue(passing_run(['--run-part', '1:2'])) + # Test that we actually select the right part + tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html'] + tests_run = get_tests_run(['--run-part', '1:2'] + tests_to_run, tests_included=True, flatten_batches=True) + self.assertEquals(['passes/error.html', 'passes/image.html'], tests_run) + + # Test that we wrap around if the number of tests is not evenly divisible by the chunk size + # (here we end up with 3 parts, each with 2 tests, and we only have 4 tests total, so the + # last part repeats the first two tests). + chunk_tests_run = get_tests_run(['--run-part', '3:3'] + tests_to_run, tests_included=True, flatten_batches=True) + self.assertEquals(['passes/error.html', 'passes/image.html'], chunk_tests_run) def test_run_singly(self): - self.assertTrue(passing_run(['--run-singly'])) + batch_tests_run = get_tests_run(['--run-singly']) + for batch in batch_tests_run: + self.assertEquals(len(batch), 1, '%s had too many tests' % ', '.join(batch)) def test_single_file(self): - # FIXME: verify # of tests run - self.assertTrue(passing_run(['passes/text.html'], tests_included=True)) + tests_run = get_tests_run(['passes/text.html'], tests_included=True, flatten_batches=True) + self.assertEquals(['passes/text.html'], tests_run) def test_test_list(self): filename = tempfile.mktemp() tmpfile = file(filename, mode='w+') tmpfile.write('passes/text.html') tmpfile.close() - self.assertTrue(passing_run(['--test-list=%s' % filename], - tests_included=True)) + tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True) + self.assertEquals(['passes/text.html'], tests_run) os.remove(filename) res, out, err, user = logging_run(['--test-list=%s' % filename], tests_included=True) @@ -271,7 +343,7 @@ class MainTest(unittest.TestCase): def get_port_for_run(args): options, parsed_args = run_webkit_tests.parse_args(args) test_port = ImageDiffTestPort(options=options, user=MockUser()) - passing_run(args=args, port_obj=test_port, tests_included=True) + passing_run(args, port_obj=test_port, tests_included=True) return test_port base_args = ['--pixel-tests', 'failures/expected/*'] @@ -287,6 +359,8 @@ class MainTest(unittest.TestCase): test_port = get_port_for_run(base_args) self.assertEqual(None, test_port.tolerance_used_for_diff_image) +MainTest = skip_if(MainTest, sys.platform == 'cygwin' and compare_version(sys, '2.6')[0] < 0, 'new-run-webkit-tests tests hang on Cygwin Python 2.5.2') + def _mocked_open(original_open, file_list): def _wrapper(name, mode, encoding): |