diff options
| author | Shimeng (Simon) Wang <swang@google.com> | 2010-12-07 17:22:45 -0800 |
|---|---|---|
| committer | Shimeng (Simon) Wang <swang@google.com> | 2010-12-22 14:15:40 -0800 |
| commit | 4576aa36e9a9671459299c7963ac95aa94beaea9 (patch) | |
| tree | 3863574e050f168c0126ecb47c83319fab0972d8 /WebKitTools/Scripts/webkitpy/layout_tests | |
| parent | 55323ac613cc31553107b68603cb627264d22bb0 (diff) | |
| download | external_webkit-4576aa36e9a9671459299c7963ac95aa94beaea9.zip external_webkit-4576aa36e9a9671459299c7963ac95aa94beaea9.tar.gz external_webkit-4576aa36e9a9671459299c7963ac95aa94beaea9.tar.bz2 | |
Merge WebKit at r73109: Initial merge by git.
Change-Id: I61f1a66d9642e3d8405d3ac6ccab2a53421c75d8
Diffstat (limited to 'WebKitTools/Scripts/webkitpy/layout_tests')
27 files changed, 879 insertions, 651 deletions
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/driver_test.py b/WebKitTools/Scripts/webkitpy/layout_tests/driver_test.py deleted file mode 100644 index 633dfe8..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/driver_test.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# -# FIXME: this is a poor attempt at a unit tests driver. We should replace -# this with something that actually uses a unit testing framework or -# at least produces output that could be useful. - -"""Simple test client for the port/Driver interface.""" - -import os -import optparse -import port - - -def run_tests(port, options, tests): - # |image_path| is a path to the image capture from the driver. - image_path = 'image_result.png' - driver = port.create_driver(image_path, None) - driver.start() - for t in tests: - uri = port.filename_to_uri(os.path.join(port.layout_tests_dir(), t)) - print "uri: " + uri - crash, timeout, checksum, output, err = \ - driver.run_test(uri, int(options.timeout), None) - print "crash: " + str(crash) - print "timeout: " + str(timeout) - print "checksum: " + str(checksum) - print 'stdout: """' - print ''.join(output) - print '"""' - print 'stderr: """' - print ''.join(err) - print '"""' - print - driver.stop() - - -if __name__ == '__main__': - # FIXME: configuration_options belong in a shared location. - configuration_options = [ - optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", help='Set the configuration to Debug'), - optparse.make_option('--release', action='store_const', const='Release', dest="configuration", help='Set the configuration to Release'), - ] - misc_options = [ - optparse.make_option('-p', '--platform', action='store', default='mac', help='Platform to test (e.g., "mac", "chromium-mac", etc.'), - optparse.make_option('--timeout', action='store', default='2000', help='test timeout in milliseconds (2000 by default)'), - optparse.make_option('--wrapper', action='store'), - optparse.make_option('--no-pixel-tests', action='store_true', default=False, help='disable pixel-to-pixel PNG comparisons'), - ] - option_list = configuration_options + misc_options - optparser = optparse.OptionParser(option_list=option_list) - options, args = optparser.parse_args() - p = port.get(options.platform, options) - run_tests(p, options, args) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py index 88f493d..fdb8da6 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py @@ -48,7 +48,11 @@ import sys import thread import threading import time -import traceback + + +from webkitpy.layout_tests.test_types import image_diff +from webkitpy.layout_tests.test_types import test_type_base +from webkitpy.layout_tests.test_types import text_diff import test_failures import test_output @@ -58,23 +62,6 @@ _log = logging.getLogger("webkitpy.layout_tests.layout_package." "dump_render_tree_thread") -def find_thread_stack(id): - """Returns a stack object that can be used to dump a stack trace for - the given thread id (or None if the id is not found).""" - for thread_id, stack in sys._current_frames().items(): - if thread_id == id: - return stack - return None - - -def log_stack(stack): - """Log a stack trace to log.error().""" - for filename, lineno, name, line in traceback.extract_stack(stack): - _log.error('File: "%s", line %d, in %s' % (filename, lineno, name)) - if line: - _log.error(' %s' % line.strip()) - - def _expected_test_output(port, filename): """Returns an expected TestOutput object.""" return test_output.TestOutput(port.expected_text(filename), @@ -82,7 +69,7 @@ def _expected_test_output(port, filename): port.expected_checksum(filename)) def _process_output(port, options, test_input, test_types, test_args, - test_output): + test_output, worker_name): """Receives the output from a DumpRenderTree process, subjects it to a number of tests, and returns a list of failure types the test produced. @@ -94,6 +81,7 @@ def _process_output(port, options, test_input, test_types, test_args, test_types: list of test types to subject the output to test_args: arguments to be passed to each test test_output: a TestOutput object containing the output of the test + worker_name: worker name for logging Returns: a TestResult object """ @@ -104,20 +92,18 @@ def _process_output(port, options, test_input, test_types, test_args, if test_output.timeout: failures.append(test_failures.FailureTimeout()) + test_name = port.relative_test_filename(test_input.filename) if test_output.crash: - _log.debug("Stacktrace for %s:\n%s" % (test_input.filename, - test_output.error)) - # Strip off "file://" since RelativeTestFilename expects - # filesystem paths. - filename = os.path.join(options.results_directory, - port.relative_test_filename( - test_input.filename)) + _log.debug("%s Stacktrace for %s:\n%s" % (worker_name, test_name, + test_output.error)) + filename = os.path.join(options.results_directory, test_name) filename = os.path.splitext(filename)[0] + "-stack.txt" port.maybe_make_directory(os.path.split(filename)[0]) with codecs.open(filename, "wb", "utf-8") as file: file.write(test_output.error) elif test_output.error: - _log.debug("Previous test output stderr lines:\n%s" % test_output.error) + _log.debug("%s %s output stderr lines:\n%s" % (worker_name, test_name, + test_output.error)) expected_test_output = _expected_test_output(port, test_input.filename) @@ -161,7 +147,7 @@ def _should_fetch_expected_checksum(options): return options.pixel_tests and not (options.new_baseline or options.reset_results) -def _run_single_test(port, options, test_input, test_types, test_args, driver): +def _run_single_test(port, options, test_input, test_types, test_args, driver, worker_name): # FIXME: Pull this into TestShellThread._run(). # The image hash is used to avoid doing an image dump if the @@ -169,23 +155,23 @@ def _run_single_test(port, options, test_input, test_types, test_args, driver): # are generating a new baseline. (Otherwise, an image from a # previous run will be copied into the baseline.""" if _should_fetch_expected_checksum(options): - image_hash_to_driver = port.expected_checksum(test_input.filename) - else: - image_hash_to_driver = None - uri = port.filename_to_uri(test_input.filename) - test_output = driver.run_test(uri, test_input.timeout, image_hash_to_driver) + test_input.image_hash = port.expected_checksum(test_input.filename) + test_output = driver.run_test(test_input) return _process_output(port, options, test_input, test_types, test_args, - test_output) + test_output, worker_name) class SingleTestThread(threading.Thread): """Thread wrapper for running a single test file.""" - def __init__(self, port, options, test_input, test_types, test_args): + def __init__(self, port, options, worker_number, worker_name, + test_input, test_types, test_args): """ Args: port: object implementing port-specific hooks options: command line argument object from optparse + worker_number: worker number for tests + worker_name: for logging test_input: Object containing the test filename and timeout test_types: A list of TestType objects to run the test output against. @@ -199,6 +185,8 @@ class SingleTestThread(threading.Thread): self._test_types = test_types self._test_args = test_args self._driver = None + self._worker_number = worker_number + self._name = worker_name def run(self): self._covered_run() @@ -206,12 +194,12 @@ class SingleTestThread(threading.Thread): def _covered_run(self): # FIXME: this is a separate routine to work around a bug # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85. - self._driver = self._port.create_driver(self._test_args.png_path, - self._options) + self._driver = self._port.create_driver(self._worker_number) self._driver.start() self._test_result = _run_single_test(self._port, self._options, self._test_input, self._test_types, - self._test_args, self._driver) + self._test_args, self._driver, + self._name) self._driver.stop() def get_test_result(self): @@ -254,29 +242,28 @@ class WatchableThread(threading.Thread): class TestShellThread(WatchableThread): - def __init__(self, port, options, filename_list_queue, result_queue, - test_types, test_args): + def __init__(self, port, options, worker_number, worker_name, + filename_list_queue, result_queue): """Initialize all the local state for this DumpRenderTree thread. Args: port: interface to port-specific hooks options: command line options argument from optparse + worker_number: identifier for a particular worker thread. + worker_name: for logging. filename_list_queue: A thread safe Queue class that contains lists of tuples of (filename, uri) pairs. result_queue: A thread safe Queue class that will contain serialized TestResult objects. - test_types: A list of TestType objects to run the test output - against. - test_args: A TestArguments object to pass to each TestType. """ WatchableThread.__init__(self) self._port = port self._options = options + self._worker_number = worker_number + self._name = worker_name self._filename_list_queue = filename_list_queue self._result_queue = result_queue self._filename_list = [] - self._test_types = test_types - self._test_args = test_args self._driver = None self._test_group_timing_stats = {} self._test_results = [] @@ -287,6 +274,12 @@ class TestShellThread(WatchableThread): self._http_lock_wait_begin = 0 self._http_lock_wait_end = 0 + self._test_types = [] + for cls in self._get_test_type_classes(): + self._test_types.append(cls(self._port, + self._options.results_directory)) + self._test_args = self._get_test_args(worker_number) + # Current group of tests we're running. self._current_group = None # Number of tests in self._current_group. @@ -294,6 +287,20 @@ class TestShellThread(WatchableThread): # Time at which we started running tests from self._current_group. self._current_group_start_time = None + def _get_test_args(self, worker_number): + """Returns the tuple of arguments for tests and for DumpRenderTree.""" + test_args = test_type_base.TestArguments() + test_args.new_baseline = self._options.new_baseline + test_args.reset_results = self._options.reset_results + + return test_args + + def _get_test_type_classes(self): + classes = [text_diff.TestTextDiff] + if self._options.pixel_tests: + classes.append(image_diff.ImageDiff) + return classes + def get_test_group_timing_stats(self): """Returns a dictionary mapping test group to a tuple of (number of tests in that group, time to run the tests)""" @@ -417,9 +424,9 @@ class TestShellThread(WatchableThread): batch_count += 1 self._num_tests += 1 if self._options.run_singly: - result = self._run_test_singly(test_input) + result = self._run_test_in_another_thread(test_input) else: - result = self._run_test(test_input) + result = self._run_test_in_this_thread(test_input) filename = test_input.filename tests_run_file.write(filename + "\n") @@ -449,7 +456,7 @@ class TestShellThread(WatchableThread): if test_runner: test_runner.update_summary(result_summary) - def _run_test_singly(self, test_input): + def _run_test_in_another_thread(self, test_input): """Run a test in a separate thread, enforcing a hard time limit. Since we can only detect the termination of a thread, not any internal @@ -461,10 +468,11 @@ class TestShellThread(WatchableThread): Returns: A TestResult - """ worker = SingleTestThread(self._port, self._options, + self._worker_number, + self._name, test_input, self._test_types, self._test_args) @@ -496,11 +504,11 @@ class TestShellThread(WatchableThread): _log.error('Cannot get results of test: %s' % test_input.filename) result = test_results.TestResult(test_input.filename, failures=[], - test_run_time=0, total_time_for_all_diffs=0, time_for_diffs=0) + test_run_time=0, total_time_for_all_diffs=0, time_for_diffs={}) return result - def _run_test(self, test_input): + def _run_test_in_this_thread(self, test_input): """Run a single test file using a shared DumpRenderTree process. Args: @@ -514,7 +522,7 @@ class TestShellThread(WatchableThread): self._next_timeout = time.time() + thread_timeout test_result = _run_single_test(self._port, self._options, test_input, self._test_types, self._test_args, - self._driver) + self._driver, self._name) self._test_results.append(test_result) return test_result @@ -527,9 +535,8 @@ class TestShellThread(WatchableThread): """ # poll() is not threadsafe and can throw OSError due to: # http://bugs.python.org/issue1731717 - if (not self._driver or self._driver.poll() is not None): - self._driver = self._port.create_driver(self._test_args.png_path, - self._options) + if not self._driver or self._driver.poll() is not None: + self._driver = self._port.create_driver(self._worker_number) self._driver.start() def _start_servers_with_lock(self): diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py deleted file mode 100644 index 63f86d9..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -""""Tests code paths not covered by the regular unit tests.""" - -import sys -import unittest - -import dump_render_tree_thread - - -class Test(unittest.TestCase): - def test_find_thread_stack_found(self): - id, stack = sys._current_frames().items()[0] - found_stack = dump_render_tree_thread.find_thread_stack(id) - self.assertNotEqual(found_stack, None) - - def test_find_thread_stack_not_found(self): - found_stack = dump_render_tree_thread.find_thread_stack(0) - self.assertEqual(found_stack, None) - - -if __name__ == '__main__': - unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py index 101d30b..b054c5b 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py @@ -129,6 +129,10 @@ class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGeneratorBase return self.PASS_RESULT # override + def _get_result_char(self, test_name): + return self._get_modifier_char(test_name) + + # override def _convert_json_to_current_version(self, results_json): archive_version = None if self.VERSION_KEY in results_json: diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py index 3267718..331e330 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py @@ -80,7 +80,7 @@ class TestResult(object): class JSONResultsGeneratorBase(object): """A JSON results generator for generic tests.""" - MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 1500 + MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 750 # Min time (seconds) that will be added to the JSON. MIN_TIME = 1 JSON_PREFIX = "ADD_RESULTS(" @@ -303,6 +303,23 @@ class JSONResultsGeneratorBase(object): return JSONResultsGenerator.PASS_RESULT + def _get_result_char(self, test_name): + """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT, + PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test result + for the given test_name. + """ + if test_name not in self._test_results_map: + return JSONResultsGenerator.NO_DATA_RESULT + + test_result = self._test_results_map[test_name] + if test_result.modifier == TestResult.DISABLED: + return JSONResultsGenerator.SKIP_RESULT + + if test_result.failed: + return JSONResultsGenerator.FAIL_RESULT + + return JSONResultsGenerator.PASS_RESULT + # FIXME: Callers should use scm.py instead. # FIXME: Identify and fix the run-time errors that were observed on Windows # chromium buildbot when we had updated this code to use scm.py once before. @@ -484,7 +501,7 @@ class JSONResultsGeneratorBase(object): tests: Dictionary containing test result entries. """ - result = self._get_modifier_char(test_name) + result = self._get_result_char(test_name) time = self._get_test_timing(test_name) if test_name not in tests: diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py index 606a613..d6275ee 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py @@ -56,15 +56,6 @@ class JSONGeneratorTest(unittest.TestCase): self._FLAKY_tests = set([]) self._FAILS_tests = set([]) - def _get_test_modifier(self, test_name): - if test_name.startswith('DISABLED_'): - return json_results_generator.JSONResultsGenerator.SKIP_RESULT - elif test_name.startswith('FLAKY_'): - return json_results_generator.JSONResultsGenerator.FLAKY_RESULT - elif test_name.startswith('FAILS_'): - return json_results_generator.JSONResultsGenerator.FAIL_RESULT - return json_results_generator.JSONResultsGenerator.PASS_RESULT - def _test_json_generation(self, passed_tests_list, failed_tests_list): tests_set = set(passed_tests_list) | set(failed_tests_list) @@ -74,9 +65,9 @@ class JSONGeneratorTest(unittest.TestCase): if t.startswith('FLAKY_')]) FAILS_tests = set([t for t in tests_set if t.startswith('FAILS_')]) - PASS_tests = tests_set ^ (DISABLED_tests | FLAKY_tests | FAILS_tests) + PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests) - passed_tests = set(passed_tests_list) ^ DISABLED_tests + passed_tests = set(passed_tests_list) - DISABLED_tests failed_tests = set(failed_tests_list) test_timings = {} @@ -180,10 +171,10 @@ class JSONGeneratorTest(unittest.TestCase): test = tests[test_name] failed = 0 - modifier = self._get_test_modifier(test_name) for result in test[JRG.RESULTS]: - if result[1] == modifier: + if result[1] == JRG.FAIL_RESULT: failed = result[0] + self.assertEqual(1, failed) timing_count = 0 diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py new file mode 100644 index 0000000..e520a9c --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py @@ -0,0 +1,197 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Module for handling messages, threads, processes, and concurrency for run-webkit-tests. + +Testing is accomplished by having a manager (TestRunner) gather all of the +tests to be run, and sending messages to a pool of workers (TestShellThreads) +to run each test. Each worker communicates with one driver (usually +DumpRenderTree) to run one test at a time and then compare the output against +what we expected to get. + +This modules provides a message broker that connects the manager to the +workers: it provides a messaging abstraction and message loops, and +handles launching threads and/or processes depending on the +requested configuration. +""" + +import logging +import sys +import time +import traceback + +import dump_render_tree_thread + +_log = logging.getLogger(__name__) + + +def get(port, options): + """Return an instance of a WorkerMessageBroker.""" + worker_model = options.worker_model + if worker_model == 'inline': + return InlineBroker(port, options) + if worker_model == 'threads': + return MultiThreadedBroker(port, options) + raise ValueError('unsupported value for --worker-model: %s' % worker_model) + + +class _WorkerState(object): + def __init__(self, name): + self.name = name + self.thread = None + + +class WorkerMessageBroker(object): + def __init__(self, port, options): + self._port = port + self._options = options + self._num_workers = int(self._options.child_processes) + + # This maps worker names to their _WorkerState values. + self._workers = {} + + def _threads(self): + return tuple([w.thread for w in self._workers.values()]) + + def start_workers(self, test_runner): + """Starts up the pool of workers for running the tests. + + Args: + test_runner: a handle to the manager/TestRunner object + """ + self._test_runner = test_runner + for worker_number in xrange(self._num_workers): + worker = _WorkerState('worker-%d' % worker_number) + worker.thread = self._start_worker(worker_number, worker.name) + self._workers[worker.name] = worker + return self._threads() + + def _start_worker(self, worker_number, worker_name): + raise NotImplementedError + + def run_message_loop(self): + """Loop processing messages until done.""" + raise NotImplementedError + + def cancel_workers(self): + """Cancel/interrupt any workers that are still alive.""" + pass + + def cleanup(self): + """Perform any necessary cleanup on shutdown.""" + pass + + +class InlineBroker(WorkerMessageBroker): + def _start_worker(self, worker_number, worker_name): + # FIXME: Replace with something that isn't a thread. + thread = dump_render_tree_thread.TestShellThread(self._port, + self._options, worker_number, worker_name, + self._test_runner._current_filename_queue, + self._test_runner._result_queue) + # Note: Don't start() the thread! If we did, it would actually + # create another thread and start executing it, and we'd no longer + # be single-threaded. + return thread + + def run_message_loop(self): + thread = self._threads()[0] + thread.run_in_main_thread(self._test_runner, + self._test_runner._current_result_summary) + self._test_runner.update() + + +class MultiThreadedBroker(WorkerMessageBroker): + def _start_worker(self, worker_number, worker_name): + thread = dump_render_tree_thread.TestShellThread(self._port, + self._options, worker_number, worker_name, + self._test_runner._current_filename_queue, + self._test_runner._result_queue) + thread.start() + return thread + + def run_message_loop(self): + threads = self._threads() + + # Loop through all the threads waiting for them to finish. + some_thread_is_alive = True + while some_thread_is_alive: + some_thread_is_alive = False + t = time.time() + for thread in threads: + exception_info = thread.exception_info() + if exception_info is not None: + # Re-raise the thread's exception here to make it + # clear that testing was aborted. Otherwise, + # the tests that did not run would be assumed + # to have passed. + raise exception_info[0], exception_info[1], exception_info[2] + + if thread.isAlive(): + some_thread_is_alive = True + next_timeout = thread.next_timeout() + if next_timeout and t > next_timeout: + log_wedged_worker(thread.getName(), thread.id()) + thread.clear_next_timeout() + + self._test_runner.update() + + if some_thread_is_alive: + time.sleep(0.01) + + def cancel_workers(self): + threads = self._threads() + for thread in threads: + thread.cancel() + + +def log_wedged_worker(name, id): + """Log information about the given worker state.""" + stack = _find_thread_stack(id) + assert(stack is not None) + _log.error("") + _log.error("%s (tid %d) is wedged" % (name, id)) + _log_stack(stack) + _log.error("") + + +def _find_thread_stack(id): + """Returns a stack object that can be used to dump a stack trace for + the given thread id (or None if the id is not found).""" + for thread_id, stack in sys._current_frames().items(): + if thread_id == id: + return stack + return None + + +def _log_stack(stack): + """Log a stack trace to log.error().""" + for filename, lineno, name, line in traceback.extract_stack(stack): + _log.error('File: "%s", line %d, in %s' % (filename, lineno, name)) + if line: + _log.error(' %s' % line.strip()) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py new file mode 100644 index 0000000..6f04fd3 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py @@ -0,0 +1,183 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging +import Queue +import sys +import thread +import threading +import time +import unittest + +from webkitpy.common import array_stream +from webkitpy.common.system import outputcapture +from webkitpy.tool import mocktool + +from webkitpy.layout_tests import run_webkit_tests + +import message_broker + + +class TestThread(threading.Thread): + def __init__(self, started_queue, stopping_queue): + threading.Thread.__init__(self) + self._thread_id = None + self._started_queue = started_queue + self._stopping_queue = stopping_queue + self._timeout = False + self._timeout_queue = Queue.Queue() + self._exception_info = None + + def id(self): + return self._thread_id + + def getName(self): + return "worker-0" + + def run(self): + self._covered_run() + + def _covered_run(self): + # FIXME: this is a separate routine to work around a bug + # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85. + self._thread_id = thread.get_ident() + try: + self._started_queue.put('') + msg = self._stopping_queue.get() + if msg == 'KeyboardInterrupt': + raise KeyboardInterrupt + elif msg == 'Exception': + raise ValueError() + elif msg == 'Timeout': + self._timeout = True + self._timeout_queue.get() + except: + self._exception_info = sys.exc_info() + + def exception_info(self): + return self._exception_info + + def next_timeout(self): + if self._timeout: + self._timeout_queue.put('done') + return time.time() - 10 + return time.time() + + def clear_next_timeout(self): + self._next_timeout = None + +class TestHandler(logging.Handler): + def __init__(self, astream): + logging.Handler.__init__(self) + self._stream = astream + + def emit(self, record): + self._stream.write(self.format(record)) + + +class MultiThreadedBrokerTest(unittest.TestCase): + class MockTestRunner(object): + def __init__(self): + pass + + def __del__(self): + pass + + def update(self): + pass + + def run_one_thread(self, msg): + runner = self.MockTestRunner() + port = None + options = mocktool.MockOptions(child_processes='1') + starting_queue = Queue.Queue() + stopping_queue = Queue.Queue() + broker = message_broker.MultiThreadedBroker(port, options) + broker._test_runner = runner + child_thread = TestThread(starting_queue, stopping_queue) + broker._workers['worker-0'] = message_broker._WorkerState('worker-0') + broker._workers['worker-0'].thread = child_thread + child_thread.start() + started_msg = starting_queue.get() + stopping_queue.put(msg) + return broker.run_message_loop() + + def test_basic(self): + interrupted = self.run_one_thread('') + self.assertFalse(interrupted) + + def test_interrupt(self): + self.assertRaises(KeyboardInterrupt, self.run_one_thread, 'KeyboardInterrupt') + + def test_timeout(self): + oc = outputcapture.OutputCapture() + oc.capture_output() + interrupted = self.run_one_thread('Timeout') + self.assertFalse(interrupted) + oc.restore_output() + + def test_exception(self): + self.assertRaises(ValueError, self.run_one_thread, 'Exception') + + +class Test(unittest.TestCase): + def test_find_thread_stack_found(self): + id, stack = sys._current_frames().items()[0] + found_stack = message_broker._find_thread_stack(id) + self.assertNotEqual(found_stack, None) + + def test_find_thread_stack_not_found(self): + found_stack = message_broker._find_thread_stack(0) + self.assertEqual(found_stack, None) + + def test_log_wedged_worker(self): + oc = outputcapture.OutputCapture() + oc.capture_output() + logger = message_broker._log + astream = array_stream.ArrayStream() + handler = TestHandler(astream) + logger.addHandler(handler) + + starting_queue = Queue.Queue() + stopping_queue = Queue.Queue() + child_thread = TestThread(starting_queue, stopping_queue) + child_thread.start() + msg = starting_queue.get() + + message_broker.log_wedged_worker(child_thread.getName(), + child_thread.id()) + stopping_queue.put('') + child_thread.join(timeout=1.0) + + self.assertFalse(astream.empty()) + self.assertFalse(child_thread.isAlive()) + oc.restore_output() + + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing.py index fb9fe6d..7a6aad1 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing.py @@ -126,7 +126,6 @@ def print_options(): ] - def parse_print_options(print_options, verbose, child_processes, is_fully_parallel): """Parse the options provided to --print and dedup and rank them. @@ -182,8 +181,8 @@ def _configure_logging(stream, verbose): log_datefmt = '%y%m%d %H:%M:%S' log_level = logging.INFO if verbose: - log_fmt = ('%(asctime)s %(process)d %(filename)s:%(lineno)-4d %(levelname)s' - '%(message)s') + log_fmt = ('%(asctime)s %(process)d %(filename)s:%(lineno)d ' + '%(levelname)s %(message)s') log_level = logging.DEBUG root = logging.getLogger() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py index 9a0f4ee..27a6a29 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py @@ -78,8 +78,9 @@ class TestUtilityFunctions(unittest.TestCase): self.assertTrue(options is not None) def test_parse_print_options(self): - def test_switches(args, verbose, child_processes, is_fully_parallel, - expected_switches_str): + def test_switches(args, expected_switches_str, + verbose=False, child_processes=1, + is_fully_parallel=False): options, args = get_options(args) if expected_switches_str: expected_switches = set(expected_switches_str.split(',')) @@ -92,28 +93,23 @@ class TestUtilityFunctions(unittest.TestCase): self.assertEqual(expected_switches, switches) # test that we default to the default set of switches - test_switches([], False, 1, False, - printing.PRINT_DEFAULT) + test_switches([], printing.PRINT_DEFAULT) # test that verbose defaults to everything - test_switches([], True, 1, False, - printing.PRINT_EVERYTHING) + test_switches([], printing.PRINT_EVERYTHING, verbose=True) # test that --print default does what it's supposed to - test_switches(['--print', 'default'], False, 1, False, - printing.PRINT_DEFAULT) + test_switches(['--print', 'default'], printing.PRINT_DEFAULT) # test that --print nothing does what it's supposed to - test_switches(['--print', 'nothing'], False, 1, False, - None) + test_switches(['--print', 'nothing'], None) # test that --print everything does what it's supposed to - test_switches(['--print', 'everything'], False, 1, False, - printing.PRINT_EVERYTHING) + test_switches(['--print', 'everything'], printing.PRINT_EVERYTHING) # this tests that '--print X' overrides '--verbose' - test_switches(['--print', 'actual'], True, 1, False, - 'actual') + test_switches(['--print', 'actual'], 'actual', verbose=True) + class Testprinter(unittest.TestCase): diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_results_uploader.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_results_uploader.py index 680b848..033c8c6 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_results_uploader.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/test_results_uploader.py @@ -27,45 +27,81 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import with_statement + +import codecs import mimetypes import socket +import urllib2 from webkitpy.common.net.networktransaction import NetworkTransaction -from webkitpy.thirdparty.autoinstalled.mechanize import Browser - def get_mime_type(filename): - return mimetypes.guess_type(filename)[0] or "text/plain" + return mimetypes.guess_type(filename)[0] or 'application/octet-stream' + + +def _encode_multipart_form_data(fields, files): + """Encode form fields for multipart/form-data. + + Args: + fields: A sequence of (name, value) elements for regular form fields. + files: A sequence of (name, filename, value) elements for data to be + uploaded as files. + Returns: + (content_type, body) ready for httplib.HTTP instance. + + Source: + http://code.google.com/p/rietveld/source/browse/trunk/upload.py + """ + BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-' + CRLF = '\r\n' + lines = [] + + for key, value in fields: + lines.append('--' + BOUNDARY) + lines.append('Content-Disposition: form-data; name="%s"' % key) + lines.append('') + if isinstance(value, unicode): + value = value.encode('utf-8') + lines.append(value) + + for key, filename, value in files: + lines.append('--' + BOUNDARY) + lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename)) + lines.append('Content-Type: %s' % get_mime_type(filename)) + lines.append('') + if isinstance(value, unicode): + value = value.encode('utf-8') + lines.append(value) + + lines.append('--' + BOUNDARY + '--') + lines.append('') + body = CRLF.join(lines) + content_type = 'multipart/form-data; boundary=%s' % BOUNDARY + return content_type, body class TestResultsUploader: def __init__(self, host): self._host = host - self._browser = Browser() def _upload_files(self, attrs, file_objs): - self._browser.open("http://%s/testfile/uploadform" % self._host) - self._browser.select_form("test_result_upload") - for (name, data) in attrs: - self._browser[name] = str(data) - - for (filename, handle) in file_objs: - self._browser.add_file(handle, get_mime_type(filename), filename, - "file") - - self._browser.submit() + url = "http://%s/testfile/upload" % self._host + content_type, data = _encode_multipart_form_data(attrs, file_objs) + headers = {"Content-Type": content_type} + request = urllib2.Request(url, data, headers) + urllib2.urlopen(request) def upload(self, params, files, timeout_seconds): - orig_timeout = socket.getdefaulttimeout() file_objs = [] - try: - file_objs = [(filename, open(path, "rb")) for (filename, path) - in files] + for filename, path in files: + with codecs.open(path, "rb") as file: + file_objs.append(('file', filename, file.read())) + orig_timeout = socket.getdefaulttimeout() + try: socket.setdefaulttimeout(timeout_seconds) NetworkTransaction(timeout_seconds=timeout_seconds).run( lambda: self._upload_files(params, file_objs)) finally: socket.setdefaulttimeout(orig_timeout) - for (filename, handle) in file_objs: - handle.close() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py index 632806f..bc5a9aa 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py @@ -384,6 +384,11 @@ class Port(object): # valid test and by printing.py to determine if baselines exist. return self._filesystem.exists(path) + def driver_cmd_line(self): + """Prints the DRT command line that will be used.""" + driver = self.create_driver(0) + return driver.cmd_line() + def update_baseline(self, path, data, encoding): """Updates the baseline for a test. @@ -487,7 +492,7 @@ class Port(object): """Relative unix-style path for a filename under the LayoutTests directory. Filenames outside the LayoutTests directory should raise an error.""" - #assert(filename.startswith(self.layout_tests_dir())) + assert filename.startswith(self.layout_tests_dir()), "%s did not start with %s" % (filename, self.layout_tests_dir()) return filename[len(self.layout_tests_dir()) + 1:] def results_directory(self): @@ -511,7 +516,7 @@ class Port(object): results_filename in a users' browser.""" return self._user.open_url(results_filename) - def create_driver(self, image_path, options): + def create_driver(self, worker_number): """Return a newly created base.Driver subclass for starting/stopping the test driver.""" raise NotImplementedError('Port.create_driver') @@ -741,7 +746,7 @@ class Port(object): def _path_to_driver(self, configuration=None): """Returns the full path to the test driver (DumpRenderTree).""" - raise NotImplementedError('Port.path_to_driver') + raise NotImplementedError('Port._path_to_driver') def _path_to_webcore_library(self): """Returns the full path to a built copy of WebCore.""" @@ -804,33 +809,26 @@ class Port(object): class Driver: """Abstract interface for the DumpRenderTree interface.""" - def __init__(self, port, png_path, options, executive): + def __init__(self, port, worker_number): """Initialize a Driver to subsequently run tests. Typically this routine will spawn DumpRenderTree in a config ready for subsequent input. port - reference back to the port object. - png_path - an absolute path for the driver to write any image - data for a test (as a PNG). If no path is provided, that - indicates that pixel test results will not be checked. - options - command line options argument from optparse - executive - reference to the process-wide Executive object - + worker_number - identifier for a particular worker/driver instance """ raise NotImplementedError('Driver.__init__') - def run_test(self, uri, timeout, checksum): + def run_test(self, test_input): """Run a single test and return the results. Note that it is okay if a test times out or crashes and leaves the driver in an indeterminate state. The upper layers of the program are responsible for cleaning up and ensuring things are okay. - uri - a full URI for the given test - timeout - number of milliseconds to wait before aborting this test. - checksum - if present, the expected checksum for the image for this - test + Args: + test_input: a TestInput object Returns a TestOutput object. """ diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py index 1e9c2b7..8d586e3 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py @@ -258,7 +258,7 @@ class VirtualTest(unittest.TestCase): self.assertVirtual(port.baseline_search_path) self.assertVirtual(port.check_build, None) self.assertVirtual(port.check_image_diff) - self.assertVirtual(port.create_driver, None, None) + self.assertVirtual(port.create_driver, 0) self.assertVirtual(port.diff_image, None, None) self.assertVirtual(port.path_to_test_expectations_file) self.assertVirtual(port.test_platform_name) @@ -282,7 +282,7 @@ class VirtualTest(unittest.TestCase): def test_virtual_driver_method(self): self.assertRaises(NotImplementedError, base.Driver, base.Port(), - "", None, None) + 0) def test_virtual_driver_methods(self): class VirtualDriver(base.Driver): @@ -290,7 +290,7 @@ class VirtualTest(unittest.TestCase): pass driver = VirtualDriver() - self.assertVirtual(driver.run_test, None, None, None) + self.assertVirtual(driver.run_test, None) self.assertVirtual(driver.poll) self.assertVirtual(driver.stop) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py index 3149290..8fe685a 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py @@ -32,6 +32,7 @@ from __future__ import with_statement import codecs +import errno import logging import os import re @@ -43,7 +44,6 @@ import tempfile import time import webbrowser -from webkitpy.common.system.executive import Executive from webkitpy.common.system.path import cygpath from webkitpy.layout_tests.layout_package import test_expectations from webkitpy.layout_tests.layout_package import test_output @@ -175,6 +175,8 @@ class ChromiumPort(base.Port): return result def driver_name(self): + if self._options.use_drt: + return "DumpRenderTree" return "test_shell" def path_from_chromium_base(self, *comps): @@ -212,13 +214,11 @@ class ChromiumPort(base.Port): if os.path.exists(cachedir): shutil.rmtree(cachedir) - def create_driver(self, image_path, options): + def create_driver(self, worker_number): """Starts a new Driver and returns a handle to it.""" - if options.use_drt and sys.platform == 'darwin': - return webkit.WebKitDriver(self, image_path, options, - executive=self._executive) - return ChromiumDriver(self, image_path, options, - executive=self._executive) + if self.get_option('use_drt') and sys.platform == 'darwin': + return webkit.WebKitDriver(self, worker_number) + return ChromiumDriver(self, worker_number) def start_helper(self): helper_path = self._path_to_helper() @@ -359,48 +359,50 @@ class ChromiumPort(base.Port): class ChromiumDriver(base.Driver): """Abstract interface for test_shell.""" - def __init__(self, port, image_path, options, executive=Executive()): + def __init__(self, port, worker_number): self._port = port - self._options = options - self._image_path = image_path - self._executive = executive - - def _driver_args(self): - driver_args = [] - if self._image_path: + self._worker_number = worker_number + self._image_path = None + if self._port.get_option('pixel_tests'): + self._image_path = os.path.join( + self._port.get_option('results_directory'), + 'png_result%s.png' % self._worker_number) + + def cmd_line(self): + cmd = self._command_wrapper(self._port.get_option('wrapper')) + cmd.append(self._port._path_to_driver()) + if self._port.get_option('pixel_tests'): # See note above in diff_image() for why we need _convert_path(). - driver_args.append("--pixel-tests=" + - self._port._convert_path(self._image_path)) + cmd.append("--pixel-tests=" + + self._port._convert_path(self._image_path)) if self._port.get_option('use_drt'): - driver_args.append('--test-shell') + cmd.append('--test-shell') else: - driver_args.append('--layout-tests') + cmd.append('--layout-tests') if self._port.get_option('startup_dialog'): - driver_args.append('--testshell-startup-dialog') + cmd.append('--testshell-startup-dialog') if self._port.get_option('gp_fault_error_box'): - driver_args.append('--gp-fault-error-box') + cmd.append('--gp-fault-error-box') - if self._options.js_flags is not None: - driver_args.append('--js-flags="' + self._options.js_flags + '"') + if self._port.get_option('js_flags') is not None: + cmd.append('--js-flags="' + self._port.get_option('js_flags') + '"') - if self._options.multiple_loads is not None and self._options.multiple_loads > 0: - driver_args.append('--multiple-loads=' + str(self._options.multiple_loads)) + if self._port.get_option('multiple_loads') > 0: + cmd.append('--multiple-loads=' + str(self._port.get_option('multiple_loads'))) if self._port.get_option('accelerated_compositing'): - driver_args.append('--enable-accelerated-compositing') + cmd.append('--enable-accelerated-compositing') if self._port.get_option('accelerated_2d_canvas'): - driver_args.append('--enable-accelerated-2d-canvas') - return driver_args + cmd.append('--enable-accelerated-2d-canvas') + return cmd def start(self): # FIXME: Should be an error to call this method twice. - cmd = self._command_wrapper(self._port.get_option('wrapper')) - cmd.append(self._port._path_to_driver()) - cmd += self._driver_args() + cmd = self.cmd_line() # We need to pass close_fds=True to work around Python bug #2320 # (otherwise we can hang when we kill DumpRenderTree when we are running @@ -454,7 +456,22 @@ class ChromiumDriver(base.Driver): else: return None - def run_test(self, uri, timeoutms, checksum): + def _output_image_with_retry(self): + # Retry a few more times because open() sometimes fails on Windows, + # raising "IOError: [Errno 13] Permission denied:" + retry_num = 50 + timeout_seconds = 5.0 + for i in range(retry_num): + try: + return self._output_image() + except IOError, e: + if e.errno == errno.EACCES: + time.sleep(timeout_seconds / retry_num) + else: + raise e + return self._output_image() + + def run_test(self, test_input): output = [] error = [] crash = False @@ -464,7 +481,9 @@ class ChromiumDriver(base.Driver): start_time = time.time() - cmd = self._test_shell_command(uri, timeoutms, checksum) + uri = self._port.filename_to_uri(test_input.filename) + cmd = self._test_shell_command(uri, test_input.timeout, + test_input.image_hash) (line, crash) = self._write_command_and_read_line(input=cmd) while not crash and line.rstrip() != "#EOF": @@ -505,9 +524,10 @@ class ChromiumDriver(base.Driver): (line, crash) = self._write_command_and_read_line(input=None) + run_time = time.time() - start_time return test_output.TestOutput( - ''.join(output), self._output_image(), actual_checksum, - crash, time.time() - start_time, timeout, ''.join(error)) + ''.join(output), self._output_image_with_retry(), actual_checksum, + crash, run_time, timeout, ''.join(error)) def stop(self): if self._proc: @@ -532,4 +552,4 @@ class ChromiumDriver(base.Driver): if self._proc.poll() is None: _log.warning('stopping test driver timed out, ' 'killing it') - self._executive.kill_process(self._proc.pid) + self._port._executive.kill_process(self._proc.pid) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py index 92a31fb..5396522 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py @@ -42,7 +42,8 @@ class ChromiumDriverTest(unittest.TestCase): def setUp(self): mock_port = Mock() - self.driver = chromium.ChromiumDriver(mock_port, image_path=None, options=None) + mock_port.get_option = lambda option_name: '' + self.driver = chromium.ChromiumDriver(mock_port, worker_number=0) def test_test_shell_command(self): expected_command = "test.html 2 checksum\n" diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/config.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/config.py index cad5e37..88f1146 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/config.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/config.py @@ -75,7 +75,6 @@ class Config(object): if configuration: flags = ["--configuration", self._FLAGS_FROM_CONFIGURATIONS[configuration]] - configuration = "" else: configuration = "" flags = ["--top-level"] @@ -133,7 +132,7 @@ class Config(object): # This code will also work if there is no SCM system at all. if not self._webkit_base_dir: abspath = os.path.abspath(__file__) - self._webkit_base_dir = abspath[0:abspath.find('WebKitTools')] + self._webkit_base_dir = abspath[0:abspath.find('WebKitTools') - 1] return self._webkit_base_dir def _script_path(self, script_name): diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/config_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/config_unittest.py index 9bea014..8ec28fc 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/config_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/config_unittest.py @@ -38,13 +38,37 @@ from webkitpy.common.system import outputcapture import config + +def mock_run_command(arg_list): + # Set this to True to test actual output (where possible). + integration_test = False + if integration_test: + return executive.Executive().run_command(arg_list) + + if 'webkit-build-directory' in arg_list[1]: + return mock_webkit_build_directory(arg_list[2:]) + return 'Error' + + +def mock_webkit_build_directory(arg_list): + if arg_list == ['--top-level']: + return '/WebKitBuild' + elif arg_list == ['--configuration', '--debug']: + return '/WebKitBuild/Debug' + elif arg_list == ['--configuration', '--release']: + return '/WebKitBuild/Release' + return 'Error' + + class ConfigTest(unittest.TestCase): def tearDown(self): config.clear_cached_configuration() - def make_config(self, output='', files={}, exit_code=0, exception=None): + def make_config(self, output='', files={}, exit_code=0, exception=None, + run_command_fn=None): e = executive_mock.MockExecutive2(output=output, exit_code=exit_code, - exception=exception) + exception=exception, + run_command_fn=run_command_fn) fs = filesystem_mock.MockFileSystem(files) return config.Config(e, fs) @@ -54,23 +78,17 @@ class ConfigTest(unittest.TestCase): c = self.make_config('foo', {'foo/Configuration': contents}) self.assertEqual(c.default_configuration(), expected) - def test_build_directory_toplevel(self): - c = self.make_config('toplevel') - self.assertEqual(c.build_directory(None), 'toplevel') + def test_build_directory(self): + # --top-level + c = self.make_config(run_command_fn=mock_run_command) + self.assertTrue(c.build_directory(None).endswith('WebKitBuild')) # Test again to check caching - self.assertEqual(c.build_directory(None), 'toplevel') - - def test_build_directory__release(self): - c = self.make_config('release') - self.assertEqual(c.build_directory('Release'), 'release') - - def test_build_directory__debug(self): - c = self.make_config('debug') - self.assertEqual(c.build_directory('Debug'), 'debug') + self.assertTrue(c.build_directory(None).endswith('WebKitBuild')) - def test_build_directory__unknown(self): - c = self.make_config("unknown") + # Test other values + self.assertTrue(c.build_directory('Release').endswith('/Release')) + self.assertTrue(c.build_directory('Debug').endswith('/Debug')) self.assertRaises(KeyError, c.build_directory, 'Unknown') def test_build_dumprendertree__success(self): @@ -168,6 +186,7 @@ class ConfigTest(unittest.TestCase): c = config.Config(executive.Executive(), filesystem.FileSystem()) base_dir = c.webkit_base_dir() self.assertTrue(base_dir) + self.assertNotEqual(base_dir[-1], '/') orig_cwd = os.getcwd() os.chdir(os.environ['HOME']) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py index 96d0d55..4ed34e6 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py @@ -95,31 +95,30 @@ class DryRunPort(object): def stop_websocket_server(self): pass - def create_driver(self, image_path, options): - return DryrunDriver(self, image_path, options, executive=None) + def create_driver(self, worker_number): + return DryrunDriver(self, worker_number) class DryrunDriver(base.Driver): """Dryrun implementation of the DumpRenderTree / Driver interface.""" - def __init__(self, port, image_path, options, executive): + def __init__(self, port, worker_number): self._port = port - self._image_path = image_path - self._executive = executive - self._layout_tests_dir = None + self._worker_number = worker_number + + def cmd_line(self): + return ['None'] def poll(self): return None - def run_test(self, uri, timeoutms, image_hash): + def run_test(self, test_input): start_time = time.time() - test_name = self._port.uri_to_test_name(uri) - path = os.path.join(self._port.layout_tests_dir(), test_name) - text_output = self._port.expected_text(path) + text_output = self._port.expected_text(test_input.filename) - if image_hash is not None: - image = self._port.expected_image(path) - hash = self._port.expected_checksum(path) + if test_input.image_hash is not None: + image = self._port.expected_image(test_input.filename) + hash = self._port.expected_checksum(test_input.filename) else: image = None hash = None diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py index 04ada50..c4b36ac 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py @@ -37,6 +37,8 @@ mock_options = mocktool.MockOptions(results_directory='layout-test-results', use_apache=True, configuration='Release') +# FIXME: This should be used for all ports, not just WebKit Mac. See +# https://bugs.webkit.org/show_bug.cgi?id=50043 . class PortTestCase(unittest.TestCase): """Tests the WebKit port implementation.""" @@ -44,6 +46,12 @@ class PortTestCase(unittest.TestCase): """Override in subclass.""" raise NotImplementedError() + def test_driver_cmd_line(self): + port = self.make_port() + if not port: + return + self.assertTrue(len(port.driver_cmd_line())) + def test_http_server(self): port = self.make_port() if not port: diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py index 0a27821..8e27f35 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py @@ -226,8 +226,8 @@ class TestPort(base.Port): def setup_test_run(self): pass - def create_driver(self, image_path, options): - return TestDriver(self, image_path, options, executive=None) + def create_driver(self, worker_number): + return TestDriver(self, worker_number) def start_http_server(self): pass @@ -281,25 +281,25 @@ WONTFIX SKIP : failures/expected/exception.html = CRASH class TestDriver(base.Driver): """Test/Dummy implementation of the DumpRenderTree interface.""" - def __init__(self, port, image_path, options, executive): + def __init__(self, port, worker_number): self._port = port - self._image_path = image_path - self._executive = executive - self._image_written = False + + def cmd_line(self): + return ['None'] def poll(self): return True - def run_test(self, uri, timeoutms, image_hash): + def run_test(self, test_input): start_time = time.time() - test_name = self._port.uri_to_test_name(uri) + test_name = self._port.relative_test_filename(test_input.filename) test = self._port._tests[test_name] if test.keyboard: raise KeyboardInterrupt if test.exception: raise ValueError('exception from ' + test_name) if test.hang: - time.sleep((float(timeoutms) * 4) / 1000.0) + time.sleep((float(test_input.timeout) * 4) / 1000.0) return test_output.TestOutput(test.actual_text, test.actual_image, test.actual_checksum, test.crash, time.time() - start_time, test.timeout, diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files.py index 3fa0fb3..2c0a7b6 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files.py @@ -78,7 +78,7 @@ def find(port, paths): # Now walk all the paths passed in on the command line and get filenames test_files = set() for path in paths_to_walk: - if os.path.isfile(path) and _has_supported_extension(path): + if os.path.isfile(path) and _is_test_file(path): test_files.add(os.path.normpath(path)) continue @@ -95,7 +95,7 @@ def find(port, paths): dirs.remove(directory) for filename in files: - if _has_supported_extension(filename): + if _is_test_file(filename): filename = os.path.join(root, filename) filename = os.path.normpath(filename) test_files.add(filename) @@ -111,3 +111,18 @@ def _has_supported_extension(filename): test on.""" extension = os.path.splitext(filename)[1] return extension in _supported_file_extensions + + +def _is_reference_html_file(filename): + """Return true if the filename points to a reference HTML file.""" + if (filename.endswith('-expected.html') or + filename.endswith('-expected-mismatch.html')): + _log.warn("Reftests are not supported - ignoring %s" % filename) + return True + return False + + +def _is_test_file(filename): + """Return true if the filename points to a test file.""" + return (_has_supported_extension(filename) and + not _is_reference_html_file(filename)) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py index c37eb92..83525c8 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py @@ -63,6 +63,13 @@ class TestFilesTest(unittest.TestCase): tests = test_files.find(port, ['userscripts/resources']) self.assertEqual(tests, set([])) + def test_is_test_file(self): + self.assertTrue(test_files._is_test_file('foo.html')) + self.assertTrue(test_files._is_test_file('foo.shtml')) + self.assertFalse(test_files._is_test_file('foo.png')) + self.assertFalse(test_files._is_test_file('foo-expected.html')) + self.assertFalse(test_files._is_test_file('foo-expected-mismatch.html')) + if __name__ == '__main__': unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py index 06797c6..09be833 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py @@ -46,8 +46,6 @@ import operator import tempfile import shutil -from webkitpy.common.system.executive import Executive - import webkitpy.common.system.ospath as ospath import webkitpy.layout_tests.layout_package.test_output as test_output import webkitpy.layout_tests.port.base as base @@ -185,9 +183,8 @@ class WebKitPort(base.Port): # This port doesn't require any specific configuration. pass - def create_driver(self, image_path, options): - return WebKitDriver(self, image_path, options, - executive=self._executive) + def create_driver(self, worker_number): + return WebKitDriver(self, worker_number) def test_base_platform_names(self): # At the moment we don't use test platform names, but we have @@ -389,40 +386,36 @@ class WebKitPort(base.Port): class WebKitDriver(base.Driver): """WebKit implementation of the DumpRenderTree interface.""" - def __init__(self, port, image_path, options, executive=Executive()): + def __init__(self, port, worker_number): + self._worker_number = worker_number self._port = port - self._image_path = image_path - self._executive = executive self._driver_tempdir = tempfile.mkdtemp(prefix='DumpRenderTree-') def __del__(self): shutil.rmtree(self._driver_tempdir) - def _driver_args(self): - driver_args = [] + def cmd_line(self): + cmd = self._command_wrapper(self._port.get_option('wrapper')) + cmd += [self._port._path_to_driver(), '-'] - if self._image_path: - driver_args.append('--pixel-tests') + if self._port.get_option('pixel_tests'): + cmd.append('--pixel-tests') if self._port.get_option('use_drt'): if self._port.get_option('accelerated_compositing'): - driver_args.append('--enable-accelerated-compositing') + cmd.append('--enable-accelerated-compositing') if self._port.get_option('accelerated_2d_canvas'): - driver_args.append('--enable-accelerated-2d-canvas') + cmd.append('--enable-accelerated-2d-canvas') - return driver_args + return cmd def start(self): - command = self._command_wrapper(self._port.get_option('wrapper')) - command += [self._port._path_to_driver(), '-'] - command += self._driver_args() - environment = self._port.setup_environ_for_server() environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path() environment['DUMPRENDERTREE_TEMP'] = self._driver_tempdir self._server_process = server_process.ServerProcess(self._port, - "DumpRenderTree", command, environment) + "DumpRenderTree", self.cmd_line(), environment) def poll(self): return self._server_process.poll() @@ -433,14 +426,15 @@ class WebKitDriver(base.Driver): return # FIXME: This function is huge. - def run_test(self, uri, timeoutms, image_hash): + def run_test(self, test_input): + uri = self._port.filename_to_uri(test_input.filename) if uri.startswith("file:///"): command = uri[7:] else: command = uri - if image_hash: - command += "'" + image_hash + if test_input.image_hash: + command += "'" + test_input.image_hash command += "\n" start_time = time.time() @@ -451,7 +445,7 @@ class WebKitDriver(base.Driver): output = str() # Use a byte array for output, even though it should be UTF-8. image = str() - timeout = int(timeoutms) / 1000.0 + timeout = int(test_input.timeout) / 1000.0 deadline = time.time() + timeout line = self._server_process.read_line(timeout) while (not self._server_process.timed_out diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py index 119de8c..f4e92a6 100755 --- a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py @@ -66,17 +66,16 @@ import traceback from layout_package import dump_render_tree_thread from layout_package import json_layout_results_generator +from layout_package import message_broker from layout_package import printing from layout_package import test_expectations from layout_package import test_failures from layout_package import test_results from layout_package import test_results_uploader -from test_types import image_diff -from test_types import text_diff -from test_types import test_type_base from webkitpy.common.system import user from webkitpy.thirdparty import simplejson +from webkitpy.tool import grammar import port @@ -102,6 +101,10 @@ class TestInput: # FIXME: filename should really be test_name as a relative path. self.filename = filename self.timeout = timeout + # The image_hash is used to avoid doing an image dump if the + # checksums match. The image_hash is set later, and only if it is needed + # for the test. + self.image_hash = None class ResultSummary(object): @@ -237,27 +240,24 @@ class TestRunner: # in DumpRenderTree. DEFAULT_TEST_TIMEOUT_MS = 6 * 1000 - def __init__(self, port, options, printer): + def __init__(self, port, options, printer, message_broker): """Initialize test runner data structures. Args: port: an object implementing port-specific options: a dictionary of command line options printer: a Printer object to record updates to. + message_broker: object used to communicate with workers. """ self._port = port self._options = options self._printer = printer + self._message_broker = message_broker # disable wss server. need to install pyOpenSSL on buildbots. # self._websocket_secure_server = websocket_server.PyWebSocket( # options.results_directory, use_tls=True, port=9323) - # a list of TestType objects - self._test_types = [text_diff.TestTextDiff] - if options.pixel_tests: - self._test_types.append(image_diff.ImageDiff) - # a set of test files, and the same tests as a list self._test_files = set() self._test_files_list = None @@ -488,7 +488,7 @@ class TestRunner: """Returns the appropriate TestInput object for the file. Mostly this is used for looking up the timeout value (in ms) to use for the given test.""" - if self._expectations.has_modifier(test_file, test_expectations.SLOW): + if self._test_is_slow(test_file): return TestInput(test_file, self._options.slow_time_out_ms) return TestInput(test_file, self._options.time_out_ms) @@ -498,23 +498,30 @@ class TestRunner: split_path = test_file.split(os.sep) return 'http' in split_path or 'websocket' in split_path - def _get_test_file_queue(self, test_files): - """Create the thread safe queue of lists of (test filenames, test URIs) - tuples. Each TestShellThread pulls a list from this queue and runs - those tests in order before grabbing the next available list. + def _test_is_slow(self, test_file): + return self._expectations.has_modifier(test_file, + test_expectations.SLOW) - Shard the lists by directory. This helps ensure that tests that depend - on each other (aka bad tests!) continue to run together as most - cross-tests dependencies tend to occur within the same directory. + def _shard_tests(self, test_files, use_real_shards): + """Groups tests into batches. + This helps ensure that tests that depend on each other (aka bad tests!) + continue to run together as most cross-tests dependencies tend to + occur within the same directory. If use_real_shards is false, we + put each (non-HTTP/websocket) test into its own shard for maximum + concurrency instead of trying to do any sort of real sharding. Return: - The Queue of lists of TestInput objects. + A list of lists of TestInput objects. """ + # FIXME: when we added http locking, we changed how this works such + # that we always lump all of the HTTP threads into a single shard. + # That will slow down experimental-fully-parallel, but it's unclear + # what the best alternative is completely revamping how we track + # when to grab the lock. test_lists = [] tests_to_http_lock = [] - if (self._options.experimental_fully_parallel or - self._is_single_threaded()): + if not use_real_shards: for test_file in test_files: test_input = self._get_test_input_for_file(test_file) if self._test_requires_lock(test_file): @@ -553,23 +560,7 @@ class TestRunner: tests_to_http_lock.reverse() test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock)) - filename_queue = Queue.Queue() - for item in test_lists: - filename_queue.put(item) - return filename_queue - - def _get_test_args(self, index): - """Returns the tuple of arguments for tests and for DumpRenderTree.""" - test_args = test_type_base.TestArguments() - test_args.png_path = None - if self._options.pixel_tests: - png_path = os.path.join(self._options.results_directory, - "png_result%s.png" % index) - test_args.png_path = png_path - test_args.new_baseline = self._options.new_baseline - test_args.reset_results = self._options.reset_results - - return test_args + return test_lists def _contains_tests(self, subdir): for test_file in self._test_files: @@ -577,39 +568,8 @@ class TestRunner: return True return False - def _instantiate_dump_render_tree_threads(self, test_files, - result_summary): - """Instantitates and starts the TestShellThread(s). - - Return: - The list of threads. - """ - filename_queue = self._get_test_file_queue(test_files) - - # Instantiate TestShellThreads and start them. - threads = [] - for i in xrange(int(self._options.child_processes)): - # Create separate TestTypes instances for each thread. - test_types = [] - for test_type in self._test_types: - test_types.append(test_type(self._port, - self._options.results_directory)) - - test_args = self._get_test_args(i) - thread = dump_render_tree_thread.TestShellThread(self._port, - self._options, filename_queue, self._result_queue, - test_types, test_args) - if self._is_single_threaded(): - thread.run_in_main_thread(self, result_summary) - else: - thread.start() - threads.append(thread) - - return threads - - def _is_single_threaded(self): - """Returns whether we should run all the tests in the main thread.""" - return int(self._options.child_processes) == 1 + def _num_workers(self): + return int(self._options.child_processes) def _run_tests(self, file_list, result_summary): """Runs the tests in the file_list. @@ -625,59 +585,48 @@ class TestRunner: in the form {filename:filename, test_run_time:test_run_time} result_summary: summary object to populate with the results """ - # FIXME: We should use webkitpy.tool.grammar.pluralize here. - plural = "" - if not self._is_single_threaded(): - plural = "s" - self._printer.print_update('Starting %s%s ...' % - (self._port.driver_name(), plural)) - threads = self._instantiate_dump_render_tree_threads(file_list, - result_summary) + + self._printer.print_update('Sharding tests ...') + num_workers = self._num_workers() + test_lists = self._shard_tests(file_list, + num_workers > 1 and not self._options.experimental_fully_parallel) + filename_queue = Queue.Queue() + for item in test_lists: + filename_queue.put(item) + + self._printer.print_update('Starting %s ...' % + grammar.pluralize('worker', num_workers)) + message_broker = self._message_broker + self._current_filename_queue = filename_queue + self._current_result_summary = result_summary + + if not self._options.dry_run: + threads = message_broker.start_workers(self) + else: + threads = {} + self._printer.print_update("Starting testing ...") + keyboard_interrupted = False + if not self._options.dry_run: + try: + message_broker.run_message_loop() + except KeyboardInterrupt: + _log.info("Interrupted, exiting") + message_broker.cancel_workers() + keyboard_interrupted = True + except: + # Unexpected exception; don't try to clean up workers. + _log.info("Exception raised, exiting") + raise - keyboard_interrupted = self._wait_for_threads_to_finish(threads, - result_summary) - (thread_timings, test_timings, individual_test_timings) = \ + thread_timings, test_timings, individual_test_timings = \ self._collect_timing_info(threads) return (keyboard_interrupted, thread_timings, test_timings, individual_test_timings) - def _wait_for_threads_to_finish(self, threads, result_summary): - keyboard_interrupted = False - try: - # Loop through all the threads waiting for them to finish. - some_thread_is_alive = True - while some_thread_is_alive: - some_thread_is_alive = False - t = time.time() - for thread in threads: - exception_info = thread.exception_info() - if exception_info is not None: - # Re-raise the thread's exception here to make it - # clear that testing was aborted. Otherwise, - # the tests that did not run would be assumed - # to have passed. - raise exception_info[0], exception_info[1], exception_info[2] - - if thread.isAlive(): - some_thread_is_alive = True - next_timeout = thread.next_timeout() - if (next_timeout and t > next_timeout): - _log_wedged_thread(thread) - thread.clear_next_timeout() - - self.update_summary(result_summary) - - if some_thread_is_alive: - time.sleep(0.01) - - except KeyboardInterrupt: - keyboard_interrupted = True - for thread in threads: - thread.cancel() - - return keyboard_interrupted + def update(self): + self.update_summary(self._current_result_summary) def _collect_timing_info(self, threads): test_timings = {} @@ -793,16 +742,18 @@ class TestRunner: self._expectations, result_summary, retry_summary) self._printer.print_unexpected_results(unexpected_results) - if self._options.record_results: + if (self._options.record_results and not self._options.dry_run and + not keyboard_interrupted): # Write the same data to log files and upload generated JSON files # to appengine server. self._upload_json_files(unexpected_results, result_summary, individual_test_timings) # Write the summary to disk (results.html) and display it if requested. - wrote_results = self._write_results_html_file(result_summary) - if self._options.show_results and wrote_results: - self._show_results_html_file() + if not self._options.dry_run: + wrote_results = self._write_results_html_file(result_summary) + if self._options.show_results and wrote_results: + self._show_results_html_file() # Now that we've completed all the processing we can, we re-raise # a KeyboardInterrupt if necessary so the caller can handle it. @@ -947,12 +898,15 @@ class TestRunner: (self._options.time_out_ms, self._options.slow_time_out_ms)) - if self._is_single_threaded(): + if self._num_workers() == 1: p.print_config("Running one %s" % self._port.driver_name()) else: p.print_config("Running %s %ss in parallel" % (self._options.child_processes, self._port.driver_name())) + p.print_config('Command line: ' + + ' '.join(self._port.driver_cmd_line())) + p.print_config("Worker model: %s" % self._options.worker_model) p.print_config("") def _print_expected_results_of_type(self, result_summary, @@ -1067,8 +1021,7 @@ class TestRunner: for test_tuple in individual_test_timings: filename = test_tuple.filename is_timeout_crash_or_slow = False - if self._expectations.has_modifier(filename, - test_expectations.SLOW): + if self._test_is_slow(filename): is_timeout_crash_or_slow = True slow_tests.append(test_tuple) @@ -1342,11 +1295,13 @@ def run(port, options, args, regular_output=sys.stderr, printer.cleanup() return 0 + broker = message_broker.get(port, options) + # We wrap any parts of the run that are slow or likely to raise exceptions # in a try/finally to ensure that we clean up the logging configuration. num_unexpected_results = -1 try: - test_runner = TestRunner(port, options, printer) + test_runner = TestRunner(port, options, printer, broker) test_runner._print_config() printer.print_update("Collecting tests ...") @@ -1375,6 +1330,7 @@ def run(port, options, args, regular_output=sys.stderr, _log.debug("Testing completed, Exit status: %d" % num_unexpected_results) finally: + broker.cleanup() printer.cleanup() return num_unexpected_results @@ -1383,8 +1339,11 @@ def run(port, options, args, regular_output=sys.stderr, def _set_up_derived_options(port_obj, options): """Sets the options values that depend on other options values.""" + if options.worker_model == 'inline': + if options.child_processes and int(options.child_processes) > 1: + _log.warning("--worker-model=inline overrides --child-processes") + options.child_processes = "1" if not options.child_processes: - # FIXME: Investigate perf/flakiness impact of using cpu_count + 1. options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES", str(port_obj.default_child_processes())) @@ -1568,6 +1527,9 @@ def parse_args(args=None): optparse.make_option("--no-build", dest="build", action="store_false", help="Don't check to see if the " "DumpRenderTree build is up-to-date."), + optparse.make_option("-n", "--dry-run", action="store_true", + default=False, + help="Do everything but actually run the tests or upload results."), # old-run-webkit-tests has --valgrind instead of wrapper. optparse.make_option("--wrapper", help="wrapper command to insert before invocations of " @@ -1607,6 +1569,9 @@ def parse_args(args=None): optparse.make_option("--child-processes", help="Number of DumpRenderTrees to run in parallel."), # FIXME: Display default number of child processes that will run. + optparse.make_option("--worker-model", action="store", + default="threads", help=("controls worker model. Valid values are " + "'inline' and 'threads' (default).")), optparse.make_option("--experimental-fully-parallel", action="store_true", default=False, help="run all tests in parallel"), @@ -1618,7 +1583,7 @@ def parse_args(args=None): # Number of times to run the set of tests (e.g. ABCABCABC) optparse.make_option("--print-last-failures", action="store_true", default=False, help="Print the tests in the last run that " - "had unexpected failures (or passes)."), + "had unexpected failures (or passes) and then exit."), optparse.make_option("--retest-last-failures", action="store_true", default=False, help="re-test the tests in the last run that " "had unexpected failures (or passes)."), @@ -1662,20 +1627,7 @@ def parse_args(args=None): old_run_webkit_tests_compat) option_parser = optparse.OptionParser(option_list=option_list) - options, args = option_parser.parse_args(args) - - return options, args - - -def _log_wedged_thread(thread): - """Log information about the given thread state.""" - id = thread.id() - stack = dump_render_tree_thread.find_thread_stack(id) - assert(stack is not None) - _log.error("") - _log.error("thread %s (%d) is wedged" % (thread.getName(), id)) - dump_render_tree_thread.log_stack(stack) - _log.error("") + return option_parser.parse_args(args) def main(): @@ -1683,6 +1635,7 @@ def main(): port_obj = port.get(options.platform, options) return run(port_obj, options, args) + if '__main__' == __name__: try: sys.exit(main()) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py index 54e1dc0..6bb741a 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py @@ -72,12 +72,14 @@ def passing_run(extra_args=None, port_obj=None, record_results=False, args.extend(['--platform', 'test']) if not record_results: args.append('--no-record-results') + if not '--child-processes' in extra_args: + args.extend(['--worker-model', 'inline']) args.extend(extra_args) if not tests_included: # We use the glob to test that globbing works. args.extend(['passes', 'http/tests', - 'http/tests/websocket/tests', + 'websocket/tests', 'failures/expected/*']) options, parsed_args = run_webkit_tests.parse_args(args) if not port_obj: @@ -92,21 +94,30 @@ def logging_run(extra_args=None, port_obj=None, tests_included=False): args = ['--no-record-results'] if not '--platform' in extra_args: args.extend(['--platform', 'test']) + if not '--child-processes' in extra_args: + args.extend(['--worker-model', 'inline']) args.extend(extra_args) if not tests_included: args.extend(['passes', 'http/tests', - 'http/tests/websocket/tests', + 'websocket/tests', 'failures/expected/*']) - options, parsed_args = run_webkit_tests.parse_args(args) - user = MockUser() - if not port_obj: - port_obj = port.get(port_name=options.platform, options=options, user=user) - buildbot_output = array_stream.ArrayStream() - regular_output = array_stream.ArrayStream() - res = run_webkit_tests.run(port_obj, options, parsed_args, - buildbot_output=buildbot_output, - regular_output=regular_output) + + oc = outputcapture.OutputCapture() + try: + oc.capture_output() + options, parsed_args = run_webkit_tests.parse_args(args) + user = MockUser() + if not port_obj: + port_obj = port.get(port_name=options.platform, options=options, + user=user) + buildbot_output = array_stream.ArrayStream() + regular_output = array_stream.ArrayStream() + res = run_webkit_tests.run(port_obj, options, parsed_args, + buildbot_output=buildbot_output, + regular_output=regular_output) + finally: + oc.restore_output() return (res, buildbot_output, regular_output, user) @@ -116,7 +127,7 @@ def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False): '--print', 'nothing', '--platform', 'test', '--no-record-results', - '--child-processes', '1'] + '--worker-model', 'inline'] args.extend(extra_args) if not tests_included: # Not including http tests since they get run out of order (that @@ -128,8 +139,8 @@ def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False): test_batches = [] class RecordingTestDriver(TestDriver): - def __init__(self, port, image_path, options): - TestDriver.__init__(self, port, image_path, options, executive=None) + def __init__(self, port, worker_number): + TestDriver.__init__(self, port, worker_number) self._current_test_batch = None def poll(self): @@ -139,16 +150,17 @@ def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False): def stop(self): self._current_test_batch = None - def run_test(self, uri, timeoutms, image_hash): + def run_test(self, test_input): if self._current_test_batch is None: self._current_test_batch = [] test_batches.append(self._current_test_batch) - self._current_test_batch.append(self._port.uri_to_test_name(uri)) - return TestDriver.run_test(self, uri, timeoutms, image_hash) + test_name = self._port.relative_test_filename(test_input.filename) + self._current_test_batch.append(test_name) + return TestDriver.run_test(self, test_input) class RecordingTestPort(TestPort): - def create_driver(self, image_path, options): - return RecordingTestDriver(self, image_path, options) + def create_driver(self, worker_number): + return RecordingTestDriver(self, worker_number) recording_port = RecordingTestPort(options=options, user=user) logging_run(extra_args=args, port_obj=recording_port, tests_included=True) @@ -189,6 +201,13 @@ class MainTest(unittest.TestCase): self.assertTrue('Running 2 DumpRenderTrees in parallel\n' in regular_output.get()) + def test_dryrun(self): + batch_tests_run = get_tests_run(['--dry-run']) + self.assertEqual(batch_tests_run, []) + + batch_tests_run = get_tests_run(['-n']) + self.assertEqual(batch_tests_run, []) + def test_exception_raised(self): self.assertRaises(ValueError, logging_run, ['failures/expected/exception.html'], tests_included=True) @@ -214,7 +233,7 @@ class MainTest(unittest.TestCase): def test_keyboard_interrupt(self): # Note that this also tests running a test marked as SKIP if # you specify it explicitly. - self.assertRaises(KeyboardInterrupt, passing_run, + self.assertRaises(KeyboardInterrupt, logging_run, ['failures/expected/keyboard.html'], tests_included=True) def test_last_results(self): @@ -359,9 +378,24 @@ class MainTest(unittest.TestCase): test_port = get_port_for_run(base_args) self.assertEqual(None, test_port.tolerance_used_for_diff_image) + def test_worker_model__inline(self): + self.assertTrue(passing_run(['--worker-model', 'inline'])) + + def test_worker_model__threads(self): + self.assertTrue(passing_run(['--worker-model', 'threads'])) + + def test_worker_model__processes(self): + self.assertRaises(ValueError, logging_run, + ['--worker-model', 'processes']) + + def test_worker_model__unknown(self): + self.assertRaises(ValueError, logging_run, + ['--worker-model', 'unknown']) + MainTest = skip_if(MainTest, sys.platform == 'cygwin' and compare_version(sys, '2.6')[0] < 0, 'new-run-webkit-tests tests hang on Cygwin Python 2.5.2') + def _mocked_open(original_open, file_list): def _wrapper(name, mode, encoding): if name.find("-expected.") != -1 and mode.find("w") != -1: @@ -439,7 +473,8 @@ class TestRunnerTest(unittest.TestCase): mock_port.relative_test_filename = lambda name: name mock_port.filename_to_uri = lambda name: name - runner = run_webkit_tests.TestRunner(port=mock_port, options=Mock(), printer=Mock()) + runner = run_webkit_tests.TestRunner(port=mock_port, options=Mock(), + printer=Mock(), message_broker=Mock()) expected_html = u"""<html> <head> <title>Layout Test Results (time)</title> @@ -453,20 +488,11 @@ class TestRunnerTest(unittest.TestCase): html = runner._results_html(["test_path"], {}, "Title", override_time="time") self.assertEqual(html, expected_html) - def queue_to_list(self, queue): - queue_list = [] - while(True): - try: - queue_list.append(queue.get_nowait()) - except Queue.Empty: - break - return queue_list - - def test_get_test_file_queue(self): - # Test that _get_test_file_queue in run_webkit_tests.TestRunner really + def test_shard_tests(self): + # Test that _shard_tests in run_webkit_tests.TestRunner really # put the http tests first in the queue. - runner = TestRunnerWrapper(port=Mock(), options=Mock(), printer=Mock()) - runner._options.experimental_fully_parallel = False + runner = TestRunnerWrapper(port=Mock(), options=Mock(), + printer=Mock(), message_broker=Mock()) test_list = [ "LayoutTests/websocket/tests/unicode.htm", @@ -487,19 +513,16 @@ class TestRunnerTest(unittest.TestCase): 'LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html', ]) - runner._options.child_processes = 1 - test_queue_for_single_thread = runner._get_test_file_queue(test_list) - runner._options.child_processes = 2 - test_queue_for_multi_thread = runner._get_test_file_queue(test_list) - - single_thread_results = self.queue_to_list(test_queue_for_single_thread) - multi_thread_results = self.queue_to_list(test_queue_for_multi_thread) + # FIXME: Ideally the HTTP tests don't have to all be in one shard. + single_thread_results = runner._shard_tests(test_list, False) + multi_thread_results = runner._shard_tests(test_list, True) self.assertEqual("tests_to_http_lock", single_thread_results[0][0]) self.assertEqual(expected_tests_to_http_lock, set(single_thread_results[0][1])) self.assertEqual("tests_to_http_lock", multi_thread_results[0][0]) self.assertEqual(expected_tests_to_http_lock, set(multi_thread_results[0][1])) + class DryrunTest(unittest.TestCase): # FIXME: it's hard to know which platforms are safe to test; the # chromium platforms require a chromium checkout, and the mac platform @@ -520,114 +543,5 @@ class DryrunTest(unittest.TestCase): '--pixel-tests'])) -class TestThread(dump_render_tree_thread.WatchableThread): - def __init__(self, started_queue, stopping_queue): - dump_render_tree_thread.WatchableThread.__init__(self) - self._started_queue = started_queue - self._stopping_queue = stopping_queue - self._timeout = False - self._timeout_queue = Queue.Queue() - - def run(self): - self._covered_run() - - def _covered_run(self): - # FIXME: this is a separate routine to work around a bug - # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85. - self._thread_id = thread.get_ident() - try: - self._started_queue.put('') - msg = self._stopping_queue.get() - if msg == 'KeyboardInterrupt': - raise KeyboardInterrupt - elif msg == 'Exception': - raise ValueError() - elif msg == 'Timeout': - self._timeout = True - self._timeout_queue.get() - except: - self._exception_info = sys.exc_info() - - def next_timeout(self): - if self._timeout: - self._timeout_queue.put('done') - return time.time() - 10 - return time.time() - - -class TestHandler(logging.Handler): - def __init__(self, astream): - logging.Handler.__init__(self) - self._stream = astream - - def emit(self, record): - self._stream.write(self.format(record)) - - -class WaitForThreadsToFinishTest(unittest.TestCase): - class MockTestRunner(run_webkit_tests.TestRunner): - def __init__(self): - pass - - def __del__(self): - pass - - def update_summary(self, result_summary): - pass - - def run_one_thread(self, msg): - runner = self.MockTestRunner() - starting_queue = Queue.Queue() - stopping_queue = Queue.Queue() - child_thread = TestThread(starting_queue, stopping_queue) - child_thread.start() - started_msg = starting_queue.get() - stopping_queue.put(msg) - threads = [child_thread] - return runner._wait_for_threads_to_finish(threads, None) - - def test_basic(self): - interrupted = self.run_one_thread('') - self.assertFalse(interrupted) - - def test_interrupt(self): - interrupted = self.run_one_thread('KeyboardInterrupt') - self.assertTrue(interrupted) - - def test_timeout(self): - oc = outputcapture.OutputCapture() - oc.capture_output() - interrupted = self.run_one_thread('Timeout') - self.assertFalse(interrupted) - oc.restore_output() - - def test_exception(self): - self.assertRaises(ValueError, self.run_one_thread, 'Exception') - - -class StandaloneFunctionsTest(unittest.TestCase): - def test_log_wedged_thread(self): - oc = outputcapture.OutputCapture() - oc.capture_output() - logger = run_webkit_tests._log - astream = array_stream.ArrayStream() - handler = TestHandler(astream) - logger.addHandler(handler) - - starting_queue = Queue.Queue() - stopping_queue = Queue.Queue() - child_thread = TestThread(starting_queue, stopping_queue) - child_thread.start() - msg = starting_queue.get() - - run_webkit_tests._log_wedged_thread(child_thread) - stopping_queue.put('') - child_thread.join(timeout=1.0) - - self.assertFalse(astream.empty()) - self.assertFalse(child_thread.isAlive()) - oc.restore_output() - - if __name__ == '__main__': unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py index 41fe9bd..da466c8 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/image_diff.py @@ -103,8 +103,8 @@ class ImageDiff(test_type_base.TestTypeBase): # If we're generating a new baseline, we pass. if test_args.new_baseline or test_args.reset_results: - self._save_baseline_files(filename, actual_test_output.image_hash, - actual_test_output.image, + self._save_baseline_files(filename, actual_test_output.image, + actual_test_output.image_hash, test_args.new_baseline) return failures diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py index 66e42ba..ca4b17d 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/test_types/text_diff.py @@ -66,7 +66,8 @@ class TestTextDiff(test_type_base.TestTypeBase): # Although all test_shell/DumpRenderTree output should be utf-8, # we do not ever decode it inside run-webkit-tests. For some tests # DumpRenderTree may not output utf-8 text (e.g. webarchives). - self._save_baseline_data(filename, output, ".txt", encoding=None, + self._save_baseline_data(filename, actual_test_output.text, + ".txt", encoding=None, generate_new_baseline=test_args.new_baseline) return failures |