diff options
Diffstat (limited to 'Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py')
| -rw-r--r-- | Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py | 322 |
1 files changed, 322 insertions, 0 deletions
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py new file mode 100644 index 0000000..96e3ee6 --- /dev/null +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py @@ -0,0 +1,322 @@ +# Copyright (C) 2011 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import logging +import threading +import time + +from webkitpy.layout_tests.port import base + +from webkitpy.layout_tests.test_types import text_diff +from webkitpy.layout_tests.test_types import image_diff + +from webkitpy.layout_tests.layout_package import test_failures +from webkitpy.layout_tests.layout_package.test_results import TestResult + + +_log = logging.getLogger(__name__) + + +class ExpectedDriverOutput: + """Groups information about an expected driver output.""" + def __init__(self, text, image, image_hash): + self.text = text + self.image = image + self.image_hash = image_hash + + +class SingleTestRunner: + + def __init__(self, options, port, worker_name, worker_number): + self._options = options + self._port = port + self._worker_name = worker_name + self._worker_number = worker_number + self._driver = None + self._test_types = [] + self.has_http_lock = False + for cls in self._get_test_type_classes(): + self._test_types.append(cls(self._port, + self._options.results_directory)) + + def cleanup(self): + self.kill_dump_render_tree() + if self.has_http_lock: + self.stop_servers_with_lock() + + def _get_test_type_classes(self): + classes = [text_diff.TestTextDiff] + if self._options.pixel_tests: + classes.append(image_diff.ImageDiff) + return classes + + def timeout(self, test_input): + # We calculate how long we expect the test to take. + # + # The DumpRenderTree watchdog uses 2.5x the timeout; we want to be + # larger than that. We also add a little more padding if we're + # running tests in a separate thread. + # + # Note that we need to convert the test timeout from a + # string value in milliseconds to a float for Python. + driver_timeout_sec = 3.0 * float(test_input.timeout) / 1000.0 + if not self._options.run_singly: + return driver_timeout_sec + + thread_padding_sec = 1.0 + thread_timeout_sec = driver_timeout_sec + thread_padding_sec + return thread_timeout_sec + + def run_test(self, test_input, timeout): + if self._options.run_singly: + return self._run_test_in_another_thread(test_input, timeout) + else: + return self._run_test_in_this_thread(test_input) + return result + + def _run_test_in_another_thread(self, test_input, thread_timeout_sec): + """Run a test in a separate thread, enforcing a hard time limit. + + Since we can only detect the termination of a thread, not any internal + state or progress, we can only run per-test timeouts when running test + files singly. + + Args: + test_input: Object containing the test filename and timeout + thread_timeout_sec: time to wait before killing the driver process. + Returns: + A TestResult + """ + worker = self + result = None + + driver = worker._port.create_driver(worker._worker_number) + driver.start() + + class SingleTestThread(threading.Thread): + def run(self): + result = worker.run(test_input, driver) + + thread = SingleTestThread() + thread.start() + thread.join(thread_timeout_sec) + if thread.isAlive(): + # If join() returned with the thread still running, the + # DumpRenderTree is completely hung and there's nothing + # more we can do with it. We have to kill all the + # DumpRenderTrees to free it up. If we're running more than + # one DumpRenderTree thread, we'll end up killing the other + # DumpRenderTrees too, introducing spurious crashes. We accept + # that tradeoff in order to avoid losing the rest of this + # thread's results. + _log.error('Test thread hung: killing all DumpRenderTrees') + + driver.stop() + + if not result: + result = TestResult(test_input.filename, failures=[], + test_run_time=0, total_time_for_all_diffs=0, time_for_diffs={}) + return result + + def _run_test_in_this_thread(self, test_input): + """Run a single test file using a shared DumpRenderTree process. + + Args: + test_input: Object containing the test filename, uri and timeout + + Returns: a TestResult object. + """ + # poll() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 + if not self._driver or self._driver.poll() is not None: + self._driver = self._port.create_driver(self._worker_number) + self._driver.start() + return self._run(self._driver, test_input) + + def _expected_driver_output(self): + return ExpectedDriverOutput(self._port.expected_text(self._filename), + self._port.expected_image(self._filename), + self._port.expected_checksum(self._filename)) + + def _should_fetch_expected_checksum(self): + return (self._options.pixel_tests and + not (self._options.new_baseline or self._options.reset_results)) + + def _driver_input(self, test_input): + self._filename = test_input.filename + self._timeout = test_input.timeout + self._testname = self._port.relative_test_filename(test_input.filename) + + # The image hash is used to avoid doing an image dump if the + # checksums match, so it should be set to a blank value if we + # are generating a new baseline. (Otherwise, an image from a + # previous run will be copied into the baseline.""" + image_hash = None + if self._should_fetch_expected_checksum(): + image_hash = self._port.expected_checksum(self._filename) + return base.DriverInput(self._filename, self._timeout, image_hash) + + def _run(self, driver, test_input): + if self._options.new_baseline or self._options.reset_results: + return self._run_rebaseline(driver, test_input) + return self._run_compare_test(driver, test_input) + + def _run_compare_test(self, driver, test_input): + driver_output = self._driver.run_test(self._driver_input(test_input)) + return self._process_output(driver_output) + + def _run_rebaseline(self, driver, test_input): + driver_output = self._driver.run_test(self._driver_input(test_input)) + failures = self._handle_error(driver_output) + # FIXME: It the test crashed or timed out, it might be bettter to avoid + # to write new baselines. + self._save_baselines(driver_output) + return TestResult(self._filename, failures, driver_output.test_time) + + def _save_baselines(self, driver_output): + # Although all test_shell/DumpRenderTree output should be utf-8, + # we do not ever decode it inside run-webkit-tests. For some tests + # DumpRenderTree may not output utf-8 text (e.g. webarchives). + self._save_baseline_data(driver_output.text, ".txt", + generate_new_baseline=self._options.new_baseline) + if self._options.pixel_tests and driver_output.image_hash: + self._save_baseline_data(driver_output.image, ".png", + generate_new_baseline=self._options.new_baseline) + self._save_baseline_data(driver_output.image_hash, ".checksum", + generate_new_baseline=self._options.new_baseline) + + def _save_baseline_data(self, data, modifier, generate_new_baseline=True): + """Saves a new baseline file into the port's baseline directory. + + The file will be named simply "<test>-expected<modifier>", suitable for + use as the expected results in a later run. + + Args: + data: result to be saved as the new baseline + modifier: type of the result file, e.g. ".txt" or ".png" + generate_new_baseline: whether to enerate a new, platform-specific + baseline, or update the existing one + """ + + port = self._port + fs = port._filesystem + if generate_new_baseline: + relative_dir = fs.dirname(self._testname) + baseline_path = port.baseline_path() + output_dir = fs.join(baseline_path, relative_dir) + output_file = fs.basename(fs.splitext(self._filename)[0] + + "-expected" + modifier) + fs.maybe_make_directory(output_dir) + output_path = fs.join(output_dir, output_file) + _log.debug('writing new baseline result "%s"' % (output_path)) + else: + output_path = port.expected_filename(self._filename, modifier) + _log.debug('resetting baseline result "%s"' % output_path) + + port.update_baseline(output_path, data) + + def _handle_error(self, driver_output): + failures = [] + fs = self._port._filesystem + if driver_output.timeout: + failures.append(test_failures.FailureTimeout()) + if driver_output.crash: + failures.append(test_failures.FailureCrash()) + _log.debug("%s Stacktrace for %s:\n%s" % (self._worker_name, self._testname, + driver_output.error)) + stack_filename = fs.join(self._options.results_directory, self._testname) + stack_filename = fs.splitext(stack_filename)[0] + "-stack.txt" + fs.maybe_make_directory(fs.dirname(stack_filename)) + fs.write_text_file(stack_filename, driver_output.error) + elif driver_output.error: + _log.debug("%s %s output stderr lines:\n%s" % (self._worker_name, self._testname, + driver_output.error)) + return failures + + def _run_test(self): + driver_output = self._driver.run_test(self._driver_input()) + return self._process_output(driver_output) + + def _process_output(self, driver_output): + """Receives the output from a DumpRenderTree process, subjects it to a + number of tests, and returns a list of failure types the test produced. + Args: + driver_output: a DriverOutput object containing the output from the driver + + Returns: a TestResult object + """ + fs = self._port._filesystem + failures = self._handle_error(driver_output) + expected_driver_output = self._expected_driver_output() + + # Check the output and save the results. + start_time = time.time() + time_for_diffs = {} + for test_type in self._test_types: + start_diff_time = time.time() + new_failures = test_type.compare_output( + self._port, self._filename, self._options, driver_output, + expected_driver_output) + # Don't add any more failures if we already have a crash, so we don't + # double-report those tests. We do double-report for timeouts since + # we still want to see the text and image output. + if not driver_output.crash: + failures.extend(new_failures) + time_for_diffs[test_type.__class__.__name__] = ( + time.time() - start_diff_time) + + total_time_for_all_diffs = time.time() - start_diff_time + return TestResult(self._filename, failures, driver_output.test_time, + total_time_for_all_diffs, time_for_diffs) + + def start_servers_with_lock(self): + _log.debug('Acquiring http lock ...') + self._port.acquire_http_lock() + _log.debug('Starting HTTP server ...') + self._port.start_http_server() + _log.debug('Starting WebSocket server ...') + self._port.start_websocket_server() + self.has_http_lock = True + + def stop_servers_with_lock(self): + """Stop the servers and release http lock.""" + if self.has_http_lock: + _log.debug('Stopping HTTP server ...') + self._port.stop_http_server() + _log.debug('Stopping WebSocket server ...') + self._port.stop_websocket_server() + _log.debug('Releasing server lock ...') + self._port.release_http_lock() + self.has_http_lock = False + + def kill_dump_render_tree(self): + """Kill the DumpRenderTree process if it's running.""" + if self._driver: + self._driver.stop() + self._driver = None |