diff options
Diffstat (limited to 'Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py')
| -rw-r--r-- | Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py | 422 |
1 files changed, 56 insertions, 366 deletions
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py index 050eefa..7ddd7b0 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py @@ -28,17 +28,11 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -"""A Thread object for running DumpRenderTree and processing URLs from a -shared queue. +"""This module implements a shared-memory, thread-based version of the worker +task in new-run-webkit-tests: it receives a list of tests from TestShellThread +and passes them one at a time to SingleTestRunner to execute.""" -Each thread runs a separate instance of the DumpRenderTree binary and validates -the output. When there are no more URLs to process in the shared queue, the -thread exits. -""" - -import copy import logging -import os import Queue import signal import sys @@ -46,199 +40,13 @@ import thread import threading import time - -from webkitpy.layout_tests.test_types import image_diff -from webkitpy.layout_tests.test_types import test_type_base -from webkitpy.layout_tests.test_types import text_diff - -import test_failures -import test_output -import test_results +from webkitpy.layout_tests.layout_package.single_test_runner import SingleTestRunner _log = logging.getLogger("webkitpy.layout_tests.layout_package." "dump_render_tree_thread") -def _expected_test_output(port, filename): - """Returns an expected TestOutput object.""" - return test_output.TestOutput(port.expected_text(filename), - port.expected_image(filename), - port.expected_checksum(filename)) - -def _process_output(port, options, test_input, test_types, test_args, - test_output, worker_name): - """Receives the output from a DumpRenderTree process, subjects it to a - number of tests, and returns a list of failure types the test produced. - - Args: - port: port-specific hooks - options: command line options argument from optparse - proc: an active DumpRenderTree process - test_input: Object containing the test filename and timeout - test_types: list of test types to subject the output to - test_args: arguments to be passed to each test - test_output: a TestOutput object containing the output of the test - worker_name: worker name for logging - - Returns: a TestResult object - """ - failures = [] - fs = port._filesystem - - if test_output.crash: - failures.append(test_failures.FailureCrash()) - if test_output.timeout: - failures.append(test_failures.FailureTimeout()) - - test_name = port.relative_test_filename(test_input.filename) - if test_output.crash: - _log.debug("%s Stacktrace for %s:\n%s" % (worker_name, test_name, - test_output.error)) - filename = fs.join(options.results_directory, test_name) - filename = fs.splitext(filename)[0] + "-stack.txt" - fs.maybe_make_directory(fs.dirname(filename)) - fs.write_text_file(filename, test_output.error) - elif test_output.error: - _log.debug("%s %s output stderr lines:\n%s" % (worker_name, test_name, - test_output.error)) - - expected_test_output = _expected_test_output(port, test_input.filename) - - # Check the output and save the results. - start_time = time.time() - time_for_diffs = {} - for test_type in test_types: - start_diff_time = time.time() - new_failures = test_type.compare_output(port, test_input.filename, - test_args, test_output, - expected_test_output) - # Don't add any more failures if we already have a crash, so we don't - # double-report those tests. We do double-report for timeouts since - # we still want to see the text and image output. - if not test_output.crash: - failures.extend(new_failures) - time_for_diffs[test_type.__class__.__name__] = ( - time.time() - start_diff_time) - - total_time_for_all_diffs = time.time() - start_diff_time - return test_results.TestResult(test_input.filename, failures, test_output.test_time, - total_time_for_all_diffs, time_for_diffs) - - -def _pad_timeout(timeout): - """Returns a safe multiple of the per-test timeout value to use - to detect hung test threads. - - """ - # When we're running one test per DumpRenderTree process, we can - # enforce a hard timeout. The DumpRenderTree watchdog uses 2.5x - # the timeout; we want to be larger than that. - return timeout * 3 - - -def _milliseconds_to_seconds(msecs): - return float(msecs) / 1000.0 - - -def _should_fetch_expected_checksum(options): - return options.pixel_tests and not (options.new_baseline or options.reset_results) - - -def _run_single_test(port, options, test_input, test_types, test_args, driver, worker_name): - # FIXME: Pull this into TestShellThread._run(). - - # The image hash is used to avoid doing an image dump if the - # checksums match, so it should be set to a blank value if we - # are generating a new baseline. (Otherwise, an image from a - # previous run will be copied into the baseline.""" - if _should_fetch_expected_checksum(options): - test_input.image_hash = port.expected_checksum(test_input.filename) - test_output = driver.run_test(test_input) - return _process_output(port, options, test_input, test_types, test_args, - test_output, worker_name) - - -class SingleTestThread(threading.Thread): - """Thread wrapper for running a single test file.""" - - def __init__(self, port, options, worker_number, worker_name, - test_input, test_types, test_args): - """ - Args: - port: object implementing port-specific hooks - options: command line argument object from optparse - worker_number: worker number for tests - worker_name: for logging - test_input: Object containing the test filename and timeout - test_types: A list of TestType objects to run the test output - against. - test_args: A TestArguments object to pass to each TestType. - """ - - threading.Thread.__init__(self) - self._port = port - self._options = options - self._test_input = test_input - self._test_types = test_types - self._test_args = test_args - self._driver = None - self._worker_number = worker_number - self._name = worker_name - - def run(self): - self._covered_run() - - def _covered_run(self): - # FIXME: this is a separate routine to work around a bug - # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85. - self._driver = self._port.create_driver(self._worker_number) - self._driver.start() - self._test_result = _run_single_test(self._port, self._options, - self._test_input, self._test_types, - self._test_args, self._driver, - self._name) - self._driver.stop() - - def get_test_result(self): - return self._test_result - - -class WatchableThread(threading.Thread): - """This class abstracts an interface used by - run_webkit_tests.TestRunner._wait_for_threads_to_finish for thread - management.""" - def __init__(self): - threading.Thread.__init__(self) - self._canceled = False - self._exception_info = None - self._next_timeout = None - self._thread_id = None - - def cancel(self): - """Set a flag telling this thread to quit.""" - self._canceled = True - - def clear_next_timeout(self): - """Mark a flag telling this thread to stop setting timeouts.""" - self._timeout = 0 - - def exception_info(self): - """If run() terminated on an uncaught exception, return it here - ((type, value, traceback) tuple). - Returns None if run() terminated normally. Meant to be called after - joining this thread.""" - return self._exception_info - - def id(self): - """Return a thread identifier.""" - return self._thread_id - - def next_timeout(self): - """Return the time the test is supposed to finish by.""" - return self._next_timeout - - -class TestShellThread(WatchableThread): +class TestShellThread(threading.Thread): def __init__(self, port, options, worker_number, worker_name, filename_list_queue, result_queue): """Initialize all the local state for this DumpRenderTree thread. @@ -253,50 +61,51 @@ class TestShellThread(WatchableThread): result_queue: A thread safe Queue class that will contain serialized TestResult objects. """ - WatchableThread.__init__(self) + threading.Thread.__init__(self) + self._canceled = False + self._exception_info = None + self._next_timeout = None + self._thread_id = None self._port = port self._options = options self._worker_number = worker_number self._name = worker_name self._filename_list_queue = filename_list_queue self._result_queue = result_queue + self._current_group = None self._filename_list = [] - self._driver = None self._test_group_timing_stats = {} self._test_results = [] self._num_tests = 0 self._start_time = 0 self._stop_time = 0 - self._have_http_lock = False self._http_lock_wait_begin = 0 self._http_lock_wait_end = 0 - self._test_types = [] - for cls in self._get_test_type_classes(): - self._test_types.append(cls(self._port, - self._options.results_directory)) - self._test_args = self._get_test_args(worker_number) + def cancel(self): + """Set a flag telling this thread to quit.""" + self._canceled = True - # Current group of tests we're running. - self._current_group = None - # Number of tests in self._current_group. - self._num_tests_in_current_group = None - # Time at which we started running tests from self._current_group. - self._current_group_start_time = None + def clear_next_timeout(self): + """Mark a flag telling this thread to stop setting timeouts.""" + self._timeout = 0 - def _get_test_args(self, worker_number): - """Returns the tuple of arguments for tests and for DumpRenderTree.""" - test_args = test_type_base.TestArguments() - test_args.new_baseline = self._options.new_baseline - test_args.reset_results = self._options.reset_results + def exception_info(self): + """If run() terminated on an uncaught exception, return it here + ((type, value, traceback) tuple). + Returns None if run() terminated normally. Meant to be called after + joining this thread.""" + return self._exception_info - return test_args + def id(self): + """Return a thread identifier.""" + return self._thread_id - def _get_test_type_classes(self): - classes = [text_diff.TestTextDiff] - if self._options.pixel_tests: - classes.append(image_diff.ImageDiff) - return classes + def next_timeout(self): + """Return the time the test is supposed to finish by.""" + if self._next_timeout: + return self._next_timeout + self._http_lock_wait_time() + return self._next_timeout def get_test_group_timing_stats(self): """Returns a dictionary mapping test group to a tuple of @@ -352,17 +161,6 @@ class TestShellThread(WatchableThread): do multi-threaded debugging.""" self._run(test_runner, result_summary) - def cancel(self): - """Clean up http lock and set a flag telling this thread to quit.""" - self._stop_servers_with_lock() - WatchableThread.cancel(self) - - def next_timeout(self): - """Return the time the test is supposed to finish by.""" - if self._next_timeout: - return self._next_timeout + self._http_lock_wait_time() - return self._next_timeout - def _http_lock_wait_time(self): """Return the time what http locking takes.""" if self._http_lock_wait_begin == 0: @@ -377,18 +175,23 @@ class TestShellThread(WatchableThread): If test_runner is not None, then we call test_runner.UpdateSummary() with the results of each test.""" + single_test_runner = SingleTestRunner(self._options, self._port, + self._name, self._worker_number) + batch_size = self._options.batch_size batch_count = 0 # Append tests we're running to the existing tests_run.txt file. # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput. tests_run_filename = self._port._filesystem.join(self._options.results_directory, - "tests_run.txt") + "tests_run%d.txt" % self._worker_number) tests_run_file = self._port._filesystem.open_text_file_for_writing(tests_run_filename, append=False) + while True: if self._canceled: _log.debug('Testing cancelled') tests_run_file.close() + single_test_runner.cleanup() return if len(self._filename_list) is 0: @@ -401,15 +204,16 @@ class TestShellThread(WatchableThread): self._current_group, self._filename_list = \ self._filename_list_queue.get_nowait() except Queue.Empty: - self._stop_servers_with_lock() - self._kill_dump_render_tree() tests_run_file.close() + single_test_runner.cleanup() return if self._current_group == "tests_to_http_lock": - self._start_servers_with_lock() - elif self._have_http_lock: - self._stop_servers_with_lock() + self._http_lock_wait_begin = time.time() + single_test_runner.start_servers_with_lock() + self._http_lock_wait_end = time.time() + elif single_test_runner.has_http_lock: + single_test_runner.stop_servers_with_lock() self._num_tests_in_current_group = len(self._filename_list) self._current_group_start_time = time.time() @@ -419,145 +223,31 @@ class TestShellThread(WatchableThread): # We have a url, run tests. batch_count += 1 self._num_tests += 1 - if self._options.run_singly: - result = self._run_test_in_another_thread(test_input) - else: - result = self._run_test_in_this_thread(test_input) - filename = test_input.filename - tests_run_file.write(filename + "\n") + timeout = single_test_runner.timeout(test_input) + result = single_test_runner.run_test(test_input, timeout) + + tests_run_file.write(test_input.filename + "\n") + test_name = self._port.relative_test_filename(test_input.filename) if result.failures: # Check and kill DumpRenderTree if we need to. - if len([1 for f in result.failures - if f.should_kill_dump_render_tree()]): - self._kill_dump_render_tree() + if any([f.should_kill_dump_render_tree() for f in result.failures]): + single_test_runner.kill_dump_render_tree() # Reset the batch count since the shell just bounced. batch_count = 0 + # Print the error message(s). - error_str = '\n'.join([' ' + f.message() for - f in result.failures]) - _log.debug("%s %s failed:\n%s" % (self.getName(), - self._port.relative_test_filename(filename), - error_str)) + _log.debug("%s %s failed:" % (self._name, test_name)) + for f in result.failures: + _log.debug("%s %s" % (self._name, f.message())) else: - _log.debug("%s %s passed" % (self.getName(), - self._port.relative_test_filename(filename))) + _log.debug("%s %s passed" % (self._name, test_name)) self._result_queue.put(result.dumps()) if batch_size > 0 and batch_count >= batch_size: # Bounce the shell and reset count. - self._kill_dump_render_tree() + single_test_runner.kill_dump_render_tree() batch_count = 0 if test_runner: test_runner.update_summary(result_summary) - - def _run_test_in_another_thread(self, test_input): - """Run a test in a separate thread, enforcing a hard time limit. - - Since we can only detect the termination of a thread, not any internal - state or progress, we can only run per-test timeouts when running test - files singly. - - Args: - test_input: Object containing the test filename and timeout - - Returns: - A TestResult - """ - worker = SingleTestThread(self._port, - self._options, - self._worker_number, - self._name, - test_input, - self._test_types, - self._test_args) - - worker.start() - - thread_timeout = _milliseconds_to_seconds( - _pad_timeout(int(test_input.timeout))) - thread._next_timeout = time.time() + thread_timeout - worker.join(thread_timeout) - if worker.isAlive(): - # If join() returned with the thread still running, the - # DumpRenderTree is completely hung and there's nothing - # more we can do with it. We have to kill all the - # DumpRenderTrees to free it up. If we're running more than - # one DumpRenderTree thread, we'll end up killing the other - # DumpRenderTrees too, introducing spurious crashes. We accept - # that tradeoff in order to avoid losing the rest of this - # thread's results. - _log.error('Test thread hung: killing all DumpRenderTrees') - if worker._driver: - worker._driver.stop() - - try: - result = worker.get_test_result() - except AttributeError, e: - # This gets raised if the worker thread has already exited. - _log.error('Cannot get results of test: %s' % test_input.filename) - # FIXME: Seems we want a unique failure type here. - result = test_results.TestResult(test_input.filename) - - return result - - def _run_test_in_this_thread(self, test_input): - """Run a single test file using a shared DumpRenderTree process. - - Args: - test_input: Object containing the test filename, uri and timeout - - Returns: a TestResult object. - """ - self._ensure_dump_render_tree_is_running() - thread_timeout = _milliseconds_to_seconds( - _pad_timeout(int(test_input.timeout))) - self._next_timeout = time.time() + thread_timeout - test_result = _run_single_test(self._port, self._options, test_input, - self._test_types, self._test_args, - self._driver, self._name) - self._test_results.append(test_result) - return test_result - - def _ensure_dump_render_tree_is_running(self): - """Start the shared DumpRenderTree, if it's not running. - - This is not for use when running tests singly, since those each start - a separate DumpRenderTree in their own thread. - - """ - # poll() is not threadsafe and can throw OSError due to: - # http://bugs.python.org/issue1731717 - if not self._driver or self._driver.poll() is not None: - self._driver = self._port.create_driver(self._worker_number) - self._driver.start() - - def _start_servers_with_lock(self): - """Acquire http lock and start the servers.""" - self._http_lock_wait_begin = time.time() - _log.debug('Acquire http lock ...') - self._port.acquire_http_lock() - _log.debug('Starting HTTP server ...') - self._port.start_http_server() - _log.debug('Starting WebSocket server ...') - self._port.start_websocket_server() - self._http_lock_wait_end = time.time() - self._have_http_lock = True - - def _stop_servers_with_lock(self): - """Stop the servers and release http lock.""" - if self._have_http_lock: - _log.debug('Stopping HTTP server ...') - self._port.stop_http_server() - _log.debug('Stopping WebSocket server ...') - self._port.stop_websocket_server() - _log.debug('Release http lock ...') - self._port.release_http_lock() - self._have_http_lock = False - - def _kill_dump_render_tree(self): - """Kill the DumpRenderTree process if it's running.""" - if self._driver: - self._driver.stop() - self._driver = None |