summaryrefslogtreecommitdiffstats
path: root/Tools/Scripts/webkitpy/layout_tests
diff options
context:
space:
mode:
Diffstat (limited to 'Tools/Scripts/webkitpy/layout_tests')
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py11
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py422
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py6
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py113
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py29
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker.py282
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker_unittest.py227
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py41
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker2.py196
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker2_unittest.py83
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py54
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py8
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py322
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py554
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py293
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_input.py4
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_output.py56
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner.py107
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner2.py129
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/worker.py104
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/base.py175
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py14
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium.py74
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py119
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py43
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py4
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py24
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py7
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py10
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/dryrun.py22
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/factory.py5
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/http_server_base.py2
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/mac.py6
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py23
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/mock_drt.py280
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py261
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py12
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/server_process.py6
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/server_process_unittest.py77
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/test.py221
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/test_files.py58
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py57
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/webkit.py53
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py36
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/websocket_server.py2
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py125
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py184
-rwxr-xr-xTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py33
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py57
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/test_types/image_diff.py60
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py63
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py2
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/test_types/text_diff.py18
-rwxr-xr-xTools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests.py2
54 files changed, 3670 insertions, 1506 deletions
diff --git a/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py b/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py
index 51dcac8..86649b6 100644
--- a/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py
+++ b/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py
@@ -36,11 +36,12 @@ import os
import subprocess
import sys
import re
-import webkitpy.common.checkout.scm as scm
-import webkitpy.common.system.executive as executive
-import webkitpy.common.system.logutils as logutils
-import webkitpy.common.system.ospath as ospath
-import webkitpy.layout_tests.port.factory as port_factory
+
+from webkitpy.common.checkout import scm
+from webkitpy.common.system import executive
+from webkitpy.common.system import logutils
+from webkitpy.common.system import ospath
+from webkitpy.layout_tests.port import factory as port_factory
_log = logutils.get_logger(__file__)
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py
index 050eefa..7ddd7b0 100644
--- a/Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py
@@ -28,17 +28,11 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-"""A Thread object for running DumpRenderTree and processing URLs from a
-shared queue.
+"""This module implements a shared-memory, thread-based version of the worker
+task in new-run-webkit-tests: it receives a list of tests from TestShellThread
+and passes them one at a time to SingleTestRunner to execute."""
-Each thread runs a separate instance of the DumpRenderTree binary and validates
-the output. When there are no more URLs to process in the shared queue, the
-thread exits.
-"""
-
-import copy
import logging
-import os
import Queue
import signal
import sys
@@ -46,199 +40,13 @@ import thread
import threading
import time
-
-from webkitpy.layout_tests.test_types import image_diff
-from webkitpy.layout_tests.test_types import test_type_base
-from webkitpy.layout_tests.test_types import text_diff
-
-import test_failures
-import test_output
-import test_results
+from webkitpy.layout_tests.layout_package.single_test_runner import SingleTestRunner
_log = logging.getLogger("webkitpy.layout_tests.layout_package."
"dump_render_tree_thread")
-def _expected_test_output(port, filename):
- """Returns an expected TestOutput object."""
- return test_output.TestOutput(port.expected_text(filename),
- port.expected_image(filename),
- port.expected_checksum(filename))
-
-def _process_output(port, options, test_input, test_types, test_args,
- test_output, worker_name):
- """Receives the output from a DumpRenderTree process, subjects it to a
- number of tests, and returns a list of failure types the test produced.
-
- Args:
- port: port-specific hooks
- options: command line options argument from optparse
- proc: an active DumpRenderTree process
- test_input: Object containing the test filename and timeout
- test_types: list of test types to subject the output to
- test_args: arguments to be passed to each test
- test_output: a TestOutput object containing the output of the test
- worker_name: worker name for logging
-
- Returns: a TestResult object
- """
- failures = []
- fs = port._filesystem
-
- if test_output.crash:
- failures.append(test_failures.FailureCrash())
- if test_output.timeout:
- failures.append(test_failures.FailureTimeout())
-
- test_name = port.relative_test_filename(test_input.filename)
- if test_output.crash:
- _log.debug("%s Stacktrace for %s:\n%s" % (worker_name, test_name,
- test_output.error))
- filename = fs.join(options.results_directory, test_name)
- filename = fs.splitext(filename)[0] + "-stack.txt"
- fs.maybe_make_directory(fs.dirname(filename))
- fs.write_text_file(filename, test_output.error)
- elif test_output.error:
- _log.debug("%s %s output stderr lines:\n%s" % (worker_name, test_name,
- test_output.error))
-
- expected_test_output = _expected_test_output(port, test_input.filename)
-
- # Check the output and save the results.
- start_time = time.time()
- time_for_diffs = {}
- for test_type in test_types:
- start_diff_time = time.time()
- new_failures = test_type.compare_output(port, test_input.filename,
- test_args, test_output,
- expected_test_output)
- # Don't add any more failures if we already have a crash, so we don't
- # double-report those tests. We do double-report for timeouts since
- # we still want to see the text and image output.
- if not test_output.crash:
- failures.extend(new_failures)
- time_for_diffs[test_type.__class__.__name__] = (
- time.time() - start_diff_time)
-
- total_time_for_all_diffs = time.time() - start_diff_time
- return test_results.TestResult(test_input.filename, failures, test_output.test_time,
- total_time_for_all_diffs, time_for_diffs)
-
-
-def _pad_timeout(timeout):
- """Returns a safe multiple of the per-test timeout value to use
- to detect hung test threads.
-
- """
- # When we're running one test per DumpRenderTree process, we can
- # enforce a hard timeout. The DumpRenderTree watchdog uses 2.5x
- # the timeout; we want to be larger than that.
- return timeout * 3
-
-
-def _milliseconds_to_seconds(msecs):
- return float(msecs) / 1000.0
-
-
-def _should_fetch_expected_checksum(options):
- return options.pixel_tests and not (options.new_baseline or options.reset_results)
-
-
-def _run_single_test(port, options, test_input, test_types, test_args, driver, worker_name):
- # FIXME: Pull this into TestShellThread._run().
-
- # The image hash is used to avoid doing an image dump if the
- # checksums match, so it should be set to a blank value if we
- # are generating a new baseline. (Otherwise, an image from a
- # previous run will be copied into the baseline."""
- if _should_fetch_expected_checksum(options):
- test_input.image_hash = port.expected_checksum(test_input.filename)
- test_output = driver.run_test(test_input)
- return _process_output(port, options, test_input, test_types, test_args,
- test_output, worker_name)
-
-
-class SingleTestThread(threading.Thread):
- """Thread wrapper for running a single test file."""
-
- def __init__(self, port, options, worker_number, worker_name,
- test_input, test_types, test_args):
- """
- Args:
- port: object implementing port-specific hooks
- options: command line argument object from optparse
- worker_number: worker number for tests
- worker_name: for logging
- test_input: Object containing the test filename and timeout
- test_types: A list of TestType objects to run the test output
- against.
- test_args: A TestArguments object to pass to each TestType.
- """
-
- threading.Thread.__init__(self)
- self._port = port
- self._options = options
- self._test_input = test_input
- self._test_types = test_types
- self._test_args = test_args
- self._driver = None
- self._worker_number = worker_number
- self._name = worker_name
-
- def run(self):
- self._covered_run()
-
- def _covered_run(self):
- # FIXME: this is a separate routine to work around a bug
- # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
- self._driver = self._port.create_driver(self._worker_number)
- self._driver.start()
- self._test_result = _run_single_test(self._port, self._options,
- self._test_input, self._test_types,
- self._test_args, self._driver,
- self._name)
- self._driver.stop()
-
- def get_test_result(self):
- return self._test_result
-
-
-class WatchableThread(threading.Thread):
- """This class abstracts an interface used by
- run_webkit_tests.TestRunner._wait_for_threads_to_finish for thread
- management."""
- def __init__(self):
- threading.Thread.__init__(self)
- self._canceled = False
- self._exception_info = None
- self._next_timeout = None
- self._thread_id = None
-
- def cancel(self):
- """Set a flag telling this thread to quit."""
- self._canceled = True
-
- def clear_next_timeout(self):
- """Mark a flag telling this thread to stop setting timeouts."""
- self._timeout = 0
-
- def exception_info(self):
- """If run() terminated on an uncaught exception, return it here
- ((type, value, traceback) tuple).
- Returns None if run() terminated normally. Meant to be called after
- joining this thread."""
- return self._exception_info
-
- def id(self):
- """Return a thread identifier."""
- return self._thread_id
-
- def next_timeout(self):
- """Return the time the test is supposed to finish by."""
- return self._next_timeout
-
-
-class TestShellThread(WatchableThread):
+class TestShellThread(threading.Thread):
def __init__(self, port, options, worker_number, worker_name,
filename_list_queue, result_queue):
"""Initialize all the local state for this DumpRenderTree thread.
@@ -253,50 +61,51 @@ class TestShellThread(WatchableThread):
result_queue: A thread safe Queue class that will contain
serialized TestResult objects.
"""
- WatchableThread.__init__(self)
+ threading.Thread.__init__(self)
+ self._canceled = False
+ self._exception_info = None
+ self._next_timeout = None
+ self._thread_id = None
self._port = port
self._options = options
self._worker_number = worker_number
self._name = worker_name
self._filename_list_queue = filename_list_queue
self._result_queue = result_queue
+ self._current_group = None
self._filename_list = []
- self._driver = None
self._test_group_timing_stats = {}
self._test_results = []
self._num_tests = 0
self._start_time = 0
self._stop_time = 0
- self._have_http_lock = False
self._http_lock_wait_begin = 0
self._http_lock_wait_end = 0
- self._test_types = []
- for cls in self._get_test_type_classes():
- self._test_types.append(cls(self._port,
- self._options.results_directory))
- self._test_args = self._get_test_args(worker_number)
+ def cancel(self):
+ """Set a flag telling this thread to quit."""
+ self._canceled = True
- # Current group of tests we're running.
- self._current_group = None
- # Number of tests in self._current_group.
- self._num_tests_in_current_group = None
- # Time at which we started running tests from self._current_group.
- self._current_group_start_time = None
+ def clear_next_timeout(self):
+ """Mark a flag telling this thread to stop setting timeouts."""
+ self._timeout = 0
- def _get_test_args(self, worker_number):
- """Returns the tuple of arguments for tests and for DumpRenderTree."""
- test_args = test_type_base.TestArguments()
- test_args.new_baseline = self._options.new_baseline
- test_args.reset_results = self._options.reset_results
+ def exception_info(self):
+ """If run() terminated on an uncaught exception, return it here
+ ((type, value, traceback) tuple).
+ Returns None if run() terminated normally. Meant to be called after
+ joining this thread."""
+ return self._exception_info
- return test_args
+ def id(self):
+ """Return a thread identifier."""
+ return self._thread_id
- def _get_test_type_classes(self):
- classes = [text_diff.TestTextDiff]
- if self._options.pixel_tests:
- classes.append(image_diff.ImageDiff)
- return classes
+ def next_timeout(self):
+ """Return the time the test is supposed to finish by."""
+ if self._next_timeout:
+ return self._next_timeout + self._http_lock_wait_time()
+ return self._next_timeout
def get_test_group_timing_stats(self):
"""Returns a dictionary mapping test group to a tuple of
@@ -352,17 +161,6 @@ class TestShellThread(WatchableThread):
do multi-threaded debugging."""
self._run(test_runner, result_summary)
- def cancel(self):
- """Clean up http lock and set a flag telling this thread to quit."""
- self._stop_servers_with_lock()
- WatchableThread.cancel(self)
-
- def next_timeout(self):
- """Return the time the test is supposed to finish by."""
- if self._next_timeout:
- return self._next_timeout + self._http_lock_wait_time()
- return self._next_timeout
-
def _http_lock_wait_time(self):
"""Return the time what http locking takes."""
if self._http_lock_wait_begin == 0:
@@ -377,18 +175,23 @@ class TestShellThread(WatchableThread):
If test_runner is not None, then we call test_runner.UpdateSummary()
with the results of each test."""
+ single_test_runner = SingleTestRunner(self._options, self._port,
+ self._name, self._worker_number)
+
batch_size = self._options.batch_size
batch_count = 0
# Append tests we're running to the existing tests_run.txt file.
# This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput.
tests_run_filename = self._port._filesystem.join(self._options.results_directory,
- "tests_run.txt")
+ "tests_run%d.txt" % self._worker_number)
tests_run_file = self._port._filesystem.open_text_file_for_writing(tests_run_filename, append=False)
+
while True:
if self._canceled:
_log.debug('Testing cancelled')
tests_run_file.close()
+ single_test_runner.cleanup()
return
if len(self._filename_list) is 0:
@@ -401,15 +204,16 @@ class TestShellThread(WatchableThread):
self._current_group, self._filename_list = \
self._filename_list_queue.get_nowait()
except Queue.Empty:
- self._stop_servers_with_lock()
- self._kill_dump_render_tree()
tests_run_file.close()
+ single_test_runner.cleanup()
return
if self._current_group == "tests_to_http_lock":
- self._start_servers_with_lock()
- elif self._have_http_lock:
- self._stop_servers_with_lock()
+ self._http_lock_wait_begin = time.time()
+ single_test_runner.start_servers_with_lock()
+ self._http_lock_wait_end = time.time()
+ elif single_test_runner.has_http_lock:
+ single_test_runner.stop_servers_with_lock()
self._num_tests_in_current_group = len(self._filename_list)
self._current_group_start_time = time.time()
@@ -419,145 +223,31 @@ class TestShellThread(WatchableThread):
# We have a url, run tests.
batch_count += 1
self._num_tests += 1
- if self._options.run_singly:
- result = self._run_test_in_another_thread(test_input)
- else:
- result = self._run_test_in_this_thread(test_input)
- filename = test_input.filename
- tests_run_file.write(filename + "\n")
+ timeout = single_test_runner.timeout(test_input)
+ result = single_test_runner.run_test(test_input, timeout)
+
+ tests_run_file.write(test_input.filename + "\n")
+ test_name = self._port.relative_test_filename(test_input.filename)
if result.failures:
# Check and kill DumpRenderTree if we need to.
- if len([1 for f in result.failures
- if f.should_kill_dump_render_tree()]):
- self._kill_dump_render_tree()
+ if any([f.should_kill_dump_render_tree() for f in result.failures]):
+ single_test_runner.kill_dump_render_tree()
# Reset the batch count since the shell just bounced.
batch_count = 0
+
# Print the error message(s).
- error_str = '\n'.join([' ' + f.message() for
- f in result.failures])
- _log.debug("%s %s failed:\n%s" % (self.getName(),
- self._port.relative_test_filename(filename),
- error_str))
+ _log.debug("%s %s failed:" % (self._name, test_name))
+ for f in result.failures:
+ _log.debug("%s %s" % (self._name, f.message()))
else:
- _log.debug("%s %s passed" % (self.getName(),
- self._port.relative_test_filename(filename)))
+ _log.debug("%s %s passed" % (self._name, test_name))
self._result_queue.put(result.dumps())
if batch_size > 0 and batch_count >= batch_size:
# Bounce the shell and reset count.
- self._kill_dump_render_tree()
+ single_test_runner.kill_dump_render_tree()
batch_count = 0
if test_runner:
test_runner.update_summary(result_summary)
-
- def _run_test_in_another_thread(self, test_input):
- """Run a test in a separate thread, enforcing a hard time limit.
-
- Since we can only detect the termination of a thread, not any internal
- state or progress, we can only run per-test timeouts when running test
- files singly.
-
- Args:
- test_input: Object containing the test filename and timeout
-
- Returns:
- A TestResult
- """
- worker = SingleTestThread(self._port,
- self._options,
- self._worker_number,
- self._name,
- test_input,
- self._test_types,
- self._test_args)
-
- worker.start()
-
- thread_timeout = _milliseconds_to_seconds(
- _pad_timeout(int(test_input.timeout)))
- thread._next_timeout = time.time() + thread_timeout
- worker.join(thread_timeout)
- if worker.isAlive():
- # If join() returned with the thread still running, the
- # DumpRenderTree is completely hung and there's nothing
- # more we can do with it. We have to kill all the
- # DumpRenderTrees to free it up. If we're running more than
- # one DumpRenderTree thread, we'll end up killing the other
- # DumpRenderTrees too, introducing spurious crashes. We accept
- # that tradeoff in order to avoid losing the rest of this
- # thread's results.
- _log.error('Test thread hung: killing all DumpRenderTrees')
- if worker._driver:
- worker._driver.stop()
-
- try:
- result = worker.get_test_result()
- except AttributeError, e:
- # This gets raised if the worker thread has already exited.
- _log.error('Cannot get results of test: %s' % test_input.filename)
- # FIXME: Seems we want a unique failure type here.
- result = test_results.TestResult(test_input.filename)
-
- return result
-
- def _run_test_in_this_thread(self, test_input):
- """Run a single test file using a shared DumpRenderTree process.
-
- Args:
- test_input: Object containing the test filename, uri and timeout
-
- Returns: a TestResult object.
- """
- self._ensure_dump_render_tree_is_running()
- thread_timeout = _milliseconds_to_seconds(
- _pad_timeout(int(test_input.timeout)))
- self._next_timeout = time.time() + thread_timeout
- test_result = _run_single_test(self._port, self._options, test_input,
- self._test_types, self._test_args,
- self._driver, self._name)
- self._test_results.append(test_result)
- return test_result
-
- def _ensure_dump_render_tree_is_running(self):
- """Start the shared DumpRenderTree, if it's not running.
-
- This is not for use when running tests singly, since those each start
- a separate DumpRenderTree in their own thread.
-
- """
- # poll() is not threadsafe and can throw OSError due to:
- # http://bugs.python.org/issue1731717
- if not self._driver or self._driver.poll() is not None:
- self._driver = self._port.create_driver(self._worker_number)
- self._driver.start()
-
- def _start_servers_with_lock(self):
- """Acquire http lock and start the servers."""
- self._http_lock_wait_begin = time.time()
- _log.debug('Acquire http lock ...')
- self._port.acquire_http_lock()
- _log.debug('Starting HTTP server ...')
- self._port.start_http_server()
- _log.debug('Starting WebSocket server ...')
- self._port.start_websocket_server()
- self._http_lock_wait_end = time.time()
- self._have_http_lock = True
-
- def _stop_servers_with_lock(self):
- """Stop the servers and release http lock."""
- if self._have_http_lock:
- _log.debug('Stopping HTTP server ...')
- self._port.stop_http_server()
- _log.debug('Stopping WebSocket server ...')
- self._port.stop_websocket_server()
- _log.debug('Release http lock ...')
- self._port.release_http_lock()
- self._have_http_lock = False
-
- def _kill_dump_render_tree(self):
- """Kill the DumpRenderTree process if it's running."""
- if self._driver:
- self._driver.stop()
- self._driver = None
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py
index 3267fb7..8226ed0 100644
--- a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py
@@ -55,8 +55,7 @@ class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGeneratorBase
def __init__(self, port, builder_name, build_name, build_number,
results_file_base_path, builder_base_url,
test_timings, expectations, result_summary, all_tests,
- generate_incremental_results=False, test_results_server=None,
- test_type="", master_name=""):
+ test_results_server=None, test_type="", master_name=""):
"""Modifies the results.json file. Grabs it off the archive directory
if it is not found locally.
@@ -67,8 +66,7 @@ class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGeneratorBase
super(JSONLayoutResultsGenerator, self).__init__(
port, builder_name, build_name, build_number, results_file_base_path,
builder_base_url, {}, port.test_repository_paths(),
- generate_incremental_results, test_results_server,
- test_type, master_name)
+ test_results_server, test_type, master_name)
self._expectations = expectations
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py
index 32ffd71..05662c2 100644
--- a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py
@@ -114,13 +114,16 @@ class JSONResultsGeneratorBase(object):
URL_FOR_TEST_LIST_JSON = \
"http://%s/testfile?builder=%s&name=%s&testlistjson=1&testtype=%s"
+ # FIXME: Remove generate_incremental_results once the reference to it in
+ # http://src.chromium.org/viewvc/chrome/trunk/tools/build/scripts/slave/gtest_slave_utils.py
+ # has been removed.
def __init__(self, port, builder_name, build_name, build_number,
results_file_base_path, builder_base_url,
test_results_map, svn_repositories=None,
- generate_incremental_results=False,
test_results_server=None,
test_type="",
- master_name=""):
+ master_name="",
+ generate_incremental_results=None):
"""Modifies the results.json file. Grabs it off the archive directory
if it is not found locally.
@@ -137,8 +140,6 @@ class JSONResultsGeneratorBase(object):
svn_repositories: A (json_field_name, svn_path) pair for SVN
repositories that tests rely on. The SVN revision will be
included in the JSON with the given json_field_name.
- generate_incremental_results: If true, generate incremental json file
- from current run results.
test_results_server: server that hosts test results json.
test_type: test type string (e.g. 'layout-tests').
master_name: the name of the buildbot master.
@@ -157,7 +158,6 @@ class JSONResultsGeneratorBase(object):
self._test_results_map = test_results_map
self._test_results = test_results_map.values()
- self._generate_incremental_results = generate_incremental_results
self._svn_repositories = svn_repositories
if not self._svn_repositories:
@@ -167,39 +167,20 @@ class JSONResultsGeneratorBase(object):
self._test_type = test_type
self._master_name = master_name
- self._json = None
self._archived_results = None
def generate_json_output(self):
- """Generates the JSON output file."""
-
- # Generate the JSON output file that has full results.
- # FIXME: stop writing out the full results file once all bots use
- # incremental results.
- if not self._json:
- self._json = self.get_json()
- if self._json:
- self._generate_json_file(self._json, self._results_file_path)
-
- # Generate the JSON output file that only has incremental results.
- if self._generate_incremental_results:
- json = self.get_json(incremental=True)
- if json:
- self._generate_json_file(
- json, self._incremental_results_file_path)
-
- def get_json(self, incremental=False):
+ json = self.get_json()
+ if json:
+ self._generate_json_file(
+ json, self._incremental_results_file_path)
+
+ def get_json(self):
"""Gets the results for the results.json file."""
results_json = {}
- if not incremental:
- if self._json:
- return self._json
-
- if self._archived_results:
- results_json = self._archived_results
if not results_json:
- results_json, error = self._get_archived_json_results(incremental)
+ results_json, error = self._get_archived_json_results()
if error:
# If there was an error don't write a results.json
# file at all as it would lose all the information on the
@@ -231,7 +212,7 @@ class JSONResultsGeneratorBase(object):
all_failing_tests = self._get_failed_test_names()
all_failing_tests.update(tests.iterkeys())
for test in all_failing_tests:
- self._insert_test_time_and_result(test, tests, incremental)
+ self._insert_test_time_and_result(test, tests)
return results_json
@@ -340,52 +321,39 @@ class JSONResultsGeneratorBase(object):
return ""
return ""
- def _get_archived_json_results(self, for_incremental=False):
- """Reads old results JSON file if it exists.
- Returns (archived_results, error) tuple where error is None if results
- were successfully read.
-
- if for_incremental is True, download JSON file that only contains test
+ def _get_archived_json_results(self):
+ """Download JSON file that only contains test
name list from test-results server. This is for generating incremental
JSON so the file generated has info for tests that failed before but
pass or are skipped from current run.
+
+ Returns (archived_results, error) tuple where error is None if results
+ were successfully read.
"""
results_json = {}
old_results = None
error = None
- if self._fs.exists(self._results_file_path) and not for_incremental:
- old_results = self._fs.read_text_file(self._results_file_path)
- elif self._builder_base_url or for_incremental:
- if for_incremental:
- if not self._test_results_server:
- # starting from fresh if no test results server specified.
- return {}, None
-
- results_file_url = (self.URL_FOR_TEST_LIST_JSON %
- (urllib2.quote(self._test_results_server),
- urllib2.quote(self._builder_name),
- self.RESULTS_FILENAME,
- urllib2.quote(self._test_type)))
- else:
- # Check if we have the archived JSON file on the buildbot
- # server.
- results_file_url = (self._builder_base_url +
- self._build_name + "/" + self.RESULTS_FILENAME)
- _log.error("Local results.json file does not exist. Grabbing "
- "it off the archive at " + results_file_url)
+ if not self._test_results_server:
+ return {}, None
- try:
- results_file = urllib2.urlopen(results_file_url)
- info = results_file.info()
- old_results = results_file.read()
- except urllib2.HTTPError, http_error:
- # A non-4xx status code means the bot is hosed for some reason
- # and we can't grab the results.json file off of it.
- if (http_error.code < 400 and http_error.code >= 500):
- error = http_error
- except urllib2.URLError, url_error:
- error = url_error
+ results_file_url = (self.URL_FOR_TEST_LIST_JSON %
+ (urllib2.quote(self._test_results_server),
+ urllib2.quote(self._builder_name),
+ self.RESULTS_FILENAME,
+ urllib2.quote(self._test_type)))
+
+ try:
+ results_file = urllib2.urlopen(results_file_url)
+ info = results_file.info()
+ old_results = results_file.read()
+ except urllib2.HTTPError, http_error:
+ # A non-4xx status code means the bot is hosed for some reason
+ # and we can't grab the results.json file off of it.
+ if (http_error.code < 400 and http_error.code >= 500):
+ error = http_error
+ except urllib2.URLError, url_error:
+ error = url_error
if old_results:
# Strip the prefix and suffix so we can get the actual JSON object.
@@ -490,7 +458,7 @@ class JSONResultsGeneratorBase(object):
int(time.time()),
self.TIME)
- def _insert_test_time_and_result(self, test_name, tests, incremental=False):
+ def _insert_test_time_and_result(self, test_name, tests):
""" Insert a test item with its results to the given tests dictionary.
Args:
@@ -514,11 +482,6 @@ class JSONResultsGeneratorBase(object):
else:
thisTest[self.TIMES] = [[1, time]]
- # Don't normalize the incremental results json because we need results
- # for tests that pass or have no data from current run.
- if not incremental:
- self._normalize_results_json(thisTest, test_name, tests)
-
def _convert_json_to_current_version(self, results_json):
"""If the JSON does not match the current version, converts it to the
current version and adds in the new version number.
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py
index ce99765..95da8fb 100644
--- a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py
@@ -94,7 +94,7 @@ class JSONGeneratorTest(unittest.TestCase):
failed_count_map = dict([(t, 1) for t in failed_tests])
# Test incremental json results
- incremental_json = generator.get_json(incremental=True)
+ incremental_json = generator.get_json()
self._verify_json_results(
tests_set,
test_timings,
@@ -106,33 +106,6 @@ class JSONGeneratorTest(unittest.TestCase):
incremental_json,
1)
- # Test aggregated json results
- generator.set_archived_results(self._json)
- json = generator.get_json(incremental=False)
- self._json = json
- self._num_runs += 1
- self._tests_set |= tests_set
- self._test_timings.update(test_timings)
- self._PASS_count += len(PASS_tests)
- self._DISABLED_count += len(DISABLED_tests)
- self._FLAKY_count += len(FLAKY_tests)
- self._fixable_count += len(DISABLED_tests | failed_tests)
-
- get = self._failed_count_map.get
- for test in failed_count_map.iterkeys():
- self._failed_count_map[test] = get(test, 0) + 1
-
- self._verify_json_results(
- self._tests_set,
- self._test_timings,
- self._failed_count_map,
- self._PASS_count,
- self._DISABLED_count,
- self._FLAKY_count,
- self._fixable_count,
- self._json,
- self._num_runs)
-
def _verify_json_results(self, tests_set, test_timings, failed_count_map,
PASS_count, DISABLED_count, FLAKY_count,
fixable_count,
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker.py
new file mode 100644
index 0000000..a0f252c
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker.py
@@ -0,0 +1,282 @@
+# Copyright (C) 2011 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Module for handling messages and concurrency for run-webkit-tests.
+
+This module implements a message broker that connects the manager
+(TestRunner2) to the workers: it provides a messaging abstraction and
+message loops (building on top of message_broker2), and handles starting
+workers by launching threads and/or processes depending on the
+requested configuration.
+
+There are a lot of classes and objects involved in a fully connected system.
+They interact more or less like:
+
+TestRunner2 --> _InlineManager ---> _InlineWorker <-> Worker
+ ^ \ / ^
+ | v v |
+ \-------------------- MessageBroker -------------/
+"""
+
+import logging
+import optparse
+import Queue
+import thread
+import threading
+import time
+
+
+# Handle Python < 2.6 where multiprocessing isn't available.
+#
+# _Multiprocessing_Process is needed so that _MultiProcessWorker
+# can be defined with or without multiprocessing.
+try:
+ import multiprocessing
+ _Multiprocessing_Process = multiprocessing.Process
+except ImportError:
+ multiprocessing = None
+ _Multiprocessing_Process = threading.Thread
+
+
+from webkitpy.layout_tests import port
+from webkitpy.layout_tests.layout_package import message_broker2
+
+
+_log = logging.getLogger(__name__)
+
+#
+# Topic names for Manager <-> Worker messaging
+#
+MANAGER_TOPIC = 'managers'
+ANY_WORKER_TOPIC = 'workers'
+
+
+def runtime_options():
+ """Return a list of optparse.Option objects for any runtime values used
+ by this module."""
+ options = [
+ optparse.make_option("--worker-model", action="store",
+ help=("controls worker model. Valid values are "
+ "'inline', 'threads', and 'processes'.")),
+ ]
+ return options
+
+
+def get(port, options, client, worker_class):
+ """Return a connection to a manager/worker message_broker
+
+ Args:
+ port - handle to layout_tests/port object for port-specific stuff
+ options - optparse argument for command-line options
+ client - message_broker2.BrokerClient implementation to dispatch
+ replies to.
+ worker_class - type of workers to create. This class must implement
+ the methods in AbstractWorker.
+ Returns:
+ A handle to an object that will talk to a message broker configured
+ for the normal manager/worker communication.
+ """
+ worker_model = options.worker_model
+ if worker_model == 'inline':
+ queue_class = Queue.Queue
+ manager_class = _InlineManager
+ elif worker_model == 'threads':
+ queue_class = Queue.Queue
+ manager_class = _ThreadedManager
+ elif worker_model == 'processes' and multiprocessing:
+ queue_class = multiprocessing.Queue
+ manager_class = _MultiProcessManager
+ else:
+ raise ValueError("unsupported value for --worker-model: %s" %
+ worker_model)
+
+ broker = message_broker2.Broker(options, queue_class)
+ return manager_class(broker, port, options, client, worker_class)
+
+
+class AbstractWorker(message_broker2.BrokerClient):
+ def __init__(self, broker_connection, worker_number, options):
+ """The constructor should be used to do any simple initialization
+ necessary, but should not do anything that creates data structures
+ that cannot be Pickled or sent across processes (like opening
+ files or sockets). Complex initialization should be done at the
+ start of the run() call.
+
+ Args:
+ broker_connection - handle to the BrokerConnection object creating
+ the worker and that can be used for messaging.
+ worker_number - identifier for this particular worker
+ options - command-line argument object from optparse"""
+
+ raise NotImplementedError
+
+ def run(self, port):
+ """Callback for the worker to start executing. Typically does any
+ remaining initialization and then calls broker_connection.run_message_loop()."""
+ raise NotImplementedError
+
+ def cancel(self):
+ """Called when possible to indicate to the worker to stop processing
+ messages and shut down. Note that workers may be stopped without this
+ method being called, so clients should not rely solely on this."""
+ raise NotImplementedError
+
+
+class _ManagerConnection(message_broker2.BrokerConnection):
+ def __init__(self, broker, options, client, worker_class):
+ """Base initialization for all Manager objects.
+
+ Args:
+ broker: handle to the message_broker2 object
+ options: command line options object
+ client: callback object (the caller)
+ worker_class: class object to use to create workers.
+ """
+ message_broker2.BrokerConnection.__init__(self, broker, client,
+ MANAGER_TOPIC, ANY_WORKER_TOPIC)
+ self._options = options
+ self._worker_class = worker_class
+
+ def start_worker(self, worker_number):
+ raise NotImplementedError
+
+
+class _InlineManager(_ManagerConnection):
+ def __init__(self, broker, port, options, client, worker_class):
+ _ManagerConnection.__init__(self, broker, options, client, worker_class)
+ self._port = port
+ self._inline_worker = None
+
+ def start_worker(self, worker_number):
+ self._inline_worker = _InlineWorkerConnection(self._broker, self._port,
+ self._client, self._worker_class, worker_number)
+ return self._inline_worker
+
+ def run_message_loop(self, delay_secs=None):
+ # Note that delay_secs is ignored in this case since we can't easily
+ # implement it.
+ self._inline_worker.run()
+ self._broker.run_all_pending(MANAGER_TOPIC, self._client)
+
+
+class _ThreadedManager(_ManagerConnection):
+ def __init__(self, broker, port, options, client, worker_class):
+ _ManagerConnection.__init__(self, broker, options, client, worker_class)
+ self._port = port
+
+ def start_worker(self, worker_number):
+ worker_connection = _ThreadedWorkerConnection(self._broker, self._port,
+ self._worker_class, worker_number)
+ worker_connection.start()
+ return worker_connection
+
+
+class _MultiProcessManager(_ManagerConnection):
+ def __init__(self, broker, port, options, client, worker_class):
+ # Note that this class does not keep a handle to the actual port
+ # object, because it isn't Picklable. Instead it keeps the port
+ # name and recreates the port in the child process from the name
+ # and options.
+ _ManagerConnection.__init__(self, broker, options, client, worker_class)
+ self._platform_name = port.real_name()
+
+ def start_worker(self, worker_number):
+ worker_connection = _MultiProcessWorkerConnection(self._broker, self._platform_name,
+ self._worker_class, worker_number, self._options)
+ worker_connection.start()
+ return worker_connection
+
+
+class _WorkerConnection(message_broker2.BrokerConnection):
+ def __init__(self, broker, worker_class, worker_number, options):
+ self._client = worker_class(self, worker_number, options)
+ self.name = self._client.name()
+ message_broker2.BrokerConnection.__init__(self, broker, self._client,
+ ANY_WORKER_TOPIC, MANAGER_TOPIC)
+
+ def yield_to_broker(self):
+ pass
+
+
+class _InlineWorkerConnection(_WorkerConnection):
+ def __init__(self, broker, port, manager_client, worker_class, worker_number):
+ _WorkerConnection.__init__(self, broker, worker_class, worker_number, port._options)
+ self._port = port
+ self._manager_client = manager_client
+
+ def run(self):
+ self._client.run(self._port)
+
+ def yield_to_broker(self):
+ self._broker.run_all_pending(MANAGER_TOPIC, self._manager_client)
+
+
+class _Thread(threading.Thread):
+ def __init__(self, worker_connection, port, client):
+ threading.Thread.__init__(self)
+ self._worker_connection = worker_connection
+ self._port = port
+ self._client = client
+
+ def run(self):
+ # FIXME: We can remove this once everyone is on 2.6.
+ if not hasattr(self, 'ident'):
+ self.ident = thread.get_ident()
+ self._client.run(self._port)
+
+
+class _ThreadedWorkerConnection(_WorkerConnection):
+ def __init__(self, broker, port, worker_class, worker_number):
+ _WorkerConnection.__init__(self, broker, worker_class, worker_number, port._options)
+ self._thread = _Thread(self, port, self._client)
+
+ def start(self):
+ self._thread.start()
+
+
+class _Process(_Multiprocessing_Process):
+ def __init__(self, worker_connection, platform_name, options, client):
+ _Multiprocessing_Process.__init__(self)
+ self._worker_connection = worker_connection
+ self._platform_name = platform_name
+ self._options = options
+ self._client = client
+
+ def run(self):
+ logging.basicConfig()
+ port_obj = port.get(self._platform_name, self._options)
+ self._client.run(port_obj)
+
+
+class _MultiProcessWorkerConnection(_WorkerConnection):
+ def __init__(self, broker, platform_name, worker_class, worker_number, options):
+ _WorkerConnection.__init__(self, broker, worker_class, worker_number, options)
+ self._proc = _Process(self, platform_name, options, self._client)
+
+ def start(self):
+ self._proc.start()
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker_unittest.py
new file mode 100644
index 0000000..ffbe081
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker_unittest.py
@@ -0,0 +1,227 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import optparse
+import Queue
+import sys
+import unittest
+
+try:
+ import multiprocessing
+except ImportError:
+ multiprocessing = None
+
+
+from webkitpy.common.system import outputcapture
+
+from webkitpy.layout_tests import port
+from webkitpy.layout_tests.layout_package import manager_worker_broker
+from webkitpy.layout_tests.layout_package import message_broker2
+
+
+class TestWorker(manager_worker_broker.AbstractWorker):
+ def __init__(self, broker_connection, worker_number, options):
+ self._broker_connection = broker_connection
+ self._options = options
+ self._worker_number = worker_number
+ self._name = 'TestWorker/%d' % worker_number
+ self._stopped = False
+
+ def handle_stop(self, src):
+ self._stopped = True
+
+ def handle_test(self, src, an_int, a_str):
+ assert an_int == 1
+ assert a_str == "hello, world"
+ self._broker_connection.post_message('test', 2, 'hi, everybody')
+
+ def is_done(self):
+ return self._stopped
+
+ def name(self):
+ return self._name
+
+ def start(self):
+ pass
+
+ def run(self, port):
+ try:
+ self._broker_connection.run_message_loop()
+ self._broker_connection.yield_to_broker()
+ self._broker_connection.post_message('done')
+ except Exception, e:
+ self._broker_connection.post_message('exception', (type(e), str(e), None))
+
+
+def get_options(worker_model):
+ option_list = manager_worker_broker.runtime_options()
+ parser = optparse.OptionParser(option_list=option_list)
+ options, args = parser.parse_args(args=['--worker-model', worker_model])
+ return options
+
+
+def make_broker(manager, worker_model):
+ options = get_options(worker_model)
+ return manager_worker_broker.get(port.get("test"), options, manager,
+ TestWorker)
+
+
+class FunctionTests(unittest.TestCase):
+ def test_get__inline(self):
+ self.assertTrue(make_broker(self, 'inline') is not None)
+
+ def test_get__threads(self):
+ self.assertTrue(make_broker(self, 'threads') is not None)
+
+ def test_get__processes(self):
+ if multiprocessing:
+ self.assertTrue(make_broker(self, 'processes') is not None)
+ else:
+ self.assertRaises(ValueError, make_broker, self, 'processes')
+
+ def test_get__unknown(self):
+ self.assertRaises(ValueError, make_broker, self, 'unknown')
+
+
+class _TestsMixin(object):
+ """Mixin class that implements a series of tests to enforce the
+ contract all implementations must follow."""
+
+ #
+ # Methods to implement the Manager side of the ClientInterface
+ #
+ def name(self):
+ return 'Tester'
+
+ def is_done(self):
+ return self._done
+
+ #
+ # Handlers for the messages the TestWorker may send.
+ #
+ def handle_done(self, src):
+ self._done = True
+
+ def handle_test(self, src, an_int, a_str):
+ self._an_int = an_int
+ self._a_str = a_str
+
+ def handle_exception(self, src, exc_info):
+ self._exception = exc_info
+ self._done = True
+
+ #
+ # Testing helper methods
+ #
+ def setUp(self):
+ self._an_int = None
+ self._a_str = None
+ self._broker = None
+ self._done = False
+ self._exception = None
+ self._worker_model = None
+
+ def make_broker(self):
+ self._broker = make_broker(self, self._worker_model)
+
+ #
+ # Actual unit tests
+ #
+ def test_done(self):
+ if not self._worker_model:
+ return
+ self.make_broker()
+ worker = self._broker.start_worker(0)
+ self._broker.post_message('test', 1, 'hello, world')
+ self._broker.post_message('stop')
+ self._broker.run_message_loop()
+ self.assertTrue(self.is_done())
+ self.assertEqual(self._an_int, 2)
+ self.assertEqual(self._a_str, 'hi, everybody')
+
+ def test_unknown_message(self):
+ if not self._worker_model:
+ return
+ self.make_broker()
+ worker = self._broker.start_worker(0)
+ self._broker.post_message('unknown')
+ self._broker.run_message_loop()
+
+ self.assertTrue(self.is_done())
+ self.assertEquals(self._exception[0], ValueError)
+ self.assertEquals(self._exception[1],
+ "TestWorker/0: received message 'unknown' it couldn't handle")
+
+
+class InlineBrokerTests(_TestsMixin, unittest.TestCase):
+ def setUp(self):
+ _TestsMixin.setUp(self)
+ self._worker_model = 'inline'
+
+
+class MultiProcessBrokerTests(_TestsMixin, unittest.TestCase):
+ def setUp(self):
+ _TestsMixin.setUp(self)
+ if multiprocessing:
+ self._worker_model = 'processes'
+ else:
+ self._worker_model = None
+
+ def queue(self):
+ return multiprocessing.Queue()
+
+
+class ThreadedBrokerTests(_TestsMixin, unittest.TestCase):
+ def setUp(self):
+ _TestsMixin.setUp(self)
+ self._worker_model = 'threads'
+
+
+class FunctionsTest(unittest.TestCase):
+ def test_runtime_options(self):
+ option_list = manager_worker_broker.runtime_options()
+ parser = optparse.OptionParser(option_list=option_list)
+ options, args = parser.parse_args([])
+ self.assertTrue(options)
+
+
+class InterfaceTest(unittest.TestCase):
+ # These tests mostly exist to pacify coverage.
+
+ # FIXME: There must be a better way to do this and also verify
+ # that classes do implement every abstract method in an interface.
+ def test_managerconnection_is_abstract(self):
+ # Test that all the base class methods are abstract and have the
+ # signature we expect.
+ broker = make_broker(self, 'inline')
+ obj = manager_worker_broker._ManagerConnection(broker._broker, None, self, None)
+ self.assertRaises(NotImplementedError, obj.start_worker, 0)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py
index 481c617..66a7aa8 100644
--- a/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py
@@ -41,9 +41,9 @@ requested configuration.
"""
import logging
-import sys
import time
-import traceback
+
+from webkitpy.common.system import stack_utils
import dump_render_tree_thread
@@ -137,6 +137,7 @@ class MultiThreadedBroker(WorkerMessageBroker):
def run_message_loop(self):
threads = self._threads()
+ wedged_threads = set()
# Loop through all the threads waiting for them to finish.
some_thread_is_alive = True
@@ -145,11 +146,15 @@ class MultiThreadedBroker(WorkerMessageBroker):
t = time.time()
for thread in threads:
if thread.isAlive():
+ if thread in wedged_threads:
+ continue
+
some_thread_is_alive = True
next_timeout = thread.next_timeout()
if next_timeout and t > next_timeout:
- log_wedged_worker(thread.getName(), thread.id())
+ stack_utils.log_thread_state(_log.error, thread.getName(), thread.id(), "is wedged")
thread.clear_next_timeout()
+ wedged_threads.add(thread)
exception_info = thread.exception_info()
if exception_info is not None:
@@ -164,34 +169,10 @@ class MultiThreadedBroker(WorkerMessageBroker):
if some_thread_is_alive:
time.sleep(0.01)
+ if wedged_threads:
+ _log.warning("All remaining threads are wedged, bailing out.")
+
def cancel_workers(self):
threads = self._threads()
for thread in threads:
thread.cancel()
-
-
-def log_wedged_worker(name, id):
- """Log information about the given worker state."""
- stack = _find_thread_stack(id)
- assert(stack is not None)
- _log.error("")
- _log.error("%s (tid %d) is wedged" % (name, id))
- _log_stack(stack)
- _log.error("")
-
-
-def _find_thread_stack(id):
- """Returns a stack object that can be used to dump a stack trace for
- the given thread id (or None if the id is not found)."""
- for thread_id, stack in sys._current_frames().items():
- if thread_id == id:
- return stack
- return None
-
-
-def _log_stack(stack):
- """Log a stack trace to log.error()."""
- for filename, lineno, name, line in traceback.extract_stack(stack):
- _log.error('File: "%s", line %d, in %s' % (filename, lineno, name))
- if line:
- _log.error(' %s' % line.strip())
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker2.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker2.py
new file mode 100644
index 0000000..ec3c970
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker2.py
@@ -0,0 +1,196 @@
+# Copyright (C) 2011 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Module for handling messaging for run-webkit-tests.
+
+This module implements a simple message broker abstraction that will be
+used to coordinate messages between the main run-webkit-tests thread
+(aka TestRunner) and the individual worker threads (previously known as
+dump_render_tree_threads).
+
+The broker simply distributes messages onto topics (named queues); the actual
+queues themselves are provided by the caller, as the queue's implementation
+requirements varies vary depending on the desired concurrency model
+(none/threads/processes).
+
+In order for shared-nothing messaging between processing to be possible,
+Messages must be picklable.
+
+The module defines one interface and two classes. Callers of this package
+must implement the BrokerClient interface, and most callers will create
+BrokerConnections as well as Brokers.
+
+The classes relate to each other as:
+
+ BrokerClient ------> BrokerConnection
+ ^ |
+ | v
+ \---------------- Broker
+
+(The BrokerClient never calls broker directly after it is created, only
+BrokerConnection. BrokerConnection passes a reference to BrokerClient to
+Broker, and Broker only invokes that reference, never talking directly to
+BrokerConnection).
+"""
+
+import cPickle
+import logging
+import Queue
+import time
+
+
+_log = logging.getLogger(__name__)
+
+
+class BrokerClient(object):
+ """Abstract base class / interface that all message broker clients must
+ implement. In addition to the methods below, by convention clients
+ implement routines of the signature type
+
+ handle_MESSAGE_NAME(self, src, ...):
+
+ where MESSAGE_NAME matches the string passed to post_message(), and
+ src indicates the name of the sender. If the message contains values in
+ the message body, those will be provided as optparams."""
+
+ def __init__(self, *optargs, **kwargs):
+ raise NotImplementedError
+
+ def is_done(self):
+ """Called from inside run_message_loop() to indicate whether to exit."""
+ raise NotImplementedError
+
+ def name(self):
+ """Return a name that identifies the client."""
+ raise NotImplementedError
+
+
+class Broker(object):
+ """Brokers provide the basic model of a set of topics. Clients can post a
+ message to any topic using post_message(), and can process messages on one
+ topic at a time using run_message_loop()."""
+
+ def __init__(self, options, queue_maker):
+ """Args:
+ options: a runtime option class from optparse
+ queue_maker: a factory method that returns objects implementing a
+ Queue interface (put()/get()).
+ """
+ self._options = options
+ self._queue_maker = queue_maker
+ self._topics = {}
+
+ def add_topic(self, topic_name):
+ if topic_name not in self._topics:
+ self._topics[topic_name] = self._queue_maker()
+
+ def _get_queue_for_topic(self, topic_name):
+ return self._topics[topic_name]
+
+ def post_message(self, client, topic_name, message_name, *message_args):
+ """Post a message to the appropriate topic name.
+
+ Messages have a name and a tuple of optional arguments. Both must be picklable."""
+ message = _Message(client.name(), topic_name, message_name, message_args)
+ queue = self._get_queue_for_topic(topic_name)
+ queue.put(_Message.dumps(message))
+
+ def run_message_loop(self, topic_name, client, delay_secs=None):
+ """Loop processing messages until client.is_done() or delay passes.
+
+ To run indefinitely, set delay_secs to None."""
+ assert delay_secs is None or delay_secs > 0
+ self._run_loop(topic_name, client, block=True, delay_secs=delay_secs)
+
+ def run_all_pending(self, topic_name, client):
+ """Process messages until client.is_done() or caller would block."""
+ self._run_loop(topic_name, client, block=False, delay_secs=None)
+
+ def _run_loop(self, topic_name, client, block, delay_secs):
+ queue = self._get_queue_for_topic(topic_name)
+ while not client.is_done():
+ try:
+ s = queue.get(block, delay_secs)
+ except Queue.Empty:
+ return
+ msg = _Message.loads(s)
+ self._dispatch_message(msg, client)
+
+ def _dispatch_message(self, message, client):
+ if not hasattr(client, 'handle_' + message.name):
+ raise ValueError(
+ "%s: received message '%s' it couldn't handle" %
+ (client.name(), message.name))
+ optargs = message.args
+ message_handler = getattr(client, 'handle_' + message.name)
+ message_handler(message.src, *optargs)
+
+
+class _Message(object):
+ @staticmethod
+ def loads(str):
+ obj = cPickle.loads(str)
+ assert(isinstance(obj, _Message))
+ return obj
+
+ def __init__(self, src, topic_name, message_name, message_args):
+ self.src = src
+ self.topic_name = topic_name
+ self.name = message_name
+ self.args = message_args
+
+ def dumps(self):
+ return cPickle.dumps(self)
+
+ def __repr__(self):
+ return ("_Message(from='%s', topic_name='%s', message_name='%s')" %
+ (self.src, self.topic_name, self.name))
+
+
+class BrokerConnection(object):
+ """BrokerConnection provides a connection-oriented facade on top of a
+ Broker, so that callers don't have to repeatedly pass the same topic
+ names over and over."""
+
+ def __init__(self, broker, client, run_topic, post_topic):
+ """Create a BrokerConnection on top of a Broker. Note that the Broker
+ is passed in rather than created so that a single Broker can be used
+ by multiple BrokerConnections."""
+ self._broker = broker
+ self._client = client
+ self._post_topic = post_topic
+ self._run_topic = run_topic
+ broker.add_topic(run_topic)
+ broker.add_topic(post_topic)
+
+ def run_message_loop(self, delay_secs=None):
+ self._broker.run_message_loop(self._run_topic, self._client, delay_secs)
+
+ def post_message(self, message_name, *message_args):
+ self._broker.post_message(self._client, self._post_topic,
+ message_name, *message_args)
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker2_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker2_unittest.py
new file mode 100644
index 0000000..0e0a88d
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker2_unittest.py
@@ -0,0 +1,83 @@
+# Copyright (C) 2011 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import unittest
+
+from webkitpy.layout_tests.layout_package import message_broker2
+
+# This file exists to test routines that aren't necessarily covered elsewhere;
+# most of the testing of message_broker2 will be covered under the tests in
+# the manager_worker_broker module.
+
+
+class MessageTest(unittest.TestCase):
+ def test__no_body(self):
+ msg = message_broker2._Message('src', 'topic_name', 'message_name', None)
+ self.assertTrue(repr(msg))
+ s = msg.dumps()
+ new_msg = message_broker2._Message.loads(s)
+ self.assertEqual(new_msg.name, 'message_name')
+ self.assertEqual(new_msg.args, None)
+ self.assertEqual(new_msg.topic_name, 'topic_name')
+ self.assertEqual(new_msg.src, 'src')
+
+ def test__body(self):
+ msg = message_broker2._Message('src', 'topic_name', 'message_name',
+ ('body', 0))
+ self.assertTrue(repr(msg))
+ s = msg.dumps()
+ new_msg = message_broker2._Message.loads(s)
+ self.assertEqual(new_msg.name, 'message_name')
+ self.assertEqual(new_msg.args, ('body', 0))
+ self.assertEqual(new_msg.topic_name, 'topic_name')
+ self.assertEqual(new_msg.src, 'src')
+
+
+class InterfaceTest(unittest.TestCase):
+ # These tests mostly exist to pacify coverage.
+
+ # FIXME: There must be a better way to do this and also verify
+ # that classes do implement every abstract method in an interface.
+
+ def test_brokerclient_is_abstract(self):
+ # Test that we can't create an instance directly.
+ self.assertRaises(NotImplementedError, message_broker2.BrokerClient)
+
+ class TestClient(message_broker2.BrokerClient):
+ def __init__(self):
+ pass
+
+ # Test that all the base class methods are abstract and have the
+ # signature we expect.
+ obj = TestClient()
+ self.assertRaises(NotImplementedError, obj.is_done)
+ self.assertRaises(NotImplementedError, obj.name)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py
index 6f04fd3..f4cb5d2 100644
--- a/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py
@@ -84,7 +84,6 @@ class TestThread(threading.Thread):
def next_timeout(self):
if self._timeout:
- self._timeout_queue.put('done')
return time.time() - 10
return time.time()
@@ -125,7 +124,12 @@ class MultiThreadedBrokerTest(unittest.TestCase):
child_thread.start()
started_msg = starting_queue.get()
stopping_queue.put(msg)
- return broker.run_message_loop()
+ res = broker.run_message_loop()
+ if msg == 'Timeout':
+ child_thread._timeout_queue.put('done')
+ child_thread.join(1.0)
+ self.assertFalse(child_thread.isAlive())
+ return res
def test_basic(self):
interrupted = self.run_one_thread('')
@@ -135,48 +139,22 @@ class MultiThreadedBrokerTest(unittest.TestCase):
self.assertRaises(KeyboardInterrupt, self.run_one_thread, 'KeyboardInterrupt')
def test_timeout(self):
+ # Because the timeout shows up as a wedged thread, this also tests
+ # log_wedged_worker().
oc = outputcapture.OutputCapture()
- oc.capture_output()
- interrupted = self.run_one_thread('Timeout')
- self.assertFalse(interrupted)
- oc.restore_output()
-
- def test_exception(self):
- self.assertRaises(ValueError, self.run_one_thread, 'Exception')
-
-
-class Test(unittest.TestCase):
- def test_find_thread_stack_found(self):
- id, stack = sys._current_frames().items()[0]
- found_stack = message_broker._find_thread_stack(id)
- self.assertNotEqual(found_stack, None)
-
- def test_find_thread_stack_not_found(self):
- found_stack = message_broker._find_thread_stack(0)
- self.assertEqual(found_stack, None)
-
- def test_log_wedged_worker(self):
- oc = outputcapture.OutputCapture()
- oc.capture_output()
+ stdout, stderr = oc.capture_output()
logger = message_broker._log
astream = array_stream.ArrayStream()
handler = TestHandler(astream)
logger.addHandler(handler)
+ interrupted = self.run_one_thread('Timeout')
+ stdout, stderr = oc.restore_output()
+ self.assertFalse(interrupted)
+ logger.handlers.remove(handler)
+ self.assertTrue('All remaining threads are wedged, bailing out.' in astream.get())
- starting_queue = Queue.Queue()
- stopping_queue = Queue.Queue()
- child_thread = TestThread(starting_queue, stopping_queue)
- child_thread.start()
- msg = starting_queue.get()
-
- message_broker.log_wedged_worker(child_thread.getName(),
- child_thread.id())
- stopping_queue.put('')
- child_thread.join(timeout=1.0)
-
- self.assertFalse(astream.empty())
- self.assertFalse(child_thread.isAlive())
- oc.restore_output()
+ def test_exception(self):
+ self.assertRaises(ValueError, self.run_one_thread, 'Exception')
if __name__ == '__main__':
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py
index 12a786e..7ab6da8 100644
--- a/Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py
@@ -144,7 +144,7 @@ class Testprinter(unittest.TestCase):
test in tests]
expectations = test_expectations.TestExpectations(
self._port, test_paths, expectations_str,
- self._port.test_platform_name(), is_debug_mode=False,
+ self._port.test_configuration(),
is_lint_mode=False)
rs = result_summary.ResultSummary(expectations, test_paths)
@@ -363,7 +363,7 @@ class Testprinter(unittest.TestCase):
def test_print_progress__detailed(self):
tests = ['passes/text.html', 'failures/expected/timeout.html',
'failures/expected/crash.html']
- expectations = 'failures/expected/timeout.html = TIMEOUT'
+ expectations = 'BUGX : failures/expected/timeout.html = TIMEOUT'
# first, test that it is disabled properly
# should still print one-line-progress
@@ -569,8 +569,8 @@ class Testprinter(unittest.TestCase):
self.assertFalse(out.empty())
expectations = """
-failures/expected/crash.html = CRASH
-failures/expected/timeout.html = TIMEOUT
+BUGX : failures/expected/crash.html = CRASH
+BUGX : failures/expected/timeout.html = TIMEOUT
"""
err.reset()
out.reset()
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py
new file mode 100644
index 0000000..96e3ee6
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py
@@ -0,0 +1,322 @@
+# Copyright (C) 2011 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import logging
+import threading
+import time
+
+from webkitpy.layout_tests.port import base
+
+from webkitpy.layout_tests.test_types import text_diff
+from webkitpy.layout_tests.test_types import image_diff
+
+from webkitpy.layout_tests.layout_package import test_failures
+from webkitpy.layout_tests.layout_package.test_results import TestResult
+
+
+_log = logging.getLogger(__name__)
+
+
+class ExpectedDriverOutput:
+ """Groups information about an expected driver output."""
+ def __init__(self, text, image, image_hash):
+ self.text = text
+ self.image = image
+ self.image_hash = image_hash
+
+
+class SingleTestRunner:
+
+ def __init__(self, options, port, worker_name, worker_number):
+ self._options = options
+ self._port = port
+ self._worker_name = worker_name
+ self._worker_number = worker_number
+ self._driver = None
+ self._test_types = []
+ self.has_http_lock = False
+ for cls in self._get_test_type_classes():
+ self._test_types.append(cls(self._port,
+ self._options.results_directory))
+
+ def cleanup(self):
+ self.kill_dump_render_tree()
+ if self.has_http_lock:
+ self.stop_servers_with_lock()
+
+ def _get_test_type_classes(self):
+ classes = [text_diff.TestTextDiff]
+ if self._options.pixel_tests:
+ classes.append(image_diff.ImageDiff)
+ return classes
+
+ def timeout(self, test_input):
+ # We calculate how long we expect the test to take.
+ #
+ # The DumpRenderTree watchdog uses 2.5x the timeout; we want to be
+ # larger than that. We also add a little more padding if we're
+ # running tests in a separate thread.
+ #
+ # Note that we need to convert the test timeout from a
+ # string value in milliseconds to a float for Python.
+ driver_timeout_sec = 3.0 * float(test_input.timeout) / 1000.0
+ if not self._options.run_singly:
+ return driver_timeout_sec
+
+ thread_padding_sec = 1.0
+ thread_timeout_sec = driver_timeout_sec + thread_padding_sec
+ return thread_timeout_sec
+
+ def run_test(self, test_input, timeout):
+ if self._options.run_singly:
+ return self._run_test_in_another_thread(test_input, timeout)
+ else:
+ return self._run_test_in_this_thread(test_input)
+ return result
+
+ def _run_test_in_another_thread(self, test_input, thread_timeout_sec):
+ """Run a test in a separate thread, enforcing a hard time limit.
+
+ Since we can only detect the termination of a thread, not any internal
+ state or progress, we can only run per-test timeouts when running test
+ files singly.
+
+ Args:
+ test_input: Object containing the test filename and timeout
+ thread_timeout_sec: time to wait before killing the driver process.
+ Returns:
+ A TestResult
+ """
+ worker = self
+ result = None
+
+ driver = worker._port.create_driver(worker._worker_number)
+ driver.start()
+
+ class SingleTestThread(threading.Thread):
+ def run(self):
+ result = worker.run(test_input, driver)
+
+ thread = SingleTestThread()
+ thread.start()
+ thread.join(thread_timeout_sec)
+ if thread.isAlive():
+ # If join() returned with the thread still running, the
+ # DumpRenderTree is completely hung and there's nothing
+ # more we can do with it. We have to kill all the
+ # DumpRenderTrees to free it up. If we're running more than
+ # one DumpRenderTree thread, we'll end up killing the other
+ # DumpRenderTrees too, introducing spurious crashes. We accept
+ # that tradeoff in order to avoid losing the rest of this
+ # thread's results.
+ _log.error('Test thread hung: killing all DumpRenderTrees')
+
+ driver.stop()
+
+ if not result:
+ result = TestResult(test_input.filename, failures=[],
+ test_run_time=0, total_time_for_all_diffs=0, time_for_diffs={})
+ return result
+
+ def _run_test_in_this_thread(self, test_input):
+ """Run a single test file using a shared DumpRenderTree process.
+
+ Args:
+ test_input: Object containing the test filename, uri and timeout
+
+ Returns: a TestResult object.
+ """
+ # poll() is not threadsafe and can throw OSError due to:
+ # http://bugs.python.org/issue1731717
+ if not self._driver or self._driver.poll() is not None:
+ self._driver = self._port.create_driver(self._worker_number)
+ self._driver.start()
+ return self._run(self._driver, test_input)
+
+ def _expected_driver_output(self):
+ return ExpectedDriverOutput(self._port.expected_text(self._filename),
+ self._port.expected_image(self._filename),
+ self._port.expected_checksum(self._filename))
+
+ def _should_fetch_expected_checksum(self):
+ return (self._options.pixel_tests and
+ not (self._options.new_baseline or self._options.reset_results))
+
+ def _driver_input(self, test_input):
+ self._filename = test_input.filename
+ self._timeout = test_input.timeout
+ self._testname = self._port.relative_test_filename(test_input.filename)
+
+ # The image hash is used to avoid doing an image dump if the
+ # checksums match, so it should be set to a blank value if we
+ # are generating a new baseline. (Otherwise, an image from a
+ # previous run will be copied into the baseline."""
+ image_hash = None
+ if self._should_fetch_expected_checksum():
+ image_hash = self._port.expected_checksum(self._filename)
+ return base.DriverInput(self._filename, self._timeout, image_hash)
+
+ def _run(self, driver, test_input):
+ if self._options.new_baseline or self._options.reset_results:
+ return self._run_rebaseline(driver, test_input)
+ return self._run_compare_test(driver, test_input)
+
+ def _run_compare_test(self, driver, test_input):
+ driver_output = self._driver.run_test(self._driver_input(test_input))
+ return self._process_output(driver_output)
+
+ def _run_rebaseline(self, driver, test_input):
+ driver_output = self._driver.run_test(self._driver_input(test_input))
+ failures = self._handle_error(driver_output)
+ # FIXME: It the test crashed or timed out, it might be bettter to avoid
+ # to write new baselines.
+ self._save_baselines(driver_output)
+ return TestResult(self._filename, failures, driver_output.test_time)
+
+ def _save_baselines(self, driver_output):
+ # Although all test_shell/DumpRenderTree output should be utf-8,
+ # we do not ever decode it inside run-webkit-tests. For some tests
+ # DumpRenderTree may not output utf-8 text (e.g. webarchives).
+ self._save_baseline_data(driver_output.text, ".txt",
+ generate_new_baseline=self._options.new_baseline)
+ if self._options.pixel_tests and driver_output.image_hash:
+ self._save_baseline_data(driver_output.image, ".png",
+ generate_new_baseline=self._options.new_baseline)
+ self._save_baseline_data(driver_output.image_hash, ".checksum",
+ generate_new_baseline=self._options.new_baseline)
+
+ def _save_baseline_data(self, data, modifier, generate_new_baseline=True):
+ """Saves a new baseline file into the port's baseline directory.
+
+ The file will be named simply "<test>-expected<modifier>", suitable for
+ use as the expected results in a later run.
+
+ Args:
+ data: result to be saved as the new baseline
+ modifier: type of the result file, e.g. ".txt" or ".png"
+ generate_new_baseline: whether to enerate a new, platform-specific
+ baseline, or update the existing one
+ """
+
+ port = self._port
+ fs = port._filesystem
+ if generate_new_baseline:
+ relative_dir = fs.dirname(self._testname)
+ baseline_path = port.baseline_path()
+ output_dir = fs.join(baseline_path, relative_dir)
+ output_file = fs.basename(fs.splitext(self._filename)[0] +
+ "-expected" + modifier)
+ fs.maybe_make_directory(output_dir)
+ output_path = fs.join(output_dir, output_file)
+ _log.debug('writing new baseline result "%s"' % (output_path))
+ else:
+ output_path = port.expected_filename(self._filename, modifier)
+ _log.debug('resetting baseline result "%s"' % output_path)
+
+ port.update_baseline(output_path, data)
+
+ def _handle_error(self, driver_output):
+ failures = []
+ fs = self._port._filesystem
+ if driver_output.timeout:
+ failures.append(test_failures.FailureTimeout())
+ if driver_output.crash:
+ failures.append(test_failures.FailureCrash())
+ _log.debug("%s Stacktrace for %s:\n%s" % (self._worker_name, self._testname,
+ driver_output.error))
+ stack_filename = fs.join(self._options.results_directory, self._testname)
+ stack_filename = fs.splitext(stack_filename)[0] + "-stack.txt"
+ fs.maybe_make_directory(fs.dirname(stack_filename))
+ fs.write_text_file(stack_filename, driver_output.error)
+ elif driver_output.error:
+ _log.debug("%s %s output stderr lines:\n%s" % (self._worker_name, self._testname,
+ driver_output.error))
+ return failures
+
+ def _run_test(self):
+ driver_output = self._driver.run_test(self._driver_input())
+ return self._process_output(driver_output)
+
+ def _process_output(self, driver_output):
+ """Receives the output from a DumpRenderTree process, subjects it to a
+ number of tests, and returns a list of failure types the test produced.
+ Args:
+ driver_output: a DriverOutput object containing the output from the driver
+
+ Returns: a TestResult object
+ """
+ fs = self._port._filesystem
+ failures = self._handle_error(driver_output)
+ expected_driver_output = self._expected_driver_output()
+
+ # Check the output and save the results.
+ start_time = time.time()
+ time_for_diffs = {}
+ for test_type in self._test_types:
+ start_diff_time = time.time()
+ new_failures = test_type.compare_output(
+ self._port, self._filename, self._options, driver_output,
+ expected_driver_output)
+ # Don't add any more failures if we already have a crash, so we don't
+ # double-report those tests. We do double-report for timeouts since
+ # we still want to see the text and image output.
+ if not driver_output.crash:
+ failures.extend(new_failures)
+ time_for_diffs[test_type.__class__.__name__] = (
+ time.time() - start_diff_time)
+
+ total_time_for_all_diffs = time.time() - start_diff_time
+ return TestResult(self._filename, failures, driver_output.test_time,
+ total_time_for_all_diffs, time_for_diffs)
+
+ def start_servers_with_lock(self):
+ _log.debug('Acquiring http lock ...')
+ self._port.acquire_http_lock()
+ _log.debug('Starting HTTP server ...')
+ self._port.start_http_server()
+ _log.debug('Starting WebSocket server ...')
+ self._port.start_websocket_server()
+ self.has_http_lock = True
+
+ def stop_servers_with_lock(self):
+ """Stop the servers and release http lock."""
+ if self.has_http_lock:
+ _log.debug('Stopping HTTP server ...')
+ self._port.stop_http_server()
+ _log.debug('Stopping WebSocket server ...')
+ self._port.stop_websocket_server()
+ _log.debug('Releasing server lock ...')
+ self._port.release_http_lock()
+ self.has_http_lock = False
+
+ def kill_dump_render_tree(self):
+ """Kill the DumpRenderTree process if it's running."""
+ if self._driver:
+ self._driver.stop()
+ self._driver = None
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py
index 806b663..494395a 100644
--- a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py
@@ -31,6 +31,7 @@
for layout tests.
"""
+import itertools
import logging
import re
@@ -84,18 +85,16 @@ def remove_pixel_failures(expected_results):
class TestExpectations:
TEST_LIST = "test_expectations.txt"
- def __init__(self, port, tests, expectations, test_platform_name,
- is_debug_mode, is_lint_mode, overrides=None):
+ def __init__(self, port, tests, expectations, test_config,
+ is_lint_mode, overrides=None):
"""Loads and parses the test expectations given in the string.
Args:
port: handle to object containing platform-specific functionality
- test: list of all of the test files
+ tests: list of all of the test files
expectations: test expectations as a string
- test_platform_name: name of the platform to match expectations
- against. Note that this may be different than
- port.test_platform_name() when is_lint_mode is True.
- is_debug_mode: whether to use the DEBUG or RELEASE modifiers
- in the expectations
+ test_config: specific values to check against when
+ parsing the file (usually port.test_config(),
+ but may be different when linting or doing other things).
is_lint_mode: If True, just parse the expectations string
looking for errors.
overrides: test expectations that are allowed to override any
@@ -104,7 +103,7 @@ class TestExpectations:
and downstream expectations).
"""
self._expected_failures = TestExpectationsFile(port, expectations,
- tests, test_platform_name, is_debug_mode, is_lint_mode,
+ tests, test_config, is_lint_mode,
overrides=overrides)
# TODO(ojan): Allow for removing skipped tests when getting the list of
@@ -197,7 +196,7 @@ class ParseError(Exception):
return '\n'.join(map(str, self.errors))
def __repr__(self):
- return 'ParseError(fatal=%s, errors=%s)' % (fatal, errors)
+ return 'ParseError(fatal=%s, errors=%s)' % (self.fatal, self.errors)
class ModifiersAndExpectations:
@@ -302,29 +301,15 @@ class TestExpectationsFile:
'fail': FAIL,
'flaky': FLAKY}
- def __init__(self, port, expectations, full_test_list, test_platform_name,
- is_debug_mode, is_lint_mode, overrides=None):
- """
- expectations: Contents of the expectations file
- full_test_list: The list of all tests to be run pending processing of
- the expections for those tests.
- test_platform_name: name of the platform to match expectations
- against. Note that this may be different than
- port.test_platform_name() when is_lint_mode is True.
- is_debug_mode: Whether we testing a test_shell built debug mode.
- is_lint_mode: Whether this is just linting test_expecatations.txt.
- overrides: test expectations that are allowed to override any
- entries in |expectations|. This is used by callers
- that need to manage two sets of expectations (e.g., upstream
- and downstream expectations).
- """
+ def __init__(self, port, expectations, full_test_list,
+ test_config, is_lint_mode, overrides=None):
+ # See argument documentation in TestExpectation(), above.
self._port = port
self._fs = port._filesystem
self._expectations = expectations
self._full_test_list = full_test_list
- self._test_platform_name = test_platform_name
- self._is_debug_mode = is_debug_mode
+ self._test_config = test_config
self._is_lint_mode = is_lint_mode
self._overrides = overrides
self._errors = []
@@ -332,7 +317,9 @@ class TestExpectationsFile:
# Maps relative test paths as listed in the expectations file to a
# list of maps containing modifiers and expectations for each time
- # the test is listed in the expectations file.
+ # the test is listed in the expectations file. We use this to
+ # keep a representation of the entire list of expectations, even
+ # invalid ones.
self._all_expectations = {}
# Maps a test to its list of expectations.
@@ -345,7 +332,8 @@ class TestExpectationsFile:
# the options minus any bug or platform strings
self._test_to_modifiers = {}
- # Maps a test to the base path that it was listed with in the list.
+ # Maps a test to the base path that it was listed with in the list and
+ # the number of matches that base path had.
self._test_list_paths = {}
self._modifier_to_tests = self._dict_of_sets(self.MODIFIERS)
@@ -372,13 +360,7 @@ class TestExpectationsFile:
def _handle_any_read_errors(self):
if len(self._errors) or len(self._non_fatal_errors):
- if self._is_debug_mode:
- build_type = 'DEBUG'
- else:
- build_type = 'RELEASE'
- _log.error('')
- _log.error("FAILURES FOR PLATFORM: %s, BUILD_TYPE: %s" %
- (self._test_platform_name.upper(), build_type))
+ _log.error("FAILURES FOR %s" % str(self._test_config))
for error in self._errors:
_log.error(error)
@@ -394,11 +376,12 @@ class TestExpectationsFile:
expectations = set([PASS])
options = []
modifiers = []
+ num_matches = 0
if self._full_test_list:
for test in self._full_test_list:
if not test in self._test_list_paths:
- self._add_test(test, modifiers, expectations, options,
- overrides_allowed=False)
+ self._add_test(test, modifiers, num_matches, expectations,
+ options, overrides_allowed=False)
def _dict_of_sets(self, strings_to_constants):
"""Takes a dict of strings->constants and returns a dict mapping
@@ -505,7 +488,8 @@ class TestExpectationsFile:
_log.info(' new: %s', new_line)
elif action == ADD_PLATFORMS_EXCEPT_THIS:
parts = line.split(':')
- new_options = parts[0]
+ _log.info('Test updated: ')
+ _log.info(' old: %s', line)
for p in self._port.test_platform_names():
p = p.upper()
# This is a temp solution for rebaselining tool.
@@ -515,13 +499,11 @@ class TestExpectationsFile:
# TODO(victorw): Remove WIN-VISTA and WIN-7 once we have
# reliable Win 7 and Win Vista buildbots setup.
if not p in (platform.upper(), 'WIN-VISTA', 'WIN-7'):
- new_options += p + ' '
- new_line = ('%s:%s' % (new_options, parts[1]))
- f_new.append(new_line)
+ new_options = parts[0] + p + ' '
+ new_line = ('%s:%s' % (new_options, parts[1]))
+ f_new.append(new_line)
+ _log.info(' new: %s', new_line)
tests_updated += 1
- _log.info('Test updated: ')
- _log.info(' old: %s', line)
- _log.info(' new: %s', new_line)
_log.info('Total tests removed: %d', tests_removed)
_log.info('Total tests updated: %d', tests_updated)
@@ -537,12 +519,15 @@ class TestExpectationsFile:
options = []
if line.find(":") is -1:
- test_and_expectation = line.split("=")
- else:
- parts = line.split(":")
- options = self._get_options_list(parts[0])
- test_and_expectation = parts[1].split('=')
+ self._add_error(lineno, "Missing a ':'", line)
+ return (None, None, None)
+ parts = line.split(':')
+
+ # FIXME: verify that there is exactly one colon in the line.
+
+ options = self._get_options_list(parts[0])
+ test_and_expectation = parts[1].split('=')
test = test_and_expectation[0].strip()
if (len(test_and_expectation) is not 2):
self._add_error(lineno, "Missing expectations.",
@@ -588,69 +573,6 @@ class TestExpectationsFile:
return REMOVE_TEST
- def _has_valid_modifiers_for_current_platform(self, options, lineno,
- test_and_expectations, modifiers):
- """Returns true if the current platform is in the options list or if
- no platforms are listed and if there are no fatal errors in the
- options list.
-
- Args:
- options: List of lowercase options.
- lineno: The line in the file where the test is listed.
- test_and_expectations: The path and expectations for the test.
- modifiers: The set to populate with modifiers.
- """
- has_any_platform = False
- has_bug_id = False
- for option in options:
- if option in self.MODIFIERS:
- modifiers.add(option)
- elif option in self._port.test_platform_names():
- has_any_platform = True
- elif re.match(r'bug\d', option) != None:
- self._add_error(lineno, 'Bug must be either BUGCR, BUGWK, or BUGV8_ for test: %s' %
- option, test_and_expectations)
- elif option.startswith('bug'):
- has_bug_id = True
- elif option not in self.BUILD_TYPES:
- self._add_error(lineno, 'Invalid modifier for test: %s' %
- option, test_and_expectations)
-
- if has_any_platform and not self._match_platform(options):
- return False
-
- if not has_bug_id and 'wontfix' not in options:
- # TODO(ojan): Turn this into an AddError call once all the
- # tests have BUG identifiers.
- self._log_non_fatal_error(lineno, 'Test lacks BUG modifier.',
- test_and_expectations)
-
- if 'release' in options or 'debug' in options:
- if self._is_debug_mode and 'debug' not in options:
- return False
- if not self._is_debug_mode and 'release' not in options:
- return False
-
- if self._is_lint_mode and 'rebaseline' in options:
- self._add_error(lineno,
- 'REBASELINE should only be used for running rebaseline.py. '
- 'Cannot be checked in.', test_and_expectations)
-
- return True
-
- def _match_platform(self, options):
- """Match the list of options against our specified platform. If any
- of the options prefix-match self._platform, return True. This handles
- the case where a test is marked WIN and the platform is WIN-VISTA.
-
- Args:
- options: list of options
- """
- for opt in options:
- if self._test_platform_name.startswith(opt):
- return True
- return False
-
def _add_to_all_expectations(self, test, options, expectations):
# Make all paths unix-style so the dashboard doesn't need to.
test = test.replace('\\', '/')
@@ -663,54 +585,43 @@ class TestExpectationsFile:
"""For each test in an expectations iterable, generate the
expectations for it."""
lineno = 0
+ matcher = ModifierMatcher(self._test_config)
for line in expectations:
lineno += 1
+ self._process_line(line, lineno, matcher, overrides_allowed)
- test_list_path, options, expectations = \
- self.parse_expectations_line(line, lineno)
- if not expectations:
- continue
+ def _process_line(self, line, lineno, matcher, overrides_allowed):
+ test_list_path, options, expectations = \
+ self.parse_expectations_line(line, lineno)
+ if not expectations:
+ return
- self._add_to_all_expectations(test_list_path,
- " ".join(options).upper(),
- " ".join(expectations).upper())
+ self._add_to_all_expectations(test_list_path,
+ " ".join(options).upper(),
+ " ".join(expectations).upper())
- modifiers = set()
- if options and not self._has_valid_modifiers_for_current_platform(
- options, lineno, test_list_path, modifiers):
- continue
+ num_matches = self._check_options(matcher, options, lineno,
+ test_list_path)
+ if num_matches == ModifierMatcher.NO_MATCH:
+ return
- expectations = self._parse_expectations(expectations, lineno,
- test_list_path)
+ expectations = self._parse_expectations(expectations, lineno,
+ test_list_path)
- if 'slow' in options and TIMEOUT in expectations:
- self._add_error(lineno,
- 'A test can not be both slow and timeout. If it times out '
- 'indefinitely, then it should be just timeout.',
- test_list_path)
+ self._check_options_against_expectations(options, expectations,
+ lineno, test_list_path)
- full_path = self._fs.join(self._port.layout_tests_dir(),
- test_list_path)
- full_path = self._fs.normpath(full_path)
- # WebKit's way of skipping tests is to add a -disabled suffix.
- # So we should consider the path existing if the path or the
- # -disabled version exists.
- if (not self._port.path_exists(full_path)
- and not self._port.path_exists(full_path + '-disabled')):
- # Log a non fatal error here since you hit this case any
- # time you update test_expectations.txt without syncing
- # the LayoutTests directory
- self._log_non_fatal_error(lineno, 'Path does not exist.',
- test_list_path)
- continue
+ if self._check_path_does_not_exist(lineno, test_list_path):
+ return
- if not self._full_test_list:
- tests = [test_list_path]
- else:
- tests = self._expand_tests(test_list_path)
+ if not self._full_test_list:
+ tests = [test_list_path]
+ else:
+ tests = self._expand_tests(test_list_path)
- self._add_tests(tests, expectations, test_list_path, lineno,
- modifiers, options, overrides_allowed)
+ modifiers = [o for o in options if o in self.MODIFIERS]
+ self._add_tests(tests, expectations, test_list_path, lineno,
+ modifiers, num_matches, options, overrides_allowed)
def _get_options_list(self, listString):
return [part.strip().lower() for part in listString.strip().split(' ')]
@@ -726,6 +637,65 @@ class TestExpectationsFile:
result.add(expectation)
return result
+ def _check_options(self, matcher, options, lineno, test_list_path):
+ match_result = self._check_syntax(matcher, options, lineno,
+ test_list_path)
+ self._check_semantics(options, lineno, test_list_path)
+ return match_result.num_matches
+
+ def _check_syntax(self, matcher, options, lineno, test_list_path):
+ match_result = matcher.match(options)
+ for error in match_result.errors:
+ self._add_error(lineno, error, test_list_path)
+ for warning in match_result.warnings:
+ self._log_non_fatal_error(lineno, warning, test_list_path)
+ return match_result
+
+ def _check_semantics(self, options, lineno, test_list_path):
+ has_wontfix = 'wontfix' in options
+ has_bug = False
+ for opt in options:
+ if opt.startswith('bug'):
+ has_bug = True
+ if re.match('bug\d+', opt):
+ self._add_error(lineno,
+ 'BUG\d+ is not allowed, must be one of '
+ 'BUGCR\d+, BUGWK\d+, BUGV8_\d+, '
+ 'or a non-numeric bug identifier.', test_list_path)
+
+ if not has_bug and not has_wontfix:
+ self._log_non_fatal_error(lineno, 'Test lacks BUG modifier.',
+ test_list_path)
+
+ if self._is_lint_mode and 'rebaseline' in options:
+ self._add_error(lineno,
+ 'REBASELINE should only be used for running rebaseline.py. '
+ 'Cannot be checked in.', test_list_path)
+
+ def _check_options_against_expectations(self, options, expectations,
+ lineno, test_list_path):
+ if 'slow' in options and TIMEOUT in expectations:
+ self._add_error(lineno,
+ 'A test can not be both SLOW and TIMEOUT. If it times out '
+ 'indefinitely, then it should be just TIMEOUT.', test_list_path)
+
+ def _check_path_does_not_exist(self, lineno, test_list_path):
+ full_path = self._fs.join(self._port.layout_tests_dir(),
+ test_list_path)
+ full_path = self._fs.normpath(full_path)
+ # WebKit's way of skipping tests is to add a -disabled suffix.
+ # So we should consider the path existing if the path or the
+ # -disabled version exists.
+ if (not self._port.path_exists(full_path)
+ and not self._port.path_exists(full_path + '-disabled')):
+ # Log a non fatal error here since you hit this case any
+ # time you update test_expectations.txt without syncing
+ # the LayoutTests directory
+ self._log_non_fatal_error(lineno, 'Path does not exist.',
+ test_list_path)
+ return True
+ return False
+
def _expand_tests(self, test_list_path):
"""Convert the test specification to an absolute, normalized
path and make sure directories end with the OS path separator."""
@@ -751,27 +721,30 @@ class TestExpectationsFile:
return result
def _add_tests(self, tests, expectations, test_list_path, lineno,
- modifiers, options, overrides_allowed):
+ modifiers, num_matches, options, overrides_allowed):
for test in tests:
- if self._already_seen_test(test, test_list_path, lineno,
- overrides_allowed):
+ if self._already_seen_better_match(test, test_list_path,
+ num_matches, lineno, overrides_allowed):
continue
self._clear_expectations_for_test(test, test_list_path)
- self._add_test(test, modifiers, expectations, options,
+ self._test_list_paths[test] = (self._fs.normpath(test_list_path),
+ num_matches, lineno)
+ self._add_test(test, modifiers, num_matches, expectations, options,
overrides_allowed)
- def _add_test(self, test, modifiers, expectations, options,
+ def _add_test(self, test, modifiers, num_matches, expectations, options,
overrides_allowed):
"""Sets the expected state for a given test.
This routine assumes the test has not been added before. If it has,
- use _ClearExpectationsForTest() to reset the state prior to
+ use _clear_expectations_for_test() to reset the state prior to
calling this.
Args:
test: test to add
modifiers: sequence of modifier keywords ('wontfix', 'slow', etc.)
+ num_matches: number of modifiers that matched the configuration
expectations: sequence of expectations (PASS, IMAGE, etc.)
options: sequence of keywords and bug identifiers.
overrides_allowed: whether we're parsing the regular expectations
@@ -828,32 +801,70 @@ class TestExpectationsFile:
if test in set_of_tests:
set_of_tests.remove(test)
- def _already_seen_test(self, test, test_list_path, lineno,
- allow_overrides):
- """Returns true if we've already seen a more precise path for this test
- than the test_list_path.
+ def _already_seen_better_match(self, test, test_list_path, num_matches,
+ lineno, overrides_allowed):
+ """Returns whether we've seen a better match already in the file.
+
+ Returns True if we've already seen a test_list_path that matches more of the test
+ than this path does
"""
+ # FIXME: See comment below about matching test configs and num_matches.
+
if not test in self._test_list_paths:
+ # We've never seen this test before.
return False
- prev_base_path = self._test_list_paths[test]
- if (prev_base_path == self._fs.normpath(test_list_path)):
- if (not allow_overrides or test in self._overridding_tests):
- if allow_overrides:
- expectation_source = "override"
- else:
- expectation_source = "expectation"
- self._add_error(lineno, 'Duplicate %s.' % expectation_source,
- test)
- return True
- else:
- # We have seen this path, but that's okay because its
- # in the overrides and the earlier path was in the
- # expectations.
- return False
+ prev_base_path, prev_num_matches, prev_lineno = self._test_list_paths[test]
+ base_path = self._fs.normpath(test_list_path)
+
+ if len(prev_base_path) > len(base_path):
+ # The previous path matched more of the test.
+ return True
+
+ if len(prev_base_path) < len(base_path):
+ # This path matches more of the test.
+ return False
+
+ if overrides_allowed and test not in self._overridding_tests:
+ # We have seen this path, but that's okay because it is
+ # in the overrides and the earlier path was in the
+ # expectations (not the overrides).
+ return False
+
+ # At this point we know we have seen a previous exact match on this
+ # base path, so we need to check the two sets of modifiers.
- # Check if we've already seen a more precise path.
- return prev_base_path.startswith(self._fs.normpath(test_list_path))
+ if overrides_allowed:
+ expectation_source = "override"
+ else:
+ expectation_source = "expectation"
+
+ # FIXME: This code was originally designed to allow lines that matched
+ # more modifiers to override lines that matched fewer modifiers.
+ # However, we currently view these as errors. If we decide to make
+ # this policy permanent, we can probably simplify this code
+ # and the ModifierMatcher code a fair amount.
+ #
+ # To use the "more modifiers wins" policy, change the "_add_error" lines for overrides
+ # to _log_non_fatal_error() and change the commented-out "return False".
+
+ if prev_num_matches == num_matches:
+ self._add_error(lineno,
+ 'Duplicate or ambiguous %s.' % expectation_source,
+ test)
+ return True
+
+ if prev_num_matches < num_matches:
+ self._add_error(lineno,
+ 'More specific entry on line %d overrides line %d' %
+ (lineno, prev_lineno), test_list_path)
+ # FIXME: return False if we want more specific to win.
+ return True
+
+ self._add_error(lineno,
+ 'More specific entry on line %d overrides line %d' %
+ (prev_lineno, lineno), test_list_path)
+ return True
def _add_error(self, lineno, msg, path):
"""Reports an error that will prevent running the tests. Does not
@@ -865,3 +876,188 @@ class TestExpectationsFile:
"""Reports an error that will not prevent running the tests. These are
still errors, but not bad enough to warrant breaking test running."""
self._non_fatal_errors.append('Line:%s %s %s' % (lineno, msg, path))
+
+
+class ModifierMatchResult(object):
+ def __init__(self, options):
+ self.num_matches = ModifierMatcher.NO_MATCH
+ self.options = options
+ self.errors = []
+ self.warnings = []
+ self.modifiers = []
+ self._matched_regexes = set()
+ self._matched_macros = set()
+
+
+class ModifierMatcher(object):
+
+ """
+ This class manages the interpretation of the "modifiers" for a given
+ line in the expectations file. Modifiers are the tokens that appear to the
+ left of the colon on a line. For example, "BUG1234", "DEBUG", and "WIN" are
+ all modifiers. This class gets what the valid modifiers are, and which
+ modifiers are allowed to exist together on a line, from the
+ TestConfiguration object that is passed in to the call.
+
+ This class detects *intra*-line errors like unknown modifiers, but
+ does not detect *inter*-line modifiers like duplicate expectations.
+
+ More importantly, this class is also used to determine if a given line
+ matches the port in question. Matches are ranked according to the number
+ of modifiers that match on a line. A line with no modifiers matches
+ everything and has a score of zero. A line with one modifier matches only
+ ports that have that modifier and gets a score of 1, and so one. Ports
+ that don't match at all get a score of -1.
+
+ Given two lines in a file that apply to the same test, if both expectations
+ match the current config, then the expectation is considered ambiguous,
+ even if one expectation matches more of the config than the other. For
+ example, in:
+
+ BUG1 RELEASE : foo.html = FAIL
+ BUG1 WIN RELEASE : foo.html = PASS
+ BUG2 WIN : bar.html = FAIL
+ BUG2 DEBUG : bar.html = PASS
+
+ lines 1 and 2 would produce an error on a Win XP Release bot (the scores
+ would be 1 and 2, respectively), and lines three and four would produce
+ a duplicate expectation on a Win Debug bot since both the 'win' and the
+ 'debug' expectations would apply (both had scores of 1).
+
+ In addition to the definitions of all of the modifiers, the class
+ supports "macros" that are expanded prior to interpretation, and "ignore
+ regexes" that can be used to skip over modifiers like the BUG* modifiers.
+ """
+ MACROS = {
+ 'mac-snowleopard': ['mac', 'snowleopard'],
+ 'mac-leopard': ['mac', 'leopard'],
+ 'win-xp': ['win', 'xp'],
+ 'win-vista': ['win', 'vista'],
+ 'win-7': ['win', 'win7'],
+ }
+
+ # We don't include the "none" modifier because it isn't actually legal.
+ REGEXES_TO_IGNORE = (['bug\w+'] +
+ TestExpectationsFile.MODIFIERS.keys()[:-1])
+ DUPLICATE_REGEXES_ALLOWED = ['bug\w+']
+
+ # Magic value returned when the options don't match.
+ NO_MATCH = -1
+
+ # FIXME: The code currently doesn't detect combinations of modifiers
+ # that are syntactically valid but semantically invalid, like
+ # 'MAC XP'. See ModifierMatchTest.test_invalid_combinations() in the
+ # _unittest.py file.
+
+ def __init__(self, test_config):
+ """Initialize a ModifierMatcher argument with the TestConfiguration it
+ should be matched against."""
+ self.test_config = test_config
+ self.allowed_configurations = test_config.all_test_configurations()
+ self.macros = self.MACROS
+
+ self.regexes_to_ignore = {}
+ for regex_str in self.REGEXES_TO_IGNORE:
+ self.regexes_to_ignore[regex_str] = re.compile(regex_str)
+
+ # Keep a set of all of the legal modifiers for quick checking.
+ self._all_modifiers = set()
+
+ # Keep a dict mapping values back to their categories.
+ self._categories_for_modifiers = {}
+ for config in self.allowed_configurations:
+ for category, modifier in config.items():
+ self._categories_for_modifiers[modifier] = category
+ self._all_modifiers.add(modifier)
+
+ def match(self, options):
+ """Checks a list of options against the config set in the constructor.
+ Options may be either actual modifier strings, "macro" strings
+ that get expanded to a list of modifiers, or strings that are allowed
+ to be ignored. All of the options must be passed in in lower case.
+
+ Returns the number of matching categories, or NO_MATCH (-1) if it
+ doesn't match or there were errors found. Matches are prioritized
+ by the number of matching categories, because the more specific
+ the options list, the more categories will match.
+
+ The results of the most recent match are available in the 'options',
+ 'modifiers', 'num_matches', 'errors', and 'warnings' properties.
+ """
+ result = ModifierMatchResult(options)
+ self._parse(result)
+ if result.errors:
+ return result
+ self._count_matches(result)
+ return result
+
+ def _parse(self, result):
+ # FIXME: Should we warn about lines having every value in a category?
+ for option in result.options:
+ self._parse_one(option, result)
+
+ def _parse_one(self, option, result):
+ if option in self._all_modifiers:
+ self._add_modifier(option, result)
+ elif option in self.macros:
+ self._expand_macro(option, result)
+ elif not self._matches_any_regex(option, result):
+ result.errors.append("Unrecognized option '%s'" % option)
+
+ def _add_modifier(self, option, result):
+ if option in result.modifiers:
+ result.errors.append("More than one '%s'" % option)
+ else:
+ result.modifiers.append(option)
+
+ def _expand_macro(self, macro, result):
+ if macro in result._matched_macros:
+ result.errors.append("More than one '%s'" % macro)
+ return
+
+ mods = []
+ for modifier in self.macros[macro]:
+ if modifier in result.options:
+ result.errors.append("Can't specify both modifier '%s' and "
+ "macro '%s'" % (modifier, macro))
+ else:
+ mods.append(modifier)
+ result._matched_macros.add(macro)
+ result.modifiers.extend(mods)
+
+ def _matches_any_regex(self, option, result):
+ for regex_str, pattern in self.regexes_to_ignore.iteritems():
+ if pattern.match(option):
+ self._handle_regex_match(regex_str, result)
+ return True
+ return False
+
+ def _handle_regex_match(self, regex_str, result):
+ if (regex_str in result._matched_regexes and
+ regex_str not in self.DUPLICATE_REGEXES_ALLOWED):
+ result.errors.append("More than one option matching '%s'" %
+ regex_str)
+ else:
+ result._matched_regexes.add(regex_str)
+
+ def _count_matches(self, result):
+ """Returns the number of modifiers that match the test config."""
+ categorized_modifiers = self._group_by_category(result.modifiers)
+ result.num_matches = 0
+ for category, modifier in self.test_config.items():
+ if category in categorized_modifiers:
+ if modifier in categorized_modifiers[category]:
+ result.num_matches += 1
+ else:
+ result.num_matches = self.NO_MATCH
+ return
+
+ def _group_by_category(self, modifiers):
+ # Returns a dict of category name -> list of modifiers.
+ modifiers_by_category = {}
+ for m in modifiers:
+ modifiers_by_category.setdefault(self._category(m), []).append(m)
+ return modifiers_by_category
+
+ def _category(self, modifier):
+ return self._categories_for_modifiers[modifier]
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py
index 8f9e5dd..05d805d 100644
--- a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py
@@ -32,6 +32,7 @@
import unittest
from webkitpy.layout_tests import port
+from webkitpy.layout_tests.port import base
from webkitpy.layout_tests.layout_package.test_expectations import *
class FunctionsTest(unittest.TestCase):
@@ -78,8 +79,11 @@ class FunctionsTest(unittest.TestCase):
class Base(unittest.TestCase):
+ # Note that all of these tests are written assuming the configuration
+ # being tested is Windows XP, Release build.
+
def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
- self._port = port.get('test', None)
+ self._port = port.get('test-win-xp', None)
self._fs = self._port._filesystem
self._exp = None
unittest.TestCase.__init__(self, testFunc)
@@ -101,16 +105,15 @@ BUG_TEST : failures/expected/text.html = TEXT
BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
-BUG_TEST WONTFIX WIN : failures/expected/image.html = IMAGE
+BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE
"""
- def parse_exp(self, expectations, overrides=None, is_lint_mode=False,
- is_debug_mode=False):
+ def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
+ test_config = self._port.test_configuration()
self._exp = TestExpectations(self._port,
tests=self.get_basic_tests(),
expectations=expectations,
- test_platform_name=self._port.test_platform_name(),
- is_debug_mode=is_debug_mode,
+ test_config=test_config,
is_lint_mode=is_lint_mode,
overrides=overrides)
@@ -119,7 +122,7 @@ BUG_TEST WONTFIX WIN : failures/expected/image.html = IMAGE
set([result]))
-class TestExpectationsTest(Base):
+class BasicTests(Base):
def test_basic(self):
self.parse_exp(self.get_basic_expectations())
self.assert_exp('failures/expected/text.html', TEXT)
@@ -127,23 +130,14 @@ class TestExpectationsTest(Base):
self.assert_exp('passes/text.html', PASS)
self.assert_exp('failures/expected/image.html', PASS)
+
+class MiscTests(Base):
def test_multiple_results(self):
self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
self.assertEqual(self._exp.get_expectations(
self.get_test('failures/expected/text.html')),
set([TEXT, CRASH]))
- def test_precedence(self):
- # This tests handling precedence of specific lines over directories
- # and tests expectations covering entire directories.
- exp_str = """
-BUGX : failures/expected/text.html = TEXT
-BUGX WONTFIX : failures/expected = IMAGE
-"""
- self.parse_exp(exp_str)
- self.assert_exp('failures/expected/text.html', TEXT)
- self.assert_exp('failures/expected/crash.html', IMAGE)
-
def test_category_expectations(self):
# This test checks unknown tests are not present in the
# expectations and that known test part of a test category is
@@ -158,20 +152,6 @@ BUGX WONTFIX : failures/expected = IMAGE
unknown_test)
self.assert_exp('failures/expected/crash.html', IMAGE)
- def test_release_mode(self):
- self.parse_exp('BUGX DEBUG : failures/expected/text.html = TEXT',
- is_debug_mode=True)
- self.assert_exp('failures/expected/text.html', TEXT)
- self.parse_exp('BUGX RELEASE : failures/expected/text.html = TEXT',
- is_debug_mode=True)
- self.assert_exp('failures/expected/text.html', PASS)
- self.parse_exp('BUGX DEBUG : failures/expected/text.html = TEXT',
- is_debug_mode=False)
- self.assert_exp('failures/expected/text.html', PASS)
- self.parse_exp('BUGX RELEASE : failures/expected/text.html = TEXT',
- is_debug_mode=False)
- self.assert_exp('failures/expected/text.html', TEXT)
-
def test_get_options(self):
self.parse_exp(self.get_basic_expectations())
self.assertEqual(self._exp.get_options(
@@ -216,7 +196,7 @@ SKIP : failures/expected/image.html""")
self.assertFalse(True, "ParseError wasn't raised")
except ParseError, e:
self.assertTrue(e.fatal)
- exp_errors = [u'Line:1 Invalid modifier for test: foo failures/expected/text.html',
+ exp_errors = [u"Line:1 Unrecognized option 'foo' failures/expected/text.html",
u"Line:2 Missing expectations. [' failures/expected/image.html']"]
self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
self.assertEqual(e.errors, exp_errors)
@@ -232,77 +212,167 @@ SKIP : failures/expected/image.html""")
self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
self.assertEqual(e.errors, exp_errors)
- def test_syntax_missing_expectation(self):
+ def test_overrides(self):
+ self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
+ "BUG_OVERRIDE : failures/expected/text.html = IMAGE")
+ self.assert_exp('failures/expected/text.html', IMAGE)
+
+ def test_overrides__duplicate(self):
+ self.assertRaises(ParseError, self.parse_exp,
+ "BUG_EXP: failures/expected/text.html = TEXT",
+ """
+BUG_OVERRIDE : failures/expected/text.html = IMAGE
+BUG_OVERRIDE : failures/expected/text.html = CRASH
+""")
+
+ def test_pixel_tests_flag(self):
+ def match(test, result, pixel_tests_enabled):
+ return self._exp.matches_an_expected_result(
+ self.get_test(test), result, pixel_tests_enabled)
+
+ self.parse_exp(self.get_basic_expectations())
+ self.assertTrue(match('failures/expected/text.html', TEXT, True))
+ self.assertTrue(match('failures/expected/text.html', TEXT, False))
+ self.assertFalse(match('failures/expected/text.html', CRASH, True))
+ self.assertFalse(match('failures/expected/text.html', CRASH, False))
+ self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
+ True))
+ self.assertTrue(match('failures/expected/image_checksum.html', PASS,
+ False))
+ self.assertTrue(match('failures/expected/crash.html', SKIP, False))
+ self.assertTrue(match('passes/text.html', PASS, False))
+
+ def test_more_specific_override_resets_skip(self):
+ self.parse_exp("BUGX SKIP : failures/expected = TEXT\n"
+ "BUGX : failures/expected/text.html = IMAGE\n")
+ self.assert_exp('failures/expected/text.html', IMAGE)
+ self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
+ 'failures/expected/text.html') in
+ self._exp.get_tests_with_result_type(SKIP))
+
+class ExpectationSyntaxTests(Base):
+ def test_missing_expectation(self):
# This is missing the expectation.
self.assertRaises(ParseError, self.parse_exp,
- 'BUG_TEST: failures/expected/text.html',
- is_debug_mode=True)
+ 'BUG_TEST: failures/expected/text.html')
- def test_syntax_invalid_option(self):
+ def test_missing_colon(self):
+ # This is missing the modifiers and the ':'
self.assertRaises(ParseError, self.parse_exp,
- 'BUG_TEST FOO: failures/expected/text.html = PASS')
+ 'failures/expected/text.html = TEXT')
- def test_syntax_invalid_expectation(self):
- # This is missing the expectation.
+ def disabled_test_too_many_colons(self):
+ # FIXME: Enable this test and fix the underlying bug.
+ self.assertRaises(ParseError, self.parse_exp,
+ 'BUG_TEST: failures/expected/text.html = PASS :')
+
+ def test_too_many_equals_signs(self):
self.assertRaises(ParseError, self.parse_exp,
- 'BUG_TEST: failures/expected/text.html = FOO')
+ 'BUG_TEST: failures/expected/text.html = TEXT = IMAGE')
+
+ def test_unrecognized_expectation(self):
+ self.assertRaises(ParseError, self.parse_exp,
+ 'BUG_TEST: failures/expected/text.html = UNKNOWN')
+
+ def test_macro(self):
+ exp_str = """
+BUG_TEST WIN-XP : failures/expected/text.html = TEXT
+"""
+ self.parse_exp(exp_str)
+ self.assert_exp('failures/expected/text.html', TEXT)
+
+
+class SemanticTests(Base):
+ def test_bug_format(self):
+ self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT')
- def test_syntax_missing_bugid(self):
+ def test_missing_bugid(self):
# This should log a non-fatal error.
self.parse_exp('SLOW : failures/expected/text.html = TEXT')
self.assertEqual(
len(self._exp._expected_failures.get_non_fatal_errors()), 1)
- def test_semantic_slow_and_timeout(self):
+ def test_slow_and_timeout(self):
# A test cannot be SLOW and expected to TIMEOUT.
self.assertRaises(ParseError, self.parse_exp,
'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT')
- def test_semantic_rebaseline(self):
+ def test_rebaseline(self):
# Can't lint a file w/ 'REBASELINE' in it.
self.assertRaises(ParseError, self.parse_exp,
'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
is_lint_mode=True)
- def test_semantic_duplicates(self):
+ def test_duplicates(self):
self.assertRaises(ParseError, self.parse_exp, """
-BUG_TEST : failures/expected/text.html = TEXT
-BUG_TEST : failures/expected/text.html = IMAGE""")
+BUG_EXP : failures/expected/text.html = TEXT
+BUG_EXP : failures/expected/text.html = IMAGE""")
self.assertRaises(ParseError, self.parse_exp,
- self.get_basic_expectations(), """
-BUG_TEST : failures/expected/text.html = TEXT
-BUG_TEST : failures/expected/text.html = IMAGE""")
+ self.get_basic_expectations(), overrides="""
+BUG_OVERRIDE : failures/expected/text.html = TEXT
+BUG_OVERRIDE : failures/expected/text.html = IMAGE""", )
- def test_semantic_missing_file(self):
+ def test_missing_file(self):
# This should log a non-fatal error.
self.parse_exp('BUG_TEST : missing_file.html = TEXT')
self.assertEqual(
len(self._exp._expected_failures.get_non_fatal_errors()), 1)
- def test_overrides(self):
- self.parse_exp(self.get_basic_expectations(), """
-BUG_OVERRIDE : failures/expected/text.html = IMAGE""")
- self.assert_exp('failures/expected/text.html', IMAGE)
+class PrecedenceTests(Base):
+ def test_file_over_directory(self):
+ # This tests handling precedence of specific lines over directories
+ # and tests expectations covering entire directories.
+ exp_str = """
+BUGX : failures/expected/text.html = TEXT
+BUGX WONTFIX : failures/expected = IMAGE
+"""
+ self.parse_exp(exp_str)
+ self.assert_exp('failures/expected/text.html', TEXT)
+ self.assert_exp('failures/expected/crash.html', IMAGE)
- def test_matches_an_expected_result(self):
+ exp_str = """
+BUGX WONTFIX : failures/expected = IMAGE
+BUGX : failures/expected/text.html = TEXT
+"""
+ self.parse_exp(exp_str)
+ self.assert_exp('failures/expected/text.html', TEXT)
+ self.assert_exp('failures/expected/crash.html', IMAGE)
- def match(test, result, pixel_tests_enabled):
- return self._exp.matches_an_expected_result(
- self.get_test(test), result, pixel_tests_enabled)
+ def test_ambiguous(self):
+ self.assertRaises(ParseError, self.parse_exp, """
+BUG_TEST RELEASE : passes/text.html = PASS
+BUG_TEST WIN : passes/text.html = FAIL
+""")
- self.parse_exp(self.get_basic_expectations())
- self.assertTrue(match('failures/expected/text.html', TEXT, True))
- self.assertTrue(match('failures/expected/text.html', TEXT, False))
- self.assertFalse(match('failures/expected/text.html', CRASH, True))
- self.assertFalse(match('failures/expected/text.html', CRASH, False))
- self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
- True))
- self.assertTrue(match('failures/expected/image_checksum.html', PASS,
- False))
- self.assertTrue(match('failures/expected/crash.html', SKIP, False))
- self.assertTrue(match('passes/text.html', PASS, False))
+ def test_more_modifiers(self):
+ exp_str = """
+BUG_TEST RELEASE : passes/text.html = PASS
+BUG_TEST WIN RELEASE : passes/text.html = TEXT
+"""
+ self.assertRaises(ParseError, self.parse_exp, exp_str)
+
+ def test_order_in_file(self):
+ exp_str = """
+BUG_TEST WIN RELEASE : passes/text.html = TEXT
+BUG_TEST RELEASE : passes/text.html = PASS
+"""
+ self.assertRaises(ParseError, self.parse_exp, exp_str)
+
+ def test_version_overrides(self):
+ exp_str = """
+BUG_TEST WIN : passes/text.html = PASS
+BUG_TEST WIN XP : passes/text.html = TEXT
+"""
+ self.assertRaises(ParseError, self.parse_exp, exp_str)
+
+ def test_macro_overrides(self):
+ exp_str = """
+BUG_TEST WIN : passes/text.html = PASS
+BUG_TEST WIN-XP : passes/text.html = TEXT
+"""
+ self.assertRaises(ParseError, self.parse_exp, exp_str)
class RebaseliningTest(Base):
@@ -327,7 +397,8 @@ BUG_TEST REBASELINE : failures/expected/text.html = TEXT
def test_remove_expand(self):
self.assertRemove('mac',
'BUGX REBASELINE : failures/expected/text.html = TEXT\n',
- 'BUGX REBASELINE WIN : failures/expected/text.html = TEXT\n')
+ 'BUGX REBASELINE WIN : failures/expected/text.html = TEXT\n'
+ 'BUGX REBASELINE WIN-XP : failures/expected/text.html = TEXT\n')
def test_remove_mac_win(self):
self.assertRemove('mac',
@@ -345,5 +416,85 @@ BUG_TEST REBASELINE : failures/expected/text.html = TEXT
'\n\n')
+class ModifierTests(unittest.TestCase):
+ def setUp(self):
+ port_obj = port.get('test-win-xp', None)
+ self.config = port_obj.test_configuration()
+ self.matcher = ModifierMatcher(self.config)
+
+ def match(self, modifiers, expected_num_matches=-1, values=None, num_errors=0):
+ matcher = self.matcher
+ if values:
+ matcher = ModifierMatcher(self.FakeTestConfiguration(values))
+ match_result = matcher.match(modifiers)
+ self.assertEqual(len(match_result.warnings), 0)
+ self.assertEqual(len(match_result.errors), num_errors)
+ self.assertEqual(match_result.num_matches, expected_num_matches,
+ 'match(%s, %s) returned -> %d, expected %d' %
+ (modifiers, str(self.config.values()),
+ match_result.num_matches, expected_num_matches))
+
+ def test_bad_match_modifier(self):
+ self.match(['foo'], num_errors=1)
+
+ def test_none(self):
+ self.match([], 0)
+
+ def test_one(self):
+ self.match(['xp'], 1)
+ self.match(['win'], 1)
+ self.match(['release'], 1)
+ self.match(['cpu'], 1)
+ self.match(['x86'], 1)
+ self.match(['leopard'], -1)
+ self.match(['gpu'], -1)
+ self.match(['debug'], -1)
+
+ def test_two(self):
+ self.match(['xp', 'release'], 2)
+ self.match(['win7', 'release'], -1)
+ self.match(['win7', 'xp'], 1)
+
+ def test_three(self):
+ self.match(['win7', 'xp', 'release'], 2)
+ self.match(['xp', 'debug', 'x86'], -1)
+ self.match(['xp', 'release', 'x86'], 3)
+ self.match(['xp', 'cpu', 'release'], 3)
+
+ def test_four(self):
+ self.match(['xp', 'release', 'cpu', 'x86'], 4)
+ self.match(['win7', 'xp', 'release', 'cpu'], 3)
+ self.match(['win7', 'xp', 'debug', 'cpu'], -1)
+
+ def test_case_insensitivity(self):
+ self.match(['Win'], num_errors=1)
+ self.match(['WIN'], num_errors=1)
+ self.match(['win'], 1)
+
+ def test_duplicates(self):
+ self.match(['release', 'release'], num_errors=1)
+ self.match(['win-xp', 'xp'], num_errors=1)
+ self.match(['win-xp', 'win-xp'], num_errors=1)
+ self.match(['xp', 'release', 'xp', 'release'], num_errors=2)
+ self.match(['rebaseline', 'rebaseline'], num_errors=1)
+
+ def test_unknown_option(self):
+ self.match(['vms'], num_errors=1)
+
+ def test_duplicate_bugs(self):
+ # BUG* regexes can appear multiple times.
+ self.match(['bugfoo', 'bugbar'], 0)
+
+ def test_invalid_combinations(self):
+ # FIXME: This should probably raise an error instead of NO_MATCH.
+ self.match(['mac', 'xp'], num_errors=0)
+
+ def test_regexes_are_ignored(self):
+ self.match(['bug123xy', 'rebaseline', 'wontfix', 'slow', 'skip'], 0)
+
+ def test_none_is_invalid(self):
+ self.match(['none'], num_errors=1)
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_input.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_input.py
index 4b027c0..0aed1dd 100644
--- a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_input.py
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_input.py
@@ -41,7 +41,3 @@ class TestInput:
# FIXME: filename should really be test_name as a relative path.
self.filename = filename
self.timeout = timeout
- # The image_hash is used to avoid doing an image dump if the
- # checksums match. The image_hash is set later, and only if it is needed
- # for the test.
- self.image_hash = None
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_output.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_output.py
deleted file mode 100644
index e809be6..0000000
--- a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_output.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# Copyright (C) 2010 Google Inc. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-# * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-# * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-# * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
-class TestOutput(object):
- """Groups information about a test output for easy passing of data.
-
- This is used not only for a actual test output, but also for grouping
- expected test output.
- """
-
- def __init__(self, text, image, image_hash,
- crash=None, test_time=None, timeout=None, error=None):
- """Initializes a TestOutput object.
-
- Args:
- text: a text output
- image: an image output
- image_hash: a string containing the checksum of the image
- crash: a boolean indicating whether the driver crashed on the test
- test_time: a time which the test has taken
- timeout: a boolean indicating whehter the test timed out
- error: any unexpected or additional (or error) text output
- """
- self.text = text
- self.image = image
- self.image_hash = image_hash
- self.crash = crash
- self.test_time = test_time
- self.timeout = timeout
- self.error = error
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner.py
index 6c07850..e3bd4ad 100644
--- a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner.py
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner.py
@@ -214,21 +214,13 @@ class TestRunner:
def lint(self):
lint_failed = False
-
- # Creating the expecations for each platform/configuration pair does
- # all the test list parsing and ensures it's correct syntax (e.g. no
- # dupes).
- for platform_name in self._port.test_platform_names():
- try:
- self.parse_expectations(platform_name, is_debug_mode=True)
- except test_expectations.ParseError:
- lint_failed = True
+ for test_configuration in self._port.all_test_configurations():
try:
- self.parse_expectations(platform_name, is_debug_mode=False)
+ self.lint_expectations(test_configuration)
except test_expectations.ParseError:
lint_failed = True
+ self._printer.write("")
- self._printer.write("")
if lint_failed:
_log.error("Lint failed.")
return -1
@@ -236,22 +228,28 @@ class TestRunner:
_log.info("Lint succeeded.")
return 0
- def parse_expectations(self, test_platform_name, is_debug_mode):
+ def lint_expectations(self, config):
+ port = self._port
+ test_expectations.TestExpectations(
+ port,
+ None,
+ port.test_expectations(),
+ config,
+ self._options.lint_test_files,
+ port.test_expectations_overrides())
+
+ def parse_expectations(self):
"""Parse the expectations from the test_list files and return a data
structure holding them. Throws an error if the test_list files have
invalid syntax."""
- if self._options.lint_test_files:
- test_files = None
- else:
- test_files = self._test_files
-
- expectations_str = self._port.test_expectations()
- overrides_str = self._port.test_expectations_overrides()
+ port = self._port
self._expectations = test_expectations.TestExpectations(
- self._port, test_files, expectations_str, test_platform_name,
- is_debug_mode, self._options.lint_test_files,
- overrides=overrides_str)
- return self._expectations
+ port,
+ self._test_files,
+ port.test_expectations(),
+ port.test_configuration(),
+ self._options.lint_test_files,
+ port.test_expectations_overrides())
# FIXME: This method is way too long and needs to be broken into pieces.
def prepare_lists_and_print_output(self):
@@ -358,9 +356,7 @@ class TestRunner:
self._test_files_list = files + skip_chunk_list
self._test_files = set(self._test_files_list)
- self._expectations = self.parse_expectations(
- self._port.test_platform_name(),
- self._options.configuration == 'Debug')
+ self.parse_expectations()
self._test_files = set(files)
self._test_files_list = files
@@ -691,6 +687,8 @@ class TestRunner:
self._expectations, result_summary, retry_summary)
self._printer.print_unexpected_results(unexpected_results)
+ # FIXME: remove record_results. It's just used for testing. There's no need
+ # for it to be a commandline argument.
if (self._options.record_results and not self._options.dry_run and
not interrupted):
# Write the same data to log files and upload generated JSON files
@@ -731,28 +729,31 @@ class TestRunner:
except Queue.Empty:
return
- expected = self._expectations.matches_an_expected_result(
- result.filename, result.type, self._options.pixel_tests)
- result_summary.add(result, expected)
- exp_str = self._expectations.get_expectations_string(
- result.filename)
- got_str = self._expectations.expectation_to_string(result.type)
- self._printer.print_test_result(result, expected, exp_str, got_str)
- self._printer.print_progress(result_summary, self._retrying,
- self._test_files_list)
-
- def interrupt_if_at_failure_limit(limit, count, message):
- if limit and count >= limit:
- raise TestRunInterruptedException(message % count)
-
- interrupt_if_at_failure_limit(
- self._options.exit_after_n_failures,
- result_summary.unexpected_failures,
- "Aborting run since %d failures were reached")
- interrupt_if_at_failure_limit(
- self._options.exit_after_n_crashes_or_timeouts,
- result_summary.unexpected_crashes_or_timeouts,
- "Aborting run since %d crashes or timeouts were reached")
+ self._update_summary_with_result(result_summary, result)
+
+ def _update_summary_with_result(self, result_summary, result):
+ expected = self._expectations.matches_an_expected_result(
+ result.filename, result.type, self._options.pixel_tests)
+ result_summary.add(result, expected)
+ exp_str = self._expectations.get_expectations_string(
+ result.filename)
+ got_str = self._expectations.expectation_to_string(result.type)
+ self._printer.print_test_result(result, expected, exp_str, got_str)
+ self._printer.print_progress(result_summary, self._retrying,
+ self._test_files_list)
+
+ def interrupt_if_at_failure_limit(limit, count, message):
+ if limit and count >= limit:
+ raise TestRunInterruptedException(message % count)
+
+ interrupt_if_at_failure_limit(
+ self._options.exit_after_n_failures,
+ result_summary.unexpected_failures,
+ "Aborting run since %d failures were reached")
+ interrupt_if_at_failure_limit(
+ self._options.exit_after_n_crashes_or_timeouts,
+ result_summary.unexpected_crashes_or_timeouts,
+ "Aborting run since %d crashes or timeouts were reached")
def _clobber_old_results(self):
# Just clobber the actual test results directories since the other
@@ -789,7 +790,7 @@ class TestRunner:
return failed_results
def _upload_json_files(self, unexpected_results, result_summary,
- individual_test_timings):
+ individual_test_timings):
"""Writes the results of the test run as JSON files into the results
dir and upload the files to the appengine server.
@@ -825,18 +826,13 @@ class TestRunner:
self._options.build_number, self._options.results_directory,
BUILDER_BASE_URL, individual_test_timings,
self._expectations, result_summary, self._test_files_list,
- not self._options.upload_full_results,
self._options.test_results_server,
"layout-tests",
self._options.master_name)
_log.debug("Finished writing JSON files.")
- json_files = ["expectations.json"]
- if self._options.upload_full_results:
- json_files.append("results.json")
- else:
- json_files.append("incremental_results.json")
+ json_files = ["expectations.json", "incremental_results.json"]
generator.upload_json_files(json_files)
@@ -844,6 +840,7 @@ class TestRunner:
"""Prints the configuration for the test run."""
p = self._printer
p.print_config("Using port '%s'" % self._port.name())
+ p.print_config("Test configuration: %s" % self._port.test_configuration())
p.print_config("Placing test results in %s" %
self._options.results_directory)
if self._options.new_baseline:
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner2.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner2.py
new file mode 100644
index 0000000..f097b83
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner2.py
@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+# Copyright (C) 2011 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+The TestRunner2 package is an alternate implementation of the TestRunner
+class that uses the manager_worker_broker module to send sets of tests to
+workers and receive their completion messages accordingly.
+"""
+
+import logging
+
+
+from webkitpy.layout_tests.layout_package import manager_worker_broker
+from webkitpy.layout_tests.layout_package import test_runner
+from webkitpy.layout_tests.layout_package import worker
+
+_log = logging.getLogger(__name__)
+
+
+class TestRunner2(test_runner.TestRunner):
+ def __init__(self, port, options, printer):
+ test_runner.TestRunner.__init__(self, port, options, printer)
+ self._all_results = []
+ self._group_stats = {}
+ self._current_result_summary = None
+ self._done = False
+
+ def is_done(self):
+ return self._done
+
+ def name(self):
+ return 'TestRunner2'
+
+ def _run_tests(self, file_list, result_summary):
+ """Runs the tests in the file_list.
+
+ Return: A tuple (keyboard_interrupted, thread_timings, test_timings,
+ individual_test_timings)
+ keyboard_interrupted is whether someone typed Ctrl^C
+ thread_timings is a list of dicts with the total runtime
+ of each thread with 'name', 'num_tests', 'total_time' properties
+ test_timings is a list of timings for each sharded subdirectory
+ of the form [time, directory_name, num_tests]
+ individual_test_timings is a list of run times for each test
+ in the form {filename:filename, test_run_time:test_run_time}
+ result_summary: summary object to populate with the results
+ """
+ self._current_result_summary = result_summary
+
+ # FIXME: shard properly.
+
+ # FIXME: should shard_tests return a list of objects rather than tuples?
+ test_lists = self._shard_tests(file_list, False)
+
+ manager_connection = manager_worker_broker.get(self._port, self._options, self, worker.Worker)
+
+ # FIXME: start all of the workers.
+ manager_connection.start_worker(0)
+
+ for test_list in test_lists:
+ manager_connection.post_message('test_list', test_list[0], test_list[1])
+
+ manager_connection.post_message('stop')
+
+ keyboard_interrupted = False
+ interrupted = False
+ if not self._options.dry_run:
+ while not self._check_if_done():
+ manager_connection.run_message_loop(delay_secs=1.0)
+
+ # FIXME: implement stats.
+ thread_timings = []
+
+ # FIXME: should this be a class instead of a tuple?
+ return (keyboard_interrupted, interrupted, thread_timings,
+ self._group_stats, self._all_results)
+
+ def _check_if_done(self):
+ """Returns true iff all the workers have either completed or wedged."""
+ # FIXME: implement to check for wedged workers.
+ return self._done
+
+ def handle_started_test(self, src, test_info, hang_timeout):
+ # FIXME: implement
+ pass
+
+ def handle_done(self, src):
+ # FIXME: implement properly to handle multiple workers.
+ self._done = True
+ pass
+
+ def handle_exception(self, src, exception_info):
+ raise exception_info
+
+ def handle_finished_list(self, src, list_name, num_tests, elapsed_time):
+ # FIXME: update stats
+ pass
+
+ def handle_finished_test(self, src, result, elapsed_time):
+ self._update_summary_with_result(self._current_result_summary, result)
+
+ # FIXME: update stats.
+ self._all_results.append(result)
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/worker.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/worker.py
new file mode 100644
index 0000000..47d4fbd
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/worker.py
@@ -0,0 +1,104 @@
+# Copyright (C) 2011 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Handle messages from the TestRunner and execute actual tests."""
+
+import logging
+import sys
+import time
+
+from webkitpy.common.system import stack_utils
+
+from webkitpy.layout_tests.layout_package import manager_worker_broker
+from webkitpy.layout_tests.layout_package import test_results
+
+
+_log = logging.getLogger(__name__)
+
+
+class Worker(manager_worker_broker.AbstractWorker):
+ def __init__(self, worker_connection, worker_number, options):
+ self._worker_connection = worker_connection
+ self._worker_number = worker_number
+ self._options = options
+ self._name = 'worker/%d' % worker_number
+ self._done = False
+ self._port = None
+
+ def _deferred_init(self, port):
+ self._port = port
+
+ def is_done(self):
+ return self._done
+
+ def name(self):
+ return self._name
+
+ def run(self, port):
+ self._deferred_init(port)
+
+ _log.debug("%s starting" % self._name)
+
+ # FIXME: need to add in error handling, better logging.
+ self._worker_connection.run_message_loop()
+ self._worker_connection.post_message('done')
+
+ def handle_test_list(self, src, list_name, test_list):
+ # FIXME: check to see if we need to get the http lock.
+
+ start_time = time.time()
+ num_tests = 0
+ for test_input in test_list:
+ self._run_test(test_input)
+ num_tests += 1
+ self._worker_connection.yield_to_broker()
+
+ elapsed_time = time.time() - start_time
+ self._worker_connection.post_message('finished_list', list_name, num_tests, elapsed_time)
+
+ # FIXME: release the lock if necessary
+
+ def handle_stop(self, src):
+ self._done = True
+
+ def _run_test(self, test_input):
+
+ # FIXME: get real timeout value from SingleTestRunner
+ test_timeout_sec = int(test_input.timeout) / 1000
+ start = time.time()
+ self._worker_connection.post_message('started_test', test_input, test_timeout_sec)
+
+ # FIXME: actually run the test.
+ result = test_results.TestResult(test_input.filename, failures=[],
+ test_run_time=0, total_time_for_all_diffs=0, time_for_diffs={})
+
+ elapsed_time = time.time() - start
+
+ # FIXME: update stats, check for failures.
+
+ self._worker_connection.post_message('finished_test', result, elapsed_time)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/base.py b/Tools/Scripts/webkitpy/layout_tests/port/base.py
index 6e5fabc..5ff4bff 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/base.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/base.py
@@ -121,15 +121,18 @@ class Port(object):
# certainly won't be available, so it's a good test to keep us
# from erroring out later.
self._pretty_patch_available = self._filesystem.exists(self._pretty_patch_path)
- self.set_option_default('configuration', None)
- if self._options.configuration is None:
+ if not hasattr(self._options, 'configuration') or self._options.configuration is None:
self._options.configuration = self.default_configuration()
+ self._test_configuration = None
def default_child_processes(self):
"""Return the number of DumpRenderTree instances to use for this
port."""
return self._executive.cpu_count()
+ def default_worker_model(self):
+ return 'old-threads'
+
def baseline_path(self):
"""Return the absolute path to the directory to store new baselines
in for this port."""
@@ -315,7 +318,7 @@ class Port(object):
path = self.expected_filename(test, '.checksum')
if not self.path_exists(path):
return None
- return self._filesystem.read_text_file(path)
+ return self._filesystem.read_binary_file(path)
def expected_image(self, test):
"""Returns the image we expect the test to produce."""
@@ -393,7 +396,7 @@ class Port(object):
driver = self.create_driver(0)
return driver.cmd_line()
- def update_baseline(self, path, data, encoding):
+ def update_baseline(self, path, data):
"""Updates the baseline for a test.
Args:
@@ -401,14 +404,8 @@ class Port(object):
the test. This function is used to update either generic or
platform-specific baselines, but we can't infer which here.
data: contents of the baseline.
- encoding: file encoding to use for the baseline.
"""
- # FIXME: remove the encoding parameter in favor of text/binary
- # functions.
- if encoding is None:
- self._filesystem.write_binary_file(path, data)
- else:
- self._filesystem.write_text_file(path, data)
+ self._filesystem.write_binary_file(path, data)
def uri_to_test_name(self, uri):
"""Return the base layout test name for a given URI.
@@ -465,6 +462,15 @@ class Port(object):
may be different (e.g., 'win-xp' instead of 'chromium-win-xp'."""
return self._name
+ def graphics_type(self):
+ """Returns whether the port uses accelerated graphics ('gpu') or not
+ ('cpu')."""
+ return 'cpu'
+
+ def real_name(self):
+ """Returns the actual name of the port, not the delegate's."""
+ return self.name()
+
def get_option(self, name, default_value=None):
# FIXME: Eventually we should not have to do a test for
# hasattr(), and we should be able to just do
@@ -496,9 +502,16 @@ class Port(object):
"""Relative unix-style path for a filename under the LayoutTests
directory. Filenames outside the LayoutTests directory should raise
an error."""
+ # FIXME: On Windows, does this return test_names with forward slashes,
+ # or windows-style relative paths?
assert filename.startswith(self.layout_tests_dir()), "%s did not start with %s" % (filename, self.layout_tests_dir())
return filename[len(self.layout_tests_dir()) + 1:]
+ def abspath_for_test(self, test_name):
+ """Returns the full path to the file for a given test name. This is the
+ inverse of relative_test_filename()."""
+ return self._filesystem.normpath(self._filesystem.join(self.layout_tests_dir(), test_name))
+
def results_directory(self):
"""Absolute path to the place to store the test results."""
raise NotImplementedError('Port.results_directory')
@@ -577,12 +590,25 @@ class Port(object):
if self._http_lock:
self._http_lock.cleanup_http_lock()
+ #
+ # TEST EXPECTATION-RELATED METHODS
+ #
+
+ def test_configuration(self):
+ """Returns the current TestConfiguration for the port."""
+ if not self._test_configuration:
+ self._test_configuration = TestConfiguration(self)
+ return self._test_configuration
+
+ def all_test_configurations(self):
+ return self.test_configuration().all_test_configurations()
+
def test_expectations(self):
"""Returns the test expectations for this port.
Basically this string should contain the equivalent of a
test_expectations file. See test_expectations.py for more details."""
- raise NotImplementedError('Port.test_expectations')
+ return self._filesystem.read_text_file(self.path_to_test_expectations_file())
def test_expectations_overrides(self):
"""Returns an optional set of overrides for the test_expectations.
@@ -593,18 +619,6 @@ class Port(object):
sync up the two repos."""
return None
- def test_base_platform_names(self):
- """Return a list of the 'base' platforms on your port. The base
- platforms represent different architectures, operating systems,
- or implementations (as opposed to different versions of a single
- platform). For example, 'mac' and 'win' might be different base
- platforms, wherease 'mac-tiger' and 'mac-leopard' might be
- different platforms. This routine is used by the rebaselining tool
- and the dashboards, and the strings correspond to the identifiers
- in your test expectations (*not* necessarily the platform names
- themselves)."""
- raise NotImplementedError('Port.base_test_platforms')
-
def test_platform_name(self):
"""Returns the string that corresponds to the given platform name
in the test expectations. This may be the same as name(), or it
@@ -810,6 +824,48 @@ class Port(object):
platform)
+class DriverInput(object):
+ """Holds the input parameters for a driver."""
+
+ def __init__(self, filename, timeout, image_hash):
+ """Initializes a DriverInput object.
+
+ Args:
+ filename: Full path to the test.
+ timeout: Timeout in msecs the driver should use while running the test
+ image_hash: A image checksum which is used to avoid doing an image dump if
+ the checksums match.
+ """
+ self.filename = filename
+ self.timeout = timeout
+ self.image_hash = image_hash
+
+
+class DriverOutput(object):
+ """Groups information about a output from driver for easy passing of data."""
+
+ def __init__(self, text, image, image_hash,
+ crash=False, test_time=None, timeout=False, error=''):
+ """Initializes a TestOutput object.
+
+ Args:
+ text: a text output
+ image: an image output
+ image_hash: a string containing the checksum of the image
+ crash: a boolean indicating whether the driver crashed on the test
+ test_time: a time which the test has taken
+ timeout: a boolean indicating whehter the test timed out
+ error: any unexpected or additional (or error) text output
+ """
+ self.text = text
+ self.image = image
+ self.image_hash = image_hash
+ self.crash = crash
+ self.test_time = test_time
+ self.timeout = timeout
+ self.error = error
+
+
class Driver:
"""Abstract interface for the DumpRenderTree interface."""
@@ -824,7 +880,7 @@ class Driver:
"""
raise NotImplementedError('Driver.__init__')
- def run_test(self, test_input):
+ def run_test(self, driver_input):
"""Run a single test and return the results.
Note that it is okay if a test times out or crashes and leaves
@@ -832,9 +888,9 @@ class Driver:
are responsible for cleaning up and ensuring things are okay.
Args:
- test_input: a TestInput object
+ driver_input: a DriverInput object
- Returns a TestOutput object.
+ Returns a DriverOutput object.
"""
raise NotImplementedError('Driver.run_test')
@@ -863,3 +919,68 @@ class Driver:
def stop(self):
raise NotImplementedError('Driver.stop')
+
+
+class TestConfiguration(object):
+ def __init__(self, port=None, os=None, version=None, architecture=None,
+ build_type=None, graphics_type=None):
+
+ # FIXME: We can get the O/S and version from test_platform_name()
+ # and version() for now, but those should go away and be cleaned up
+ # with more generic methods like operation_system() and os_version()
+ # or something. Note that we need to strip the leading '-' off the
+ # version string if it is present.
+ if port:
+ port_version = port.version()
+ self.os = os or port.test_platform_name().replace(port_version, '')
+ self.version = version or port_version[1:]
+ self.architecture = architecture or 'x86'
+ self.build_type = build_type or port._options.configuration.lower()
+ self.graphics_type = graphics_type or port.graphics_type()
+
+ def items(self):
+ return self.__dict__.items()
+
+ def keys(self):
+ return self.__dict__.keys()
+
+ def __str__(self):
+ return ("<%(os)s, %(version)s, %(build_type)s, %(graphics_type)s>" %
+ self.__dict__)
+
+ def __repr__(self):
+ return "TestConfig(os='%(os)s', version='%(version)s', architecture='%(architecture)s', build_type='%(build_type)s', graphics_type='%(graphics_type)s')" % self.__dict__
+
+ def values(self):
+ """Returns the configuration values of this instance as a tuple."""
+ return self.__dict__.values()
+
+ def all_test_configurations(self):
+ """Returns a sequence of the TestConfigurations the port supports."""
+ # By default, we assume we want to test every graphics type in
+ # every configuration on every system.
+ test_configurations = []
+ for system in self.all_systems():
+ for build_type in self.all_build_types():
+ for graphics_type in self.all_graphics_types():
+ test_configurations.append(TestConfiguration(
+ os=system[0],
+ version=system[1],
+ architecture=system[2],
+ build_type=build_type,
+ graphics_type=graphics_type))
+ return test_configurations
+
+ def all_systems(self):
+ return (('mac', 'leopard', 'x86'),
+ ('mac', 'snowleopard', 'x86'),
+ ('win', 'xp', 'x86'),
+ ('win', 'vista', 'x86'),
+ ('win', 'win7', 'x86'),
+ ('linux', 'hardy', 'x86'))
+
+ def all_build_types(self):
+ return ('debug', 'release')
+
+ def all_graphics_types(self):
+ return ('cpu', 'gpu')
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py
index 72f2d05..ef90484 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py
@@ -224,19 +224,6 @@ class PortTest(unittest.TestCase):
port = base.Port()
self.assertEqual(port.get_option('foo', 'bar'), 'bar')
- def test_set_option_default__unset(self):
- port = base.Port()
- port.set_option_default('foo', 'bar')
- self.assertEqual(port.get_option('foo'), 'bar')
-
- def test_set_option_default__set(self):
- options, args = optparse.OptionParser().parse_args([])
- options.foo = 'bar'
- port = base.Port(options=options)
- # This call should have no effect.
- port.set_option_default('foo', 'new_bar')
- self.assertEqual(port.get_option('foo'), 'bar')
-
def test_name__unset(self):
port = base.Port()
self.assertEqual(port.name(), None)
@@ -263,7 +250,6 @@ class VirtualTest(unittest.TestCase):
self.assertVirtual(port.test_platform_name)
self.assertVirtual(port.results_directory)
self.assertVirtual(port.test_expectations)
- self.assertVirtual(port.test_base_platform_names)
self.assertVirtual(port.test_platform_name)
self.assertVirtual(port.test_platforms)
self.assertVirtual(port.test_platform_name_to_name, None)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py
index ad1bea6..7d56fa2 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py
@@ -41,16 +41,9 @@ import webbrowser
from webkitpy.common.system import executive
from webkitpy.common.system.path import cygpath
from webkitpy.layout_tests.layout_package import test_expectations
-from webkitpy.layout_tests.layout_package import test_output
-
-import base
-import http_server
-
-# Chromium DRT on OSX uses WebKitDriver.
-if sys.platform == 'darwin':
- import webkit
-
-import websocket_server
+from webkitpy.layout_tests.port import base
+from webkitpy.layout_tests.port import http_server
+from webkitpy.layout_tests.port import websocket_server
_log = logging.getLogger("webkitpy.layout_tests.port.chromium")
@@ -176,8 +169,6 @@ class ChromiumPort(base.Port):
return result
def driver_name(self):
- if self._options.use_test_shell:
- return "test_shell"
return "DumpRenderTree"
def path_from_chromium_base(self, *comps):
@@ -189,7 +180,7 @@ class ChromiumPort(base.Port):
if offset == -1:
self._chromium_base_dir = self._filesystem.join(
abspath[0:abspath.find('Tools')],
- 'WebKit', 'chromium')
+ 'Source', 'WebKit', 'chromium')
else:
self._chromium_base_dir = abspath[0:offset]
return self._filesystem.join(self._chromium_base_dir, *comps)
@@ -217,8 +208,6 @@ class ChromiumPort(base.Port):
def create_driver(self, worker_number):
"""Starts a new Driver and returns a handle to it."""
- if not self.get_option('use_test_shell') and sys.platform == 'darwin':
- return webkit.WebKitDriver(self, worker_number)
return ChromiumDriver(self, worker_number)
def start_helper(self):
@@ -241,9 +230,6 @@ class ChromiumPort(base.Port):
# http://bugs.python.org/issue1731717
self._helper.wait()
- def test_base_platform_names(self):
- return ('linux', 'mac', 'win')
-
def test_expectations(self):
"""Returns the test expectations for this port.
@@ -273,15 +259,14 @@ class ChromiumPort(base.Port):
all_test_files.update(extra_test_files)
expectations = test_expectations.TestExpectations(
- self, all_test_files, expectations_str, test_platform_name,
- is_debug_mode, is_lint_mode=False, overrides=overrides_str)
+ self, all_test_files, expectations_str, self.test_configuration(),
+ is_lint_mode=False, overrides=overrides_str)
tests_dir = self.layout_tests_dir()
return [self.relative_test_filename(test)
for test in expectations.get_tests_with_result_type(test_expectations.SKIP)]
def test_platform_names(self):
- return self.test_base_platform_names() + ('win-xp',
- 'win-vista', 'win-7')
+ return ('mac', 'win', 'linux', 'win-xp', 'win-vista', 'win-7')
def test_platform_name_to_name(self, test_platform_name):
if test_platform_name in self.test_platform_names():
@@ -340,13 +325,11 @@ class ChromiumPort(base.Port):
def _path_to_image_diff(self):
binary_name = 'ImageDiff'
- if self.get_option('use_test_shell'):
- binary_name = 'image_diff'
return self._build_path(self.get_option('configuration'), binary_name)
class ChromiumDriver(base.Driver):
- """Abstract interface for test_shell."""
+ """Abstract interface for DRT."""
def __init__(self, port, worker_number):
self._port = port
@@ -365,10 +348,7 @@ class ChromiumDriver(base.Driver):
cmd.append("--pixel-tests=" +
self._port._convert_path(self._image_path))
- if self._port.get_option('use_test_shell'):
- cmd.append('--layout-tests')
- else:
- cmd.append('--test-shell')
+ cmd.append('--test-shell')
if self._port.get_option('startup_dialog'):
cmd.append('--testshell-startup-dialog')
@@ -385,14 +365,12 @@ class ChromiumDriver(base.Driver):
if self._port.get_option('stress_deopt'):
cmd.append('--stress-deopt')
- # test_shell does not support accelerated compositing.
- if not self._port.get_option("use_test_shell"):
- if self._port.get_option('accelerated_compositing'):
- cmd.append('--enable-accelerated-compositing')
- if self._port.get_option('accelerated_2d_canvas'):
- cmd.append('--enable-accelerated-2d-canvas')
- if self._port.get_option('enable_hardware_gpu'):
- cmd.append('--enable-hardware-gpu')
+ if self._port.get_option('accelerated_compositing'):
+ cmd.append('--enable-accelerated-compositing')
+ if self._port.get_option('accelerated_2d_canvas'):
+ cmd.append('--enable-accelerated-2d-canvas')
+ if self._port.get_option('enable_hardware_gpu'):
+ cmd.append('--enable-hardware-gpu')
return cmd
def start(self):
@@ -420,17 +398,17 @@ class ChromiumDriver(base.Driver):
try:
if input:
if isinstance(input, unicode):
- # TestShell expects utf-8
+ # DRT expects utf-8
input = input.encode("utf-8")
self._proc.stdin.write(input)
# DumpRenderTree text output is always UTF-8. However some tests
# (e.g. webarchive) may spit out binary data instead of text so we
- # don't bother to decode the output (for either DRT or test_shell).
+ # don't bother to decode the output.
line = self._proc.stdout.readline()
# We could assert() here that line correctly decodes as UTF-8.
return (line, False)
except IOError, e:
- _log.error("IOError communicating w/ test_shell: " + str(e))
+ _log.error("IOError communicating w/ DRT: " + str(e))
return (None, True)
def _test_shell_command(self, uri, timeoutms, checksum):
@@ -465,7 +443,7 @@ class ChromiumDriver(base.Driver):
raise e
return self._output_image()
- def run_test(self, test_input):
+ def run_test(self, driver_input):
output = []
error = []
crash = False
@@ -475,9 +453,9 @@ class ChromiumDriver(base.Driver):
start_time = time.time()
- uri = self._port.filename_to_uri(test_input.filename)
- cmd = self._test_shell_command(uri, test_input.timeout,
- test_input.image_hash)
+ uri = self._port.filename_to_uri(driver_input.filename)
+ cmd = self._test_shell_command(uri, driver_input.timeout,
+ driver_input.image_hash)
(line, crash) = self._write_command_and_read_line(input=cmd)
while not crash and line.rstrip() != "#EOF":
@@ -485,7 +463,7 @@ class ChromiumDriver(base.Driver):
if line == '' and self.poll() is not None:
# This is hex code 0xc000001d, which is used for abrupt
# termination. This happens if we hit ctrl+c from the prompt
- # and we happen to be waiting on test_shell.
+ # and we happen to be waiting on DRT.
# sdoyon: Not sure for which OS and in what circumstances the
# above code is valid. What works for me under Linux to detect
# ctrl+c is for the subprocess returncode to be negative
@@ -519,7 +497,7 @@ class ChromiumDriver(base.Driver):
(line, crash) = self._write_command_and_read_line(input=None)
run_time = time.time() - start_time
- return test_output.TestOutput(
+ return base.DriverOutput(
''.join(output), self._output_image_with_retry(), actual_checksum,
crash, run_time, timeout, ''.join(error))
@@ -532,8 +510,8 @@ class ChromiumDriver(base.Driver):
if sys.platform not in ('win32', 'cygwin'):
# Closing stdin/stdout/stderr hangs sometimes on OS X,
# (see __init__(), above), and anyway we don't want to hang
- # the harness if test_shell is buggy, so we wait a couple
- # seconds to give test_shell a chance to clean up, but then
+ # the harness if DRT is buggy, so we wait a couple
+ # seconds to give DRT a chance to clean up, but then
# force-kill the process if necessary.
KILL_TIMEOUT = 3.0
timeout = time.time() + KILL_TIMEOUT
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py
index b88d8aa..e8c75c4 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py
@@ -30,64 +30,64 @@ import chromium_linux
import chromium_mac
import chromium_win
+from webkitpy.layout_tests.port import test_files
-def get(**kwargs):
+
+def get(platform=None, port_name='chromium-gpu', **kwargs):
"""Some tests have slightly different results when run while using
hardware acceleration. In those cases, we prepend an additional directory
to the baseline paths."""
- port_name = kwargs.get('port_name', None)
+ platform = platform or sys.platform
if port_name == 'chromium-gpu':
- if sys.platform in ('cygwin', 'win32'):
+ if platform in ('cygwin', 'win32'):
port_name = 'chromium-gpu-win'
- elif sys.platform == 'linux2':
+ elif platform == 'linux2':
port_name = 'chromium-gpu-linux'
- elif sys.platform == 'darwin':
+ elif platform == 'darwin':
port_name = 'chromium-gpu-mac'
else:
- raise NotImplementedError('unsupported platform: %s' %
- sys.platform)
+ raise NotImplementedError('unsupported platform: %s' % platform)
if port_name == 'chromium-gpu-linux':
- return ChromiumGpuLinuxPort(**kwargs)
-
- if port_name.startswith('chromium-gpu-mac'):
- return ChromiumGpuMacPort(**kwargs)
-
- if port_name.startswith('chromium-gpu-win'):
- return ChromiumGpuWinPort(**kwargs)
-
+ return ChromiumGpuLinuxPort(port_name=port_name, **kwargs)
+ if port_name == 'chromium-gpu-mac':
+ return ChromiumGpuMacPort(port_name=port_name, **kwargs)
+ if port_name == 'chromium-gpu-win':
+ return ChromiumGpuWinPort(port_name=port_name, **kwargs)
raise NotImplementedError('unsupported port: %s' % port_name)
-def _set_gpu_options(options):
- if options:
- if options.accelerated_compositing is None:
- options.accelerated_compositing = True
- if options.accelerated_2d_canvas is None:
- options.accelerated_2d_canvas = True
+# FIXME: These should really be a mixin class.
- # FIXME: Remove this after http://codereview.chromium.org/5133001/ is enabled
- # on the bots.
- if options.builder_name is not None and not ' - GPU' in options.builder_name:
- options.builder_name = options.builder_name + ' - GPU'
+def _set_gpu_options(port):
+ if port.get_option('accelerated_compositing') is None:
+ port._options.accelerated_compositing = True
+ if port.get_option('accelerated_2d_canvas') is None:
+ port._options.accelerated_2d_canvas = True
+ # FIXME: Remove this after http://codereview.chromium.org/5133001/ is enabled
+ # on the bots.
+ if port.get_option('builder_name') is not None and not ' - GPU' in port._options.builder_name:
+ port._options.builder_name += ' - GPU'
-def _gpu_overrides(port):
- try:
- overrides_path = port.path_from_chromium_base('webkit', 'tools',
- 'layout_tests', 'test_expectations_gpu.txt')
- except AssertionError:
- return None
- if not port._filesystem.exists(overrides_path):
- return None
- return port._filesystem.read_text_file(overrides_path)
+
+def _tests(port, paths):
+ if not paths:
+ paths = ['compositing', 'platform/chromium/compositing']
+ if not port.name().startswith('chromium-gpu-mac'):
+ # Canvas is not yet accelerated on the Mac, so there's no point
+ # in running the tests there.
+ paths += ['fast/canvas', 'canvas/philip']
+ # invalidate_rect.html tests a bug in the compositor.
+ # See https://bugs.webkit.org/show_bug.cgi?id=53117
+ paths += ['plugins/invalidate_rect.html']
+ return test_files.find(port, paths)
class ChromiumGpuLinuxPort(chromium_linux.ChromiumLinuxPort):
- def __init__(self, **kwargs):
- kwargs.setdefault('port_name', 'chromium-gpu-linux')
- _set_gpu_options(kwargs.get('options'))
- chromium_linux.ChromiumLinuxPort.__init__(self, **kwargs)
+ def __init__(self, port_name='chromium-gpu-linux', **kwargs):
+ chromium_linux.ChromiumLinuxPort.__init__(self, port_name=port_name, **kwargs)
+ _set_gpu_options(self)
def baseline_search_path(self):
# Mimic the Linux -> Win expectations fallback in the ordinary Chromium port.
@@ -97,19 +97,18 @@ class ChromiumGpuLinuxPort(chromium_linux.ChromiumLinuxPort):
def default_child_processes(self):
return 1
- def path_to_test_expectations_file(self):
- return self.path_from_webkit_base('LayoutTests', 'platform',
- 'chromium-gpu', 'test_expectations.txt')
+ def graphics_type(self):
+ return 'gpu'
+
+ def tests(self, paths):
+ return _tests(self, paths)
- def test_expectations_overrides(self):
- return _gpu_overrides(self)
class ChromiumGpuMacPort(chromium_mac.ChromiumMacPort):
- def __init__(self, **kwargs):
- kwargs.setdefault('port_name', 'chromium-gpu-mac')
- _set_gpu_options(kwargs.get('options'))
- chromium_mac.ChromiumMacPort.__init__(self, **kwargs)
+ def __init__(self, port_name='chromium-gpu-mac', **kwargs):
+ chromium_mac.ChromiumMacPort.__init__(self, port_name=port_name, **kwargs)
+ _set_gpu_options(self)
def baseline_search_path(self):
return (map(self._webkit_baseline_path, ['chromium-gpu-mac', 'chromium-gpu']) +
@@ -118,19 +117,18 @@ class ChromiumGpuMacPort(chromium_mac.ChromiumMacPort):
def default_child_processes(self):
return 1
- def path_to_test_expectations_file(self):
- return self.path_from_webkit_base('LayoutTests', 'platform',
- 'chromium-gpu', 'test_expectations.txt')
+ def graphics_type(self):
+ return 'gpu'
+
+ def tests(self, paths):
+ return _tests(self, paths)
- def test_expectations_overrides(self):
- return _gpu_overrides(self)
class ChromiumGpuWinPort(chromium_win.ChromiumWinPort):
- def __init__(self, **kwargs):
- kwargs.setdefault('port_name', 'chromium-gpu-win' + self.version())
- _set_gpu_options(kwargs.get('options'))
- chromium_win.ChromiumWinPort.__init__(self, **kwargs)
+ def __init__(self, port_name='chromium-gpu-win', **kwargs):
+ chromium_win.ChromiumWinPort.__init__(self, port_name=port_name, **kwargs)
+ _set_gpu_options(self)
def baseline_search_path(self):
return (map(self._webkit_baseline_path, ['chromium-gpu-win', 'chromium-gpu']) +
@@ -139,9 +137,8 @@ class ChromiumGpuWinPort(chromium_win.ChromiumWinPort):
def default_child_processes(self):
return 1
- def path_to_test_expectations_file(self):
- return self.path_from_webkit_base('LayoutTests', 'platform',
- 'chromium-gpu', 'test_expectations.txt')
+ def graphics_type(self):
+ return 'gpu'
- def test_expectations_overrides(self):
- return _gpu_overrides(self)
+ def tests(self, paths):
+ return _tests(self, paths)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py
index 0bfb127..96962ec 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py
@@ -40,20 +40,34 @@ class ChromiumGpuTest(unittest.TestCase):
def test_get_chromium_gpu_win(self):
self.assertOverridesWorked('chromium-gpu-win')
- def assertOverridesWorked(self, port_name):
+ def test_get_chromium_gpu__on_linux(self):
+ self.assertOverridesWorked('chromium-gpu-linux', 'chromium-gpu', 'linux2')
+
+ def test_get_chromium_gpu__on_mac(self):
+ self.assertOverridesWorked('chromium-gpu-mac', 'chromium-gpu', 'darwin')
+
+ def test_get_chromium_gpu__on_win(self):
+ self.assertOverridesWorked('chromium-gpu-win', 'chromium-gpu', 'win32')
+ self.assertOverridesWorked('chromium-gpu-win', 'chromium-gpu', 'cygwin')
+
+ def assertOverridesWorked(self, port_name, input_name=None, platform=None):
# test that we got the right port
mock_options = mocktool.MockOptions(accelerated_compositing=None,
accelerated_2d_canvas=None,
builder_name='foo',
child_processes=None)
- port = chromium_gpu.get(port_name=port_name, options=mock_options)
+ if input_name and platform:
+ port = chromium_gpu.get(platform=platform, port_name=input_name,
+ options=mock_options)
+ else:
+ port = chromium_gpu.get(port_name=port_name, options=mock_options)
self.assertTrue(port._options.accelerated_compositing)
self.assertTrue(port._options.accelerated_2d_canvas)
self.assertEqual(port.default_child_processes(), 1)
self.assertEqual(port._options.builder_name, 'foo - GPU')
- # we use startswith() instead of Equal to gloss over platform versions.
- self.assertTrue(port.name().startswith(port_name))
+ # We don't support platform-specific versions of the GPU port yet.
+ self.assertEqual(port.name(), port_name)
# test that it has the right directories in front of the search path.
paths = port.baseline_search_path()
@@ -64,9 +78,24 @@ class ChromiumGpuTest(unittest.TestCase):
else:
self.assertEqual(port._webkit_baseline_path('chromium-gpu'), paths[1])
- # Test that we have the right expectations file.
- self.assertTrue('chromium-gpu' in
- port.path_to_test_expectations_file())
+
+ # Test that we're limiting to the correct directories.
+ # These two tests are picked mostly at random, but we make sure they
+ # exist separately from being filtered out by the port.
+ files = port.tests(None)
+
+ path = port.abspath_for_test('compositing/checkerboard.html')
+ self.assertTrue(port._filesystem.exists(path))
+ self.assertTrue(path in files)
+
+ path = port.abspath_for_test('fast/html/keygen.html')
+ self.assertTrue(port._filesystem.exists(path))
+ self.assertFalse(path in files)
+ if port_name.startswith('chromium-gpu-mac'):
+ path = port.abspath_for_test('fast/canvas/set-colors.html')
+ self.assertTrue(port._filesystem.exists(path))
+ self.assertFalse(path in files)
+
if __name__ == '__main__':
unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py
index c1c85f8..c3c5a21 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py
@@ -85,7 +85,7 @@ class ChromiumLinuxPort(chromium.ChromiumPort):
base = self.path_from_chromium_base()
if self._filesystem.exists(self._filesystem.join(base, 'sconsbuild')):
return self._filesystem.join(base, 'sconsbuild', *comps)
- if self._filesystem.exists(self._filesystem.join(base, 'out', *comps)) or self.get_option('use_test_shell'):
+ if self._filesystem.exists(self._filesystem.join(base, 'out', *comps)):
return self._filesystem.join(base, 'out', *comps)
base = self.path_from_webkit_base()
if self._filesystem.exists(self._filesystem.join(base, 'sconsbuild')):
@@ -153,8 +153,6 @@ class ChromiumLinuxPort(chromium.ChromiumPort):
if not configuration:
configuration = self.get_option('configuration')
binary_name = 'DumpRenderTree'
- if self.get_option('use_test_shell'):
- binary_name = 'test_shell'
return self._build_path(configuration, binary_name)
def _path_to_helper(self):
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py
index 5360ab3..17862a2 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py
@@ -69,18 +69,18 @@ class ChromiumMacPort(chromium.ChromiumPort):
return result
def default_child_processes(self):
- # FIXME: we need to run single-threaded for now. See
- # https://bugs.webkit.org/show_bug.cgi?id=38553. Unfortunately this
- # routine is called right before the logger is configured, so if we
- # try to _log.warning(), it gets thrown away.
- import sys
- sys.stderr.write("Defaulting to one child - see https://bugs.webkit.org/show_bug.cgi?id=38553\n")
- return 1
+ if self.get_option('worker_model') == 'old-threads':
+ # FIXME: we need to run single-threaded for now. See
+ # https://bugs.webkit.org/show_bug.cgi?id=38553. Unfortunately this
+ # routine is called right before the logger is configured, so if we
+ # try to _log.warning(), it gets thrown away.
+ import sys
+ sys.stderr.write("Defaulting to one child - see https://bugs.webkit.org/show_bug.cgi?id=38553\n")
+ return 1
+
+ return chromium.ChromiumPort.default_child_processes(self)
def driver_name(self):
- """name for this port's equivalent of DumpRenderTree."""
- if self.get_option('use_test_shell'):
- return "TestShell"
return "DumpRenderTree"
def test_platform_name(self):
@@ -110,7 +110,7 @@ class ChromiumMacPort(chromium.ChromiumPort):
*comps)
path = self.path_from_chromium_base('xcodebuild', *comps)
- if self._filesystem.exists(path) or self.get_option('use_test_shell'):
+ if self._filesystem.exists(path):
return path
return self.path_from_webkit_base(
'Source', 'WebKit', 'chromium', 'xcodebuild', *comps)
@@ -154,8 +154,6 @@ class ChromiumMacPort(chromium.ChromiumPort):
def _path_to_helper(self):
binary_name = 'LayoutTestHelper'
- if self.get_option('use_test_shell'):
- binary_name = 'layout_test_helper'
return self._build_path(self.get_option('configuration'), binary_name)
def _path_to_wdiff(self):
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py
index 6c8987b..b89c8cc 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py
@@ -116,13 +116,6 @@ class ChromiumPortTest(unittest.TestCase):
port = ChromiumPortTest.TestMacPort(options=mock_options)
self.assertTrue(port._path_to_image_diff().endswith(
'/xcodebuild/default/ImageDiff'))
- mock_options = mocktool.MockOptions(use_test_shell=True)
- port = ChromiumPortTest.TestLinuxPort(options=mock_options)
- self.assertTrue(port._path_to_image_diff().endswith(
- '/out/default/image_diff'), msg=port._path_to_image_diff())
- port = ChromiumPortTest.TestMacPort(options=mock_options)
- self.assertTrue(port._path_to_image_diff().endswith(
- '/xcodebuild/default/image_diff'))
# FIXME: Figure out how this is going to work on Windows.
#port = chromium_win.ChromiumWinPort('test-port', options=MockOptions())
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py
index 14f2777..f4cbf80 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py
@@ -113,9 +113,9 @@ class ChromiumWinPort(chromium.ChromiumPort):
if self._filesystem.exists(p):
return p
p = self.path_from_chromium_base('chrome', *comps)
- if self._filesystem.exists(p) or self.get_option('use_test_shell'):
+ if self._filesystem.exists(p):
return p
- return self._filesystem.join(self.path_from_webkit_base(), 'WebKit', 'chromium', *comps)
+ return self._filesystem.join(self.path_from_webkit_base(), 'Source', 'WebKit', 'chromium', *comps)
def _lighttpd_path(self, *comps):
return self.path_from_chromium_base('third_party', 'lighttpd', 'win',
@@ -141,20 +141,14 @@ class ChromiumWinPort(chromium.ChromiumPort):
if not configuration:
configuration = self.get_option('configuration')
binary_name = 'DumpRenderTree.exe'
- if self.get_option('use_test_shell'):
- binary_name = 'test_shell.exe'
return self._build_path(configuration, binary_name)
def _path_to_helper(self):
binary_name = 'LayoutTestHelper.exe'
- if self.get_option('use_test_shell'):
- binary_name = 'layout_test_helper.exe'
return self._build_path(self.get_option('configuration'), binary_name)
def _path_to_image_diff(self):
binary_name = 'ImageDiff.exe'
- if self.get_option('use_test_shell'):
- binary_name = 'image_diff.exe'
return self._build_path(self.get_option('configuration'), binary_name)
def _path_to_wdiff(self):
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py b/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py
index 4ed34e6..6b3bd51 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py
@@ -50,8 +50,6 @@ import os
import sys
import time
-from webkitpy.layout_tests.layout_package import test_output
-
import base
import factory
@@ -71,6 +69,12 @@ class DryRunPort(object):
def __getattr__(self, name):
return getattr(self.__delegate, name)
+ def acquire_http_lock(self):
+ pass
+
+ def release_http_lock(self):
+ pass
+
def check_build(self, needs_http):
return True
@@ -112,18 +116,18 @@ class DryrunDriver(base.Driver):
def poll(self):
return None
- def run_test(self, test_input):
+ def run_test(self, driver_input):
start_time = time.time()
- text_output = self._port.expected_text(test_input.filename)
+ text_output = self._port.expected_text(driver_input.filename)
- if test_input.image_hash is not None:
- image = self._port.expected_image(test_input.filename)
- hash = self._port.expected_checksum(test_input.filename)
+ if driver_input.image_hash is not None:
+ image = self._port.expected_image(driver_input.filename)
+ hash = self._port.expected_checksum(driver_input.filename)
else:
image = None
hash = None
- return test_output.TestOutput(text_output, image, hash, False,
- time.time() - start_time, False, None)
+ return base.DriverOutput(text_output, image, hash, False,
+ time.time() - start_time, False, '')
def start(self):
pass
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/factory.py b/Tools/Scripts/webkitpy/layout_tests/port/factory.py
index 6935744..7ae6eb6 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/factory.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/factory.py
@@ -70,12 +70,15 @@ def _get_kwargs(**kwargs):
raise NotImplementedError('unknown port; sys.platform = "%s"' %
sys.platform)
- if port_to_use == 'test':
+ if port_to_use.startswith('test'):
import test
maker = test.TestPort
elif port_to_use.startswith('dryrun'):
import dryrun
maker = dryrun.DryRunPort
+ elif port_to_use.startswith('mock-'):
+ import mock_drt
+ maker = mock_drt.MockDRTPort
elif port_to_use.startswith('mac'):
import mac
maker = mac.MacPort
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/http_server_base.py b/Tools/Scripts/webkitpy/layout_tests/port/http_server_base.py
index 52a0403..2a43e81 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/http_server_base.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/http_server_base.py
@@ -67,7 +67,7 @@ class HttpServerBase(object):
url = 'http%s://127.0.0.1:%d/' % (http_suffix, mapping['port'])
try:
- response = urllib.urlopen(url)
+ response = urllib.urlopen(url, proxies={})
_log.debug("Server running at %s" % url)
except IOError, e:
_log.debug("Server NOT running at %s: %s" % (url, e))
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mac.py b/Tools/Scripts/webkitpy/layout_tests/port/mac.py
index 0622196..1398ed3 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/mac.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/mac.py
@@ -33,9 +33,7 @@ import os
import platform
import signal
-import webkitpy.common.system.ospath as ospath
-import webkitpy.layout_tests.port.server_process as server_process
-from webkitpy.layout_tests.port.webkit import WebKitPort, WebKitDriver
+from webkitpy.layout_tests.port.webkit import WebKitPort
_log = logging.getLogger("webkitpy.layout_tests.port.mac")
@@ -52,7 +50,7 @@ class MacPort(WebKitPort):
# four threads in parallel.
# See https://bugs.webkit.org/show_bug.cgi?id=36622
child_processes = WebKitPort.default_child_processes(self)
- if child_processes > 4:
+ if self.get_option('worker_model') == 'old-threads' and child_processes > 4:
return 4
return child_processes
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py
index d383a4c..ef04679 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py
@@ -35,23 +35,31 @@ import port_testcase
class MacTest(port_testcase.PortTestCase):
- def make_port(self, options=port_testcase.mock_options):
+ def make_port(self, port_name=None, options=port_testcase.mock_options):
if sys.platform != 'darwin':
return None
- port_obj = mac.MacPort(options=options)
+ port_obj = mac.MacPort(port_name=port_name, options=options)
port_obj._options.results_directory = port_obj.results_directory()
port_obj._options.configuration = 'Release'
return port_obj
- def test_skipped_file_paths(self):
- port = self.make_port()
+ def assert_skipped_files_for_version(self, port_name, expected_paths):
+ port = self.make_port(port_name)
if not port:
return
skipped_paths = port._skipped_file_paths()
# FIXME: _skipped_file_paths should return WebKit-relative paths.
# So to make it unit testable, we strip the WebKit directory from the path.
relative_paths = [path[len(port.path_from_webkit_base()):] for path in skipped_paths]
- self.assertEqual(relative_paths, ['LayoutTests/platform/mac-leopard/Skipped', 'LayoutTests/platform/mac/Skipped'])
+ self.assertEqual(relative_paths, expected_paths)
+
+ def test_skipped_file_paths(self):
+ self.assert_skipped_files_for_version('mac',
+ ['/LayoutTests/platform/mac/Skipped'])
+ self.assert_skipped_files_for_version('mac-snowleopard',
+ ['/LayoutTests/platform/mac-snowleopard/Skipped', '/LayoutTests/platform/mac/Skipped'])
+ self.assert_skipped_files_for_version('mac-leopard',
+ ['/LayoutTests/platform/mac-leopard/Skipped', '/LayoutTests/platform/mac/Skipped'])
example_skipped_file = u"""
# <rdar://problem/5647952> fast/events/mouseout-on-window.html needs mac DRT to issue mouse out events
@@ -69,12 +77,11 @@ svg/batik/text/smallFonts.svg
"svg/batik/text/smallFonts.svg",
]
- def test_skipped_file_paths(self):
+ def test_tests_from_skipped_file_contents(self):
port = self.make_port()
if not port:
return
- skipped_file = StringIO.StringIO(self.example_skipped_file)
- self.assertEqual(port._tests_from_skipped_file(skipped_file), self.example_skipped_tests)
+ self.assertEqual(port._tests_from_skipped_file_contents(self.example_skipped_file), self.example_skipped_tests)
if __name__ == '__main__':
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mock_drt.py b/Tools/Scripts/webkitpy/layout_tests/port/mock_drt.py
new file mode 100644
index 0000000..1147846
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/mock_drt.py
@@ -0,0 +1,280 @@
+#!/usr/bin/env python
+# Copyright (C) 2011 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the Google name nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+This is an implementation of the Port interface that overrides other
+ports and changes the Driver binary to "MockDRT".
+"""
+
+import logging
+import optparse
+import os
+import sys
+
+from webkitpy.common.system import filesystem
+
+from webkitpy.layout_tests.port import base
+from webkitpy.layout_tests.port import factory
+
+_log = logging.getLogger(__name__)
+
+
+class MockDRTPort(object):
+ """MockPort implementation of the Port interface."""
+
+ def __init__(self, **kwargs):
+ prefix = 'mock-'
+ if 'port_name' in kwargs:
+ kwargs['port_name'] = kwargs['port_name'][len(prefix):]
+ self.__delegate = factory.get(**kwargs)
+ self.__real_name = prefix + self.__delegate.name()
+
+ def real_name(self):
+ return self.__real_name
+
+ def __getattr__(self, name):
+ return getattr(self.__delegate, name)
+
+ def acquire_http_lock(self):
+ pass
+
+ def release_http_lock(self):
+ pass
+
+ def check_build(self, needs_http):
+ return True
+
+ def check_sys_deps(self, needs_http):
+ return True
+
+ def driver_cmd_line(self):
+ driver = self.create_driver(0)
+ return driver.cmd_line()
+
+ def _path_to_driver(self):
+ return os.path.abspath(__file__)
+
+ def create_driver(self, worker_number):
+ # We need to create a driver object as the delegate would, but
+ # overwrite the path to the driver binary in its command line. We do
+ # this by actually overwriting its cmd_line() method with a proxy
+ # method that splices in the mock_drt path and command line arguments
+ # in place of the actual path to the driver binary.
+
+ def overriding_cmd_line():
+ cmd = self.__original_driver_cmd_line()
+ index = cmd.index(self.__delegate._path_to_driver())
+ cmd[index:index + 1] = [sys.executable, self._path_to_driver(),
+ '--platform', self.name()]
+ return cmd
+
+ delegated_driver = self.__delegate.create_driver(worker_number)
+ self.__original_driver_cmd_line = delegated_driver.cmd_line
+ delegated_driver.cmd_line = overriding_cmd_line
+ return delegated_driver
+
+ def start_helper(self):
+ pass
+
+ def start_http_server(self):
+ pass
+
+ def start_websocket_server(self):
+ pass
+
+ def stop_helper(self):
+ pass
+
+ def stop_http_server(self):
+ pass
+
+ def stop_websocket_server(self):
+ pass
+
+
+def main(argv, fs, stdin, stdout, stderr):
+ """Run the tests."""
+
+ options, args = parse_options(argv)
+ if options.chromium:
+ drt = MockChromiumDRT(options, args, fs, stdin, stdout, stderr)
+ else:
+ drt = MockDRT(options, args, fs, stdin, stdout, stderr)
+ return drt.run()
+
+
+def parse_options(argv):
+ # FIXME: We have to do custom arg parsing instead of using the optparse
+ # module. First, Chromium and non-Chromium DRTs have a different argument
+ # syntax. Chromium uses --pixel-tests=<path>, and non-Chromium uses
+ # --pixel-tests as a boolean flag. Second, we don't want to have to list
+ # every command line flag DRT accepts, but optparse complains about
+ # unrecognized flags. At some point it might be good to share a common
+ # DRT options class between this file and webkit.py and chromium.py
+ # just to get better type checking.
+ platform_index = argv.index('--platform')
+ platform = argv[platform_index + 1]
+
+ pixel_tests = False
+ pixel_path = None
+ chromium = False
+ if platform.startswith('chromium'):
+ chromium = True
+ for arg in argv:
+ if arg.startswith('--pixel-tests'):
+ pixel_tests = True
+ pixel_path = arg[len('--pixel-tests='):]
+ else:
+ pixel_tests = '--pixel-tests' in argv
+ options = base.DummyOptions(chromium=chromium,
+ platform=platform,
+ pixel_tests=pixel_tests,
+ pixel_path=pixel_path)
+ return (options, [])
+
+
+# FIXME: Should probably change this to use DriverInput after
+# https://bugs.webkit.org/show_bug.cgi?id=53004 lands.
+class _DRTInput(object):
+ def __init__(self, line):
+ vals = line.strip().split("'")
+ if len(vals) == 1:
+ self.uri = vals[0]
+ self.checksum = None
+ else:
+ self.uri = vals[0]
+ self.checksum = vals[1]
+
+
+class MockDRT(object):
+ def __init__(self, options, args, filesystem, stdin, stdout, stderr):
+ self._options = options
+ self._args = args
+ self._filesystem = filesystem
+ self._stdout = stdout
+ self._stdin = stdin
+ self._stderr = stderr
+
+ port_name = None
+ if options.platform:
+ port_name = options.platform
+ self._port = factory.get(port_name, options=options, filesystem=filesystem)
+
+ def run(self):
+ while True:
+ line = self._stdin.readline()
+ if not line:
+ break
+ self.run_one_test(self.parse_input(line))
+ return 0
+
+ def parse_input(self, line):
+ return _DRTInput(line)
+
+ def run_one_test(self, test_input):
+ port = self._port
+ if test_input.uri.startswith('http'):
+ test_name = port.uri_to_test_name(test_input.uri)
+ test_path = self._filesystem.join(port.layout_tests_dir(), test_name)
+ else:
+ test_path = test_input.uri
+
+ actual_text = port.expected_text(test_path)
+ if self._options.pixel_tests and test_input.checksum:
+ actual_checksum = port.expected_checksum(test_path)
+ actual_image = port.expected_image(test_path)
+
+ self._stdout.write('Content-Type: text/plain\n')
+
+ # FIXME: Note that we don't ensure there is a trailing newline!
+ # This mirrors actual (Mac) DRT behavior but is a bug.
+ self._stdout.write(actual_text)
+ self._stdout.write('#EOF\n')
+
+ if self._options.pixel_tests and test_input.checksum:
+ self._stdout.write('\n')
+ self._stdout.write('ActualHash: %s\n' % actual_checksum)
+ self._stdout.write('ExpectedHash: %s\n' % test_input.checksum)
+ if actual_checksum != test_input.checksum:
+ self._stdout.write('Content-Type: image/png\n')
+ self._stdout.write('Content-Length: %s\n\n' % len(actual_image))
+ self._stdout.write(actual_image)
+ self._stdout.write('#EOF\n')
+ self._stdout.flush()
+ self._stderr.flush()
+
+
+# FIXME: Should probably change this to use DriverInput after
+# https://bugs.webkit.org/show_bug.cgi?id=53004 lands.
+class _ChromiumDRTInput(_DRTInput):
+ def __init__(self, line):
+ vals = line.strip().split()
+ if len(vals) == 3:
+ self.uri, self.timeout, self.checksum = vals
+ else:
+ self.uri = vals[0]
+ self.timeout = vals[1]
+ self.checksum = None
+
+
+class MockChromiumDRT(MockDRT):
+ def parse_input(self, line):
+ return _ChromiumDRTInput(line)
+
+ def run_one_test(self, test_input):
+ port = self._port
+ test_name = self._port.uri_to_test_name(test_input.uri)
+ test_path = self._filesystem.join(port.layout_tests_dir(), test_name)
+
+ actual_text = port.expected_text(test_path)
+ actual_image = ''
+ actual_checksum = ''
+ if self._options.pixel_tests and test_input.checksum:
+ actual_checksum = port.expected_checksum(test_path)
+ if actual_checksum != test_input.checksum:
+ actual_image = port.expected_image(test_path)
+
+ self._stdout.write("#URL:%s\n" % test_input.uri)
+ if self._options.pixel_tests and test_input.checksum:
+ self._stdout.write("#MD5:%s\n" % actual_checksum)
+ self._filesystem.write_binary_file(self._options.pixel_path,
+ actual_image)
+ self._stdout.write(actual_text)
+
+ # FIXME: (See above FIXME as well). Chromium DRT appears to always
+ # ensure the text output has a trailing newline. Mac DRT does not.
+ if not actual_text.endswith('\n'):
+ self._stdout.write('\n')
+ self._stdout.write('#EOF\n')
+ self._stdout.flush()
+
+
+if __name__ == '__main__':
+ fs = filesystem.FileSystem()
+ sys.exit(main(sys.argv[1:], fs, sys.stdin, sys.stdout, sys.stderr))
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py
new file mode 100644
index 0000000..1506315
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py
@@ -0,0 +1,261 @@
+#!/usr/bin/env python
+# Copyright (C) 2011 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit tests for MockDRT."""
+
+import unittest
+
+from webkitpy.common import newstringio
+
+from webkitpy.layout_tests.port import mock_drt
+from webkitpy.layout_tests.port import factory
+from webkitpy.layout_tests.port import port_testcase
+from webkitpy.layout_tests.port import test
+
+
+class MockDRTPortTest(port_testcase.PortTestCase):
+ def make_port(self):
+ return mock_drt.MockDRTPort()
+
+ def test_port_name_in_constructor(self):
+ self.assertTrue(mock_drt.MockDRTPort(port_name='mock-test'))
+
+ def test_acquire_http_lock(self):
+ # Only checking that no exception is raised.
+ self.make_port().acquire_http_lock()
+
+ def test_release_http_lock(self):
+ # Only checking that no exception is raised.
+ self.make_port().release_http_lock()
+
+ def test_check_build(self):
+ port = self.make_port()
+ self.assertTrue(port.check_build(True))
+
+ def test_check_sys_deps(self):
+ port = self.make_port()
+ self.assertTrue(port.check_sys_deps(True))
+
+ def test_start_helper(self):
+ # Only checking that no exception is raised.
+ self.make_port().start_helper()
+
+ def test_start_http_server(self):
+ # Only checking that no exception is raised.
+ self.make_port().start_http_server()
+
+ def test_start_websocket_server(self):
+ # Only checking that no exception is raised.
+ self.make_port().start_websocket_server()
+
+ def test_stop_helper(self):
+ # Only checking that no exception is raised.
+ self.make_port().stop_helper()
+
+ def test_stop_http_server(self):
+ # Only checking that no exception is raised.
+ self.make_port().stop_http_server()
+
+ def test_stop_websocket_server(self):
+ # Only checking that no exception is raised.
+ self.make_port().stop_websocket_server()
+
+
+class MockDRTTest(unittest.TestCase):
+ def to_path(self, port, test_name):
+ return port._filesystem.join(port.layout_tests_dir(), test_name)
+
+ def input_line(self, port, test_name, checksum=None):
+ url = port.filename_to_uri(self.to_path(port, test_name))
+ # FIXME: we shouldn't have to work around platform-specific issues
+ # here.
+ if url.startswith('file:////'):
+ url = url[len('file:////') - 1:]
+ if url.startswith('file:///'):
+ url = url[len('file:///') - 1:]
+
+ if checksum:
+ return url + "'" + checksum + '\n'
+ return url + '\n'
+
+ def extra_args(self, pixel_tests):
+ if pixel_tests:
+ return ['--pixel-tests', '-']
+ return ['-']
+
+ def make_drt(self, options, args, filesystem, stdin, stdout, stderr):
+ return mock_drt.MockDRT(options, args, filesystem, stdin, stdout, stderr)
+
+ def make_input_output(self, port, test_name, pixel_tests,
+ expected_checksum, drt_output, drt_input=None):
+ path = self.to_path(port, test_name)
+ if pixel_tests:
+ if not expected_checksum:
+ expected_checksum = port.expected_checksum(path)
+ if not drt_input:
+ drt_input = self.input_line(port, test_name, expected_checksum)
+ text_output = port.expected_text(path)
+
+ if not drt_output:
+ drt_output = self.expected_output(port, test_name, pixel_tests,
+ text_output, expected_checksum)
+ return (drt_input, drt_output)
+
+ def expected_output(self, port, test_name, pixel_tests, text_output, expected_checksum):
+ if pixel_tests and expected_checksum:
+ return ['Content-Type: text/plain\n',
+ text_output,
+ '#EOF\n',
+ '\n',
+ 'ActualHash: %s\n' % expected_checksum,
+ 'ExpectedHash: %s\n' % expected_checksum,
+ '#EOF\n']
+ else:
+ return ['Content-Type: text/plain\n',
+ text_output,
+ '#EOF\n',
+ '#EOF\n']
+
+ def assertTest(self, test_name, pixel_tests, expected_checksum=None,
+ drt_output=None, filesystem=None):
+ platform = 'test'
+ filesystem = filesystem or test.unit_test_filesystem()
+ port = factory.get(platform, filesystem=filesystem)
+ drt_input, drt_output = self.make_input_output(port, test_name,
+ pixel_tests, expected_checksum, drt_output)
+
+ args = ['--platform', 'test'] + self.extra_args(pixel_tests)
+ stdin = newstringio.StringIO(drt_input)
+ stdout = newstringio.StringIO()
+ stderr = newstringio.StringIO()
+ options, args = mock_drt.parse_options(args)
+
+ drt = self.make_drt(options, args, filesystem, stdin, stdout, stderr)
+ res = drt.run()
+
+ self.assertEqual(res, 0)
+
+ # We use the StringIO.buflist here instead of getvalue() because
+ # the StringIO might be a mix of unicode/ascii and 8-bit strings.
+ self.assertEqual(stdout.buflist, drt_output)
+ self.assertEqual(stderr.getvalue(), '')
+
+ def test_main(self):
+ filesystem = test.unit_test_filesystem()
+ stdin = newstringio.StringIO()
+ stdout = newstringio.StringIO()
+ stderr = newstringio.StringIO()
+ res = mock_drt.main(['--platform', 'test'] + self.extra_args(False),
+ filesystem, stdin, stdout, stderr)
+ self.assertEqual(res, 0)
+ self.assertEqual(stdout.getvalue(), '')
+ self.assertEqual(stderr.getvalue(), '')
+ self.assertEqual(filesystem.written_files, {})
+
+ def test_pixeltest_passes(self):
+ # This also tests that we handle HTTP: test URLs properly.
+ self.assertTest('http/tests/passes/text.html', True)
+
+ def test_pixeltest__fails(self):
+ self.assertTest('failures/expected/checksum.html', pixel_tests=True,
+ expected_checksum='wrong-checksum',
+ drt_output=['Content-Type: text/plain\n',
+ 'checksum-txt',
+ '#EOF\n',
+ '\n',
+ 'ActualHash: checksum-checksum\n',
+ 'ExpectedHash: wrong-checksum\n',
+ 'Content-Type: image/png\n',
+ 'Content-Length: 13\n\n',
+ 'checksum\x8a-png',
+ '#EOF\n'])
+
+ def test_textonly(self):
+ self.assertTest('passes/image.html', False)
+
+
+class MockChromiumDRTTest(MockDRTTest):
+ def extra_args(self, pixel_tests):
+ if pixel_tests:
+ return ['--pixel-tests=/tmp/png_result0.png']
+ return []
+
+ def make_drt(self, options, args, filesystem, stdin, stdout, stderr):
+ options.chromium = True
+
+ # We have to set these by hand because --platform test won't trigger
+ # the Chromium code paths.
+ options.pixel_path = '/tmp/png_result0.png'
+ options.pixel_tests = True
+
+ return mock_drt.MockChromiumDRT(options, args, filesystem, stdin, stdout, stderr)
+
+ def input_line(self, port, test_name, checksum=None):
+ url = port.filename_to_uri(self.to_path(port, test_name))
+ if checksum:
+ return url + ' 6000 ' + checksum + '\n'
+ return url + ' 6000\n'
+
+ def expected_output(self, port, test_name, pixel_tests, text_output, expected_checksum):
+ url = port.filename_to_uri(self.to_path(port, test_name))
+ if pixel_tests and expected_checksum:
+ return ['#URL:%s\n' % url,
+ '#MD5:%s\n' % expected_checksum,
+ text_output,
+ '\n',
+ '#EOF\n']
+ else:
+ return ['#URL:%s\n' % url,
+ text_output,
+ '\n',
+ '#EOF\n']
+
+ def test_pixeltest__fails(self):
+ filesystem = test.unit_test_filesystem()
+ self.assertTest('failures/expected/checksum.html', pixel_tests=True,
+ expected_checksum='wrong-checksum',
+ drt_output=['#URL:file:///test.checkout/LayoutTests/failures/expected/checksum.html\n',
+ '#MD5:checksum-checksum\n',
+ 'checksum-txt',
+ '\n',
+ '#EOF\n'],
+ filesystem=filesystem)
+ self.assertEquals(filesystem.written_files,
+ {'/tmp/png_result0.png': 'checksum\x8a-png'})
+
+ def test_chromium_parse_options(self):
+ options, args = mock_drt.parse_options(['--platform', 'chromium-mac',
+ '--pixel-tests=/tmp/png_result0.png'])
+ self.assertTrue(options.chromium)
+ self.assertTrue(options.pixel_tests)
+ self.assertEquals(options.pixel_path, '/tmp/png_result0.png')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py b/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py
index 0b03b4c..4146d40 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py
@@ -88,3 +88,15 @@ class PortTestCase(unittest.TestCase):
return
port.start_websocket_server()
port.stop_websocket_server()
+
+ def test_test_configuration(self):
+ port = self.make_port()
+ if not port:
+ return
+ self.assertTrue(port.test_configuration())
+
+ def test_all_test_configurations(self):
+ port = self.make_port()
+ if not port:
+ return
+ self.assertTrue(len(port.all_test_configurations()) > 0)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/server_process.py b/Tools/Scripts/webkitpy/layout_tests/port/server_process.py
index 5a0a40c..7974f94 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/server_process.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/server_process.py
@@ -115,7 +115,11 @@ class ServerProcess:
if is not already running."""
if not self._proc:
self._start()
- self._proc.stdin.write(input)
+ try:
+ self._proc.stdin.write(input)
+ except IOError, e:
+ self.stop()
+ self.crashed = True
def read_line(self, timeout):
"""Read a single line from the subprocess, waiting until the deadline.
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/server_process_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/server_process_unittest.py
new file mode 100644
index 0000000..f3429cb
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/server_process_unittest.py
@@ -0,0 +1,77 @@
+# Copyright (C) 2011 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import unittest
+
+from webkitpy.layout_tests.port import server_process
+
+
+class MockFile(object):
+ def __init__(self, server_process):
+ self._server_process = server_process
+
+ def fileno(self):
+ return 1
+
+ def write(self, line):
+ self._server_process.broken_pipes.append(self)
+ raise IOError
+
+ def close(self):
+ pass
+
+
+class MockProc(object):
+ def __init__(self, server_process):
+ self.stdin = MockFile(server_process)
+ self.stdout = MockFile(server_process)
+ self.stderr = MockFile(server_process)
+ self.pid = 1
+
+ def poll(self):
+ return 1
+
+
+class FakeServerProcess(server_process.ServerProcess):
+ def _start(self):
+ self._proc = MockProc(self)
+ self.stdin = self._proc.stdin
+ self.broken_pipes = []
+
+
+class TestServerProcess(unittest.TestCase):
+ def test_broken_pipe(self):
+ server_process = FakeServerProcess(port_obj=None, name="test", cmd=["test"])
+ server_process.write("should break")
+ self.assertTrue(server_process.crashed)
+ self.assertEquals(server_process._proc, None)
+ self.assertEquals(server_process.broken_pipes, [server_process.stdin])
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/test.py b/Tools/Scripts/webkitpy/layout_tests/port/test.py
index 5df5c2d..b94c378 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/test.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/test.py
@@ -33,8 +33,7 @@ from __future__ import with_statement
import time
from webkitpy.common.system import filesystem_mock
-
-from webkitpy.layout_tests.layout_package import test_output
+from webkitpy.tool import mocktool
import base
@@ -51,9 +50,17 @@ class TestInstance:
self.keyboard = False
self.error = ''
self.timeout = False
- self.actual_text = self.base + '-txt\n'
- self.actual_checksum = self.base + '-checksum\n'
- self.actual_image = self.base + '-png\n'
+
+ # The values of each field are treated as raw byte strings. They
+ # will be converted to unicode strings where appropriate using
+ # MockFileSystem.read_text_file().
+ self.actual_text = self.base + '-txt'
+ self.actual_checksum = self.base + '-checksum'
+
+ # We add the '\x8a' for the image file to prevent the value from
+ # being treated as UTF-8 (the character is invalid)
+ self.actual_image = self.base + '\x8a' + '-png'
+
self.expected_text = self.actual_text
self.expected_checksum = self.actual_checksum
self.expected_image = self.actual_image
@@ -84,53 +91,44 @@ class TestList:
def unit_test_list():
tests = TestList()
tests.add('failures/expected/checksum.html',
- actual_checksum='checksum_fail-checksum')
+ actual_checksum='checksum_fail-checksum')
tests.add('failures/expected/crash.html', crash=True)
tests.add('failures/expected/exception.html', exception=True)
tests.add('failures/expected/timeout.html', timeout=True)
tests.add('failures/expected/hang.html', hang=True)
- tests.add('failures/expected/missing_text.html',
- expected_text=None)
+ tests.add('failures/expected/missing_text.html', expected_text=None)
tests.add('failures/expected/image.html',
- actual_image='image_fail-png',
- expected_image='image-png')
+ actual_image='image_fail-png',
+ expected_image='image-png')
tests.add('failures/expected/image_checksum.html',
- actual_checksum='image_checksum_fail-checksum',
- actual_image='image_checksum_fail-png')
- tests.add('failures/expected/keyboard.html',
- keyboard=True)
- tests.add('failures/expected/missing_check.html',
- expected_checksum=None)
- tests.add('failures/expected/missing_image.html',
- expected_image=None)
- tests.add('failures/expected/missing_text.html',
- expected_text=None)
+ actual_checksum='image_checksum_fail-checksum',
+ actual_image='image_checksum_fail-png')
+ tests.add('failures/expected/keyboard.html', keyboard=True)
+ tests.add('failures/expected/missing_check.html', expected_checksum=None)
+ tests.add('failures/expected/missing_image.html', expected_image=None)
+ tests.add('failures/expected/missing_text.html', expected_text=None)
tests.add('failures/expected/newlines_leading.html',
- expected_text="\nfoo\n",
- actual_text="foo\n")
+ expected_text="\nfoo\n", actual_text="foo\n")
tests.add('failures/expected/newlines_trailing.html',
- expected_text="foo\n\n",
- actual_text="foo\n")
+ expected_text="foo\n\n", actual_text="foo\n")
tests.add('failures/expected/newlines_with_excess_CR.html',
- expected_text="foo\r\r\r\n",
- actual_text="foo\n")
- tests.add('failures/expected/text.html',
- actual_text='text_fail-png')
+ expected_text="foo\r\r\r\n", actual_text="foo\n")
+ tests.add('failures/expected/text.html', actual_text='text_fail-png')
tests.add('failures/unexpected/crash.html', crash=True)
tests.add('failures/unexpected/text-image-checksum.html',
- actual_text='text-image-checksum_fail-txt',
- actual_checksum='text-image-checksum_fail-checksum')
+ actual_text='text-image-checksum_fail-txt',
+ actual_checksum='text-image-checksum_fail-checksum')
tests.add('failures/unexpected/timeout.html', timeout=True)
tests.add('http/tests/passes/text.html')
tests.add('http/tests/ssl/text.html')
tests.add('passes/error.html', error='stuff going to stderr')
tests.add('passes/image.html')
tests.add('passes/platform_image.html')
+
# Text output files contain "\r\n" on Windows. This may be
# helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
tests.add('passes/text.html',
- expected_text='\nfoo\n\n',
- actual_text='\nfoo\r\n\r\r\n')
+ expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n')
tests.add('websocket/tests/passes/text.html')
return tests
@@ -184,6 +182,9 @@ WONTFIX SKIP : failures/expected/keyboard.html = CRASH
WONTFIX SKIP : failures/expected/exception.html = CRASH
"""
+ # Add in a file should be ignored by test_files.find().
+ files[LAYOUT_TEST_DIR + 'userscripts/resources/iframe.html'] = 'iframe'
+
fs = filesystem_mock.MockFileSystem(files)
fs._tests = test_list
return fs
@@ -192,30 +193,31 @@ WONTFIX SKIP : failures/expected/exception.html = CRASH
class TestPort(base.Port):
"""Test implementation of the Port interface."""
- def __init__(self, **kwargs):
- # FIXME: what happens if we're not passed in the test filesystem
- # and the tests don't match what's in the filesystem?
- #
- # We'll leave as is for now to avoid unnecessary dependencies while
- # converting all of the unit tests over to using
- # unit_test_filesystem(). If things get out of sync the tests should
- # fail in fairly obvious ways. Eventually we want to just do:
- #
- # assert kwargs['filesystem']._tests
- # self._tests = kwargs['filesystem']._tests
+ def __init__(self, port_name=None, user=None, filesystem=None, **kwargs):
+ if not filesystem:
+ filesystem = unit_test_filesystem()
+
+ assert filesystem._tests
+ self._tests = filesystem._tests
+
+ if not user:
+ user = mocktool.MockUser()
- if 'filesystem' not in kwargs or kwargs['filesystem'] is None:
- kwargs['filesystem'] = unit_test_filesystem()
- self._tests = kwargs['filesystem']._tests
- else:
- self._tests = unit_test_list()
+ if not port_name or port_name == 'test':
+ port_name = 'test-mac'
- kwargs.setdefault('port_name', 'test')
- base.Port.__init__(self, **kwargs)
+ self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt'
+ base.Port.__init__(self, port_name=port_name, filesystem=filesystem, user=user,
+ **kwargs)
+
+ def _path_to_driver(self):
+ # This routine shouldn't normally be called, but it is called by
+ # the mock_drt Driver. We return something, but make sure it's useless.
+ return 'junk'
def baseline_path(self):
- return self._filesystem.join(self.layout_tests_dir(), 'platform',
- self.name() + self.version())
+ # We don't bother with a fallback path.
+ return self._filesystem.join(self.layout_tests_dir(), 'platform', self.name())
def baseline_search_path(self):
return [self.baseline_path()]
@@ -223,11 +225,14 @@ class TestPort(base.Port):
def check_build(self, needs_http):
return True
+ def default_configuration(self):
+ return 'Release'
+
def diff_image(self, expected_contents, actual_contents,
diff_filename=None):
diffed = actual_contents != expected_contents
if diffed and diff_filename:
- self._filesystem.write_text_file(diff_filename,
+ self._filesystem.write_binary_file(diff_filename,
"< %s\n---\n> %s\n" % (expected_contents, actual_contents))
return diffed
@@ -261,23 +266,98 @@ class TestPort(base.Port):
def stop_websocket_server(self):
pass
- def test_base_platform_names(self):
- return ('mac', 'win')
-
- def test_expectations(self):
- return self._filesystem.read_text_file(LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt')
+ def path_to_test_expectations_file(self):
+ return self._expectations_path
def test_platform_name(self):
- return 'mac'
+ name_map = {
+ 'test-mac': 'mac',
+ 'test-win': 'win',
+ 'test-win-xp': 'win-xp',
+ }
+ return name_map[self._name]
def test_platform_names(self):
- return self.test_base_platform_names()
+ return ('mac', 'win', 'win-xp')
def test_platform_name_to_name(self, test_platform_name):
- return test_platform_name
+ name_map = {
+ 'mac': 'test-mac',
+ 'win': 'test-win',
+ 'win-xp': 'test-win-xp',
+ }
+ return name_map[test_platform_name]
+
+ # FIXME: These next two routines are copied from base.py with
+ # the calls to path.abspath_to_uri() removed. We shouldn't have
+ # to do this.
+ def filename_to_uri(self, filename):
+ """Convert a test file (which is an absolute path) to a URI."""
+ LAYOUTTEST_HTTP_DIR = "http/tests/"
+ LAYOUTTEST_WEBSOCKET_DIR = "http/tests/websocket/tests/"
+
+ relative_path = self.relative_test_filename(filename)
+ port = None
+ use_ssl = False
+
+ if (relative_path.startswith(LAYOUTTEST_WEBSOCKET_DIR)
+ or relative_path.startswith(LAYOUTTEST_HTTP_DIR)):
+ relative_path = relative_path[len(LAYOUTTEST_HTTP_DIR):]
+ port = 8000
+
+ # Make http/tests/local run as local files. This is to mimic the
+ # logic in run-webkit-tests.
+ #
+ # TODO(dpranke): remove the media reference and the SSL reference?
+ if (port and not relative_path.startswith("local/") and
+ not relative_path.startswith("media/")):
+ if relative_path.startswith("ssl/"):
+ port += 443
+ protocol = "https"
+ else:
+ protocol = "http"
+ return "%s://127.0.0.1:%u/%s" % (protocol, port, relative_path)
+
+ return "file://" + self._filesystem.abspath(filename)
+
+ def uri_to_test_name(self, uri):
+ """Return the base layout test name for a given URI.
+
+ This returns the test name for a given URI, e.g., if you passed in
+ "file:///src/LayoutTests/fast/html/keygen.html" it would return
+ "fast/html/keygen.html".
+
+ """
+ test = uri
+ if uri.startswith("file:///"):
+ prefix = "file://" + self.layout_tests_dir() + "/"
+ return test[len(prefix):]
+
+ if uri.startswith("http://127.0.0.1:8880/"):
+ # websocket tests
+ return test.replace('http://127.0.0.1:8880/', '')
+
+ if uri.startswith("http://"):
+ # regular HTTP test
+ return test.replace('http://127.0.0.1:8000/', 'http/tests/')
+
+ if uri.startswith("https://"):
+ return test.replace('https://127.0.0.1:8443/', 'http/tests/')
+
+ raise NotImplementedError('unknown url type: %s' % uri)
def version(self):
- return ''
+ version_map = {
+ 'test-win-xp': '-xp',
+ 'test-win': '-7',
+ 'test-mac': '-leopard',
+ }
+ return version_map[self._name]
+
+ def test_configuration(self):
+ if not self._test_configuration:
+ self._test_configuration = TestTestConfiguration(self)
+ return self._test_configuration
class TestDriver(base.Driver):
@@ -287,7 +367,7 @@ class TestDriver(base.Driver):
self._port = port
def cmd_line(self):
- return ['None']
+ return [self._port._path_to_driver()]
def poll(self):
return True
@@ -302,13 +382,20 @@ class TestDriver(base.Driver):
raise ValueError('exception from ' + test_name)
if test.hang:
time.sleep((float(test_input.timeout) * 4) / 1000.0)
- return test_output.TestOutput(test.actual_text, test.actual_image,
- test.actual_checksum, test.crash,
- time.time() - start_time, test.timeout,
- test.error)
+ return base.DriverOutput(test.actual_text, test.actual_image,
+ test.actual_checksum, test.crash,
+ time.time() - start_time, test.timeout,
+ test.error)
def start(self):
pass
def stop(self):
pass
+
+
+class TestTestConfiguration(base.TestConfiguration):
+ def all_systems(self):
+ return (('mac', 'leopard', 'x86'),
+ ('win', 'xp', 'x86'),
+ ('win', 'win7', 'x86'))
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/test_files.py b/Tools/Scripts/webkitpy/layout_tests/port/test_files.py
index 41d918f..534462a 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/test_files.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/test_files.py
@@ -49,37 +49,47 @@ _supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.xhtmlmp
_skipped_directories = set(['.svn', '_svn', 'resources', 'script-tests'])
-def find(port, paths):
- """Finds the set of tests under port.layout_tests_dir().
+def find(port, paths=None):
+ """Finds the set of tests under a given list of sub-paths.
Args:
- paths: a list of command line paths relative to the layout_tests_dir()
- to limit the search to. glob patterns are ok.
+ paths: a list of path expressions relative to port.layout_tests_dir()
+ to search. Glob patterns are ok, as are path expressions with
+ forward slashes on Windows. If paths is empty, we look at
+ everything under the layout_tests_dir().
+ """
+ paths = paths or ['*']
+ filesystem = port._filesystem
+ return normalized_find(filesystem, normalize(filesystem, port.layout_tests_dir(), paths))
+
+
+def normalize(filesystem, base_dir, paths):
+ return [filesystem.normpath(filesystem.join(base_dir, path)) for path in paths]
+
+
+def normalized_find(filesystem, paths):
+ """Finds the set of tests under the list of paths.
+
+ Args:
+ paths: a list of absolute path expressions to search.
+ Glob patterns are ok.
"""
- fs = port._filesystem
gather_start_time = time.time()
paths_to_walk = set()
- # if paths is empty, provide a pre-defined list.
- if paths:
- _log.debug("Gathering tests from: %s relative to %s" % (paths, port.layout_tests_dir()))
- for path in paths:
- # If there's an * in the name, assume it's a glob pattern.
- path = fs.join(port.layout_tests_dir(), path)
- if path.find('*') > -1:
- filenames = fs.glob(path)
- paths_to_walk.update(filenames)
- else:
- paths_to_walk.add(path)
- else:
- _log.debug("Gathering tests from: %s" % port.layout_tests_dir())
- paths_to_walk.add(port.layout_tests_dir())
+ for path in paths:
+ # If there's an * in the name, assume it's a glob pattern.
+ if path.find('*') > -1:
+ filenames = filesystem.glob(path)
+ paths_to_walk.update(filenames)
+ else:
+ paths_to_walk.add(path)
# FIXME: I'm not sure there's much point in this being a set. A list would
# probably be faster.
test_files = set()
for path in paths_to_walk:
- files = fs.files_under(path, _skipped_directories, _is_test_file)
+ files = filesystem.files_under(path, _skipped_directories, _is_test_file)
test_files.update(set(files))
gather_time = time.time() - gather_start_time
@@ -88,10 +98,10 @@ def find(port, paths):
return test_files
-def _has_supported_extension(fs, filename):
+def _has_supported_extension(filesystem, filename):
"""Return true if filename is one of the file extensions we want to run a
test on."""
- extension = fs.splitext(filename)[1]
+ extension = filesystem.splitext(filename)[1]
return extension in _supported_file_extensions
@@ -104,7 +114,7 @@ def _is_reference_html_file(filename):
return False
-def _is_test_file(fs, dirname, filename):
+def _is_test_file(filesystem, dirname, filename):
"""Return true if the filename points to a test file."""
- return (_has_supported_extension(fs, filename) and
+ return (_has_supported_extension(filesystem, filename) and
not _is_reference_html_file(filename))
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py
index a68950a..a29ba49 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py
@@ -26,44 +26,41 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+import sys
import unittest
-import base
+from webkitpy.layout_tests.port import test
import test_files
-
class TestFilesTest(unittest.TestCase):
def test_find_no_paths_specified(self):
- port = base.Port()
+ port = test.TestPort()
layout_tests_dir = port.layout_tests_dir()
- port.layout_tests_dir = lambda: port._filesystem.join(layout_tests_dir,
- 'fast', 'html')
tests = test_files.find(port, [])
- self.assertNotEqual(tests, 0)
+ self.assertNotEqual(len(tests), 0)
def test_find_one_test(self):
- port = base.Port()
- # This is just a test picked at random but known to exist.
- tests = test_files.find(port, ['fast/html/keygen.html'])
+ port = test.TestPort()
+ tests = test_files.find(port, ['failures/expected/image.html'])
self.assertEqual(len(tests), 1)
def test_find_glob(self):
- port = base.Port()
- tests = test_files.find(port, ['fast/html/key*'])
- self.assertEqual(len(tests), 1)
+ port = test.TestPort()
+ tests = test_files.find(port, ['failures/expected/im*'])
+ self.assertEqual(len(tests), 2)
def test_find_with_skipped_directories(self):
- port = base.Port()
+ port = test.TestPort()
tests = port.tests('userscripts')
- self.assertTrue('userscripts/resources/frame1.html' not in tests)
+ self.assertTrue('userscripts/resources/iframe.html' not in tests)
def test_find_with_skipped_directories_2(self):
- port = base.Port()
+ port = test.TestPort()
tests = test_files.find(port, ['userscripts/resources'])
self.assertEqual(tests, set([]))
def test_is_test_file(self):
- port = base.Port()
+ port = test.TestPort()
fs = port._filesystem
self.assertTrue(test_files._is_test_file(fs, '', 'foo.html'))
self.assertTrue(test_files._is_test_file(fs, '', 'foo.shtml'))
@@ -72,5 +69,33 @@ class TestFilesTest(unittest.TestCase):
self.assertFalse(test_files._is_test_file(fs, '', 'foo-expected-mismatch.html'))
+class MockWinFileSystem(object):
+ def join(self, *paths):
+ return '\\'.join(paths)
+
+ def normpath(self, path):
+ return path.replace('/', '\\')
+
+
+class TestWinNormalize(unittest.TestCase):
+ def assert_filesystem_normalizes(self, filesystem):
+ self.assertEquals(test_files.normalize(filesystem, "c:\\foo",
+ ['fast/html', 'fast/canvas/*', 'compositing/foo.html']),
+ ['c:\\foo\\fast\html', 'c:\\foo\\fast\canvas\*', 'c:\\foo\compositing\\foo.html'])
+
+ def test_mocked_win(self):
+ # This tests test_files.normalize, using portable behavior emulating
+ # what we think Windows is supposed to do. This test will run on all
+ # platforms.
+ self.assert_filesystem_normalizes(MockWinFileSystem())
+
+ def test_win(self):
+ # This tests the actual windows platform, to ensure we get the same
+ # results that we get in test_mocked_win().
+ if sys.platform != 'win':
+ return
+ self.assert_filesystem_normalizes(FileSystem())
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/webkit.py b/Tools/Scripts/webkitpy/layout_tests/port/webkit.py
index 577acd4..65a047d 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/webkit.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/webkit.py
@@ -40,10 +40,9 @@ import sys
import time
import webbrowser
-import webkitpy.common.system.ospath as ospath
-import webkitpy.layout_tests.layout_package.test_output as test_output
-import webkitpy.layout_tests.port.base as base
-import webkitpy.layout_tests.port.server_process as server_process
+from webkitpy.common.system import ospath
+from webkitpy.layout_tests.port import base
+from webkitpy.layout_tests.port import server_process
_log = logging.getLogger("webkitpy.layout_tests.port.webkit")
@@ -57,7 +56,8 @@ class WebKitPort(base.Port):
# FIXME: disable pixel tests until they are run by default on the
# build machines.
- self.set_option_default('pixel_tests', False)
+ if not hasattr(self._options, "pixel_tests") or self._options.pixel_tests == None:
+ self._options.pixel_tests = False
def baseline_path(self):
return self._webkit_baseline_path(self._name)
@@ -120,9 +120,9 @@ class WebKitPort(base.Port):
return self._diff_image_reply(sp, diff_filename)
def _diff_image_request(self, expected_contents, actual_contents):
- # FIXME: use self.get_option('tolerance') and
- # self.set_option_default('tolerance', 0.1) once that behaves correctly
- # with default values.
+ # FIXME: There needs to be a more sane way of handling default
+ # values for options so that you can distinguish between a default
+ # value of None and a default value that wasn't set.
if self.get_option('tolerance') is not None:
tolerance = self.get_option('tolerance')
else:
@@ -159,7 +159,7 @@ class WebKitPort(base.Port):
if m.group(2) == 'passed':
result = False
elif output and diff_filename:
- self._filesystem.write_text_file(diff_filename, output)
+ self._filesystem.write_binary_file(diff_filename, output)
elif sp.timed_out:
_log.error("ImageDiff timed out")
elif sp.crashed:
@@ -179,11 +179,6 @@ class WebKitPort(base.Port):
def create_driver(self, worker_number):
return WebKitDriver(self, worker_number)
- def test_base_platform_names(self):
- # At the moment we don't use test platform names, but we have
- # to return something.
- return ('mac', 'win')
-
def _tests_for_other_platforms(self):
raise NotImplementedError('WebKitPort._tests_for_other_platforms')
# The original run-webkit-tests builds up a "whitelist" of tests to
@@ -283,9 +278,9 @@ class WebKitPort(base.Port):
unsupported_feature_tests = self._skipped_tests_for_unsupported_features()
return disabled_feature_tests + webarchive_tests + unsupported_feature_tests
- def _tests_from_skipped_file(self, skipped_file):
+ def _tests_from_skipped_file_contents(self, skipped_file_contents):
tests_to_skip = []
- for line in skipped_file.readlines():
+ for line in skipped_file_contents.split('\n'):
line = line.strip()
if line.startswith('#') or not len(line):
continue
@@ -301,7 +296,8 @@ class WebKitPort(base.Port):
if not self._filesystem.exists(filename):
_log.warn("Failed to open Skipped file: %s" % filename)
continue
- skipped_file = self._filesystem.read_text_file(filename)
+ skipped_file_contents = self._filesystem.read_text_file(filename)
+ tests_to_skip.extend(self._tests_from_skipped_file_contents(skipped_file_contents))
return tests_to_skip
def test_expectations(self):
@@ -335,8 +331,7 @@ class WebKitPort(base.Port):
return self._name + self.version()
def test_platform_names(self):
- return self.test_base_platform_names() + (
- 'mac-tiger', 'mac-leopard', 'mac-snowleopard')
+ return ('mac', 'win', 'mac-tiger', 'mac-leopard', 'mac-snowleopard')
def _build_path(self, *comps):
return self._filesystem.join(self._config.build_directory(
@@ -409,15 +404,15 @@ class WebKitDriver(base.Driver):
return
# FIXME: This function is huge.
- def run_test(self, test_input):
- uri = self._port.filename_to_uri(test_input.filename)
+ def run_test(self, driver_input):
+ uri = self._port.filename_to_uri(driver_input.filename)
if uri.startswith("file:///"):
command = uri[7:]
else:
command = uri
- if test_input.image_hash:
- command += "'" + test_input.image_hash
+ if driver_input.image_hash:
+ command += "'" + driver_input.image_hash
command += "\n"
start_time = time.time()
@@ -428,7 +423,7 @@ class WebKitDriver(base.Driver):
output = str() # Use a byte array for output, even though it should be UTF-8.
image = str()
- timeout = int(test_input.timeout) / 1000.0
+ timeout = int(driver_input.timeout) / 1000.0
deadline = time.time() + timeout
line = self._server_process.read_line(timeout)
while (not self._server_process.timed_out
@@ -475,11 +470,11 @@ class WebKitDriver(base.Driver):
# FIXME: This seems like the wrong section of code to be doing
# this reset in.
self._server_process.error = ""
- return test_output.TestOutput(output, image, actual_image_hash,
- self._server_process.crashed,
- time.time() - start_time,
- self._server_process.timed_out,
- error)
+ return base.DriverOutput(output, image, actual_image_hash,
+ self._server_process.crashed,
+ time.time() - start_time,
+ self._server_process.timed_out,
+ error)
def stop(self):
if self._server_process:
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py
index 7b68310..c72a411 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python
# Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged
+# Copyright (C) 2010 Google Inc. All rights reserved.
#
# All rights reserved.
#
@@ -26,13 +27,19 @@
import unittest
+from webkitpy.common.system import filesystem_mock
+
from webkitpy.layout_tests.port.webkit import WebKitPort
class TestWebKitPort(WebKitPort):
- def __init__(self, symbol_list=None, feature_list=None):
+ def __init__(self, symbol_list=None, feature_list=None,
+ expectations_file=None, skips_file=None, **kwargs):
self.symbol_list = symbol_list
self.feature_list = feature_list
+ self.expectations_file = expectations_file
+ self.skips_file = skips_file
+ WebKitPort.__init__(self, **kwargs)
def _runtime_feature_list(self):
return self.feature_list
@@ -46,7 +53,14 @@ class TestWebKitPort(WebKitPort):
def _tests_for_disabled_features(self):
return ["accessibility", ]
+ def path_to_test_expectations_file(self):
+ if self.expectations_file:
+ return self.expectations_file
+ return WebKitPort.path_to_test_expectations_file(self)
+
def _skipped_file_paths(self):
+ if self.skips_file:
+ return [self.skips_file]
return []
class WebKitPortTest(unittest.TestCase):
@@ -66,3 +80,23 @@ class WebKitPortTest(unittest.TestCase):
def test_skipped_layout_tests(self):
self.assertEqual(TestWebKitPort(None, None).skipped_layout_tests(),
set(["media", "accessibility"]))
+
+ def test_test_expectations(self):
+ # Check that we read both the expectations file and anything in a
+ # Skipped file, and that we include the feature and platform checks.
+ files = {
+ '/tmp/test_expectations.txt': 'BUG_TESTEXPECTATIONS SKIP : fast/html/article-element.html = FAIL\n',
+ '/tmp/Skipped': 'fast/html/keygen.html',
+ }
+ mock_fs = filesystem_mock.MockFileSystem(files)
+ port = TestWebKitPort(expectations_file='/tmp/test_expectations.txt',
+ skips_file='/tmp/Skipped', filesystem=mock_fs)
+ self.assertEqual(port.test_expectations(),
+ """BUG_TESTEXPECTATIONS SKIP : fast/html/article-element.html = FAIL
+BUG_SKIPPED SKIP : fast/html/keygen.html = FAIL
+BUG_SKIPPED SKIP : media = FAIL
+BUG_SKIPPED SKIP : accessibility = FAIL""")
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/websocket_server.py b/Tools/Scripts/webkitpy/layout_tests/port/websocket_server.py
index 926bc04..713ad21 100644
--- a/Tools/Scripts/webkitpy/layout_tests/port/websocket_server.py
+++ b/Tools/Scripts/webkitpy/layout_tests/port/websocket_server.py
@@ -73,7 +73,7 @@ def url_is_alive(url):
wait_time = 10
while wait_time > 0:
try:
- response = urllib.urlopen(url)
+ response = urllib.urlopen(url, proxies={})
# Server is up and responding.
return True
except IOError:
diff --git a/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py b/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py
index c852186..567975c 100644
--- a/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py
+++ b/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py
@@ -47,18 +47,17 @@ import optparse
import re
import sys
import time
-import urllib
-import zipfile
from webkitpy.common.checkout import scm
+from webkitpy.common.system import zipfileset
from webkitpy.common.system import path
+from webkitpy.common.system import urlfetcher
from webkitpy.common.system.executive import ScriptError
-import port
-from layout_package import test_expectations
+from webkitpy.layout_tests import port
+from webkitpy.layout_tests.layout_package import test_expectations
-_log = logging.getLogger("webkitpy.layout_tests."
- "rebaseline_chromium_webkit_tests")
+_log = logging.getLogger(__name__)
BASELINE_SUFFIXES = ['.txt', '.png', '.checksum']
REBASELINE_PLATFORM_ORDER = ['mac', 'win', 'win-xp', 'win-vista', 'linux']
@@ -142,7 +141,7 @@ class Rebaseliner(object):
REVISION_REGEX = r'<a href=\"(\d+)/\">'
- def __init__(self, running_port, target_port, platform, options):
+ def __init__(self, running_port, target_port, platform, options, url_fetcher, zip_factory, scm):
"""
Args:
running_port: the Port the script is running on.
@@ -150,14 +149,20 @@ class Rebaseliner(object):
configuration information like the test_expectations.txt
file location and the list of test platforms.
platform: the test platform to rebaseline
- options: the command-line options object."""
+ options: the command-line options object.
+ url_fetcher: object that can fetch objects from URLs
+ zip_factory: optional object that can fetch zip files from URLs
+ scm: scm object for adding new baselines
+ """
self._platform = platform
self._options = options
self._port = running_port
self._filesystem = running_port._filesystem
self._target_port = target_port
+
self._rebaseline_port = port.get(
- self._target_port.test_platform_name_to_name(platform), options)
+ self._target_port.test_platform_name_to_name(platform), options,
+ filesystem=self._filesystem)
self._rebaselining_tests = []
self._rebaselined_tests = []
@@ -170,10 +175,11 @@ class Rebaseliner(object):
test_expectations.TestExpectations(self._rebaseline_port,
None,
expectations_str,
- self._platform,
- False,
+ self._rebaseline_port.test_configuration(),
False)
- self._scm = scm.default_scm()
+ self._url_fetcher = url_fetcher
+ self._zip_factory = zip_factory
+ self._scm = scm
def run(self, backup):
"""Run rebaseline process."""
@@ -192,8 +198,11 @@ class Rebaseliner(object):
log_dashed_string('Extracting and adding new baselines',
self._platform)
if not self._extract_and_add_new_baselines(archive_file):
+ archive_file.close()
return False
+ archive_file.close()
+
log_dashed_string('Updating rebaselined tests in file',
self._platform)
self._update_rebaselined_tests_in_file(backup)
@@ -254,9 +263,7 @@ class Rebaseliner(object):
_log.debug('Url to retrieve revision: "%s"', url)
- f = urllib.urlopen(url)
- content = f.read()
- f.close()
+ content = self._url_fetcher.fetch(url)
revisions = re.findall(self.REVISION_REGEX, content)
if not revisions:
@@ -313,33 +320,24 @@ class Rebaseliner(object):
return archive_url
def _download_buildbot_archive(self):
- """Download layout test archive file from buildbot.
-
- Returns:
- True if download succeeded or
- False otherwise.
- """
-
+ """Download layout test archive file from buildbot and return a handle to it."""
url = self._get_archive_url()
if url is None:
return None
- fn = urllib.urlretrieve(url)[0]
- _log.info('Archive downloaded and saved to file: "%s"', fn)
- return fn
+ archive_file = zipfileset.ZipFileSet(url, filesystem=self._filesystem,
+ zip_factory=self._zip_factory)
+ _log.info('Archive downloaded')
+ return archive_file
- def _extract_and_add_new_baselines(self, archive_file):
- """Extract new baselines from archive and add them to SVN repository.
-
- Args:
- archive_file: full path to the archive file.
+ def _extract_and_add_new_baselines(self, zip_file):
+ """Extract new baselines from the zip file and add them to SVN repository.
Returns:
List of tests that have been rebaselined or
None on failure.
"""
- zip_file = zipfile.ZipFile(archive_file, 'r')
zip_namelist = zip_file.namelist()
_log.debug('zip file namelist:')
@@ -419,7 +417,6 @@ class Rebaseliner(object):
test_no += 1
zip_file.close()
- self._filesystem.remove(archive_file)
return self._rebaselined_tests
@@ -857,18 +854,9 @@ def parse_options(args):
return (options, target_options)
-def main():
- """Main function to produce new baselines."""
-
- (options, target_options) = parse_options(sys.argv[1:])
-
- # We need to create three different Port objects over the life of this
- # script. |target_port_obj| is used to determine configuration information:
- # location of the expectations file, names of ports to rebaseline, etc.
- # |port_obj| is used for runtime functionality like actually diffing
- # Then we create a rebaselining port to actual find and manage the
- # baselines.
- target_port_obj = port.get(None, target_options)
+def main(args):
+ """Bootstrap function that sets up the object references we need and calls real_main()."""
+ options, target_options = parse_options(args)
# Set up our logging format.
log_level = logging.INFO
@@ -879,20 +867,53 @@ def main():
'%(levelname)s %(message)s'),
datefmt='%y%m%d %H:%M:%S')
+ target_port_obj = port.get(None, target_options)
host_port_obj = get_host_port_object(options)
- if not host_port_obj:
- sys.exit(1)
+ if not host_port_obj or not target_port_obj:
+ return 1
+
+ url_fetcher = urlfetcher.UrlFetcher(host_port_obj._filesystem)
+ scm_obj = scm.default_scm()
+
+ # We use the default zip factory method.
+ zip_factory = None
+
+ return real_main(options, target_options, host_port_obj, target_port_obj, url_fetcher,
+ zip_factory, scm_obj)
+
+def real_main(options, target_options, host_port_obj, target_port_obj, url_fetcher,
+ zip_factory, scm_obj):
+ """Main function to produce new baselines. The Rebaseliner object uses two
+ different Port objects - one to represent the machine the object is running
+ on, and one to represent the port whose expectations are being updated.
+ E.g., you can run the script on a mac and rebaseline the 'win' port.
+
+ Args:
+ options: command-line argument used for the host_port_obj (see below)
+ target_options: command_line argument used for the target_port_obj.
+ This object may have slightly different values than |options|.
+ host_port_obj: a Port object for the platform the script is running
+ on. This is used to produce image and text diffs, mostly, and
+ is usually acquired from get_host_port_obj().
+ target_port_obj: a Port obj representing the port getting rebaselined.
+ This is used to find the expectations file, the baseline paths,
+ etc.
+ url_fetcher: object used to download the build archives from the bots
+ zip_factory: factory function used to create zip file objects for
+ the archives.
+ scm_obj: object used to add new baselines to the source control system.
+ """
# Verify 'platforms' option is valid.
if not options.platforms:
_log.error('Invalid "platforms" option. --platforms must be '
'specified in order to rebaseline.')
- sys.exit(1)
+ return 1
platforms = [p.strip().lower() for p in options.platforms.split(',')]
for platform in platforms:
if not platform in REBASELINE_PLATFORM_ORDER:
_log.error('Invalid platform: "%s"' % (platform))
- sys.exit(1)
+ return 1
# Adjust the platform order so rebaseline tool is running at the order of
# 'mac', 'win' and 'linux'. This is in same order with layout test baseline
@@ -909,7 +930,8 @@ def main():
backup = options.backup
for platform in rebaseline_platforms:
rebaseliner = Rebaseliner(host_port_obj, target_port_obj,
- platform, options)
+ platform, options, url_fetcher, zip_factory,
+ scm_obj)
_log.info('')
log_dashed_string('Rebaseline started', platform)
@@ -934,7 +956,8 @@ def main():
html_generator.show_html()
log_dashed_string('Rebaselining result comparison done', None)
- sys.exit(0)
+ return 0
+
if '__main__' == __name__:
- main()
+ sys.exit(main(sys.argv[1:]))
diff --git a/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py b/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py
index 50c0204..730220b 100644
--- a/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py
@@ -32,11 +32,14 @@
import unittest
from webkitpy.tool import mocktool
+from webkitpy.common.system import urlfetcher_mock
from webkitpy.common.system import filesystem_mock
+from webkitpy.common.system import zipfileset_mock
+from webkitpy.common.system import outputcapture
from webkitpy.common.system.executive import Executive, ScriptError
-import port
-import rebaseline_chromium_webkit_tests
+from webkitpy.layout_tests import port
+from webkitpy.layout_tests import rebaseline_chromium_webkit_tests
class MockPort(object):
@@ -53,6 +56,57 @@ def get_mock_get(config_expectations):
return mock_get
+ARCHIVE_URL = 'http://localhost/layout_test_results'
+
+
+def test_options():
+ return mocktool.MockOptions(configuration=None,
+ backup=False,
+ html_directory='/tmp',
+ archive_url=ARCHIVE_URL,
+ force_archive_url=None,
+ webkit_canary=True,
+ use_drt=False,
+ target_platform='chromium',
+ verbose=False,
+ quiet=False,
+ platforms='mac,win,win-xp')
+
+
+def test_host_port_and_filesystem(options, expectations):
+ filesystem = port.unit_test_filesystem()
+ host_port_obj = port.get('test', options, filesystem=filesystem,
+ user=mocktool.MockUser())
+
+ expectations_path = host_port_obj.path_to_test_expectations_file()
+ filesystem.write_text_file(expectations_path, expectations)
+ return (host_port_obj, filesystem)
+
+
+def test_url_fetcher(filesystem):
+ urls = {
+ ARCHIVE_URL + '/Webkit_Mac10_5/': '<a href="1/"><a href="2/">',
+ ARCHIVE_URL + '/Webkit_Win/': '<a href="1/">',
+ }
+ return urlfetcher_mock.make_fetcher_cls(urls)(filesystem)
+
+
+def test_zip_factory():
+ ziphashes = {
+ ARCHIVE_URL + '/Webkit_Mac10_5/2/layout-test-results.zip': {
+ 'layout-test-results/failures/expected/image-actual.txt': 'new-image-txt',
+ 'layout-test-results/failures/expected/image-actual.checksum': 'new-image-checksum',
+ 'layout-test-results/failures/expected/image-actual.png': 'new-image-png',
+ },
+ ARCHIVE_URL + '/Webkit_Win/1/layout-test-results.zip': {
+ 'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt',
+ 'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum',
+ 'layout-test-results/failures/expected/image-actual.png': 'win-image-png',
+ },
+ }
+ return zipfileset_mock.make_factory(ziphashes)
+
+
class TestGetHostPortObject(unittest.TestCase):
def assert_result(self, release_present, debug_present, valid_port_obj):
# Tests whether we get a valid port object returned when we claim
@@ -60,9 +114,8 @@ class TestGetHostPortObject(unittest.TestCase):
port.get = get_mock_get({'Release': release_present,
'Debug': debug_present})
options = mocktool.MockOptions(configuration=None,
- html_directory=None)
- port_obj = rebaseline_chromium_webkit_tests.get_host_port_object(
- options)
+ html_directory='/tmp')
+ port_obj = rebaseline_chromium_webkit_tests.get_host_port_object(options)
if valid_port_obj:
self.assertNotEqual(port_obj, None)
else:
@@ -84,18 +137,7 @@ class TestGetHostPortObject(unittest.TestCase):
port.get = old_get
-class TestRebaseliner(unittest.TestCase):
- def make_rebaseliner(self):
- options = mocktool.MockOptions(configuration=None,
- html_directory=None)
- filesystem = filesystem_mock.MockFileSystem()
- host_port_obj = port.get('test', options, filesystem=filesystem)
- target_options = options
- target_port_obj = port.get('test', target_options, filesystem=filesystem)
- platform = 'test'
- return rebaseline_chromium_webkit_tests.Rebaseliner(
- host_port_obj, target_port_obj, platform, options)
-
+class TestOptions(unittest.TestCase):
def test_parse_options(self):
(options, target_options) = rebaseline_chromium_webkit_tests.parse_options([])
self.assertTrue(target_options.chromium)
@@ -105,39 +147,113 @@ class TestRebaseliner(unittest.TestCase):
self.assertFalse(hasattr(target_options, 'chromium'))
self.assertEqual(options.tolerance, 0)
+
+class TestRebaseliner(unittest.TestCase):
+ def make_rebaseliner(self, expectations):
+ options = test_options()
+ host_port_obj, filesystem = test_host_port_and_filesystem(options, expectations)
+
+ target_options = options
+ target_port_obj = port.get('test', target_options,
+ filesystem=filesystem)
+ target_port_obj._expectations = expectations
+ platform = target_port_obj.test_platform_name()
+
+ url_fetcher = test_url_fetcher(filesystem)
+ zip_factory = test_zip_factory()
+ mock_scm = mocktool.MockSCM()
+ rebaseliner = rebaseline_chromium_webkit_tests.Rebaseliner(host_port_obj,
+ target_port_obj, platform, options, url_fetcher, zip_factory, mock_scm)
+ return rebaseliner, filesystem
+
def test_noop(self):
# this method tests that was can at least instantiate an object, even
# if there is nothing to do.
- rebaseliner = self.make_rebaseliner()
- self.assertNotEqual(rebaseliner, None)
+ rebaseliner, filesystem = self.make_rebaseliner("")
+ rebaseliner.run(False)
+ self.assertEqual(len(filesystem.written_files), 1)
+
+ def test_one_platform(self):
+ rebaseliner, filesystem = self.make_rebaseliner(
+ "BUGX REBASELINE MAC : failures/expected/image.html = IMAGE")
+ rebaseliner.run(False)
+ # We expect to have written 12 files over the course of this rebaseline:
+ # *) 3 files in /__im_tmp for the extracted archive members
+ # *) 3 new baselines under '/test.checkout/LayoutTests'
+ # *) 4 files in /tmp for the new and old baselines in the result file
+ # (-{old,new}.{txt,png}
+ # *) 1 text diff in /tmp for the result file (-diff.txt). We don't
+ # create image diffs (FIXME?) and don't display the checksums.
+ # *) 1 updated test_expectations file
+ self.assertEqual(len(filesystem.written_files), 12)
+ self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test/test_expectations.txt'], '')
+ self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac/failures/expected/image-expected.checksum'], 'new-image-checksum')
+ self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac/failures/expected/image-expected.png'], 'new-image-png')
+ self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac/failures/expected/image-expected.txt'], 'new-image-txt')
+
+ def test_all_platforms(self):
+ rebaseliner, filesystem = self.make_rebaseliner(
+ "BUGX REBASELINE : failures/expected/image.html = IMAGE")
+ rebaseliner.run(False)
+ # See comment in test_one_platform for an explanation of the 12 written tests.
+ # Note that even though the rebaseline is marked for all platforms, each
+ # rebaseliner only ever does one.
+ self.assertEqual(len(filesystem.written_files), 12)
+ self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test/test_expectations.txt'],
+ 'BUGX REBASELINE WIN : failures/expected/image.html = IMAGE\n'
+ 'BUGX REBASELINE WIN-XP : failures/expected/image.html = IMAGE\n')
+ self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac/failures/expected/image-expected.checksum'], 'new-image-checksum')
+ self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac/failures/expected/image-expected.png'], 'new-image-png')
+ self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac/failures/expected/image-expected.txt'], 'new-image-txt')
def test_diff_baselines_txt(self):
- rebaseliner = self.make_rebaseliner()
- output = rebaseliner._port.expected_text(
- rebaseliner._port._filesystem.join(rebaseliner._port.layout_tests_dir(),
- 'passes/text.html'))
+ rebaseliner, filesystem = self.make_rebaseliner("")
+ port = rebaseliner._port
+ output = port.expected_text(
+ port._filesystem.join(port.layout_tests_dir(), 'passes/text.html'))
self.assertFalse(rebaseliner._diff_baselines(output, output,
is_image=False))
def test_diff_baselines_png(self):
- rebaseliner = self.make_rebaseliner()
- image = rebaseliner._port.expected_image(
- rebaseliner._port._filesystem.join(rebaseliner._port.layout_tests_dir(),
- 'passes/image.html'))
+ rebaseliner, filesystem = self.make_rebaseliner('')
+ port = rebaseliner._port
+ image = port.expected_image(
+ port._filesystem.join(port.layout_tests_dir(), 'passes/image.html'))
self.assertFalse(rebaseliner._diff_baselines(image, image,
is_image=True))
+class TestRealMain(unittest.TestCase):
+ def test_all_platforms(self):
+ expectations = "BUGX REBASELINE : failures/expected/image.html = IMAGE"
+
+ options = test_options()
+
+ host_port_obj, filesystem = test_host_port_and_filesystem(options, expectations)
+ url_fetcher = test_url_fetcher(filesystem)
+ zip_factory = test_zip_factory()
+ mock_scm = mocktool.MockSCM()
+ oc = outputcapture.OutputCapture()
+ oc.capture_output()
+ rebaseline_chromium_webkit_tests.real_main(options, options, host_port_obj,
+ host_port_obj, url_fetcher, zip_factory, mock_scm)
+ oc.restore_output()
+
+ # We expect to have written 35 files over the course of this rebaseline:
+ # *) 11 files * 3 ports for the new baselines and the diffs (see breakdown
+ # under test_one_platform, above)
+ # *) the updated test_expectations file
+ # *) the rebaseline results html file
+ self.assertEqual(len(filesystem.written_files), 35)
+ self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test/test_expectations.txt'], '')
+
+
class TestHtmlGenerator(unittest.TestCase):
def make_generator(self, files, tests):
options = mocktool.MockOptions(configuration=None, html_directory='/tmp')
- host_port = port.get('test', options, filesystem=filesystem_mock.MockFileSystem(files))
- generator = rebaseline_chromium_webkit_tests.HtmlGenerator(
- host_port,
- target_port=None,
- options=options,
- platforms=['mac'],
- rebaselining_tests=tests)
+ host_port = port.get('test', options, filesystem=port.unit_test_filesystem(files))
+ generator = rebaseline_chromium_webkit_tests.HtmlGenerator(host_port,
+ target_port=None, options=options, platforms=['mac'], rebaselining_tests=tests)
return generator, host_port
def test_generate_baseline_links(self):
diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
index 17b6e89..2d55b93 100755
--- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
+++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
@@ -30,8 +30,6 @@
"""Run layout tests."""
-from __future__ import with_statement
-
import errno
import logging
import optparse
@@ -41,6 +39,7 @@ import sys
from layout_package import printing
from layout_package import test_runner
+from layout_package import test_runner2
from webkitpy.common.system import user
from webkitpy.thirdparty import simplejson
@@ -89,7 +88,11 @@ def run(port, options, args, regular_output=sys.stderr,
# in a try/finally to ensure that we clean up the logging configuration.
num_unexpected_results = -1
try:
- runner = test_runner.TestRunner(port, options, printer)
+ if options.worker_model in ('inline', 'threads', 'processes'):
+ runner = test_runner2.TestRunner2(port, options, printer)
+ else:
+ runner = test_runner.TestRunner(port, options, printer)
+
runner._print_config()
printer.print_update("Collecting tests ...")
@@ -100,11 +103,11 @@ def run(port, options, args, regular_output=sys.stderr,
return -1
raise
- printer.print_update("Parsing expectations ...")
if options.lint_test_files:
return runner.lint()
- runner.parse_expectations(port.test_platform_name(),
- options.configuration == 'Debug')
+
+ printer.print_update("Parsing expectations ...")
+ runner.parse_expectations()
printer.print_update("Checking build ...")
if not port.check_build(runner.needs_http()):
@@ -128,9 +131,12 @@ def _set_up_derived_options(port_obj, options):
# We return a list of warnings to print after the printer is initialized.
warnings = []
- if options.worker_model == 'old-inline':
+ if options.worker_model is None:
+ options.worker_model = port_obj.default_worker_model()
+
+ if options.worker_model in ('inline', 'old-inline'):
if options.child_processes and int(options.child_processes) > 1:
- warnings.append("--worker-model=old-inline overrides --child-processes")
+ warnings.append("--worker-model=%s overrides --child-processes" % options.worker_model)
options.child_processes = "1"
if not options.child_processes:
options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
@@ -226,9 +232,6 @@ def parse_args(args=None):
optparse.make_option("--nocheck-sys-deps", action="store_true",
default=False,
help="Don't check the system dependencies (themes)"),
- optparse.make_option("--use-test-shell", action="store_true",
- default=False,
- help="Use test_shell instead of DRT"),
optparse.make_option("--accelerated-compositing",
action="store_true",
help="Use hardware-accelated compositing for rendering"),
@@ -368,8 +371,8 @@ def parse_args(args=None):
help="Number of DumpRenderTrees to run in parallel."),
# FIXME: Display default number of child processes that will run.
optparse.make_option("--worker-model", action="store",
- default="old-threads", help=("controls worker model. Valid values "
- "are 'old-inline', 'old-threads'.")),
+ default=None, help=("controls worker model. Valid values are 'old-inline', "
+ "'old-threads', 'inline', 'threads', and 'processes'.")),
optparse.make_option("--experimental-fully-parallel",
action="store_true", default=False,
help="run all tests in parallel"),
@@ -415,10 +418,6 @@ def parse_args(args=None):
optparse.make_option("--test-results-server", default="",
help=("If specified, upload results json files to this appengine "
"server.")),
- optparse.make_option("--upload-full-results",
- action="store_true",
- default=False,
- help="If true, upload full json results to server."),
]
option_list = (configuration_options + print_options +
diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py
index 677becd..84f5718 100644
--- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py
@@ -45,7 +45,7 @@ import unittest
from webkitpy.common import array_stream
from webkitpy.common.system import outputcapture
from webkitpy.common.system import filesystem_mock
-from webkitpy.common.system import user
+from webkitpy.tool import mocktool
from webkitpy.layout_tests import port
from webkitpy.layout_tests import run_webkit_tests
from webkitpy.layout_tests.layout_package import dump_render_tree_thread
@@ -56,14 +56,6 @@ from webkitpy.test.skip import skip_if
from webkitpy.thirdparty.mock import Mock
-class MockUser():
- def __init__(self):
- self.url = None
-
- def open_url(self, url):
- self.url = url
-
-
def parse_args(extra_args=None, record_results=False, tests_included=False,
print_nothing=True):
extra_args = extra_args or []
@@ -93,7 +85,7 @@ def passing_run(extra_args=None, port_obj=None, record_results=False,
tests_included)
if not port_obj:
port_obj = port.get(port_name=options.platform, options=options,
- user=MockUser(), filesystem=filesystem)
+ user=mocktool.MockUser(), filesystem=filesystem)
res = run_webkit_tests.run(port_obj, options, parsed_args)
return res == 0
@@ -103,7 +95,7 @@ def logging_run(extra_args=None, port_obj=None, record_results=False, tests_incl
record_results=record_results,
tests_included=tests_included,
print_nothing=False)
- user = MockUser()
+ user = mocktool.MockUser()
if not port_obj:
port_obj = port.get(port_name=options.platform, options=options,
user=user, filesystem=filesystem)
@@ -135,7 +127,7 @@ def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False,
extra_args = ['passes', 'failures'] + extra_args
options, parsed_args = parse_args(extra_args, tests_included=True)
- user = MockUser()
+ user = mocktool.MockUser()
test_batches = []
@@ -216,7 +208,8 @@ class MainTest(unittest.TestCase):
def test_full_results_html(self):
# FIXME: verify html?
- self.assertTrue(passing_run(['--full-results-html']))
+ res, out, err, user = logging_run(['--full-results-html'])
+ self.assertEqual(res, 0)
def test_help_printing(self):
res, out, err, user = logging_run(['--help-printing'])
@@ -256,7 +249,7 @@ class MainTest(unittest.TestCase):
def test_lint_test_files__errors(self):
options, parsed_args = parse_args(['--lint-test-files'])
- user = MockUser()
+ user = mocktool.MockUser()
port_obj = port.get(options.platform, options=options, user=user)
port_obj.test_expectations = lambda: "# syntax error"
res, out, err = run_and_capture(port_obj, options, parsed_args)
@@ -352,7 +345,7 @@ class MainTest(unittest.TestCase):
self.assertEqual(res, 3)
self.assertFalse(out.empty())
self.assertFalse(err.empty())
- self.assertEqual(user.url, '/tmp/layout-test-results/results.html')
+ self.assertEqual(user.opened_urls, ['/tmp/layout-test-results/results.html'])
def test_exit_after_n_failures(self):
# Unexpected failures should result in tests stopping.
@@ -414,7 +407,7 @@ class MainTest(unittest.TestCase):
with fs.mkdtemp() as tmpdir:
res, out, err, user = logging_run(['--results-directory=' + str(tmpdir)],
tests_included=True, filesystem=fs)
- self.assertEqual(user.url, fs.join(tmpdir, 'results.html'))
+ self.assertEqual(user.opened_urls, [fs.join(tmpdir, 'results.html')])
def test_results_directory_default(self):
# We run a configuration that should fail, to generate output, then
@@ -422,7 +415,7 @@ class MainTest(unittest.TestCase):
# This is the default location.
res, out, err, user = logging_run(tests_included=True)
- self.assertEqual(user.url, '/tmp/layout-test-results/results.html')
+ self.assertEqual(user.opened_urls, ['/tmp/layout-test-results/results.html'])
def test_results_directory_relative(self):
# We run a configuration that should fail, to generate output, then
@@ -430,7 +423,7 @@ class MainTest(unittest.TestCase):
res, out, err, user = logging_run(['--results-directory=foo'],
tests_included=True)
- self.assertEqual(user.url, '/tmp/foo/results.html')
+ self.assertEqual(user.opened_urls, ['/tmp/foo/results.html'])
def test_tolerance(self):
class ImageDiffTestPort(TestPort):
@@ -441,7 +434,7 @@ class MainTest(unittest.TestCase):
def get_port_for_run(args):
options, parsed_args = run_webkit_tests.parse_args(args)
- test_port = ImageDiffTestPort(options=options, user=MockUser())
+ test_port = ImageDiffTestPort(options=options, user=mocktool.MockUser())
passing_run(args, port_obj=test_port, tests_included=True)
return test_port
@@ -459,11 +452,27 @@ class MainTest(unittest.TestCase):
self.assertEqual(None, test_port.tolerance_used_for_diff_image)
def test_worker_model__inline(self):
+ self.assertTrue(passing_run(['--worker-model', 'inline']))
+
+ def test_worker_model__old_inline_with_child_processes(self):
+ res, out, err, user = logging_run(['--worker-model', 'old-inline',
+ '--child-processes', '2'])
+ self.assertEqual(res, 0)
+ self.assertTrue('--worker-model=old-inline overrides --child-processes\n' in err.get())
+
+ def test_worker_model__old_inline(self):
self.assertTrue(passing_run(['--worker-model', 'old-inline']))
- def test_worker_model__threads(self):
+ def test_worker_model__old_threads(self):
self.assertTrue(passing_run(['--worker-model', 'old-threads']))
+ def test_worker_model__processes(self):
+ if compare_version(sys, '2.6')[0] >= 0:
+ self.assertTrue(passing_run(['--worker-model', 'processes']))
+
+ def test_worker_model__threads(self):
+ self.assertTrue(passing_run(['--worker-model', 'threads']))
+
def test_worker_model__unknown(self):
self.assertRaises(ValueError, logging_run,
['--worker-model', 'unknown'])
@@ -491,7 +500,7 @@ class RebaselineTest(unittest.TestCase):
'failures/expected/missing_image.html'],
tests_included=True, filesystem=fs)
file_list = fs.written_files.keys()
- file_list.remove('/tmp/layout-test-results/tests_run.txt')
+ file_list.remove('/tmp/layout-test-results/tests_run0.txt')
self.assertEqual(len(file_list), 6)
self.assertBaselines(file_list,
"/passes/image")
@@ -508,12 +517,12 @@ class RebaselineTest(unittest.TestCase):
'failures/expected/missing_image.html'],
tests_included=True, filesystem=fs)
file_list = fs.written_files.keys()
- file_list.remove('/tmp/layout-test-results/tests_run.txt')
+ file_list.remove('/tmp/layout-test-results/tests_run0.txt')
self.assertEqual(len(file_list), 6)
self.assertBaselines(file_list,
- "/platform/test/passes/image")
+ "/platform/test-mac/passes/image")
self.assertBaselines(file_list,
- "/platform/test/failures/expected/missing_image")
+ "/platform/test-mac/failures/expected/missing_image")
class DryrunTest(unittest.TestCase):
diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/image_diff.py b/Tools/Scripts/webkitpy/layout_tests/test_types/image_diff.py
index 44605d2..1d7e107 100644
--- a/Tools/Scripts/webkitpy/layout_tests/test_types/image_diff.py
+++ b/Tools/Scripts/webkitpy/layout_tests/test_types/image_diff.py
@@ -49,23 +49,6 @@ _log = logging.getLogger("webkitpy.layout_tests.test_types.image_diff")
class ImageDiff(test_type_base.TestTypeBase):
- def _save_baseline_files(self, filename, image, image_hash,
- generate_new_baseline):
- """Saves new baselines for the PNG and checksum.
-
- Args:
- filename: test filename
- image: a image output
- image_hash: a checksum of the image
- generate_new_baseline: whether to generate a new, platform-specific
- baseline, or update the existing one
- """
- self._save_baseline_data(filename, image, ".png", encoding=None,
- generate_new_baseline=generate_new_baseline)
- self._save_baseline_data(filename, image_hash, ".checksum",
- encoding="ascii",
- generate_new_baseline=generate_new_baseline)
-
def _copy_image(self, filename, actual_image, expected_image):
self.write_output_files(filename, '.png',
output=actual_image, expected=expected_image,
@@ -85,54 +68,47 @@ class ImageDiff(test_type_base.TestTypeBase):
self.FILENAME_SUFFIX_COMPARE)
return port.diff_image(actual_image, expected_image, diff_filename)
- def compare_output(self, port, filename, test_args, actual_test_output,
- expected_test_output):
+ def compare_output(self, port, filename, options, actual_driver_output,
+ expected_driver_output):
"""Implementation of CompareOutput that checks the output image and
checksum against the expected files from the LayoutTest directory.
"""
failures = []
# If we didn't produce a hash file, this test must be text-only.
- if actual_test_output.image_hash is None:
- return failures
-
- # If we're generating a new baseline, we pass.
- if test_args.new_baseline or test_args.reset_results:
- self._save_baseline_files(filename, actual_test_output.image,
- actual_test_output.image_hash,
- test_args.new_baseline)
+ if actual_driver_output.image_hash is None:
return failures
- if not expected_test_output.image:
+ if not expected_driver_output.image:
# Report a missing expected PNG file.
- self._copy_image(filename, actual_test_output.image, expected_image=None)
- self._copy_image_hash(filename, actual_test_output.image_hash,
- expected_test_output.image_hash)
+ self._copy_image(filename, actual_driver_output.image, expected_image=None)
+ self._copy_image_hash(filename, actual_driver_output.image_hash,
+ expected_driver_output.image_hash)
failures.append(test_failures.FailureMissingImage())
return failures
- if not expected_test_output.image_hash:
+ if not expected_driver_output.image_hash:
# Report a missing expected checksum file.
- self._copy_image(filename, actual_test_output.image,
- expected_test_output.image)
- self._copy_image_hash(filename, actual_test_output.image_hash,
+ self._copy_image(filename, actual_driver_output.image,
+ expected_driver_output.image)
+ self._copy_image_hash(filename, actual_driver_output.image_hash,
expected_image_hash=None)
failures.append(test_failures.FailureMissingImageHash())
return failures
- if actual_test_output.image_hash == expected_test_output.image_hash:
+ if actual_driver_output.image_hash == expected_driver_output.image_hash:
# Hash matched (no diff needed, okay to return).
return failures
- self._copy_image(filename, actual_test_output.image,
- expected_test_output.image)
- self._copy_image_hash(filename, actual_test_output.image_hash,
- expected_test_output.image_hash)
+ self._copy_image(filename, actual_driver_output.image,
+ expected_driver_output.image)
+ self._copy_image_hash(filename, actual_driver_output.image_hash,
+ expected_driver_output.image_hash)
# Even though we only use the result in one codepath below but we
# still need to call CreateImageDiff for other codepaths.
images_are_different = self._create_diff_image(port, filename,
- actual_test_output.image,
- expected_test_output.image)
+ actual_driver_output.image,
+ expected_driver_output.image)
if not images_are_different:
failures.append(test_failures.FailureImageHashIncorrect())
else:
diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py b/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py
index ad65016..09bfc31 100644
--- a/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py
+++ b/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py
@@ -28,8 +28,6 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Defines the interface TestTypeBase which other test types inherit from.
-
-Also defines the TestArguments "struct" to pass them additional arguments.
"""
import cgi
@@ -39,21 +37,6 @@ import logging
_log = logging.getLogger("webkitpy.layout_tests.test_types.test_type_base")
-class TestArguments(object):
- """Struct-like wrapper for additional arguments needed by
- specific tests."""
- # Whether to save new baseline results.
- new_baseline = False
-
- # Path to the actual PNG file generated by pixel tests
- png_path = None
-
- # Value of checksum generated by pixel tests.
- hash = None
-
- # Whether to use wdiff to generate by-word diffs.
- wdiff = False
-
# Python bug workaround. See the wdiff code in WriteOutputFiles for an
# explanation.
_wdiff_available = True
@@ -87,39 +70,6 @@ class TestTypeBase(object):
self._port.relative_test_filename(filename))
fs.maybe_make_directory(fs.dirname(output_filename))
- def _save_baseline_data(self, filename, data, modifier, encoding,
- generate_new_baseline=True):
- """Saves a new baseline file into the port's baseline directory.
-
- The file will be named simply "<test>-expected<modifier>", suitable for
- use as the expected results in a later run.
-
- Args:
- filename: path to the test file
- data: result to be saved as the new baseline
- modifier: type of the result file, e.g. ".txt" or ".png"
- encoding: file encoding (none, "utf-8", etc.)
- generate_new_baseline: whether to enerate a new, platform-specific
- baseline, or update the existing one
- """
-
- port = self._port
- fs = self._port._filesystem
- if generate_new_baseline:
- relative_dir = fs.dirname(port.relative_test_filename(filename))
- baseline_path = port.baseline_path()
- output_dir = fs.join(baseline_path, relative_dir)
- output_file = fs.basename(fs.splitext(filename)[0] +
- self.FILENAME_SUFFIX_EXPECTED + modifier)
- fs.maybe_make_directory(output_dir)
- output_path = fs.join(output_dir, output_file)
- _log.debug('writing new baseline result "%s"' % (output_path))
- else:
- output_path = port.expected_filename(filename, modifier)
- _log.debug('resetting baseline result "%s"' % output_path)
-
- port.update_baseline(output_path, data, encoding)
-
def output_filename(self, filename, modifier):
"""Returns a filename inside the output dir that contains modifier.
@@ -139,8 +89,8 @@ class TestTypeBase(object):
self._port.relative_test_filename(filename))
return fs.splitext(output_filename)[0] + modifier
- def compare_output(self, port, filename, test_args, actual_test_output,
- expected_test_output):
+ def compare_output(self, port, filename, options, actual_driver_output,
+ expected_driver_output):
"""Method that compares the output from the test with the
expected value.
@@ -149,12 +99,11 @@ class TestTypeBase(object):
Args:
port: object implementing port-specific information and methods
filename: absolute filename to test file
- test_args: a TestArguments object holding optional additional
- arguments
- actual_test_output: a TestOutput object which represents actual test
+ options: command line argument object from optparse
+ actual_driver_output: a DriverOutput object which represents actual test
output
- expected_test_output: a TestOutput object which represents a expected
- test output
+ expected_driver_output: a ExpectedDriverOutput object which represents a
+ expected test output
Return:
a list of TestFailure objects, empty if the test passes
diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py b/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py
index 5dbfcb6..7af4d9c 100644
--- a/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py
+++ b/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py
@@ -40,7 +40,7 @@ class Test(unittest.TestCase):
test_type = test_type_base.TestTypeBase(None, None)
self.assertRaises(NotImplementedError, test_type.compare_output,
None, "foo.txt", '',
- test_type_base.TestArguments(), 'Debug')
+ {}, 'Debug')
if __name__ == '__main__':
diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/text_diff.py b/Tools/Scripts/webkitpy/layout_tests/test_types/text_diff.py
index 7b7febe..07c3d03 100644
--- a/Tools/Scripts/webkitpy/layout_tests/test_types/text_diff.py
+++ b/Tools/Scripts/webkitpy/layout_tests/test_types/text_diff.py
@@ -53,26 +53,16 @@ class TestTextDiff(test_type_base.TestTypeBase):
# the normalized text expectation files.
return output.replace("\r\r\n", "\r\n").replace("\r\n", "\n")
- def compare_output(self, port, filename, test_args, actual_test_output,
- expected_test_output):
+ def compare_output(self, port, filename, options, actual_driver_output,
+ expected_driver_output):
"""Implementation of CompareOutput that checks the output text against
the expected text from the LayoutTest directory."""
failures = []
- # If we're generating a new baseline, we pass.
- if test_args.new_baseline or test_args.reset_results:
- # Although all test_shell/DumpRenderTree output should be utf-8,
- # we do not ever decode it inside run-webkit-tests. For some tests
- # DumpRenderTree may not output utf-8 text (e.g. webarchives).
- self._save_baseline_data(filename, actual_test_output.text,
- ".txt", encoding=None,
- generate_new_baseline=test_args.new_baseline)
- return failures
-
# Normalize text to diff
- actual_text = self._get_normalized_output_text(actual_test_output.text)
+ actual_text = self._get_normalized_output_text(actual_driver_output.text)
# Assuming expected_text is already normalized.
- expected_text = expected_test_output.text
+ expected_text = expected_driver_output.text
# Write output files for new tests, too.
if port.compare_text(actual_text, expected_text):
diff --git a/Tools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests.py b/Tools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests.py
index f4c8098..7267aa6 100755
--- a/Tools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests.py
+++ b/Tools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests.py
@@ -31,7 +31,7 @@ import optparse
import os
import re
import sys
-import webkitpy.common.checkout.scm as scm
+from webkitpy.common.checkout import scm
_log = logging.getLogger("webkitpy.layout_tests."
"update-webgl-conformance-tests")