diff options
Diffstat (limited to 'Tools/Scripts/webkitpy/layout_tests')
52 files changed, 1810 insertions, 1056 deletions
diff --git a/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py b/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py index 86649b6..5472dfe 100644 --- a/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py +++ b/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py @@ -55,9 +55,7 @@ def port_fallbacks(): back on. All platforms fall back on 'base'. """ fallbacks = {_BASE_PLATFORM: []} - platform_dir = os.path.join(scm.find_checkout_root(), 'LayoutTests', - 'platform') - for port_name in os.listdir(platform_dir): + for port_name in port_factory.all_port_names(): try: platforms = port_factory.get(port_name).baseline_search_path() except NotImplementedError: @@ -67,6 +65,7 @@ def port_fallbacks(): continue fallbacks[port_name] = [os.path.basename(p) for p in platforms][1:] fallbacks[port_name].append(_BASE_PLATFORM) + return fallbacks @@ -119,6 +118,15 @@ def cluster_file_hashes(glob_pattern): return parse_git_output(git_output, glob_pattern) +def dirname_to_platform(dirname): + if dirname == 'chromium-linux': + return 'chromium-linux-x86' + elif dirname == 'chromium-win': + return 'chromium-win-win7' + elif dirname == 'chromium-mac': + return 'chromium-mac-snowleopard' + return dirname + def extract_platforms(paths): """Extracts the platforms from a list of paths matching ^platform/(.*?)/. Args: @@ -130,7 +138,7 @@ def extract_platforms(paths): for path in paths: match = re.match(r'^platform/(.*?)/', path) if match: - platform = match.group(1) + platform = dirname_to_platform(match.group(1)) else: platform = _BASE_PLATFORM platforms[platform] = path @@ -154,10 +162,11 @@ def has_intermediate_results(test, fallbacks, matching_platform, path_exists: Optional parameter that allows us to stub out os.path.exists for testing. """ - for platform in fallbacks: + for dirname in fallbacks: + platform = dirname_to_platform(dirname) if platform == matching_platform: return False - test_path = os.path.join('LayoutTests', 'platform', platform, test) + test_path = os.path.join('LayoutTests', 'platform', dirname, test) if path_exists(test_path): return True return False @@ -199,7 +208,10 @@ def find_dups(hashes, port_fallbacks, relative_to): # See if any of the platforms are redundant with each other. for platform in platforms.keys(): - for fallback in port_fallbacks[platform]: + if platform not in port_factory.all_port_names(): + continue + for dirname in port_fallbacks[platform]: + fallback = dirname_to_platform(dirname) if fallback not in platforms.keys(): continue # We have to verify that there isn't an intermediate result @@ -215,7 +227,7 @@ def find_dups(hashes, port_fallbacks, relative_to): yield { 'test': test, 'platform': platform, - 'fallback': fallback, + 'fallback': dirname, 'path': path, } diff --git a/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py b/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py index 47dc8a2..29ff8d5 100644 --- a/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py @@ -95,7 +95,7 @@ class ListDuplicatesTest(unittest.TestCase): # intermediate results. (False, ('fast/foo-expected.txt', ['chromium-win', 'chromium', 'base'], - 'chromium-win', + 'chromium-win-win7', lambda path: True)), # Since chromium-win has a result, we have an intermediate result. (True, ('fast/foo-expected.txt', @@ -159,12 +159,12 @@ class ListDuplicatesTest(unittest.TestCase): self.assertEquals({'test': 'animage.png', 'path': 'LayoutTests/platform/chromium-linux/animage.png', 'fallback': 'chromium-win', - 'platform': 'chromium-linux'}, + 'platform': 'chromium-linux-x86'}, result[0]) self.assertEquals({'test': 'foo-expected.txt', 'path': 'LayoutTests/platform/chromium-linux/foo-expected.txt', 'fallback': 'chromium-win', - 'platform': 'chromium-linux'}, + 'platform': 'chromium-linux-x86'}, result[1]) result = deduplicate_tests.deduplicate('*.txt') @@ -174,7 +174,7 @@ class ListDuplicatesTest(unittest.TestCase): self.assertEquals({'test': 'foo-expected.txt', 'path': 'LayoutTests/platform/chromium-linux/foo-expected.txt', 'fallback': 'chromium-win', - 'platform': 'chromium-linux'}, + 'platform': 'chromium-linux-x86'}, result[0]) result = deduplicate_tests.deduplicate('*.png') @@ -184,7 +184,7 @@ class ListDuplicatesTest(unittest.TestCase): self.assertEquals({'test': 'animage.png', 'path': 'LayoutTests/platform/chromium-linux/animage.png', 'fallback': 'chromium-win', - 'platform': 'chromium-linux'}, + 'platform': 'chromium-linux-x86'}, result[0]) def test_get_relative_test_path(self): diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py index 7ddd7b0..83b2215 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py @@ -40,13 +40,13 @@ import thread import threading import time -from webkitpy.layout_tests.layout_package.single_test_runner import SingleTestRunner +from webkitpy.layout_tests.layout_package import worker_mixin _log = logging.getLogger("webkitpy.layout_tests.layout_package." "dump_render_tree_thread") -class TestShellThread(threading.Thread): +class TestShellThread(threading.Thread, worker_mixin.WorkerMixin): def __init__(self, port, options, worker_number, worker_name, filename_list_queue, result_queue): """Initialize all the local state for this DumpRenderTree thread. @@ -130,6 +130,7 @@ class TestShellThread(threading.Thread): def run(self): """Delegate main work to a helper method and watch for uncaught exceptions.""" + self._covered_run() def _covered_run(self): @@ -175,23 +176,16 @@ class TestShellThread(threading.Thread): If test_runner is not None, then we call test_runner.UpdateSummary() with the results of each test.""" - single_test_runner = SingleTestRunner(self._options, self._port, - self._name, self._worker_number) - - batch_size = self._options.batch_size - batch_count = 0 - # Append tests we're running to the existing tests_run.txt file. - # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput. - tests_run_filename = self._port._filesystem.join(self._options.results_directory, - "tests_run%d.txt" % self._worker_number) - tests_run_file = self._port._filesystem.open_text_file_for_writing(tests_run_filename, append=False) + # Initialize the real state of the WorkerMixin now that we're executing + # in the child thread. Technically, we could have called this during + # __init__(), but we wait until now to match Worker.run(). + self.safe_init(self._port) while True: if self._canceled: _log.debug('Testing cancelled') - tests_run_file.close() - single_test_runner.cleanup() + self.cleanup() return if len(self._filename_list) is 0: @@ -204,16 +198,15 @@ class TestShellThread(threading.Thread): self._current_group, self._filename_list = \ self._filename_list_queue.get_nowait() except Queue.Empty: - tests_run_file.close() - single_test_runner.cleanup() + self.cleanup() return if self._current_group == "tests_to_http_lock": self._http_lock_wait_begin = time.time() - single_test_runner.start_servers_with_lock() + self.start_servers_with_lock() self._http_lock_wait_end = time.time() - elif single_test_runner.has_http_lock: - single_test_runner.stop_servers_with_lock() + elif self._has_http_lock: + self.stop_servers_with_lock() self._num_tests_in_current_group = len(self._filename_list) self._current_group_start_time = time.time() @@ -221,33 +214,13 @@ class TestShellThread(threading.Thread): test_input = self._filename_list.pop() # We have a url, run tests. - batch_count += 1 self._num_tests += 1 - timeout = single_test_runner.timeout(test_input) - result = single_test_runner.run_test(test_input, timeout) - - tests_run_file.write(test_input.filename + "\n") - test_name = self._port.relative_test_filename(test_input.filename) - if result.failures: - # Check and kill DumpRenderTree if we need to. - if any([f.should_kill_dump_render_tree() for f in result.failures]): - single_test_runner.kill_dump_render_tree() - # Reset the batch count since the shell just bounced. - batch_count = 0 - - # Print the error message(s). - _log.debug("%s %s failed:" % (self._name, test_name)) - for f in result.failures: - _log.debug("%s %s" % (self._name, f.message())) - else: - _log.debug("%s %s passed" % (self._name, test_name)) - self._result_queue.put(result.dumps()) + result = self.run_test_with_timeout(test_input, self.timeout(test_input)) - if batch_size > 0 and batch_count >= batch_size: - # Bounce the shell and reset count. - single_test_runner.kill_dump_render_tree() - batch_count = 0 + self.clean_up_after_test(test_input, result) + self._test_results.append(result) + self._result_queue.put(result.dumps()) if test_runner: test_runner.update_summary(result_summary) diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py index 8226ed0..19b02e8 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py @@ -42,10 +42,10 @@ class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGeneratorBase # Additional JSON fields. WONTFIX = "wontfixCounts" - # Note that we omit test_expectations.FAIL from this list because - # it should never show up (it's a legacy input expectation, never - # an output expectation). - FAILURE_TO_CHAR = {test_expectations.CRASH: "C", + FAILURE_TO_CHAR = {test_expectations.PASS: json_results_generator.JSONResultsGeneratorBase.PASS_RESULT, + test_expectations.SKIP: json_results_generator.JSONResultsGeneratorBase.SKIP_RESULT, + test_expectations.FAIL: "Y", + test_expectations.CRASH: "C", test_expectations.TIMEOUT: "T", test_expectations.IMAGE: "I", test_expectations.TEXT: "F", diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py index 05662c2..e7f804f 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py @@ -42,6 +42,26 @@ import webkitpy.thirdparty.simplejson as simplejson _log = logging.getLogger("webkitpy.layout_tests.layout_package.json_results_generator") +_JSON_PREFIX = "ADD_RESULTS(" +_JSON_SUFFIX = ");" + + +def strip_json_wrapper(json_content): + return json_content[len(_JSON_PREFIX):len(json_content) - len(_JSON_SUFFIX)] + + +def load_json(filesystem, file_path): + content = filesystem.read_text_file(file_path) + content = strip_json_wrapper(content) + return simplejson.loads(content) + + +def write_json(filesystem, json_object, file_path): + # Specify separators in order to get compact encoding. + json_data = simplejson.dumps(json_object, separators=(',', ':')) + json_string = _JSON_PREFIX + json_data + _JSON_SUFFIX + filesystem.write_text_file(file_path, json_string) + # FIXME: We already have a TestResult class in test_results.py class TestResult(object): """A simple class that represents a single test result.""" @@ -80,8 +100,6 @@ class JSONResultsGeneratorBase(object): MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 750 # Min time (seconds) that will be added to the JSON. MIN_TIME = 1 - JSON_PREFIX = "ADD_RESULTS(" - JSON_SUFFIX = ");" # Note that in non-chromium tests those chars are used to indicate # test modifiers (FAILS, FLAKY, etc) but not actual test results. @@ -109,6 +127,7 @@ class JSONResultsGeneratorBase(object): ALL_FIXABLE_COUNT = "allFixableCount" RESULTS_FILENAME = "results.json" + FULL_RESULTS_FILENAME = "full_results.json" INCREMENTAL_RESULTS_FILENAME = "incremental_results.json" URL_FOR_TEST_LIST_JSON = \ @@ -151,10 +170,6 @@ class JSONResultsGeneratorBase(object): self._build_number = build_number self._builder_base_url = builder_base_url self._results_directory = results_file_base_path - self._results_file_path = self._fs.join(results_file_base_path, - self.RESULTS_FILENAME) - self._incremental_results_file_path = self._fs.join( - results_file_base_path, self.INCREMENTAL_RESULTS_FILENAME) self._test_results_map = test_results_map self._test_results = test_results_map.values() @@ -172,8 +187,26 @@ class JSONResultsGeneratorBase(object): def generate_json_output(self): json = self.get_json() if json: - self._generate_json_file( - json, self._incremental_results_file_path) + file_path = self._fs.join(self._results_directory, self.INCREMENTAL_RESULTS_FILENAME) + write_json(self._fs, json, file_path) + + def generate_full_results_file(self): + # Use the same structure as the compacted version of TestRunner.summarize_results. + # For now we only include the times as this is only used for treemaps and + # expected/actual don't make sense for gtests. + results = {} + results['version'] = 1 + + tests = {} + + for test in self._test_results_map: + time_seconds = self._test_results_map[test].time + tests[test] = {} + tests[test]['time_ms'] = int(1000 * time_seconds) + + results['tests'] = tests + file_path = self._fs.join(self._results_directory, self.FULL_RESULTS_FILENAME) + write_json(self._fs, results, file_path) def get_json(self): """Gets the results for the results.json file.""" @@ -249,12 +282,6 @@ class JSONResultsGeneratorBase(object): _log.info("JSON files uploaded.") - def _generate_json_file(self, json, file_path): - # Specify separators in order to get compact encoding. - json_data = simplejson.dumps(json, separators=(',', ':')) - json_string = self.JSON_PREFIX + json_data + self.JSON_SUFFIX - self._fs.write_text_file(file_path, json_string) - def _get_test_timing(self, test_name): """Returns test timing data (elapsed time) in second for the given test_name.""" @@ -357,8 +384,7 @@ class JSONResultsGeneratorBase(object): if old_results: # Strip the prefix and suffix so we can get the actual JSON object. - old_results = old_results[len(self.JSON_PREFIX): - len(old_results) - len(self.JSON_SUFFIX)] + old_results = strip_json_wrapper(old_results) try: results_json = simplejson.loads(old_results) diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py index 95da8fb..9d786d9 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py @@ -106,6 +106,10 @@ class JSONGeneratorTest(unittest.TestCase): incremental_json, 1) + # We don't verify the results here, but at least we make sure the code runs without errors. + generator.generate_json_output() + generator.generate_full_results_file() + def _verify_json_results(self, tests_set, test_timings, failed_count_map, PASS_count, DISABLED_count, FLAKY_count, fixable_count, diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker.py index a0f252c..4886c30 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker.py @@ -52,17 +52,13 @@ import time # Handle Python < 2.6 where multiprocessing isn't available. -# -# _Multiprocessing_Process is needed so that _MultiProcessWorker -# can be defined with or without multiprocessing. try: import multiprocessing - _Multiprocessing_Process = multiprocessing.Process except ImportError: multiprocessing = None - _Multiprocessing_Process = threading.Thread +from webkitpy.common.system import stack_utils from webkitpy.layout_tests import port from webkitpy.layout_tests.layout_package import message_broker2 @@ -219,6 +215,18 @@ class _WorkerConnection(message_broker2.BrokerConnection): message_broker2.BrokerConnection.__init__(self, broker, self._client, ANY_WORKER_TOPIC, MANAGER_TOPIC) + def cancel(self): + raise NotImplementedError + + def is_alive(self): + raise NotImplementedError + + def join(self, timeout): + raise NotImplementedError + + def log_wedged_worker(self, test_name): + raise NotImplementedError + def yield_to_broker(self): pass @@ -226,11 +234,26 @@ class _WorkerConnection(message_broker2.BrokerConnection): class _InlineWorkerConnection(_WorkerConnection): def __init__(self, broker, port, manager_client, worker_class, worker_number): _WorkerConnection.__init__(self, broker, worker_class, worker_number, port._options) + self._alive = False self._port = port self._manager_client = manager_client + def cancel(self): + self._client.cancel() + + def is_alive(self): + return self._alive + + def join(self, timeout): + assert not self._alive + + def log_wedged_worker(self, test_name): + assert False, "_InlineWorkerConnection.log_wedged_worker() called" + def run(self): + self._alive = True self._client.run(self._port) + self._alive = False def yield_to_broker(self): self._broker.run_all_pending(MANAGER_TOPIC, self._manager_client) @@ -243,6 +266,12 @@ class _Thread(threading.Thread): self._port = port self._client = client + def cancel(self): + return self._client.cancel() + + def log_wedged_worker(self, test_name): + stack_utils.log_thread_state(_log.error, self._client.name(), self.ident, " is wedged on test %s" % test_name) + def run(self): # FIXME: We can remove this once everyone is on 2.6. if not hasattr(self, 'ident'): @@ -255,22 +284,40 @@ class _ThreadedWorkerConnection(_WorkerConnection): _WorkerConnection.__init__(self, broker, worker_class, worker_number, port._options) self._thread = _Thread(self, port, self._client) + def cancel(self): + return self._thread.cancel() + + def is_alive(self): + # FIXME: Change this to is_alive once everyone is on 2.6. + return self._thread.isAlive() + + def join(self, timeout): + return self._thread.join(timeout) + + def log_wedged_worker(self, test_name): + return self._thread.log_wedged_worker(test_name) + def start(self): self._thread.start() -class _Process(_Multiprocessing_Process): - def __init__(self, worker_connection, platform_name, options, client): - _Multiprocessing_Process.__init__(self) - self._worker_connection = worker_connection - self._platform_name = platform_name - self._options = options - self._client = client +if multiprocessing: - def run(self): - logging.basicConfig() - port_obj = port.get(self._platform_name, self._options) - self._client.run(port_obj) + class _Process(multiprocessing.Process): + def __init__(self, worker_connection, platform_name, options, client): + multiprocessing.Process.__init__(self) + self._worker_connection = worker_connection + self._platform_name = platform_name + self._options = options + self._client = client + + def log_wedged_worker(self, test_name): + _log.error("%s (pid %d) is wedged on test %s" % (self.name, self.pid, test_name)) + + def run(self): + logging.basicConfig() + port_obj = port.get(self._platform_name, self._options) + self._client.run(port_obj) class _MultiProcessWorkerConnection(_WorkerConnection): @@ -278,5 +325,17 @@ class _MultiProcessWorkerConnection(_WorkerConnection): _WorkerConnection.__init__(self, broker, worker_class, worker_number, options) self._proc = _Process(self, platform_name, options, self._client) + def cancel(self): + return self._proc.terminate() + + def is_alive(self): + return self._proc.is_alive() + + def join(self, timeout): + return self._proc.join(timeout) + + def log_wedged_worker(self, test_name): + return self._proc.log_wedged_worker(test_name) + def start(self): self._proc.start() diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker_unittest.py index ffbe081..c32f880 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/manager_worker_broker_unittest.py @@ -43,14 +43,34 @@ from webkitpy.layout_tests import port from webkitpy.layout_tests.layout_package import manager_worker_broker from webkitpy.layout_tests.layout_package import message_broker2 +# In order to reliably control when child workers are starting and stopping, +# we use a pair of global variables to hold queues used for messaging. Ideally +# we wouldn't need globals, but we can't pass these through a lexical closure +# because those can't be Pickled and sent to a subprocess, and we'd prefer not +# to have to pass extra arguments to the worker in the start_worker() call. +starting_queue = None +stopping_queue = None + + +def make_broker(manager, worker_model, start_queue=None, stop_queue=None): + global starting_queue + global stopping_queue + starting_queue = start_queue + stopping_queue = stop_queue + options = get_options(worker_model) + return manager_worker_broker.get(port.get("test"), options, manager, _TestWorker) + -class TestWorker(manager_worker_broker.AbstractWorker): +class _TestWorker(manager_worker_broker.AbstractWorker): def __init__(self, broker_connection, worker_number, options): self._broker_connection = broker_connection self._options = options self._worker_number = worker_number self._name = 'TestWorker/%d' % worker_number self._stopped = False + self._canceled = False + self._starting_queue = starting_queue + self._stopping_queue = stopping_queue def handle_stop(self, src): self._stopped = True @@ -61,15 +81,20 @@ class TestWorker(manager_worker_broker.AbstractWorker): self._broker_connection.post_message('test', 2, 'hi, everybody') def is_done(self): - return self._stopped + return self._stopped or self._canceled def name(self): return self._name - def start(self): - pass + def cancel(self): + self._canceled = True def run(self, port): + if self._starting_queue: + self._starting_queue.put('') + + if self._stopping_queue: + self._stopping_queue.get() try: self._broker_connection.run_message_loop() self._broker_connection.yield_to_broker() @@ -85,11 +110,6 @@ def get_options(worker_model): return options -def make_broker(manager, worker_model): - options = get_options(worker_model) - return manager_worker_broker.get(port.get("test"), options, manager, - TestWorker) - class FunctionTests(unittest.TestCase): def test_get__inline(self): @@ -99,6 +119,10 @@ class FunctionTests(unittest.TestCase): self.assertTrue(make_broker(self, 'threads') is not None) def test_get__processes(self): + # This test sometimes fails on Windows. See <http://webkit.org/b/55087>. + if sys.platform in ('cygwin', 'win32'): + return + if multiprocessing: self.assertTrue(make_broker(self, 'processes') is not None) else: @@ -112,18 +136,12 @@ class _TestsMixin(object): """Mixin class that implements a series of tests to enforce the contract all implementations must follow.""" - # - # Methods to implement the Manager side of the ClientInterface - # def name(self): return 'Tester' def is_done(self): return self._done - # - # Handlers for the messages the TestWorker may send. - # def handle_done(self, src): self._done = True @@ -135,9 +153,6 @@ class _TestsMixin(object): self._exception = exc_info self._done = True - # - # Testing helper methods - # def setUp(self): self._an_int = None self._a_str = None @@ -146,33 +161,58 @@ class _TestsMixin(object): self._exception = None self._worker_model = None - def make_broker(self): - self._broker = make_broker(self, self._worker_model) + def make_broker(self, starting_queue=None, stopping_queue=None): + self._broker = make_broker(self, self._worker_model, starting_queue, + stopping_queue) + + def test_cancel(self): + self.make_broker() + worker = self._broker.start_worker(0) + worker.cancel() + self._broker.post_message('test', 1, 'hello, world') + worker.join(0.5) + self.assertFalse(worker.is_alive()) - # - # Actual unit tests - # def test_done(self): - if not self._worker_model: - return self.make_broker() worker = self._broker.start_worker(0) self._broker.post_message('test', 1, 'hello, world') self._broker.post_message('stop') self._broker.run_message_loop() + worker.join(0.5) + self.assertFalse(worker.is_alive()) self.assertTrue(self.is_done()) self.assertEqual(self._an_int, 2) self.assertEqual(self._a_str, 'hi, everybody') + def test_log_wedged_worker(self): + starting_queue = self.queue() + stopping_queue = self.queue() + self.make_broker(starting_queue, stopping_queue) + oc = outputcapture.OutputCapture() + oc.capture_output() + try: + worker = self._broker.start_worker(0) + starting_queue.get() + worker.log_wedged_worker('test_name') + stopping_queue.put('') + self._broker.post_message('stop') + self._broker.run_message_loop() + worker.join(0.5) + self.assertFalse(worker.is_alive()) + self.assertTrue(self.is_done()) + finally: + oc.restore_output() + def test_unknown_message(self): - if not self._worker_model: - return self.make_broker() worker = self._broker.start_worker(0) self._broker.post_message('unknown') self._broker.run_message_loop() + worker.join(0.5) self.assertTrue(self.is_done()) + self.assertFalse(worker.is_alive()) self.assertEquals(self._exception[0], ValueError) self.assertEquals(self._exception[1], "TestWorker/0: received message 'unknown' it couldn't handle") @@ -183,17 +223,22 @@ class InlineBrokerTests(_TestsMixin, unittest.TestCase): _TestsMixin.setUp(self) self._worker_model = 'inline' + def test_log_wedged_worker(self): + self.make_broker() + worker = self._broker.start_worker(0) + self.assertRaises(AssertionError, worker.log_wedged_worker, None) -class MultiProcessBrokerTests(_TestsMixin, unittest.TestCase): - def setUp(self): - _TestsMixin.setUp(self) - if multiprocessing: + +# FIXME: https://bugs.webkit.org/show_bug.cgi?id=54520. +if multiprocessing and sys.platform not in ('cygwin', 'win32'): + + class MultiProcessBrokerTests(_TestsMixin, unittest.TestCase): + def setUp(self): + _TestsMixin.setUp(self) self._worker_model = 'processes' - else: - self._worker_model = None - def queue(self): - return multiprocessing.Queue() + def queue(self): + return multiprocessing.Queue() class ThreadedBrokerTests(_TestsMixin, unittest.TestCase): @@ -201,6 +246,9 @@ class ThreadedBrokerTests(_TestsMixin, unittest.TestCase): _TestsMixin.setUp(self) self._worker_model = 'threads' + def queue(self): + return Queue.Queue() + class FunctionsTest(unittest.TestCase): def test_runtime_options(self): @@ -222,6 +270,16 @@ class InterfaceTest(unittest.TestCase): obj = manager_worker_broker._ManagerConnection(broker._broker, None, self, None) self.assertRaises(NotImplementedError, obj.start_worker, 0) + def test_workerconnection_is_abstract(self): + # Test that all the base class methods are abstract and have the + # signature we expect. + broker = make_broker(self, 'inline') + obj = manager_worker_broker._WorkerConnection(broker._broker, _TestWorker, 0, None) + self.assertRaises(NotImplementedError, obj.cancel) + self.assertRaises(NotImplementedError, obj.is_alive) + self.assertRaises(NotImplementedError, obj.join, None) + self.assertRaises(NotImplementedError, obj.log_wedged_worker, None) + if __name__ == '__main__': unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py index 7ab6da8..8f63edd 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py @@ -513,8 +513,7 @@ class Testprinter(unittest.TestCase): retry.add(self.get_result('passes/text.html'), True) retry.add(self.get_result('failures/expected/timeout.html'), True) retry.add(self.get_result('failures/expected/crash.html'), True) - unexpected_results = test_runner.summarize_unexpected_results( - self._port, exp, rs, retry) + unexpected_results = test_runner.summarize_results(self._port, exp, rs, retry, test_timings={}, only_unexpected=True) return unexpected_results tests = ['passes/text.html', 'failures/expected/timeout.html', diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/result_summary.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/result_summary.py index 80fd6ac..0cee4f1 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/result_summary.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/result_summary.py @@ -81,7 +81,7 @@ class ResultSummary(object): if expected: self.expected += 1 else: - self.unexpected_results[result.filename] = result.type + self.unexpected_results[result.filename] = result self.unexpected += 1 if len(result.failures): self.unexpected_failures += 1 diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py index 96e3ee6..d755f67 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py @@ -28,21 +28,22 @@ import logging -import threading import time from webkitpy.layout_tests.port import base - -from webkitpy.layout_tests.test_types import text_diff -from webkitpy.layout_tests.test_types import image_diff - from webkitpy.layout_tests.layout_package import test_failures +from webkitpy.layout_tests.layout_package import test_result_writer from webkitpy.layout_tests.layout_package.test_results import TestResult _log = logging.getLogger(__name__) +def run_single_test(port, options, test_input, driver, worker_name): + runner = SingleTestRunner(options, port, driver, test_input, worker_name) + return runner.run() + + class ExpectedDriverOutput: """Groups information about an expected driver output.""" def __init__(self, text, image, image_hash): @@ -53,111 +54,14 @@ class ExpectedDriverOutput: class SingleTestRunner: - def __init__(self, options, port, worker_name, worker_number): + def __init__(self, options, port, driver, test_input, worker_name): self._options = options self._port = port + self._driver = driver + self._filename = test_input.filename + self._timeout = test_input.timeout self._worker_name = worker_name - self._worker_number = worker_number - self._driver = None - self._test_types = [] - self.has_http_lock = False - for cls in self._get_test_type_classes(): - self._test_types.append(cls(self._port, - self._options.results_directory)) - - def cleanup(self): - self.kill_dump_render_tree() - if self.has_http_lock: - self.stop_servers_with_lock() - - def _get_test_type_classes(self): - classes = [text_diff.TestTextDiff] - if self._options.pixel_tests: - classes.append(image_diff.ImageDiff) - return classes - - def timeout(self, test_input): - # We calculate how long we expect the test to take. - # - # The DumpRenderTree watchdog uses 2.5x the timeout; we want to be - # larger than that. We also add a little more padding if we're - # running tests in a separate thread. - # - # Note that we need to convert the test timeout from a - # string value in milliseconds to a float for Python. - driver_timeout_sec = 3.0 * float(test_input.timeout) / 1000.0 - if not self._options.run_singly: - return driver_timeout_sec - - thread_padding_sec = 1.0 - thread_timeout_sec = driver_timeout_sec + thread_padding_sec - return thread_timeout_sec - - def run_test(self, test_input, timeout): - if self._options.run_singly: - return self._run_test_in_another_thread(test_input, timeout) - else: - return self._run_test_in_this_thread(test_input) - return result - - def _run_test_in_another_thread(self, test_input, thread_timeout_sec): - """Run a test in a separate thread, enforcing a hard time limit. - - Since we can only detect the termination of a thread, not any internal - state or progress, we can only run per-test timeouts when running test - files singly. - - Args: - test_input: Object containing the test filename and timeout - thread_timeout_sec: time to wait before killing the driver process. - Returns: - A TestResult - """ - worker = self - result = None - - driver = worker._port.create_driver(worker._worker_number) - driver.start() - - class SingleTestThread(threading.Thread): - def run(self): - result = worker.run(test_input, driver) - - thread = SingleTestThread() - thread.start() - thread.join(thread_timeout_sec) - if thread.isAlive(): - # If join() returned with the thread still running, the - # DumpRenderTree is completely hung and there's nothing - # more we can do with it. We have to kill all the - # DumpRenderTrees to free it up. If we're running more than - # one DumpRenderTree thread, we'll end up killing the other - # DumpRenderTrees too, introducing spurious crashes. We accept - # that tradeoff in order to avoid losing the rest of this - # thread's results. - _log.error('Test thread hung: killing all DumpRenderTrees') - - driver.stop() - - if not result: - result = TestResult(test_input.filename, failures=[], - test_run_time=0, total_time_for_all_diffs=0, time_for_diffs={}) - return result - - def _run_test_in_this_thread(self, test_input): - """Run a single test file using a shared DumpRenderTree process. - - Args: - test_input: Object containing the test filename, uri and timeout - - Returns: a TestResult object. - """ - # poll() is not threadsafe and can throw OSError due to: - # http://bugs.python.org/issue1731717 - if not self._driver or self._driver.poll() is not None: - self._driver = self._port.create_driver(self._worker_number) - self._driver.start() - return self._run(self._driver, test_input) + self._testname = port.relative_test_filename(test_input.filename) def _expected_driver_output(self): return ExpectedDriverOutput(self._port.expected_text(self._filename), @@ -168,11 +72,7 @@ class SingleTestRunner: return (self._options.pixel_tests and not (self._options.new_baseline or self._options.reset_results)) - def _driver_input(self, test_input): - self._filename = test_input.filename - self._timeout = test_input.timeout - self._testname = self._port.relative_test_filename(test_input.filename) - + def _driver_input(self): # The image hash is used to avoid doing an image dump if the # checksums match, so it should be set to a blank value if we # are generating a new baseline. (Otherwise, an image from a @@ -182,17 +82,21 @@ class SingleTestRunner: image_hash = self._port.expected_checksum(self._filename) return base.DriverInput(self._filename, self._timeout, image_hash) - def _run(self, driver, test_input): + def run(self): if self._options.new_baseline or self._options.reset_results: - return self._run_rebaseline(driver, test_input) - return self._run_compare_test(driver, test_input) + return self._run_rebaseline() + return self._run_compare_test() - def _run_compare_test(self, driver, test_input): - driver_output = self._driver.run_test(self._driver_input(test_input)) - return self._process_output(driver_output) + def _run_compare_test(self): + driver_output = self._driver.run_test(self._driver_input()) + expected_driver_output = self._expected_driver_output() + test_result = self._compare_output(driver_output, expected_driver_output) + test_result_writer.write_test_result(self._port, self._options.results_directory, self._filename, + driver_output, expected_driver_output, test_result.failures) + return test_result - def _run_rebaseline(self, driver, test_input): - driver_output = self._driver.run_test(self._driver_input(test_input)) + def _run_rebaseline(self): + driver_output = self._driver.run_test(self._driver_input()) failures = self._handle_error(driver_output) # FIXME: It the test crashed or timed out, it might be bettter to avoid # to write new baselines. @@ -223,7 +127,7 @@ class SingleTestRunner: generate_new_baseline: whether to enerate a new, platform-specific baseline, or update the existing one """ - + assert data is not None port = self._port fs = port._filesystem if generate_new_baseline: @@ -250,6 +154,7 @@ class SingleTestRunner: failures.append(test_failures.FailureCrash()) _log.debug("%s Stacktrace for %s:\n%s" % (self._worker_name, self._testname, driver_output.error)) + # FIXME: Use test_result_writer module. stack_filename = fs.join(self._options.results_directory, self._testname) stack_filename = fs.splitext(stack_filename)[0] + "-stack.txt" fs.maybe_make_directory(fs.dirname(stack_filename)) @@ -259,64 +164,49 @@ class SingleTestRunner: driver_output.error)) return failures - def _run_test(self): - driver_output = self._driver.run_test(self._driver_input()) - return self._process_output(driver_output) - - def _process_output(self, driver_output): - """Receives the output from a DumpRenderTree process, subjects it to a - number of tests, and returns a list of failure types the test produced. - Args: - driver_output: a DriverOutput object containing the output from the driver - - Returns: a TestResult object - """ - fs = self._port._filesystem - failures = self._handle_error(driver_output) - expected_driver_output = self._expected_driver_output() + def _compare_output(self, driver_output, expected_driver_output): + failures = [] + failures.extend(self._handle_error(driver_output)) - # Check the output and save the results. - start_time = time.time() - time_for_diffs = {} - for test_type in self._test_types: - start_diff_time = time.time() - new_failures = test_type.compare_output( - self._port, self._filename, self._options, driver_output, - expected_driver_output) - # Don't add any more failures if we already have a crash, so we don't - # double-report those tests. We do double-report for timeouts since - # we still want to see the text and image output. - if not driver_output.crash: - failures.extend(new_failures) - time_for_diffs[test_type.__class__.__name__] = ( - time.time() - start_diff_time) + if driver_output.crash: + # Don't continue any more if we already have a crash. + # In case of timeouts, we continue since we still want to see the text and image output. + return TestResult(self._filename, failures, driver_output.test_time) - total_time_for_all_diffs = time.time() - start_diff_time - return TestResult(self._filename, failures, driver_output.test_time, - total_time_for_all_diffs, time_for_diffs) + failures.extend(self._compare_text(driver_output.text, expected_driver_output.text)) + if self._options.pixel_tests: + failures.extend(self._compare_image(driver_output, expected_driver_output)) + return TestResult(self._filename, failures, driver_output.test_time) - def start_servers_with_lock(self): - _log.debug('Acquiring http lock ...') - self._port.acquire_http_lock() - _log.debug('Starting HTTP server ...') - self._port.start_http_server() - _log.debug('Starting WebSocket server ...') - self._port.start_websocket_server() - self.has_http_lock = True + def _compare_text(self, actual_text, expected_text): + failures = [] + if self._port.compare_text(self._get_normalized_output_text(actual_text), + # Assuming expected_text is already normalized. + expected_text): + if expected_text == '': + failures.append(test_failures.FailureMissingResult()) + else: + failures.append(test_failures.FailureTextMismatch()) + return failures - def stop_servers_with_lock(self): - """Stop the servers and release http lock.""" - if self.has_http_lock: - _log.debug('Stopping HTTP server ...') - self._port.stop_http_server() - _log.debug('Stopping WebSocket server ...') - self._port.stop_websocket_server() - _log.debug('Releasing server lock ...') - self._port.release_http_lock() - self.has_http_lock = False + def _get_normalized_output_text(self, output): + """Returns the normalized text output, i.e. the output in which + the end-of-line characters are normalized to "\n".""" + # Running tests on Windows produces "\r\n". The "\n" part is helpfully + # changed to "\r\n" by our system (Python/Cygwin), resulting in + # "\r\r\n", when, in fact, we wanted to compare the text output with + # the normalized text expectation files. + return output.replace("\r\r\n", "\r\n").replace("\r\n", "\n") - def kill_dump_render_tree(self): - """Kill the DumpRenderTree process if it's running.""" - if self._driver: - self._driver.stop() - self._driver = None + def _compare_image(self, driver_output, expected_driver_outputs): + failures = [] + # If we didn't produce a hash file, this test must be text-only. + if driver_output.image_hash is None: + return failures + if not expected_driver_outputs.image: + failures.append(test_failures.FailureMissingImage()) + elif not expected_driver_outputs.image_hash: + failures.append(test_failures.FailureMissingImageHash()) + elif driver_output.image_hash != expected_driver_outputs.image_hash: + failures.append(test_failures.FailureImageHashMismatch()) + return failures diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py index 494395a..132ccc2 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py @@ -242,10 +242,6 @@ class TestExpectationsFile: SKIP: Doesn't run the test. SLOW: The test takes a long time to run, but does not timeout indefinitely. WONTFIX: For tests that we never intend to pass on a given platform. - DEBUG: Expectations apply only to the debug build. - RELEASE: Expectations apply only to release build. - LINUX/WIN/WIN-XP/WIN-VISTA/WIN-7/MAC: Expectations apply only to these - platforms. Notes: -A test cannot be both SLOW and TIMEOUT @@ -496,9 +492,9 @@ class TestExpectationsFile: # Do not add tags WIN-7 and WIN-VISTA to test expectations # if the original line does not specify the platform # option. - # TODO(victorw): Remove WIN-VISTA and WIN-7 once we have + # TODO(victorw): Remove WIN-VISTA and WIN-WIN7 once we have # reliable Win 7 and Win Vista buildbots setup. - if not p in (platform.upper(), 'WIN-VISTA', 'WIN-7'): + if not p in (platform.upper(), 'WIN-VISTA', 'WIN-WIN7'): new_options = parts[0] + p + ' ' new_line = ('%s:%s' % (new_options, parts[1])) f_new.append(new_line) diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py new file mode 100644 index 0000000..882da91 --- /dev/null +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py @@ -0,0 +1,195 @@ +# Copyright (C) 2011 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import logging +import os + +from webkitpy.layout_tests.layout_package import test_failures + + +_log = logging.getLogger(__name__) + + +def write_test_result(port, root_output_dir, filename, driver_output, + expected_driver_output, failures): + """Write the test result to the result output directory.""" + checksums_mismatch_but_images_are_same = False + imagehash_mismatch_failure = None + writer = TestResultWriter(port, root_output_dir, filename) + for failure in failures: + # FIXME: Instead of this long 'if' block, each failure class might + # have a responsibility for writing a test result. + if isinstance(failure, (test_failures.FailureMissingResult, + test_failures.FailureTextMismatch)): + writer.write_text_files(driver_output.text, expected_driver_output.text) + writer.create_text_diff_and_write_result(driver_output.text, expected_driver_output.text) + elif isinstance(failure, test_failures.FailureMissingImage): + writer.write_image_files(driver_output.image, expected_image=None) + writer.write_image_hashes(driver_output.image_hash, expected_driver_output.image_hash) + elif isinstance(failure, test_failures.FailureMissingImageHash): + writer.write_image_files(driver_output.image, expected_driver_output.image) + writer.write_image_hashes(driver_output.image_hash, expected_image_hash=None) + elif isinstance(failure, test_failures.FailureImageHashMismatch): + writer.write_image_files(driver_output.image, expected_driver_output.image) + writer.write_image_hashes(driver_output.image_hash, expected_driver_output.image_hash) + images_are_different = writer.create_image_diff_and_write_result( + driver_output.image, expected_driver_output.image) + if not images_are_different: + checksums_mismatch_but_images_are_same = True + imagehash_mismatch_failure = failure + elif isinstance(failure, test_failures.FailureCrash): + writer.write_crash_report(driver_output.error) + else: + assert isinstance(failure, (test_failures.FailureTimeout,)) + + # FIXME: This is an ugly hack to handle FailureImageHashIncorrect case. + # Ideally, FailureImageHashIncorrect case should be detected before this + # function is called. But it requires calling create_diff_image() to detect + # whether two images are same or not. So we need this hack until we have a better approach. + if checksums_mismatch_but_images_are_same: + # Replace FailureImageHashMismatch with FailureImageHashIncorrect. + failures.remove(imagehash_mismatch_failure) + failures.append(test_failures.FailureImageHashIncorrect()) + + +class TestResultWriter(object): + """A class which handles all writing operations to the result directory.""" + + # Filename pieces when writing failures to the test results directory. + FILENAME_SUFFIX_ACTUAL = "-actual" + FILENAME_SUFFIX_EXPECTED = "-expected" + FILENAME_SUFFIX_DIFF = "-diff" + FILENAME_SUFFIX_WDIFF = "-wdiff.html" + FILENAME_SUFFIX_PRETTY_PATCH = "-pretty-diff.html" + FILENAME_SUFFIX_IMAGE_DIFF = "-diff.png" + + def __init__(self, port, root_output_dir, filename): + self._port = port + self._root_output_dir = root_output_dir + self._filename = filename + self._testname = port.relative_test_filename(filename) + + def _make_output_directory(self): + """Creates the output directory (if needed) for a given test filename.""" + fs = self._port._filesystem + output_filename = fs.join(self._root_output_dir, self._testname) + self._port.maybe_make_directory(fs.dirname(output_filename)) + + def output_filename(self, modifier): + """Returns a filename inside the output dir that contains modifier. + + For example, if test name is "fast/dom/foo.html" and modifier is "-expected.txt", + the return value is "/<path-to-root-output-dir>/fast/dom/foo-expected.txt". + + Args: + modifier: a string to replace the extension of filename with + + Return: + The absolute path to the output filename + """ + fs = self._port._filesystem + output_filename = fs.join(self._root_output_dir, self._testname) + return fs.splitext(output_filename)[0] + modifier + + def write_output_files(self, file_type, output, expected): + """Writes the test output, the expected output in the results directory. + + The full output filename of the actual, for example, will be + <filename>-actual<file_type> + For instance, + my_test-actual.txt + + Args: + file_type: A string describing the test output file type, e.g. ".txt" + output: A string containing the test output + expected: A string containing the expected test output + """ + self._make_output_directory() + actual_filename = self.output_filename(self.FILENAME_SUFFIX_ACTUAL + file_type) + expected_filename = self.output_filename(self.FILENAME_SUFFIX_EXPECTED + file_type) + + fs = self._port._filesystem + if output: + fs.write_binary_file(actual_filename, output) + if expected: + fs.write_binary_file(expected_filename, expected) + + def write_crash_report(self, error): + """Write crash information.""" + fs = self._port._filesystem + filename = self.output_filename("-stack.txt") + fs.maybe_make_directory(fs.dirname(filename)) + fs.write_text_file(filename, error) + + def write_text_files(self, actual_text, expected_text): + self.write_output_files(".txt", actual_text, expected_text) + + def create_text_diff_and_write_result(self, actual_text, expected_text): + # FIXME: This function is actually doing the diffs as well as writing results. + # It might be better to extract code which does 'diff' and make it a separate function. + if not actual_text or not expected_text: + return + + self._make_output_directory() + file_type = '.txt' + actual_filename = self.output_filename(self.FILENAME_SUFFIX_ACTUAL + file_type) + expected_filename = self.output_filename(self.FILENAME_SUFFIX_EXPECTED + file_type) + fs = self._port._filesystem + # We treat diff output as binary. Diff output may contain multiple files + # in conflicting encodings. + diff = self._port.diff_text(expected_text, actual_text, expected_filename, actual_filename) + diff_filename = self.output_filename(self.FILENAME_SUFFIX_DIFF + file_type) + fs.write_binary_file(diff_filename, diff) + + # Shell out to wdiff to get colored inline diffs. + wdiff = self._port.wdiff_text(expected_filename, actual_filename) + wdiff_filename = self.output_filename(self.FILENAME_SUFFIX_WDIFF) + fs.write_binary_file(wdiff_filename, wdiff) + + # Use WebKit's PrettyPatch.rb to get an HTML diff. + pretty_patch = self._port.pretty_patch_text(diff_filename) + pretty_patch_filename = self.output_filename(self.FILENAME_SUFFIX_PRETTY_PATCH) + fs.write_binary_file(pretty_patch_filename, pretty_patch) + + def write_image_files(self, actual_image, expected_image): + self.write_output_files('.png', actual_image, expected_image) + + def write_image_hashes(self, actual_image_hash, expected_image_hash): + self.write_output_files('.checksum', actual_image_hash, expected_image_hash) + + def create_image_diff_and_write_result(self, actual_image, expected_image): + """Writes the visual diff of the expected/actual PNGs. + + Returns True if the images are different. + """ + # FIXME: This function is actually doing the diff as well as writing a result. + # It might be better to extract 'diff' code and make it a separate function. + # To do so, we have to change port.diff_image() as well. + diff_filename = self.output_filename(self.FILENAME_SUFFIX_IMAGE_DIFF) + return self._port.diff_image(actual_image, expected_image, diff_filename) diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py index 055f65b..b3840af 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py @@ -38,22 +38,17 @@ class TestResult(object): def loads(str): return cPickle.loads(str) - def __init__(self, filename, failures=None, test_run_time=None, total_time_for_all_diffs=None, time_for_diffs=None): + def __init__(self, filename, failures=None, test_run_time=None): self.filename = filename self.failures = failures or [] self.test_run_time = test_run_time or 0 - self.total_time_for_all_diffs = total_time_for_all_diffs or 0 - self.time_for_diffs = time_for_diffs or {} # FIXME: Why is this a dictionary? - # FIXME: Setting this in the constructor makes this class hard to mutate. self.type = test_failures.determine_result_type(failures) def __eq__(self, other): return (self.filename == other.filename and self.failures == other.failures and - self.test_run_time == other.test_run_time and - self.time_for_diffs == other.time_for_diffs and - self.total_time_for_all_diffs == other.total_time_for_all_diffs) + self.test_run_time == other.test_run_time) def __ne__(self, other): return not (self == other) diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results_unittest.py index c8fcf64..81107d3 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results_unittest.py @@ -37,15 +37,11 @@ class Test(unittest.TestCase): self.assertEqual(result.filename, 'foo') self.assertEqual(result.failures, []) self.assertEqual(result.test_run_time, 0) - self.assertEqual(result.total_time_for_all_diffs, 0) - self.assertEqual(result.time_for_diffs, {}) def test_loads(self): result = TestResult(filename='foo', failures=[], - test_run_time=1.1, - total_time_for_all_diffs=0.5, - time_for_diffs={}) + test_run_time=1.1) s = result.dumps() new_result = TestResult.loads(s) self.assertTrue(isinstance(new_result, TestResult)) diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner.py index e3bd4ad..0859f68 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner.py @@ -37,6 +37,7 @@ create a final report. from __future__ import with_statement +import copy import errno import logging import math @@ -45,17 +46,17 @@ import random import sys import time -from result_summary import ResultSummary -from test_input import TestInput - -import dump_render_tree_thread -import json_layout_results_generator -import message_broker -import printing -import test_expectations -import test_failures -import test_results -import test_results_uploader +from webkitpy.layout_tests.layout_package import dump_render_tree_thread +from webkitpy.layout_tests.layout_package import json_layout_results_generator +from webkitpy.layout_tests.layout_package import json_results_generator +from webkitpy.layout_tests.layout_package import message_broker +from webkitpy.layout_tests.layout_package import printing +from webkitpy.layout_tests.layout_package import test_expectations +from webkitpy.layout_tests.layout_package import test_failures +from webkitpy.layout_tests.layout_package import test_results +from webkitpy.layout_tests.layout_package import test_results_uploader +from webkitpy.layout_tests.layout_package.result_summary import ResultSummary +from webkitpy.layout_tests.layout_package.test_input import TestInput from webkitpy.thirdparty import simplejson from webkitpy.tool import grammar @@ -68,8 +69,7 @@ BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/" TestExpectationsFile = test_expectations.TestExpectationsFile -def summarize_unexpected_results(port_obj, expectations, result_summary, - retry_summary): +def summarize_results(port_obj, expectations, result_summary, retry_summary, test_timings, only_unexpected): """Summarize any unexpected results as a dict. FIXME: split this data structure into a separate class? @@ -79,6 +79,8 @@ def summarize_unexpected_results(port_obj, expectations, result_summary, expectations: test_expectations.TestExpectations object result_summary: summary object from initial test runs retry_summary: summary object from final test run of retried tests + test_timings: a list of TestResult objects which contain test runtimes in seconds + only_unexpected: whether to return a summary only for the unexpected results Returns: A dictionary containing a summary of the unexpected results from the run, with the following fields: @@ -88,11 +90,13 @@ def summarize_unexpected_results(port_obj, expectations, result_summary, 'num_regressions': # of non-flaky failures 'num_flaky': # of flaky failures 'num_passes': # of unexpected passes - 'tests': a dict of tests -> {'expected': '...', 'actual': '...'} + 'tests': a dict of tests -> {'expected': '...', 'actual': '...', 'time_ms': ...} """ results = {} results['version'] = 1 + test_timings_map = dict((test_result.filename, test_result.test_run_time) for test_result in test_timings) + tbe = result_summary.tests_by_expectation tbt = result_summary.tests_by_timeline results['fixable'] = len(tbt[test_expectations.NOW] - @@ -104,31 +108,36 @@ def summarize_unexpected_results(port_obj, expectations, result_summary, num_flaky = 0 num_regressions = 0 keywords = {} - for k, v in TestExpectationsFile.EXPECTATIONS.iteritems(): - keywords[v] = k.upper() + for expecation_string, expectation_enum in TestExpectationsFile.EXPECTATIONS.iteritems(): + keywords[expectation_enum] = expecation_string.upper() + + for modifier_string, modifier_enum in TestExpectationsFile.MODIFIERS.iteritems(): + keywords[modifier_enum] = modifier_string.upper() tests = {} - for filename, result in result_summary.unexpected_results.iteritems(): + original_results = result_summary.unexpected_results if only_unexpected else result_summary.results + + for filename, result in original_results.iteritems(): # Note that if a test crashed in the original run, we ignore # whether or not it crashed when we retried it (if we retried it), # and always consider the result not flaky. test = port_obj.relative_test_filename(filename) expected = expectations.get_expectations_string(filename) - actual = [keywords[result]] + result_type = result.type + actual = [keywords[result_type]] - if result == test_expectations.PASS: + if result_type == test_expectations.PASS: num_passes += 1 - elif result == test_expectations.CRASH: + elif result_type == test_expectations.CRASH: num_regressions += 1 - else: + elif filename in result_summary.unexpected_results: if filename not in retry_summary.unexpected_results: - actual.extend(expectations.get_expectations_string( - filename).split(" ")) + actual.extend(expectations.get_expectations_string(filename).split(" ")) num_flaky += 1 else: - retry_result = retry_summary.unexpected_results[filename] - if result != retry_result: - actual.append(keywords[retry_result]) + retry_result_type = retry_summary.unexpected_results[filename].type + if result_type != retry_result_type: + actual.append(keywords[retry_result_type]) num_flaky += 1 else: num_regressions += 1 @@ -137,6 +146,10 @@ def summarize_unexpected_results(port_obj, expectations, result_summary, tests[test]['expected'] = expected tests[test]['actual'] = " ".join(actual) + if filename in test_timings_map: + time_seconds = test_timings_map[filename] + tests[test]['time_ms'] = int(1000 * time_seconds) + results['tests'] = tests results['num_passes'] = num_passes results['num_flaky'] = num_flaky @@ -150,6 +163,9 @@ class TestRunInterruptedException(Exception): def __init__(self, reason): self.reason = reason + def __reduce__(self): + return self.__class__, (self.reason,) + class TestRunner: """A class for managing running a series of tests on a series of layout @@ -683,17 +699,19 @@ class TestRunner: result_summary.expected, result_summary.unexpected) - unexpected_results = summarize_unexpected_results(self._port, - self._expectations, result_summary, retry_summary) + unexpected_results = summarize_results(self._port, + self._expectations, result_summary, retry_summary, individual_test_timings, only_unexpected=True) self._printer.print_unexpected_results(unexpected_results) # FIXME: remove record_results. It's just used for testing. There's no need # for it to be a commandline argument. if (self._options.record_results and not self._options.dry_run and - not interrupted): + not keyboard_interrupted): # Write the same data to log files and upload generated JSON files # to appengine server. - self._upload_json_files(unexpected_results, result_summary, + summarized_results = summarize_results(self._port, + self._expectations, result_summary, retry_summary, individual_test_timings, only_unexpected=False) + self._upload_json_files(unexpected_results, summarized_results, result_summary, individual_test_timings) # Write the summary to disk (results.html) and display it if requested. @@ -782,14 +800,22 @@ class TestRunner: """ failed_results = {} for test, result in result_summary.unexpected_results.iteritems(): - if (result == test_expectations.PASS or - result == test_expectations.CRASH and not include_crashes): + if (result.type == test_expectations.PASS or + result.type == test_expectations.CRASH and not include_crashes): continue - failed_results[test] = result + failed_results[test] = result.type return failed_results - def _upload_json_files(self, unexpected_results, result_summary, + def _char_for_result(self, result): + result = result.lower() + if result in TestExpectationsFile.EXPECTATIONS: + result_enum_value = TestExpectationsFile.EXPECTATIONS[result] + else: + result_enum_value = TestExpectationsFile.MODIFIERS[result] + return json_layout_results_generator.JSONLayoutResultsGenerator.FAILURE_TO_CHAR[result_enum_value] + + def _upload_json_files(self, unexpected_results, summarized_results, result_summary, individual_test_timings): """Writes the results of the test run as JSON files into the results dir and upload the files to the appengine server. @@ -803,19 +829,22 @@ class TestRunner: Args: unexpected_results: dict of unexpected results + summarized_results: dict of results result_summary: full summary object individual_test_timings: list of test times (used by the flakiness dashboard). """ - results_directory = self._options.results_directory - _log.debug("Writing JSON files in %s." % results_directory) - unexpected_json_path = self._fs.join(results_directory, "unexpected_results.json") - with self._fs.open_text_file_for_writing(unexpected_json_path) as file: - simplejson.dump(unexpected_results, file, sort_keys=True, indent=2) + _log.debug("Writing JSON files in %s." % self._options.results_directory) + + unexpected_json_path = self._fs.join(self._options.results_directory, "unexpected_results.json") + json_results_generator.write_json(self._fs, unexpected_results, unexpected_json_path) + + full_results_path = self._fs.join(self._options.results_directory, "full_results.json") + json_results_generator.write_json(self._fs, summarized_results, full_results_path) # Write a json file of the test_expectations.txt file for the layout # tests dashboard. - expectations_path = self._fs.join(results_directory, "expectations.json") + expectations_path = self._fs.join(self._options.results_directory, "expectations.json") expectations_json = \ self._expectations.get_expectations_json_for_all_platforms() self._fs.write_text_file(expectations_path, @@ -832,7 +861,7 @@ class TestRunner: _log.debug("Finished writing JSON files.") - json_files = ["expectations.json", "incremental_results.json"] + json_files = ["expectations.json", "incremental_results.json", "full_results.json"] generator.upload_json_files(json_files) @@ -930,34 +959,9 @@ class TestRunner: Args: individual_test_timings: List of TestResults for all tests. """ - test_types = [] # Unit tests don't actually produce any timings. - if individual_test_timings: - test_types = individual_test_timings[0].time_for_diffs.keys() - times_for_dump_render_tree = [] - times_for_diff_processing = [] - times_per_test_type = {} - for test_type in test_types: - times_per_test_type[test_type] = [] - - for test_stats in individual_test_timings: - times_for_dump_render_tree.append(test_stats.test_run_time) - times_for_diff_processing.append( - test_stats.total_time_for_all_diffs) - time_for_diffs = test_stats.time_for_diffs - for test_type in test_types: - times_per_test_type[test_type].append( - time_for_diffs[test_type]) - - self._print_statistics_for_test_timings( - "PER TEST TIME IN TESTSHELL (seconds):", - times_for_dump_render_tree) - self._print_statistics_for_test_timings( - "PER TEST DIFF PROCESSING TIMES (seconds):", - times_for_diff_processing) - for test_type in test_types: - self._print_statistics_for_test_timings( - "PER TEST TIMES BY TEST TYPE: %s" % test_type, - times_per_test_type[test_type]) + times_for_dump_render_tree = [test_stats.test_run_time for test_stats in individual_test_timings] + self._print_statistics_for_test_timings("PER TEST TIME IN TESTSHELL (seconds):", + times_for_dump_render_tree) def _print_individual_test_times(self, individual_test_timings, result_summary): diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner2.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner2.py index f097b83..0522d39 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner2.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner2.py @@ -34,25 +34,65 @@ workers and receive their completion messages accordingly. """ import logging +import time +from webkitpy.tool import grammar from webkitpy.layout_tests.layout_package import manager_worker_broker from webkitpy.layout_tests.layout_package import test_runner from webkitpy.layout_tests.layout_package import worker + _log = logging.getLogger(__name__) +class _WorkerState(object): + """A class for the TestRunner/manager to use to track the current state + of the workers.""" + def __init__(self, number, worker_connection): + self.worker_connection = worker_connection + self.number = number + self.done = False + self.current_test_name = None + self.next_timeout = None + self.wedged = False + self.stats = {} + self.stats['name'] = worker_connection.name + self.stats['num_tests'] = 0 + self.stats['total_time'] = 0 + + def __repr__(self): + return "_WorkerState(" + str(self.__dict__) + ")" + + class TestRunner2(test_runner.TestRunner): def __init__(self, port, options, printer): test_runner.TestRunner.__init__(self, port, options, printer) self._all_results = [] self._group_stats = {} self._current_result_summary = None - self._done = False + + # This maps worker names to the state we are tracking for each of them. + self._worker_states = {} def is_done(self): - return self._done + worker_states = self._worker_states.values() + return worker_states and all(self._worker_is_done(worker_state) for worker_state in worker_states) + + def _worker_is_done(self, worker_state): + t = time.time() + if worker_state.done or worker_state.wedged: + return True + + next_timeout = worker_state.next_timeout + WEDGE_PADDING = 40.0 + if next_timeout and t > next_timeout + WEDGE_PADDING: + _log.error('') + worker_state.worker_connection.log_wedged_worker(worker_state.current_test_name) + _log.error('') + worker_state.wedged = True + return True + return False def name(self): return 'TestRunner2' @@ -60,8 +100,9 @@ class TestRunner2(test_runner.TestRunner): def _run_tests(self, file_list, result_summary): """Runs the tests in the file_list. - Return: A tuple (keyboard_interrupted, thread_timings, test_timings, - individual_test_timings) + Return: A tuple (interrupted, keyboard_interrupted, thread_timings, + test_timings, individual_test_timings) + interrupted is whether the run was interrupted keyboard_interrupted is whether someone typed Ctrl^C thread_timings is a list of dicts with the total runtime of each thread with 'name', 'num_tests', 'total_time' properties @@ -72,58 +113,118 @@ class TestRunner2(test_runner.TestRunner): result_summary: summary object to populate with the results """ self._current_result_summary = result_summary + self._all_results = [] + self._group_stats = {} + self._worker_states = {} + + num_workers = self._num_workers() + keyboard_interrupted = False + interrupted = False + thread_timings = [] + + self._printer.print_update('Sharding tests ...') + test_lists = self._shard_tests(file_list, + num_workers > 1 and not self._options.experimental_fully_parallel) + _log.debug("Using %d shards" % len(test_lists)) - # FIXME: shard properly. + manager_connection = manager_worker_broker.get(self._port, self._options, + self, worker.Worker) - # FIXME: should shard_tests return a list of objects rather than tuples? - test_lists = self._shard_tests(file_list, False) + if self._options.dry_run: + return (keyboard_interrupted, interrupted, thread_timings, + self._group_stats, self._all_results) - manager_connection = manager_worker_broker.get(self._port, self._options, self, worker.Worker) + self._printer.print_update('Starting %s ...' % + grammar.pluralize('worker', num_workers)) + for worker_number in xrange(num_workers): + worker_connection = manager_connection.start_worker(worker_number) + worker_state = _WorkerState(worker_number, worker_connection) + self._worker_states[worker_connection.name] = worker_state - # FIXME: start all of the workers. - manager_connection.start_worker(0) + # FIXME: If we start workers up too quickly, DumpRenderTree appears + # to thrash on something and time out its first few tests. Until + # we can figure out what's going on, sleep a bit in between + # workers. + time.sleep(0.1) + self._printer.print_update("Starting testing ...") for test_list in test_lists: manager_connection.post_message('test_list', test_list[0], test_list[1]) - manager_connection.post_message('stop') + # We post one 'stop' message for each worker. Because the stop message + # are sent after all of the tests, and because each worker will stop + # reading messsages after receiving a stop, we can be sure each + # worker will get a stop message and hence they will all shut down. + for i in xrange(num_workers): + manager_connection.post_message('stop') - keyboard_interrupted = False - interrupted = False - if not self._options.dry_run: - while not self._check_if_done(): + try: + while not self.is_done(): + # We loop with a timeout in order to be able to detect wedged threads. manager_connection.run_message_loop(delay_secs=1.0) - # FIXME: implement stats. - thread_timings = [] + if any(worker_state.wedged for worker_state in self._worker_states.values()): + _log.error('') + _log.error('Remaining workers are wedged, bailing out.') + _log.error('') + else: + _log.debug('No wedged threads') + + # Make sure all of the workers have shut down (if possible). + for worker_state in self._worker_states.values(): + if not worker_state.wedged and worker_state.worker_connection.is_alive(): + worker_state.worker_connection.join(0.5) + assert not worker_state.worker_connection.is_alive() + + except KeyboardInterrupt: + _log.info("Interrupted, exiting") + self.cancel_workers() + keyboard_interrupted = True + except test_runner.TestRunInterruptedException, e: + _log.info(e.reason) + self.cancel_workers() + interrupted = True + except: + # Unexpected exception; don't try to clean up workers. + _log.info("Exception raised, exiting") + raise + + thread_timings = [worker_state.stats for worker_state in self._worker_states.values()] # FIXME: should this be a class instead of a tuple? - return (keyboard_interrupted, interrupted, thread_timings, + return (interrupted, keyboard_interrupted, thread_timings, self._group_stats, self._all_results) - def _check_if_done(self): - """Returns true iff all the workers have either completed or wedged.""" - # FIXME: implement to check for wedged workers. - return self._done + def cancel_workers(self): + for worker_state in self._worker_states.values(): + worker_state.worker_connection.cancel() - def handle_started_test(self, src, test_info, hang_timeout): - # FIXME: implement - pass + def handle_started_test(self, source, test_info, hang_timeout): + worker_state = self._worker_states[source] + worker_state.current_test_name = self._port.relative_test_filename(test_info.filename) + worker_state.next_timeout = time.time() + hang_timeout - def handle_done(self, src): - # FIXME: implement properly to handle multiple workers. - self._done = True - pass + def handle_done(self, source): + worker_state = self._worker_states[source] + worker_state.done = True - def handle_exception(self, src, exception_info): - raise exception_info + def handle_exception(self, source, exception_info): + exception_type, exception_value, exception_traceback = exception_info + raise exception_type, exception_value, exception_traceback - def handle_finished_list(self, src, list_name, num_tests, elapsed_time): - # FIXME: update stats - pass + def handle_finished_list(self, source, list_name, num_tests, elapsed_time): + self._group_stats[list_name] = (num_tests, elapsed_time) - def handle_finished_test(self, src, result, elapsed_time): - self._update_summary_with_result(self._current_result_summary, result) + def handle_finished_test(self, source, result, elapsed_time): + worker_state = self._worker_states[source] + worker_state.next_timeout = None + worker_state.current_test_name = None + worker_state.stats['total_time'] += elapsed_time + worker_state.stats['num_tests'] += 1 + + if worker_state.wedged: + # This shouldn't happen if we have our timeouts tuned properly. + _log.error("%s unwedged", w.name) - # FIXME: update stats. self._all_results.append(result) + self._update_summary_with_result(self._current_result_summary, result) diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/worker.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/worker.py index 47d4fbd..bf77526 100644 --- a/Tools/Scripts/webkitpy/layout_tests/layout_package/worker.py +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/worker.py @@ -35,41 +35,63 @@ import time from webkitpy.common.system import stack_utils from webkitpy.layout_tests.layout_package import manager_worker_broker -from webkitpy.layout_tests.layout_package import test_results +from webkitpy.layout_tests.layout_package import worker_mixin _log = logging.getLogger(__name__) -class Worker(manager_worker_broker.AbstractWorker): +class Worker(manager_worker_broker.AbstractWorker, worker_mixin.WorkerMixin): def __init__(self, worker_connection, worker_number, options): self._worker_connection = worker_connection self._worker_number = worker_number self._options = options self._name = 'worker/%d' % worker_number self._done = False + self._canceled = False self._port = None - def _deferred_init(self, port): - self._port = port + def __del__(self): + self.cleanup() + + def cancel(self): + """Attempt to abort processing (best effort).""" + self._canceled = True def is_done(self): - return self._done + return self._done or self._canceled def name(self): return self._name def run(self, port): - self._deferred_init(port) + self.safe_init(port) + exception_msg = "" _log.debug("%s starting" % self._name) - # FIXME: need to add in error handling, better logging. - self._worker_connection.run_message_loop() - self._worker_connection.post_message('done') + try: + self._worker_connection.run_message_loop() + if not self.is_done(): + raise AssertionError("%s: ran out of messages in worker queue." + % self._name) + except KeyboardInterrupt: + exception_msg = ", interrupted" + except: + exception_msg = ", exception raised" + finally: + _log.debug("%s done%s" % (self._name, exception_msg)) + if exception_msg: + exception_type, exception_value, exception_traceback = sys.exc_info() + stack_utils.log_traceback(_log.error, exception_traceback) + # FIXME: Figure out how to send a message with a traceback. + self._worker_connection.post_message('exception', + (exception_type, exception_value, None)) + self._worker_connection.post_message('done') def handle_test_list(self, src, list_name, test_list): - # FIXME: check to see if we need to get the http lock. + if list_name == "tests_to_http_lock": + self.start_servers_with_lock() start_time = time.time() num_tests = 0 @@ -81,24 +103,20 @@ class Worker(manager_worker_broker.AbstractWorker): elapsed_time = time.time() - start_time self._worker_connection.post_message('finished_list', list_name, num_tests, elapsed_time) - # FIXME: release the lock if necessary + if self._has_http_lock: + self.stop_servers_with_lock() def handle_stop(self, src): self._done = True def _run_test(self, test_input): - - # FIXME: get real timeout value from SingleTestRunner - test_timeout_sec = int(test_input.timeout) / 1000 + test_timeout_sec = self.timeout(test_input) start = time.time() self._worker_connection.post_message('started_test', test_input, test_timeout_sec) - # FIXME: actually run the test. - result = test_results.TestResult(test_input.filename, failures=[], - test_run_time=0, total_time_for_all_diffs=0, time_for_diffs={}) + result = self.run_test_with_timeout(test_input, test_timeout_sec) elapsed_time = time.time() - start - - # FIXME: update stats, check for failures. - self._worker_connection.post_message('finished_test', result, elapsed_time) + + self.clean_up_after_test(test_input, result) diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/worker_mixin.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/worker_mixin.py new file mode 100644 index 0000000..7876f91 --- /dev/null +++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/worker_mixin.py @@ -0,0 +1,208 @@ +# Copyright (C) 2011 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import logging +import sys +import threading +import time + +from webkitpy.layout_tests.layout_package import single_test_runner +from webkitpy.layout_tests.layout_package import test_results + +_log = logging.getLogger(__name__) + + +class WorkerMixin(object): + """This class holds logic common to Worker and TestShellThread that + doesn't directly have to do with running the tests (which is + SingleTestRunner's responsibility. This class cannot stand on its own.""" + def __init__(self): + assert False, "WorkerMixin can't be directly instantiated" + + def safe_init(self, port): + """This method should only be called when it is is safe for the mixin + to create state that can't be Pickled. + + This routine exists so that the mixin can be created and then marshaled + across into a child process.""" + self._port = port + self._filesystem = port._filesystem + self._batch_count = 0 + self._batch_size = self._options.batch_size + self._driver = None + tests_run_filename = self._filesystem.join(self._options.results_directory, + "tests_run%d.txt" % self._worker_number) + self._tests_run_file = self._filesystem.open_text_file_for_writing(tests_run_filename) + + # FIXME: it's goofy that we have to track this at all, but it's due to + # the awkward logic in TestShellThread._run(). When we remove that + # file, we should rewrite this code so that caller keeps track of whether + # the lock is held. + self._has_http_lock = False + + def cleanup(self): + if self._driver: + self.kill_driver() + if self._has_http_lock: + self.stop_servers_with_lock() + if self._tests_run_file: + self._tests_run_file.close() + self._tests_run_file = None + + def timeout(self, test_input): + """Compute the appropriate timeout value for a test.""" + # The DumpRenderTree watchdog uses 2.5x the timeout; we want to be + # larger than that. We also add a little more padding if we're + # running tests in a separate thread. + # + # Note that we need to convert the test timeout from a + # string value in milliseconds to a float for Python. + driver_timeout_sec = 3.0 * float(test_input.timeout) / 1000.0 + if not self._options.run_singly: + return driver_timeout_sec + + thread_padding_sec = 1.0 + thread_timeout_sec = driver_timeout_sec + thread_padding_sec + return thread_timeout_sec + + def start_servers_with_lock(self): + _log.debug('Acquiring http lock ...') + self._port.acquire_http_lock() + _log.debug('Starting HTTP server ...') + self._port.start_http_server() + _log.debug('Starting WebSocket server ...') + self._port.start_websocket_server() + self._has_http_lock = True + + def stop_servers_with_lock(self): + if self._has_http_lock: + _log.debug('Stopping HTTP server ...') + self._port.stop_http_server() + _log.debug('Stopping WebSocket server ...') + self._port.stop_websocket_server() + _log.debug('Releasing server lock ...') + self._port.release_http_lock() + self._has_http_lock = False + + def kill_driver(self): + if self._driver: + self._driver.stop() + self._driver = None + + def run_test_with_timeout(self, test_input, timeout): + if self._options.run_singly: + return self._run_test_in_another_thread(test_input, timeout) + else: + return self._run_test_in_this_thread(test_input) + return result + + def clean_up_after_test(self, test_input, result): + self._batch_count += 1 + self._tests_run_file.write(test_input.filename + "\n") + test_name = self._port.relative_test_filename(test_input.filename) + + if result.failures: + # Check and kill DumpRenderTree if we need to. + if any([f.should_kill_dump_render_tree() for f in result.failures]): + self.kill_driver() + # Reset the batch count since the shell just bounced. + self._batch_count = 0 + + # Print the error message(s). + _log.debug("%s %s failed:" % (self._name, test_name)) + for f in result.failures: + _log.debug("%s %s" % (self._name, f.message())) + else: + _log.debug("%s %s passed" % (self._name, test_name)) + + if self._batch_size > 0 and self._batch_count >= self._batch_size: + # Bounce the shell and reset count. + self.kill_driver() + self._batch_count = 0 + + def _run_test_in_another_thread(self, test_input, thread_timeout_sec): + """Run a test in a separate thread, enforcing a hard time limit. + + Since we can only detect the termination of a thread, not any internal + state or progress, we can only run per-test timeouts when running test + files singly. + + Args: + test_input: Object containing the test filename and timeout + thread_timeout_sec: time to wait before killing the driver process. + Returns: + A TestResult + """ + worker = self + result = None + + driver = worker._port.create_driver(worker._worker_number) + driver.start() + + class SingleTestThread(threading.Thread): + def run(self): + result = worker._run_single_test(driver, test_input) + + thread = SingleTestThread() + thread.start() + thread.join(thread_timeout_sec) + if thread.isAlive(): + # If join() returned with the thread still running, the + # DumpRenderTree is completely hung and there's nothing + # more we can do with it. We have to kill all the + # DumpRenderTrees to free it up. If we're running more than + # one DumpRenderTree thread, we'll end up killing the other + # DumpRenderTrees too, introducing spurious crashes. We accept + # that tradeoff in order to avoid losing the rest of this + # thread's results. + _log.error('Test thread hung: killing all DumpRenderTrees') + + driver.stop() + + if not result: + result = test_results.TestResult(test_input.filename, failures=[], test_run_time=0) + return result + + def _run_test_in_this_thread(self, test_input): + """Run a single test file using a shared DumpRenderTree process. + + Args: + test_input: Object containing the test filename, uri and timeout + + Returns: a TestResult object. + """ + # poll() is not threadsafe and can throw OSError due to: + # http://bugs.python.org/issue1731717 + if not self._driver or self._driver.poll() is not None: + self._driver = self._port.create_driver(self._worker_number) + self._driver.start() + return self._run_single_test(self._driver, test_input) + + def _run_single_test(self, driver, test_input): + return single_test_runner.run_single_test(self._port, self._options, + test_input, driver, self._name) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/apache_http_server.py b/Tools/Scripts/webkitpy/layout_tests/port/apache_http_server.py index 46617f6..05a2338 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/apache_http_server.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/apache_http_server.py @@ -75,6 +75,7 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): test_dir = self._port_obj.layout_tests_dir() js_test_resources_dir = self._cygwin_safe_join(test_dir, "fast", "js", "resources") + media_resources_dir = self._cygwin_safe_join(test_dir, "media") mime_types_path = self._cygwin_safe_join(test_dir, "http", "conf", "mime.types") cert_file = self._cygwin_safe_join(test_dir, "http", "conf", @@ -92,6 +93,7 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): '-f', "\"%s\"" % self._get_apache_config_file_path(test_dir, output_dir), '-C', "\'DocumentRoot \"%s\"\'" % document_root, '-c', "\'Alias /js-test-resources \"%s\"'" % js_test_resources_dir, + '-c', "\'Alias /media-resources \"%s\"'" % media_resources_dir, '-C', "\'Listen %s\'" % "127.0.0.1:8000", '-C', "\'Listen %s\'" % "127.0.0.1:8081", '-c', "\'TypesConfig \"%s\"\'" % mime_types_path, @@ -194,6 +196,7 @@ class LayoutTestApacheHttpd(http_server_base.HttpServerBase): # FIXME: We should not need to be joining shell arguments into strings. # shell=True is a trail of tears. # Note: Not thread safe: http://bugs.python.org/issue2320 + _log.debug('Starting http server, cmd="%s"' % str(self._start_cmd)) self._httpd_proc = subprocess.Popen(self._start_cmd, stderr=subprocess.PIPE, shell=True) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/base.py b/Tools/Scripts/webkitpy/layout_tests/port/base.py index 5ff4bff..247a260 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/base.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/base.py @@ -38,6 +38,12 @@ import shlex import sys import time +# Handle Python < 2.6 where multiprocessing isn't available. +try: + import multiprocessing +except ImportError: + multiprocessing = None + import apache_http_server import config as port_config import http_lock @@ -83,6 +89,7 @@ class Port(object): config=None, **kwargs): self._name = port_name + self._architecture = 'x86' self._options = options if self._options is None: # FIXME: Ideally we'd have a package-wide way to get a @@ -124,6 +131,7 @@ class Port(object): if not hasattr(self._options, 'configuration') or self._options.configuration is None: self._options.configuration = self.default_configuration() self._test_configuration = None + self._multiprocessing_is_available = (multiprocessing is not None) def default_child_processes(self): """Return the number of DumpRenderTree instances to use for this @@ -356,9 +364,8 @@ class Port(object): # Make http/tests/local run as local files. This is to mimic the # logic in run-webkit-tests. # - # TODO(dpranke): remove the media reference and the SSL reference? - if (port and not relative_path.startswith("local/") and - not relative_path.startswith("media/")): + # TODO(dpranke): remove the SSL reference? + if (port and not relative_path.startswith("local/")): if relative_path.startswith("ssl/"): port += 443 protocol = "https" @@ -641,9 +648,12 @@ class Port(object): "chromium-mac" on the Chromium ports.""" raise NotImplementedError('Port.test_platform_name_to_name') + def architecture(self): + return self._architecture + def version(self): """Returns a string indicating the version of a given platform, e.g. - '-leopard' or '-xp'. + 'leopard' or 'xp'. This is used to help identify the exact port when parsing test expectations, determining search paths, and logging information.""" @@ -891,6 +901,7 @@ class Driver: driver_input: a DriverInput object Returns a DriverOutput object. + Note that DriverOutput.image will be '' (empty string) if a test crashes. """ raise NotImplementedError('Driver.run_test') @@ -928,13 +939,12 @@ class TestConfiguration(object): # FIXME: We can get the O/S and version from test_platform_name() # and version() for now, but those should go away and be cleaned up # with more generic methods like operation_system() and os_version() - # or something. Note that we need to strip the leading '-' off the - # version string if it is present. + # or something. if port: port_version = port.version() - self.os = os or port.test_platform_name().replace(port_version, '') - self.version = version or port_version[1:] - self.architecture = architecture or 'x86' + self.os = os or port.test_platform_name().replace('-' + port_version, '') + self.version = version or port_version + self.architecture = architecture or port.architecture() self.build_type = build_type or port._options.configuration.lower() self.graphics_type = graphics_type or port.graphics_type() @@ -945,7 +955,7 @@ class TestConfiguration(object): return self.__dict__.keys() def __str__(self): - return ("<%(os)s, %(version)s, %(build_type)s, %(graphics_type)s>" % + return ("<%(os)s, %(version)s, %(architecture)s, %(build_type)s, %(graphics_type)s>" % self.__dict__) def __repr__(self): @@ -977,7 +987,8 @@ class TestConfiguration(object): ('win', 'xp', 'x86'), ('win', 'vista', 'x86'), ('win', 'win7', 'x86'), - ('linux', 'hardy', 'x86')) + ('linux', 'hardy', 'x86'), + ('linux', 'hardy', 'x86_64')) def all_build_types(self): return ('debug', 'release') diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py index 7d56fa2..baf1893 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py @@ -423,10 +423,10 @@ class ChromiumDriver(base.Driver): def _output_image(self): """Returns the image output which driver generated.""" png_path = self._image_path - if png_path and self._port._filesystem.isfile(png_path): + if png_path and self._port._filesystem.exists(png_path): return self._port._filesystem.read_binary_file(png_path) else: - return None + return '' def _output_image_with_retry(self): # Retry a few more times because open() sometimes fails on Windows, @@ -443,6 +443,11 @@ class ChromiumDriver(base.Driver): raise e return self._output_image() + def _clear_output_image(self): + png_path = self._image_path + if png_path and self._port._filesystem.exists(png_path): + self._port._filesystem.remove(png_path) + def run_test(self, driver_input): output = [] error = [] @@ -450,7 +455,7 @@ class ChromiumDriver(base.Driver): timeout = False actual_uri = None actual_checksum = None - + self._clear_output_image() start_time = time.time() uri = self._port.filename_to_uri(driver_input.filename) @@ -497,9 +502,10 @@ class ChromiumDriver(base.Driver): (line, crash) = self._write_command_and_read_line(input=None) run_time = time.time() - start_time - return base.DriverOutput( - ''.join(output), self._output_image_with_retry(), actual_checksum, - crash, run_time, timeout, ''.join(error)) + output_image = self._output_image_with_retry() + assert output_image is not None + return base.DriverOutput(''.join(output), output_image, actual_checksum, + crash, run_time, timeout, ''.join(error)) def stop(self): if self._proc: diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py index e8c75c4..167f23e 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py @@ -48,11 +48,11 @@ def get(platform=None, port_name='chromium-gpu', **kwargs): else: raise NotImplementedError('unsupported platform: %s' % platform) - if port_name == 'chromium-gpu-linux': + if port_name.startswith('chromium-gpu-linux'): return ChromiumGpuLinuxPort(port_name=port_name, **kwargs) - if port_name == 'chromium-gpu-mac': + if port_name.startswith('chromium-gpu-mac'): return ChromiumGpuMacPort(port_name=port_name, **kwargs) - if port_name == 'chromium-gpu-win': + if port_name.startswith('chromium-gpu-win'): return ChromiumGpuWinPort(port_name=port_name, **kwargs) raise NotImplementedError('unsupported port: %s' % port_name) @@ -73,7 +73,7 @@ def _set_gpu_options(port): def _tests(port, paths): if not paths: - paths = ['compositing', 'platform/chromium/compositing'] + paths = ['compositing', 'platform/chromium/compositing', 'media'] if not port.name().startswith('chromium-gpu-mac'): # Canvas is not yet accelerated on the Mac, so there's no point # in running the tests there. @@ -84,11 +84,26 @@ def _tests(port, paths): return test_files.find(port, paths) +def _test_platform_names(self): + return ('mac', 'win', 'linux') + + +def _test_platform_name_to_name(self, test_platform_name): + if test_platform_name in self.test_platform_names(): + return 'chromium-gpu-' + test_platform_name + raise ValueError('Unsupported test_platform_name: %s' % + test_platform_name) + + class ChromiumGpuLinuxPort(chromium_linux.ChromiumLinuxPort): def __init__(self, port_name='chromium-gpu-linux', **kwargs): chromium_linux.ChromiumLinuxPort.__init__(self, port_name=port_name, **kwargs) _set_gpu_options(self) + def baseline_path(self): + # GPU baselines aren't yet versioned. + return self._webkit_baseline_path('chromium-gpu-linux') + def baseline_search_path(self): # Mimic the Linux -> Win expectations fallback in the ordinary Chromium port. return (map(self._webkit_baseline_path, ['chromium-gpu-linux', 'chromium-gpu-win', 'chromium-gpu']) + @@ -103,6 +118,14 @@ class ChromiumGpuLinuxPort(chromium_linux.ChromiumLinuxPort): def tests(self, paths): return _tests(self, paths) + def test_platform_name(self): + return 'linux' + + def test_platform_names(self): + return _test_platform_names(self) + + def test_platform_name_to_name(self, name): + return _test_platform_name_to_name(self, name) class ChromiumGpuMacPort(chromium_mac.ChromiumMacPort): @@ -110,6 +133,10 @@ class ChromiumGpuMacPort(chromium_mac.ChromiumMacPort): chromium_mac.ChromiumMacPort.__init__(self, port_name=port_name, **kwargs) _set_gpu_options(self) + def baseline_path(self): + # GPU baselines aren't yet versioned. + return self._webkit_baseline_path('chromium-gpu-mac') + def baseline_search_path(self): return (map(self._webkit_baseline_path, ['chromium-gpu-mac', 'chromium-gpu']) + chromium_mac.ChromiumMacPort.baseline_search_path(self)) @@ -123,6 +150,14 @@ class ChromiumGpuMacPort(chromium_mac.ChromiumMacPort): def tests(self, paths): return _tests(self, paths) + def test_platform_name(self): + return 'mac' + + def test_platform_names(self): + return _test_platform_names(self) + + def test_platform_name_to_name(self, name): + return _test_platform_name_to_name(self, name) class ChromiumGpuWinPort(chromium_win.ChromiumWinPort): @@ -130,6 +165,10 @@ class ChromiumGpuWinPort(chromium_win.ChromiumWinPort): chromium_win.ChromiumWinPort.__init__(self, port_name=port_name, **kwargs) _set_gpu_options(self) + def baseline_path(self): + # GPU baselines aren't yet versioned. + return self._webkit_baseline_path('chromium-gpu-win') + def baseline_search_path(self): return (map(self._webkit_baseline_path, ['chromium-gpu-win', 'chromium-gpu']) + chromium_win.ChromiumWinPort.baseline_search_path(self)) @@ -142,3 +181,12 @@ class ChromiumGpuWinPort(chromium_win.ChromiumWinPort): def tests(self, paths): return _tests(self, paths) + + def test_platform_name(self): + return 'win' + + def test_platform_names(self): + return _test_platform_names(self) + + def test_platform_name_to_name(self, name): + return _test_platform_name_to_name(self, name) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py index 96962ec..9c5dae7 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py @@ -29,6 +29,7 @@ import unittest from webkitpy.tool import mocktool import chromium_gpu +from webkitpy.layout_tests.port import factory class ChromiumGpuTest(unittest.TestCase): def test_get_chromium_gpu_linux(self): @@ -66,8 +67,7 @@ class ChromiumGpuTest(unittest.TestCase): self.assertEqual(port.default_child_processes(), 1) self.assertEqual(port._options.builder_name, 'foo - GPU') - # We don't support platform-specific versions of the GPU port yet. - self.assertEqual(port.name(), port_name) + self.assertTrue(port.name().startswith(port_name)) # test that it has the right directories in front of the search path. paths = port.baseline_search_path() @@ -96,6 +96,31 @@ class ChromiumGpuTest(unittest.TestCase): self.assertTrue(port._filesystem.exists(path)) self.assertFalse(path in files) + def test_chromium_gpu__vista(self): + port = factory.get('chromium-gpu-win-vista') + self.assertEquals(port.name(), 'chromium-gpu-win-vista') + self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-gpu-win')) + + def test_chromium_gpu__xp(self): + port = factory.get('chromium-gpu-win-xp') + self.assertEquals(port.name(), 'chromium-gpu-win-xp') + self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-gpu-win')) + + def test_chromium_gpu__win7(self): + port = factory.get('chromium-gpu-win-win7') + self.assertEquals(port.name(), 'chromium-gpu-win-win7') + self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-gpu-win')) + + def test_chromium_gpu__leopard(self): + port = factory.get('chromium-gpu-mac-leopard') + self.assertEquals(port.name(), 'chromium-gpu-mac-leopard') + self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-gpu-mac')) + + def test_chromium_gpu__snowleopard(self): + port = factory.get('chromium-gpu-mac-snowleopard') + self.assertEquals(port.name(), 'chromium-gpu-mac-snowleopard') + self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-gpu-mac')) + if __name__ == '__main__': unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py index c3c5a21..a8e1bb2 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py @@ -40,13 +40,62 @@ _log = logging.getLogger("webkitpy.layout_tests.port.chromium_linux") class ChromiumLinuxPort(chromium.ChromiumPort): """Chromium Linux implementation of the Port class.""" - - def __init__(self, **kwargs): - kwargs.setdefault('port_name', 'chromium-linux') - chromium.ChromiumPort.__init__(self, **kwargs) + SUPPORTED_ARCHITECTURES = ('x86', 'x86_64') + + FALLBACK_PATHS = { + 'x86_64': ['chromium-linux-x86_64', 'chromium-linux', 'chromium-win', 'chromium', 'win', 'mac'], + 'x86': ['chromium-linux', 'chromium-win', 'chromium', 'win', 'mac'], + } + + def __init__(self, port_name=None, rebaselining=False, **kwargs): + port_name = port_name or 'chromium-linux' + chromium.ChromiumPort.__init__(self, port_name=port_name, **kwargs) + + # We re-set the port name once the base object is fully initialized + # in order to be able to find the DRT binary properly. + if port_name.endswith('-linux') and not rebaselining: + self._architecture = self._determine_architecture() + # FIXME: this is an ugly hack to avoid renaming the GPU port. + if port_name == 'chromium-linux': + port_name = port_name + '-' + self._architecture + elif rebaselining: + self._architecture = 'x86' + else: + base, arch = port_name.rsplit('-', 1) + assert base in ('chromium-linux', 'chromium-gpu-linux') + self._architecture = arch + assert self._architecture in self.SUPPORTED_ARCHITECTURES + assert port_name in ('chromium-linux', 'chromium-gpu-linux', + 'chromium-linux-x86', 'chromium-linux-x86_64') + self._name = port_name + + def _determine_architecture(self): + driver_path = self._path_to_driver() + file_output = '' + if self._filesystem.exists(driver_path): + file_output = self._executive.run_command(['file', driver_path], + return_stderr=True) + + if 'ELF 32-bit LSB executable' in file_output: + return 'x86' + if 'ELF 64-bit LSB executable' in file_output: + return 'x86_64' + if file_output: + _log.warning('Could not determine architecture from "file" output: %s' % file_output) + + # We don't know what the architecture is; default to 'x86' because + # maybe we're rebaselining and the binary doesn't actually exist, + # or something else weird is going on. It's okay to do this because + # if we actually try to use the binary, check_build() should fail. + return 'x86' + + def baseline_path(self): + if self._architecture == 'x86_64': + return self._webkit_baseline_path(self._name) + return self._webkit_baseline_path('chromium-linux') def baseline_search_path(self): - port_names = ["chromium-linux", "chromium-win", "chromium", "win", "mac"] + port_names = self.FALLBACK_PATHS[self._architecture] return map(self._webkit_baseline_path, port_names) def check_build(self, needs_http): @@ -65,13 +114,18 @@ class ChromiumLinuxPort(chromium.ChromiumPort): 'LinuxBuildInstructions') return result + def default_worker_model(self): + if self._multiprocessing_is_available: + return 'processes' + return 'old-threads' + def test_platform_name(self): # We use 'linux' instead of 'chromium-linux' in test_expectations.txt. return 'linux' def version(self): - # We don't have different versions on linux. - return '' + # FIXME: add support for Lucid. + return 'hardy' # # PROTECTED METHODS diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux_unittest.py new file mode 100644 index 0000000..05ec067 --- /dev/null +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux_unittest.py @@ -0,0 +1,104 @@ +# Copyright (C) 2011 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.system import executive_mock +from webkitpy.common.system import filesystem_mock + +from webkitpy.layout_tests.port import chromium_linux +from webkitpy.layout_tests.port import port_testcase + + +class ChromiumLinuxPortTest(port_testcase.PortTestCase): + def port_maker(self, platform): + if platform != 'linux': + return None + return chromium_linux.ChromiumLinuxPort + + def assert_architecture(self, port_name=None, file_output=None, + expected_architecture=None): + filesystem = filesystem_mock.MockFileSystem() + filesystem.exists = lambda x: 'DumpRenderTree' in x + executive = None + if file_output: + executive = executive_mock.MockExecutive2(file_output) + + port = chromium_linux.ChromiumLinuxPort(port_name=port_name, + executive=executive, filesystem=filesystem) + self.assertEquals(port.architecture(), expected_architecture) + if expected_architecture == 'x86': + self.assertTrue(port.baseline_path().endswith('chromium-linux')) + self.assertTrue(port.baseline_search_path()[0].endswith('chromium-linux')) + else: + self.assertTrue(port.baseline_path().endswith('chromium-linux-x86_64')) + self.assertTrue(port.baseline_search_path()[0].endswith('chromium-linux-x86_64')) + self.assertTrue(port.baseline_search_path()[1].endswith('chromium-linux')) + + def test_architectures(self): + self.assert_architecture(port_name='chromium-linux-x86', + expected_architecture='x86') + self.assert_architecture(port_name='chromium-linux-x86_64', + expected_architecture='x86_64') + self.assert_architecture(file_output='ELF 32-bit LSB executable', + expected_architecture='x86') + self.assert_architecture(file_output='ELF 64-bit LSB executable', + expected_architecture='x86_64') + + def test_check_illegal_port_names(self): + # FIXME: Check that, for now, these are illegal port names. + # Eventually we should be able to do the right thing here. + self.assertRaises(AssertionError, chromium_linux.ChromiumLinuxPort, + port_name='chromium-x86-linux') + self.assertRaises(AssertionError, chromium_linux.ChromiumLinuxPort, + port_name='chromium-linux-x86-gpu') + + def test_determine_architecture_fails(self): + # Test that we default to 'x86' if the driver doesn't exist. + filesystem = filesystem_mock.MockFileSystem() + port = chromium_linux.ChromiumLinuxPort(filesystem=filesystem) + self.assertEquals(port.architecture(), 'x86') + + # Test that we default to 'x86' on an unknown architecture. + filesystem = filesystem_mock.MockFileSystem() + filesystem.exists = lambda x: True + executive = executive_mock.MockExecutive2('win32') + port = chromium_linux.ChromiumLinuxPort(filesystem=filesystem, + executive=executive) + self.assertEquals(port.architecture(), 'x86') + + # Test that we raise errors if something weird happens. + filesystem = filesystem_mock.MockFileSystem() + filesystem.exists = lambda x: True + executive = executive_mock.MockExecutive2(exception=AssertionError) + self.assertRaises(AssertionError, chromium_linux.ChromiumLinuxPort, + filesystem=filesystem, executive=executive) + + +if __name__ == '__main__': + unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py index 17862a2..78a6682 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py @@ -31,10 +31,10 @@ import logging import os -import platform import signal -import chromium +from webkitpy.layout_tests.port import mac +from webkitpy.layout_tests.port import chromium from webkitpy.common.system.executive import Executive @@ -43,20 +43,51 @@ _log = logging.getLogger("webkitpy.layout_tests.port.chromium_mac") class ChromiumMacPort(chromium.ChromiumPort): """Chromium Mac implementation of the Port class.""" + SUPPORTED_OS_VERSIONS = ('leopard', 'snowleopard') + + FALLBACK_PATHS = { + 'leopard': ['chromium-mac-leopard', 'chromium-mac-snowleopard', 'chromium-mac', 'chromium', + 'mac-leopard', 'mac-snowleopard', 'mac'], + 'snowleopard': ['chromium-mac-snowleopard', 'chromium-mac', 'chromium', + 'mac-snowleopard', 'mac'], + '': ['chromium-mac', 'chromium', 'mac'], + } + + def __init__(self, port_name=None, os_version_string=None, rebaselining=False, **kwargs): + # We're a little generic here because this code is reused by the + # 'google-chrome' port as well as the 'mock-' and 'dryrun-' ports. + port_name = port_name or 'chromium-mac' + + if port_name.endswith('-mac'): + # FIXME: The rebaselining flag is an ugly hack that lets us create an + # "chromium-mac" port that is not version-specific. It should only be + # used by rebaseline-chromium-webkit-tests to explicitly put files into + # the generic directory. In theory we shouldn't need this, because + # the newest mac port should be using 'chromium-mac' as the baseline + # directory. However, we also don't have stable SL bots :( + # + # When we remove this FIXME, we also need to remove '' as a valid + # fallback key in self.FALLBACK_PATHS. + if rebaselining: + self._version = '' + else: + self._version = mac.os_version(os_version_string, self.SUPPORTED_OS_VERSIONS) + port_name = port_name + '-' + self._version + else: + self._version = port_name[port_name.index('-mac-') + 5:] + assert self._version in self.SUPPORTED_OS_VERSIONS + + chromium.ChromiumPort.__init__(self, port_name=port_name, **kwargs) - def __init__(self, **kwargs): - kwargs.setdefault('port_name', 'chromium-mac') - chromium.ChromiumPort.__init__(self, **kwargs) + def baseline_path(self): + if self.version() == 'snowleopard': + # We treat Snow Leopard as the newest version of mac, + # so it gets the base dir. + return self._webkit_baseline_path('chromium-mac') + return self._webkit_baseline_path(self.name()) def baseline_search_path(self): - port_names = [ - "chromium-mac" + self.version(), - "chromium-mac", - "chromium", - "mac" + self.version(), - "mac", - ] - return map(self._webkit_baseline_path, port_names) + return map(self._webkit_baseline_path, self.FALLBACK_PATHS[self._version]) def check_build(self, needs_http): result = chromium.ChromiumPort.check_build(self, needs_http) @@ -80,25 +111,22 @@ class ChromiumMacPort(chromium.ChromiumPort): return chromium.ChromiumPort.default_child_processes(self) + def default_worker_model(self): + if self._multiprocessing_is_available: + return 'processes' + return 'old-threads' + def driver_name(self): return "DumpRenderTree" def test_platform_name(self): # We use 'mac' instead of 'chromium-mac' + + # FIXME: Get rid of this method after rebaseline_chromium_webkit_tests dies. return 'mac' def version(self): - # FIXME: It's strange that this string is -version, not just version. - os_version_string = platform.mac_ver()[0] # e.g. "10.5.6" - if not os_version_string: - return '-leopard' - release_version = int(os_version_string.split('.')[1]) - # we don't support 'tiger' or earlier releases - if release_version == 5: - return '-leopard' - elif release_version == 6: - return '-snowleopard' - return '' + return self._version # # PROTECTED METHODS diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py index d63faa0..12011c6 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py @@ -26,15 +26,65 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import chromium_mac import unittest from webkitpy.thirdparty.mock import Mock +from webkitpy.layout_tests.port import chromium_mac +from webkitpy.layout_tests.port import port_testcase -class ChromiumMacPortTest(unittest.TestCase): + +class ChromiumMacPortTest(port_testcase.PortTestCase): + def port_maker(self, platform): + if platform != 'darwin': + return None + return chromium_mac.ChromiumMacPort def test_check_wdiff_install(self): port = chromium_mac.ChromiumMacPort() + # Currently is always true, just logs if missing. self.assertTrue(port._check_wdiff_install()) + + def assert_name(self, port_name, os_version_string, expected): + port = chromium_mac.ChromiumMacPort(port_name=port_name, + os_version_string=os_version_string) + self.assertEquals(expected, port.name()) + + def test_versions(self): + port = chromium_mac.ChromiumMacPort() + self.assertTrue(port.name() in ('chromium-mac-leopard', 'chromium-mac-snowleopard')) + + self.assert_name(None, '10.5.3', 'chromium-mac-leopard') + self.assert_name('chromium-mac', '10.5.3', 'chromium-mac-leopard') + self.assert_name('chromium-mac-leopard', '10.5.3', 'chromium-mac-leopard') + self.assert_name('chromium-mac-leopard', '10.6.3', 'chromium-mac-leopard') + + self.assert_name(None, '10.6.3', 'chromium-mac-snowleopard') + self.assert_name('chromium-mac', '10.6.3', 'chromium-mac-snowleopard') + self.assert_name('chromium-mac-snowleopard', '10.5.3', 'chromium-mac-snowleopard') + self.assert_name('chromium-mac-snowleopard', '10.6.3', 'chromium-mac-snowleopard') + + self.assertRaises(KeyError, self.assert_name, None, '10.7.1', 'chromium-mac-leopard') + self.assertRaises(AssertionError, self.assert_name, None, '10.4.1', 'chromium-mac-leopard') + + def test_generic_rebaselining_port(self): + port = chromium_mac.ChromiumMacPort(rebaselining=True) + self.assertEquals(port.name(), 'chromium-mac') + self.assertEquals(port.version(), '') + self.assertEquals(port.baseline_path(), port._webkit_baseline_path(port.name())) + + port = chromium_mac.ChromiumMacPort(port_name='chromium-mac-leopard', rebaselining=True) + self.assertEquals(port.name(), 'chromium-mac-leopard') + self.assertEquals(port.baseline_path(), port._webkit_baseline_path(port.name())) + + def test_baseline_path(self): + port = chromium_mac.ChromiumMacPort(port_name='chromium-mac-leopard') + self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-mac-leopard')) + + port = chromium_mac.ChromiumMacPort(port_name='chromium-mac-snowleopard') + self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-mac')) + + +if __name__ == '__main__': + unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py index b89c8cc..b287875 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py @@ -90,8 +90,8 @@ class ChromiumPortTest(unittest.TestCase): class TestMacPort(chromium_mac.ChromiumMacPort): def __init__(self, options): chromium_mac.ChromiumMacPort.__init__(self, - port_name='test-port', - options=options) + options=options, + filesystem=filesystem_mock.MockFileSystem()) def default_configuration(self): self.default_configuration_called = True @@ -100,7 +100,6 @@ class ChromiumPortTest(unittest.TestCase): class TestLinuxPort(chromium_linux.ChromiumLinuxPort): def __init__(self, options): chromium_linux.ChromiumLinuxPort.__init__(self, - port_name='test-port', options=options, filesystem=filesystem_mock.MockFileSystem()) @@ -108,16 +107,27 @@ class ChromiumPortTest(unittest.TestCase): self.default_configuration_called = True return 'default' + class TestWinPort(chromium_win.ChromiumWinPort): + def __init__(self, options): + chromium_win.ChromiumWinPort.__init__(self, + options=options, + filesystem=filesystem_mock.MockFileSystem()) + + def default_configuration(self): + self.default_configuration_called = True + return 'default' + def test_path_to_image_diff(self): mock_options = mocktool.MockOptions() port = ChromiumPortTest.TestLinuxPort(options=mock_options) self.assertTrue(port._path_to_image_diff().endswith( - '/out/default/ImageDiff'), msg=port._path_to_image_diff()) + '/out/default/ImageDiff')) port = ChromiumPortTest.TestMacPort(options=mock_options) self.assertTrue(port._path_to_image_diff().endswith( '/xcodebuild/default/ImageDiff')) - # FIXME: Figure out how this is going to work on Windows. - #port = chromium_win.ChromiumWinPort('test-port', options=MockOptions()) + port = ChromiumPortTest.TestWinPort(options=mock_options) + self.assertTrue(port._path_to_image_diff().endswith( + '/default/ImageDiff.exe')) def test_skipped_layout_tests(self): mock_options = mocktool.MockOptions() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py index f4cbf80..e7c6e49 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py @@ -37,12 +37,60 @@ import chromium _log = logging.getLogger("webkitpy.layout_tests.port.chromium_win") +def os_version(windows_version=None): + if not windows_version: + if hasattr(sys, 'getwindowsversion'): + windows_version = tuple(sys.getwindowsversion()[:2]) + else: + # Make up something for testing. + windows_version = (5, 1) + + version_strings = { + (6, 1): 'win7', + (6, 0): 'vista', + (5, 1): 'xp', + } + return version_strings[windows_version] + + class ChromiumWinPort(chromium.ChromiumPort): """Chromium Win implementation of the Port class.""" - - def __init__(self, **kwargs): - kwargs.setdefault('port_name', 'chromium-win' + self.version()) - chromium.ChromiumPort.__init__(self, **kwargs) + # FIXME: Figure out how to unify this with base.TestConfiguration.all_systems()? + SUPPORTED_VERSIONS = ('xp', 'vista', 'win7') + + # FIXME: Do we need mac-snowleopard here, like the base win port? + FALLBACK_PATHS = { + 'xp': ['chromium-win-xp', 'chromium-win-vista', 'chromium-win', 'chromium', 'win', 'mac'], + 'vista': ['chromium-win-vista', 'chromium-win', 'chromium', 'win', 'mac'], + 'win7': ['chromium-win', 'chromium', 'win', 'mac'], + '': ['chromium-win', 'chromium', 'win', 'mac'], + } + + def __init__(self, port_name=None, windows_version=None, rebaselining=False, **kwargs): + # We're a little generic here because this code is reused by the + # 'google-chrome' port as well as the 'mock-' and 'dryrun-' ports. + port_name = port_name or 'chromium-win' + + if port_name.endswith('-win'): + # FIXME: The rebaselining flag is an ugly hack that lets us create an + # "chromium-win" port that is not version-specific. It should only be + # used by rebaseline-chromium-webkit-tests to explicitly put files into + # the generic directory. In theory we shouldn't need this, because + # the newest win port should be using 'chromium-win' as the baseline + # directory. However, we also don't have stable Win 7 bots :( + # + # When we remove this FIXME, we also need to remove '' as a valid + # fallback key in self.FALLBACK_PATHS. + if rebaselining: + self._version = '' + else: + self._version = os_version(windows_version) + port_name = port_name + '-' + self._version + else: + self._version = port_name[port_name.index('-win-') + 5:] + assert self._version in self.SUPPORTED_VERSIONS + + chromium.ChromiumPort.__init__(self, port_name=port_name, **kwargs) def setup_environ_for_server(self): env = chromium.ChromiumPort.setup_environ_for_server(self) @@ -54,21 +102,21 @@ class ChromiumWinPort(chromium.ChromiumPort): # python executable to run cgi program. env["CYGWIN_PATH"] = self.path_from_chromium_base( "third_party", "cygwin", "bin") - if (sys.platform == "win32" and self.get_option('register_cygwin')): + if (sys.platform in ("cygwin", "win32") and self.get_option('register_cygwin')): setup_mount = self.path_from_chromium_base("third_party", "cygwin", "setup_mount.bat") self._executive.run_command([setup_mount]) return env + def baseline_path(self): + if self.version() == 'win7': + # Win 7 is the newest version of windows, so it gets the base dir. + return self._webkit_baseline_path('chromium-win') + return self._webkit_baseline_path(self.name()) + def baseline_search_path(self): - port_names = [] - if self._name.endswith('-win-xp'): - port_names.append("chromium-win-xp") - if self._name.endswith('-win-xp') or self._name.endswith('-win-vista'): - port_names.append("chromium-win-vista") - # FIXME: This may need to include mac-snowleopard like win.py. - port_names.extend(["chromium-win", "chromium", "win", "mac"]) + port_names = self.FALLBACK_PATHS[self.version()] return map(self._webkit_baseline_path, port_names) def check_build(self, needs_http): @@ -87,19 +135,14 @@ class ChromiumWinPort(chromium.ChromiumPort): def test_platform_name(self): # We return 'win-xp', not 'chromium-win-xp' here, for convenience. - return 'win' + self.version() + + # FIXME: Get rid of this method after rebaseline_chromium_webkit_tests dies. + if self.version() == '': + return 'win' + return 'win-' + self.version() def version(self): - if not hasattr(sys, 'getwindowsversion'): - return '' - winver = sys.getwindowsversion() - if winver[0] == 6 and (winver[1] == 1): - return '-7' - if winver[0] == 6 and (winver[1] == 0): - return '-vista' - if winver[0] == 5 and (winver[1] == 1 or winver[1] == 2): - return '-xp' - return '' + return self._version # # PROTECTED ROUTINES diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_win_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win_unittest.py index d677589..8ea7060 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/chromium_win_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win_unittest.py @@ -29,16 +29,19 @@ import os import sys import unittest -import chromium_win + from webkitpy.common.system import outputcapture from webkitpy.tool import mocktool +from webkitpy.layout_tests.port import chromium_win +from webkitpy.layout_tests.port import port_testcase -class ChromiumWinTest(unittest.TestCase): +class ChromiumWinTest(port_testcase.PortTestCase): class RegisterCygwinOption(object): def __init__(self): self.register_cygwin = True + self.results_directory = '/' def setUp(self): self.orig_platform = sys.platform @@ -47,11 +50,26 @@ class ChromiumWinTest(unittest.TestCase): sys.platform = self.orig_platform self._port = None + def port_maker(self, platform): + if platform not in ('cygwin', 'win32'): + return None + return chromium_win.ChromiumWinPort + def _mock_path_from_chromium_base(self, *comps): return self._port._filesystem.join("/chromium/src", *comps) + def test_default_worker_model(self): + port = self.make_port() + if not port: + return + + self.assertEqual(port.default_worker_model(), 'old-threads') + def test_setup_environ_for_server(self): - port = chromium_win.ChromiumWinPort() + port = self.make_port() + if not port: + return + port._executive = mocktool.MockExecutive(should_log=True) self._port = port port.path_from_chromium_base = self._mock_path_from_chromium_base @@ -62,16 +80,74 @@ class ChromiumWinTest(unittest.TestCase): self.assertNotEqual(env["PATH"], os.environ["PATH"]) def test_setup_environ_for_server_register_cygwin(self): - sys.platform = "win32" - port = chromium_win.ChromiumWinPort( - options=ChromiumWinTest.RegisterCygwinOption()) + port = self.make_port(options=ChromiumWinTest.RegisterCygwinOption()) + if not port: + return + port._executive = mocktool.MockExecutive(should_log=True) port.path_from_chromium_base = self._mock_path_from_chromium_base self._port = port setup_mount = self._mock_path_from_chromium_base("third_party", - "cygwin", - "setup_mount.bat") + "cygwin", + "setup_mount.bat") expected_stderr = "MOCK run_command: %s\n" % [setup_mount] output = outputcapture.OutputCapture() output.assert_outputs(self, port.setup_environ_for_server, expected_stderr=expected_stderr) + + def assert_name(self, port_name, windows_version, expected): + port = chromium_win.ChromiumWinPort(port_name=port_name, + windows_version=windows_version) + self.assertEquals(expected, port.name()) + + def test_versions(self): + port = chromium_win.ChromiumWinPort() + self.assertTrue(port.name() in ('chromium-win-xp', 'chromium-win-vista', 'chromium-win-win7')) + + self.assert_name(None, (5, 1), 'chromium-win-xp') + self.assert_name('chromium-win', (5, 1), 'chromium-win-xp') + self.assert_name('chromium-win-xp', (5, 1), 'chromium-win-xp') + self.assert_name('chromium-win-xp', (6, 0), 'chromium-win-xp') + self.assert_name('chromium-win-xp', (6, 1), 'chromium-win-xp') + + self.assert_name(None, (6, 0), 'chromium-win-vista') + self.assert_name('chromium-win', (6, 0), 'chromium-win-vista') + self.assert_name('chromium-win-vista', (5, 1), 'chromium-win-vista') + self.assert_name('chromium-win-vista', (6, 0), 'chromium-win-vista') + self.assert_name('chromium-win-vista', (6, 1), 'chromium-win-vista') + + self.assert_name(None, (6, 1), 'chromium-win-win7') + self.assert_name('chromium-win', (6, 1), 'chromium-win-win7') + self.assert_name('chromium-win-win7', (5, 1), 'chromium-win-win7') + self.assert_name('chromium-win-win7', (6, 0), 'chromium-win-win7') + self.assert_name('chromium-win-win7', (6, 1), 'chromium-win-win7') + + self.assertRaises(KeyError, self.assert_name, None, (4, 0), 'chromium-win-xp') + self.assertRaises(KeyError, self.assert_name, None, (5, 0), 'chromium-win-xp') + self.assertRaises(KeyError, self.assert_name, None, (5, 2), 'chromium-win-xp') + self.assertRaises(KeyError, self.assert_name, None, (7, 1), 'chromium-win-xp') + + def test_generic_rebaselining_port(self): + port = chromium_win.ChromiumWinPort(rebaselining=True) + self.assertEquals(port.name(), 'chromium-win') + self.assertEquals(port.version(), '') + self.assertEquals(port.baseline_path(), port._webkit_baseline_path(port.name())) + + port = chromium_win.ChromiumWinPort(port_name='chromium-win-xp', rebaselining=True) + self.assertEquals(port.name(), 'chromium-win-xp') + self.assertEquals(port.baseline_path(), port._webkit_baseline_path(port.name())) + + def test_baseline_path(self): + port = chromium_win.ChromiumWinPort(port_name='chromium-win-xp') + self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-win-xp')) + + port = chromium_win.ChromiumWinPort(port_name='chromium-win-vista') + self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-win-vista')) + + port = chromium_win.ChromiumWinPort(port_name='chromium-win-win7') + self.assertEquals(port.baseline_path(), port._webkit_baseline_path('chromium-win')) + + + +if __name__ == '__main__': + unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/config_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/config_unittest.py index 2cce3cc..8f1e0d4 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/config_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/config_unittest.py @@ -189,7 +189,10 @@ class ConfigTest(unittest.TestCase): self.assertNotEqual(base_dir[-1], '/') orig_cwd = os.getcwd() - os.chdir(os.environ['HOME']) + if sys.platform == 'win32': + os.chdir(os.environ['USERPROFILE']) + else: + os.chdir(os.environ['HOME']) c = config.Config(executive.Executive(), filesystem.FileSystem()) try: base_dir_2 = c.webkit_base_dir() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/factory.py b/Tools/Scripts/webkitpy/layout_tests/port/factory.py index 7ae6eb6..683dba3 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/factory.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/factory.py @@ -32,9 +32,26 @@ import sys -ALL_PORT_NAMES = ['test', 'dryrun', 'mac', 'win', 'gtk', 'qt', 'chromium-mac', - 'chromium-linux', 'chromium-win', 'google-chrome-win', - 'google-chrome-mac', 'google-chrome-linux32', 'google-chrome-linux64'] + +def all_port_names(): + """Return a list of all valid, fully-specified, "real" port names. + + This is the list of directories that are used as actual baseline_paths() + by real ports. This does not include any "fake" names like "test" + or "mock-mac", and it does not include any directories that are not .""" + # FIXME: There's probably a better way to generate this list ... + return ['chromium-gpu-linux', + 'chromium-gpu-mac-snowleopard', 'chromium-gpu-mac-leopard', + 'chromium-gpu-win-xp', 'chromium-gpu-win-vista', 'chromium-gpu-win-win7', + 'chromium-linux-x86_64', 'chromium-linux-x86', + 'chromium-mac-leopard', 'chromium-mac-snowleopard', + 'chromium-win-xp', 'chromium-win-vista', 'chromium-win-win7', + 'google-chrome-linux32', 'google-chrome-linux64', + 'gtk', + 'mac-tiger', 'mac-leopard', 'mac-snowleopard', 'mac-wk2', + 'qt-linux', 'qt-mac', 'qt-win', 'qt-wk2', + 'win-xp', 'win', 'win-wk2', + ] def get(port_name=None, options=None, **kwargs): @@ -109,8 +126,3 @@ def _get_kwargs(**kwargs): else: raise NotImplementedError('unsupported port: %s' % port_to_use) return maker(**kwargs) - -def get_all(options=None): - """Returns all the objects implementing the Port interface.""" - return dict([(port_name, get(port_name, options=options)) - for port_name in ALL_PORT_NAMES]) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py index 978a557..e4a2cd4 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py @@ -152,24 +152,6 @@ class FactoryTest(unittest.TestCase): self.assert_platform_port("cygwin", self.chromium_options, chromium_win.ChromiumWinPort) - def test_get_all_ports(self): - ports = factory.get_all() - for name in factory.ALL_PORT_NAMES: - self.assertTrue(name in ports.keys()) - self.assert_port("test", test.TestPort, ports["test"]) - self.assert_port("dryrun-test", dryrun.DryRunPort, ports["dryrun"]) - self.assert_port("dryrun-mac", dryrun.DryRunPort, ports["dryrun"]) - self.assert_port("mac", mac.MacPort, ports["mac"]) - self.assert_port("win", win.WinPort, ports["win"]) - self.assert_port("gtk", gtk.GtkPort, ports["gtk"]) - self.assert_port("qt", qt.QtPort, ports["qt"]) - self.assert_port("chromium-mac", chromium_mac.ChromiumMacPort, - ports["chromium-mac"]) - self.assert_port("chromium-linux", chromium_linux.ChromiumLinuxPort, - ports["chromium-linux"]) - self.assert_port("chromium-win", chromium_win.ChromiumWinPort, - ports["chromium-win"]) - def test_unknown_specified(self): # Test what happens when you specify an unknown port. orig_platform = sys.platform diff --git a/Tools/Scripts/webkitpy/layout_tests/port/google_chrome.py b/Tools/Scripts/webkitpy/layout_tests/port/google_chrome.py index ae90374..811e7e0 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/google_chrome.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/google_chrome.py @@ -45,6 +45,8 @@ def GetGoogleChromePort(**kwargs): """Some tests have slightly different results when compiled as Google Chrome vs Chromium. In those cases, we prepend an additional directory to to the baseline paths.""" + # FIXME: This whole routine is a tremendous hack that needs to be cleaned up. + port_name = kwargs['port_name'] del kwargs['port_name'] if port_name == 'google-chrome-linux32': @@ -62,7 +64,10 @@ def GetGoogleChromePort(**kwargs): return _test_expectations_overrides(self, chromium_linux.ChromiumLinuxPort) - return GoogleChromeLinux32Port(**kwargs) + def architecture(self): + return 'x86' + + return GoogleChromeLinux32Port(port_name='chromium-linux-x86', **kwargs) elif port_name == 'google-chrome-linux64': import chromium_linux @@ -78,7 +83,12 @@ def GetGoogleChromePort(**kwargs): return _test_expectations_overrides(self, chromium_linux.ChromiumLinuxPort) - return GoogleChromeLinux64Port(**kwargs) + def architecture(self): + return 'x86_64' + + # We use chromium-linux-x86 here in order to skip over the linux-x86_64 + # baselines. + return GoogleChromeLinux64Port(port_name='chromium-linux-x86', **kwargs) elif port_name.startswith('google-chrome-mac'): import chromium_mac diff --git a/Tools/Scripts/webkitpy/layout_tests/port/http_server.py b/Tools/Scripts/webkitpy/layout_tests/port/http_server.py index bd75e27..752b099 100755 --- a/Tools/Scripts/webkitpy/layout_tests/port/http_server.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/http_server.py @@ -74,9 +74,13 @@ class Lighttpd(http_server_base.HttpServerBase): self._port_obj.layout_tests_dir(), 'http', 'tests') self._js_test_resource = os.path.join( self._port_obj.layout_tests_dir(), 'fast', 'js', 'resources') + self._media_resource = os.path.join( + self._port_obj.layout_tests_dir(), 'media') + except: self._webkit_tests = None self._js_test_resource = None + self._media_resource = None # Self generated certificate for SSL server (for client cert get # <base-path>\chrome\test\data\ssl\certs\root_ca_cert.crt) @@ -147,6 +151,10 @@ class Lighttpd(http_server_base.HttpServerBase): f.write(('alias.url = ( "/js-test-resources" => "%s" )\n\n') % (self._js_test_resource)) + # Setup a link to where the media resources are stored. + f.write(('alias.url += ( "/media-resources" => "%s" )\n\n') % + (self._media_resource)) + # dump out of virtual host config at the bottom. if self._root: if self._port: @@ -199,7 +207,7 @@ class Lighttpd(http_server_base.HttpServerBase): os.path.join(tmp_module_path, lib_file)) env = self._port_obj.setup_environ_for_server() - _log.debug('Starting http server') + _log.debug('Starting http server, cmd="%s"' % str(start_cmd)) # FIXME: Should use Executive.run_command self._process = subprocess.Popen(start_cmd, env=env) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mac.py b/Tools/Scripts/webkitpy/layout_tests/port/mac.py index 1398ed3..0168ec7 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/mac.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/mac.py @@ -38,12 +38,50 @@ from webkitpy.layout_tests.port.webkit import WebKitPort _log = logging.getLogger("webkitpy.layout_tests.port.mac") +def os_version(os_version_string=None, supported_versions=None): + # We only support Tiger, Leopard, and Snow Leopard. + if not os_version_string: + if hasattr(platform, 'mac_ver') and platform.mac_ver()[0]: + os_version_string = platform.mac_ver()[0] + else: + # Make up something for testing. + os_version_string = "10.5.6" + release_version = int(os_version_string.split('.')[1]) + version_strings = { + 4: 'tiger', + 5: 'leopard', + 6: 'snowleopard', + } + version_string = version_strings[release_version] + if supported_versions: + assert version_string in supported_versions + return version_string + + class MacPort(WebKitPort): """WebKit Mac implementation of the Port class.""" + # FIXME: 'wk2' probably shouldn't be a version, it should probably be + # a modifier, like 'chromium-gpu' is to 'chromium'. + SUPPORTED_VERSIONS = ('tiger', 'leopard', 'snowleopard', 'wk2') + + FALLBACK_PATHS = { + 'tiger': ['mac-tiger', 'mac-leopard', 'mac-snowleopard', 'mac'], + 'leopard': ['mac-leopard', 'mac-snowleopard', 'mac'], + 'snowleopard': ['mac-snowleopard', 'mac'], + 'wk2': ['mac-wk2', 'mac'], + } + + def __init__(self, port_name=None, os_version_string=None, **kwargs): + port_name = port_name or 'mac' + + if port_name == 'mac': + self._version = os_version(os_version_string) + port_name = port_name + '-' + self._version + else: + self._version = port_name[4:] + assert self._version in self.SUPPORTED_VERSIONS - def __init__(self, **kwargs): - kwargs.setdefault('port_name', 'mac' + self.version()) - WebKitPort.__init__(self, **kwargs) + WebKitPort.__init__(self, port_name=port_name, **kwargs) def default_child_processes(self): # FIXME: new-run-webkit-tests is unstable on Mac running more than @@ -54,16 +92,13 @@ class MacPort(WebKitPort): return 4 return child_processes + def default_worker_model(self): + if self._multiprocessing_is_available: + return 'processes' + return 'old-threads' + def baseline_search_path(self): - port_names = [] - if self._name == 'mac-tiger': - port_names.append("mac-tiger") - if self._name in ('mac-tiger', 'mac-leopard'): - port_names.append("mac-leopard") - if self._name in ('mac-tiger', 'mac-leopard', 'mac-snowleopard'): - port_names.append("mac-snowleopard") - port_names.append("mac") - return map(self._webkit_baseline_path, port_names) + return map(self._webkit_baseline_path, self.FALLBACK_PATHS[self._version]) def path_to_test_expectations_file(self): return self.path_from_webkit_base('LayoutTests', 'platform', @@ -73,7 +108,7 @@ class MacPort(WebKitPort): # FIXME: This method will need to be made work for non-mac # platforms and moved into base.Port. skipped_files = [] - if self._name in ('mac-tiger', 'mac-leopard', 'mac-snowleopard'): + if self._name in ('mac-leopard', 'mac-snowleopard'): skipped_files.append(self._filesystem.join( self._webkit_baseline_path(self._name), 'Skipped')) skipped_files.append(self._filesystem.join(self._webkit_baseline_path('mac'), @@ -81,20 +116,10 @@ class MacPort(WebKitPort): return skipped_files def test_platform_name(self): - return 'mac' + self.version() + return 'mac-' + self.version() def version(self): - os_version_string = platform.mac_ver()[0] # e.g. "10.5.6" - if not os_version_string: - return '-leopard' - release_version = int(os_version_string.split('.')[1]) - if release_version == 4: - return '-tiger' - elif release_version == 5: - return '-leopard' - elif release_version == 6: - return '-snowleopard' - return '' + return self._version def _build_java_test_support(self): java_tests_path = self._filesystem.join(self.layout_tests_dir(), "java") diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py index ef04679..4586a23 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py @@ -30,23 +30,18 @@ import StringIO import sys import unittest -import mac -import port_testcase +from webkitpy.layout_tests.port import mac +from webkitpy.layout_tests.port import port_testcase class MacTest(port_testcase.PortTestCase): - def make_port(self, port_name=None, options=port_testcase.mock_options): - if sys.platform != 'darwin': + def port_maker(self, platform): + if platform != 'darwin': return None - port_obj = mac.MacPort(port_name=port_name, options=options) - port_obj._options.results_directory = port_obj.results_directory() - port_obj._options.configuration = 'Release' - return port_obj + return mac.MacPort def assert_skipped_files_for_version(self, port_name, expected_paths): - port = self.make_port(port_name) - if not port: - return + port = mac.MacPort(port_name=port_name) skipped_paths = port._skipped_file_paths() # FIXME: _skipped_file_paths should return WebKit-relative paths. # So to make it unit testable, we strip the WebKit directory from the path. @@ -54,8 +49,11 @@ class MacTest(port_testcase.PortTestCase): self.assertEqual(relative_paths, expected_paths) def test_skipped_file_paths(self): - self.assert_skipped_files_for_version('mac', - ['/LayoutTests/platform/mac/Skipped']) + # We skip this on win32 because we use '/' as the dir separator and it's + # not worth making platform-independent. + if sys.platform == 'win32': + return None + self.assert_skipped_files_for_version('mac-snowleopard', ['/LayoutTests/platform/mac-snowleopard/Skipped', '/LayoutTests/platform/mac/Skipped']) self.assert_skipped_files_for_version('mac-leopard', @@ -78,11 +76,40 @@ svg/batik/text/smallFonts.svg ] def test_tests_from_skipped_file_contents(self): - port = self.make_port() - if not port: - return + port = mac.MacPort() self.assertEqual(port._tests_from_skipped_file_contents(self.example_skipped_file), self.example_skipped_tests) + def assert_name(self, port_name, os_version_string, expected): + port = mac.MacPort(port_name=port_name, + os_version_string=os_version_string) + self.assertEquals(expected, port.name()) + + def test_versions(self): + port = self.make_port() + if port: + self.assertTrue(port.name() in ('mac-tiger', 'mac-leopard', 'mac-snowleopard')) + + self.assert_name(None, '10.4.8', 'mac-tiger') + self.assert_name('mac', '10.4.8', 'mac-tiger') + self.assert_name('mac-tiger', '10.4.8', 'mac-tiger') + self.assert_name('mac-tiger', '10.5.3', 'mac-tiger') + self.assert_name('mac-tiger', '10.6.3', 'mac-tiger') + + self.assert_name(None, '10.5.3', 'mac-leopard') + self.assert_name('mac', '10.5.3', 'mac-leopard') + self.assert_name('mac-leopard', '10.4.8', 'mac-leopard') + self.assert_name('mac-leopard', '10.5.3', 'mac-leopard') + self.assert_name('mac-leopard', '10.6.3', 'mac-leopard') + + self.assert_name(None, '10.6.3', 'mac-snowleopard') + self.assert_name('mac', '10.6.3', 'mac-snowleopard') + self.assert_name('mac-snowleopard', '10.4.3', 'mac-snowleopard') + self.assert_name('mac-snowleopard', '10.5.3', 'mac-snowleopard') + self.assert_name('mac-snowleopard', '10.6.3', 'mac-snowleopard') + + self.assertRaises(KeyError, self.assert_name, None, '10.7.1', 'mac-leopard') + self.assertRaises(KeyError, self.assert_name, None, '10.3.1', 'mac-leopard') + if __name__ == '__main__': unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py index 1506315..71de14b 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py @@ -29,6 +29,7 @@ """Unit tests for MockDRT.""" +import sys import unittest from webkitpy.common import newstringio @@ -41,8 +42,15 @@ from webkitpy.layout_tests.port import test class MockDRTPortTest(port_testcase.PortTestCase): def make_port(self): + if sys.platform == 'win32': + # We use this because the 'win' port doesn't work yet. + return mock_drt.MockDRTPort(port_name='mock-chromium-win') return mock_drt.MockDRTPort() + def test_default_worker_model(self): + # only overridding the default test; we don't care about this one. + pass + def test_port_name_in_constructor(self): self.assertTrue(mock_drt.MockDRTPort(port_name='mock-test')) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py b/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py index 4146d40..d37fdc0 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py @@ -28,8 +28,15 @@ """Unit testing base class for Port implementations.""" +import sys import unittest +# Handle Python < 2.6 where multiprocessing isn't available. +try: + import multiprocessing +except ImportError: + multiprocessing = None + from webkitpy.tool import mocktool mock_options = mocktool.MockOptions(results_directory='layout-test-results', use_apache=True, @@ -40,10 +47,34 @@ mock_options = mocktool.MockOptions(results_directory='layout-test-results', class PortTestCase(unittest.TestCase): """Tests the WebKit port implementation.""" - def make_port(self, options=mock_options): - """Override in subclass.""" + def port_maker(self, platform): + """Override to return the class object of the port to be tested, + or None if a valid port object cannot be constructed on the specified + platform.""" raise NotImplementedError() + def make_port(self, options=mock_options): + """This routine should be used for tests that should only be run + when we can create a full, valid port object.""" + maker = self.port_maker(sys.platform) + if not maker: + return None + + port = maker(options=options) + if hasattr(options, "results_directory"): + port._options.results_directory = port.results_directory() + return port + + def test_default_worker_model(self): + port = self.make_port() + if not port: + return + + if multiprocessing: + self.assertEqual(port.default_worker_model(), 'processes') + else: + self.assertEqual(port.default_worker_model(), 'old-threads') + def test_driver_cmd_line(self): port = self.make_port() if not port: @@ -100,3 +131,9 @@ class PortTestCase(unittest.TestCase): if not port: return self.assertTrue(len(port.all_test_configurations()) > 0) + + def test_baseline_search_path(self): + port = self.make_port() + if not port: + return + self.assertTrue(port.baseline_path() in port.baseline_search_path()) diff --git a/Tools/Scripts/webkitpy/layout_tests/port/qt.py b/Tools/Scripts/webkitpy/layout_tests/port/qt.py index 1695b60..e159bf7 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/qt.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/qt.py @@ -114,6 +114,6 @@ class QtPort(WebKitPort): return None def setup_environ_for_server(self): - env = webkit.WebKitPort.setup_environ_for_server(self) + env = WebKitPort.setup_environ_for_server(self) env['QTWEBKIT_PLUGIN_PATH'] = self._build_path('lib/plugins') return env diff --git a/Tools/Scripts/webkitpy/layout_tests/port/test.py b/Tools/Scripts/webkitpy/layout_tests/port/test.py index b94c378..d323ed5 100644 --- a/Tools/Scripts/webkitpy/layout_tests/port/test.py +++ b/Tools/Scripts/webkitpy/layout_tests/port/test.py @@ -348,9 +348,9 @@ class TestPort(base.Port): def version(self): version_map = { - 'test-win-xp': '-xp', - 'test-win': '-7', - 'test-mac': '-leopard', + 'test-win-xp': 'xp', + 'test-win': 'win7', + 'test-mac': 'leopard', } return version_map[self._name] diff --git a/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py b/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py index 567975c..24b8c97 100644 --- a/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py +++ b/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py @@ -66,11 +66,17 @@ ARCHIVE_DIR_NAME_DICT = {'win': 'Webkit_Win__deps_', 'win-xp': 'Webkit_Win__deps_', 'mac': 'Webkit_Mac10_5__deps_', 'linux': 'Webkit_Linux__deps_', + 'win-canary': 'Webkit_Win', 'win-vista-canary': 'webkit-dbg-vista', 'win-xp-canary': 'Webkit_Win', 'mac-canary': 'Webkit_Mac10_5', - 'linux-canary': 'Webkit_Linux'} + 'linux-canary': 'Webkit_Linux', + + 'gpu-mac-canary': 'Webkit_Mac10_5_-_GPU', + 'gpu-win-canary': 'Webkit_Win_-_GPU', + 'gpu-linux-canary': 'Webkit_Linux_-_GPU', +} def log_dashed_string(text, platform, logging_level=logging.INFO): @@ -160,9 +166,11 @@ class Rebaseliner(object): self._filesystem = running_port._filesystem self._target_port = target_port + # FIXME: See the comments in chromium_{win,mac}.py about why we need + # the 'rebaselining' keyword. self._rebaseline_port = port.get( self._target_port.test_platform_name_to_name(platform), options, - filesystem=self._filesystem) + filesystem=self._filesystem, rebaselining=True) self._rebaselining_tests = [] self._rebaselined_tests = [] @@ -274,7 +282,7 @@ class Rebaseliner(object): _log.info('Latest revision: "%s"', revisions[len(revisions) - 1]) return revisions[len(revisions) - 1] - def _get_archive_dir_name(self, platform, webkit_canary): + def _get_archive_dir_name(self, platform): """Get name of the layout test archive directory. Returns: @@ -282,9 +290,6 @@ class Rebaseliner(object): None on failure """ - if webkit_canary: - platform += '-canary' - if platform in ARCHIVE_DIR_NAME_DICT: return ARCHIVE_DIR_NAME_DICT[platform] else: @@ -303,8 +308,13 @@ class Rebaseliner(object): if self._options.force_archive_url: return self._options.force_archive_url - dir_name = self._get_archive_dir_name(self._platform, - self._options.webkit_canary) + platform = self._platform + if self._options.webkit_canary: + platform += '-canary' + if self._options.gpu: + platform = 'gpu-' + platform + + dir_name = self._get_archive_dir_name(platform) if not dir_name: return None @@ -620,6 +630,7 @@ class HtmlGenerator(object): self._html_directory = options.html_directory self._port = port self._target_port = target_port + self._options = options self._platforms = platforms self._rebaselining_tests = rebaselining_tests self._filesystem = port._filesystem @@ -802,8 +813,12 @@ def parse_options(args): action='store_true', help='Suppress result HTML viewing') + option_parser.add_option('-g', '--gpu', + action='store_true', default=False, + help='Rebaseline the GPU versions') + option_parser.add_option('-p', '--platforms', - default='mac,win,win-xp,win-vista,linux', + default=None, help=('Comma delimited list of platforms ' 'that need rebaselining.')) @@ -845,6 +860,11 @@ def parse_options(args): '("mac", "chromium", "qt", etc.). Defaults ' 'to "chromium".')) options = option_parser.parse_args(args)[0] + if options.platforms == None: + if options.gpu: + options.platforms = 'mac,win,linux' + else: + options.platforms = 'mac,win,win-xp,win-vista,linux' target_options = copy.copy(options) if options.target_platform == 'chromium': @@ -867,7 +887,10 @@ def main(args): '%(levelname)s %(message)s'), datefmt='%y%m%d %H:%M:%S') - target_port_obj = port.get(None, target_options) + target_port_name = None + if options.gpu and options.target_platform == 'chromium': + target_port_name = 'chromium-gpu' + target_port_obj = port.get(target_port_name, target_options) host_port_obj = get_host_port_object(options) if not host_port_obj or not target_port_obj: return 1 diff --git a/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py b/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py index 730220b..c50e1c4 100644 --- a/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py @@ -70,7 +70,8 @@ def test_options(): target_platform='chromium', verbose=False, quiet=False, - platforms='mac,win,win-xp') + platforms='mac,win,win-xp', + gpu=False) def test_host_port_and_filesystem(options, expectations): diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py index 2d55b93..d27ea1e 100755 --- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py +++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py @@ -37,6 +37,7 @@ import os import signal import sys +from layout_package import json_results_generator from layout_package import printing from layout_package import test_runner from layout_package import test_runner2 @@ -171,11 +172,9 @@ def _gather_unexpected_results(filesystem, options): """Returns the unexpected results from the previous run, if any.""" last_unexpected_results = [] if options.print_last_failures or options.retest_last_failures: - unexpected_results_filename = filesystem.join( - options.results_directory, "unexpected_results.json") + unexpected_results_filename = filesystem.join(options.results_directory, "unexpected_results.json") if filesystem.exists(unexpected_results_filename): - content = filesystem.read_text_file(unexpected_results_filename) - results = simplejson.loads(content) + results = json_results_generator.load_json(filesystem, unexpected_results_filename) last_unexpected_results = results['tests'].keys() return last_unexpected_results @@ -234,7 +233,7 @@ def parse_args(args=None): help="Don't check the system dependencies (themes)"), optparse.make_option("--accelerated-compositing", action="store_true", - help="Use hardware-accelated compositing for rendering"), + help="Use hardware-accelerated compositing for rendering"), optparse.make_option("--no-accelerated-compositing", action="store_false", dest="accelerated_compositing", @@ -376,12 +375,12 @@ def parse_args(args=None): optparse.make_option("--experimental-fully-parallel", action="store_true", default=False, help="run all tests in parallel"), - optparse.make_option("--exit-after-n-failures", type="int", nargs=1, + optparse.make_option("--exit-after-n-failures", type="int", default=500, help="Exit after the first N failures instead of running all " "tests"), optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int", - nargs=1, help="Exit after the first N crashes instead of running " - "all tests"), + default=20, help="Exit after the first N crashes instead of " + "running all tests"), # FIXME: consider: --iterations n # Number of times to run the set of tests (e.g. ABCABCABC) optparse.make_option("--print-last-failures", action="store_true", diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py index 84f5718..3fe7b14 100644 --- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py @@ -42,6 +42,11 @@ import time import threading import unittest +try: + import multiprocessing +except ImportError: + multiprocessing = None + from webkitpy.common import array_stream from webkitpy.common.system import outputcapture from webkitpy.common.system import filesystem_mock @@ -231,6 +236,11 @@ class MainTest(unittest.TestCase): self.assertRaises(KeyboardInterrupt, logging_run, ['failures/expected/keyboard.html'], tests_included=True) + def test_keyboard_interrupt_inline_worker_model(self): + self.assertRaises(KeyboardInterrupt, logging_run, + ['failures/expected/keyboard.html', '--worker-model', 'inline'], + tests_included=True) + def test_last_results(self): fs = port.unit_test_filesystem() # We do a logging run here instead of a passing run in order to @@ -347,6 +357,18 @@ class MainTest(unittest.TestCase): self.assertFalse(err.empty()) self.assertEqual(user.opened_urls, ['/tmp/layout-test-results/results.html']) + def test_exit_after_n_failures_upload(self): + fs = port.unit_test_filesystem() + (res, buildbot_output, regular_output, user) = logging_run([ + 'failures/unexpected/text-image-checksum.html', + 'passes/text.html', + '--exit-after-n-failures', '1', + ], + tests_included=True, + record_results=True, + filesystem=fs) + self.assertTrue('/tmp/layout-test-results/incremental_results.json' in fs.files) + def test_exit_after_n_failures(self): # Unexpected failures should result in tests stopping. tests_run = get_tests_run([ @@ -399,6 +421,17 @@ class MainTest(unittest.TestCase): flatten_batches=True) self.assertEquals(['failures/expected/crash.html', 'passes/text.html'], tests_run) + def test_exit_after_n_crashes_inline_worker_model(self): + tests_run = get_tests_run([ + 'failures/unexpected/timeout.html', + 'passes/text.html', + '--exit-after-n-crashes-or-timeouts', '1', + '--worker-model', 'inline', + ], + tests_included=True, + flatten_batches=True) + self.assertEquals(['failures/unexpected/timeout.html'], tests_run) + def test_results_directory_absolute(self): # We run a configuration that should fail, to generate output, then # look for what the output results url was. @@ -467,9 +500,15 @@ class MainTest(unittest.TestCase): self.assertTrue(passing_run(['--worker-model', 'old-threads'])) def test_worker_model__processes(self): - if compare_version(sys, '2.6')[0] >= 0: + # FIXME: remove this when we fix test-webkitpy to work properly + # with the multiprocessing module (bug 54520). + if multiprocessing and sys.platform not in ('cygwin', 'win32'): self.assertTrue(passing_run(['--worker-model', 'processes'])) + def test_worker_model__processes_and_dry_run(self): + if multiprocessing and sys.platform not in ('cygwin', 'win32'): + self.assertTrue(passing_run(['--worker-model', 'processes', '--dry-run'])) + def test_worker_model__threads(self): self.assertTrue(passing_run(['--worker-model', 'threads'])) diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/__init__.py b/Tools/Scripts/webkitpy/layout_tests/test_types/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/Tools/Scripts/webkitpy/layout_tests/test_types/__init__.py +++ /dev/null diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/image_diff.py b/Tools/Scripts/webkitpy/layout_tests/test_types/image_diff.py deleted file mode 100644 index 1d7e107..0000000 --- a/Tools/Scripts/webkitpy/layout_tests/test_types/image_diff.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Compares the image output of a test to the expected image output. - -Compares hashes for the generated and expected images. If the output doesn't -match, returns FailureImageHashMismatch and outputs both hashes into the layout -test results directory. -""" - -import errno -import logging - -from webkitpy.layout_tests.layout_package import test_failures -from webkitpy.layout_tests.test_types import test_type_base - -# Cache whether we have the image_diff executable available. -_compare_available = True -_compare_msg_printed = False - -_log = logging.getLogger("webkitpy.layout_tests.test_types.image_diff") - - -class ImageDiff(test_type_base.TestTypeBase): - - def _copy_image(self, filename, actual_image, expected_image): - self.write_output_files(filename, '.png', - output=actual_image, expected=expected_image, - encoding=None, print_text_diffs=False) - - def _copy_image_hash(self, filename, actual_image_hash, expected_image_hash): - self.write_output_files(filename, '.checksum', - actual_image_hash, expected_image_hash, - encoding="ascii", print_text_diffs=False) - - def _create_diff_image(self, port, filename, actual_image, expected_image): - """Creates the visual diff of the expected/actual PNGs. - - Returns True if the images are different. - """ - diff_filename = self.output_filename(filename, - self.FILENAME_SUFFIX_COMPARE) - return port.diff_image(actual_image, expected_image, diff_filename) - - def compare_output(self, port, filename, options, actual_driver_output, - expected_driver_output): - """Implementation of CompareOutput that checks the output image and - checksum against the expected files from the LayoutTest directory. - """ - failures = [] - - # If we didn't produce a hash file, this test must be text-only. - if actual_driver_output.image_hash is None: - return failures - - if not expected_driver_output.image: - # Report a missing expected PNG file. - self._copy_image(filename, actual_driver_output.image, expected_image=None) - self._copy_image_hash(filename, actual_driver_output.image_hash, - expected_driver_output.image_hash) - failures.append(test_failures.FailureMissingImage()) - return failures - if not expected_driver_output.image_hash: - # Report a missing expected checksum file. - self._copy_image(filename, actual_driver_output.image, - expected_driver_output.image) - self._copy_image_hash(filename, actual_driver_output.image_hash, - expected_image_hash=None) - failures.append(test_failures.FailureMissingImageHash()) - return failures - - if actual_driver_output.image_hash == expected_driver_output.image_hash: - # Hash matched (no diff needed, okay to return). - return failures - - self._copy_image(filename, actual_driver_output.image, - expected_driver_output.image) - self._copy_image_hash(filename, actual_driver_output.image_hash, - expected_driver_output.image_hash) - - # Even though we only use the result in one codepath below but we - # still need to call CreateImageDiff for other codepaths. - images_are_different = self._create_diff_image(port, filename, - actual_driver_output.image, - expected_driver_output.image) - if not images_are_different: - failures.append(test_failures.FailureImageHashIncorrect()) - else: - failures.append(test_failures.FailureImageHashMismatch()) - - return failures diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py b/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py deleted file mode 100644 index 09bfc31..0000000 --- a/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Defines the interface TestTypeBase which other test types inherit from. -""" - -import cgi -import errno -import logging - -_log = logging.getLogger("webkitpy.layout_tests.test_types.test_type_base") - - -# Python bug workaround. See the wdiff code in WriteOutputFiles for an -# explanation. -_wdiff_available = True - - -class TestTypeBase(object): - - # Filename pieces when writing failures to the test results directory. - FILENAME_SUFFIX_ACTUAL = "-actual" - FILENAME_SUFFIX_EXPECTED = "-expected" - FILENAME_SUFFIX_DIFF = "-diff" - FILENAME_SUFFIX_WDIFF = "-wdiff.html" - FILENAME_SUFFIX_PRETTY_PATCH = "-pretty-diff.html" - FILENAME_SUFFIX_COMPARE = "-diff.png" - - def __init__(self, port, root_output_dir): - """Initialize a TestTypeBase object. - - Args: - port: object implementing port-specific information and methods - root_output_dir: The unix style path to the output dir. - """ - self._root_output_dir = root_output_dir - self._port = port - - def _make_output_directory(self, filename): - """Creates the output directory (if needed) for a given test - filename.""" - fs = self._port._filesystem - output_filename = fs.join(self._root_output_dir, - self._port.relative_test_filename(filename)) - fs.maybe_make_directory(fs.dirname(output_filename)) - - def output_filename(self, filename, modifier): - """Returns a filename inside the output dir that contains modifier. - - For example, if filename is c:/.../fast/dom/foo.html and modifier is - "-expected.txt", the return value is - c:/cygwin/tmp/layout-test-results/fast/dom/foo-expected.txt - - Args: - filename: absolute filename to test file - modifier: a string to replace the extension of filename with - - Return: - The absolute windows path to the output filename - """ - fs = self._port._filesystem - output_filename = fs.join(self._root_output_dir, - self._port.relative_test_filename(filename)) - return fs.splitext(output_filename)[0] + modifier - - def compare_output(self, port, filename, options, actual_driver_output, - expected_driver_output): - """Method that compares the output from the test with the - expected value. - - This is an abstract method to be implemented by all sub classes. - - Args: - port: object implementing port-specific information and methods - filename: absolute filename to test file - options: command line argument object from optparse - actual_driver_output: a DriverOutput object which represents actual test - output - expected_driver_output: a ExpectedDriverOutput object which represents a - expected test output - - Return: - a list of TestFailure objects, empty if the test passes - """ - raise NotImplementedError - - def _write_into_file_at_path(self, file_path, contents, encoding): - """This method assumes that byte_array is already encoded - into the right format.""" - fs = self._port._filesystem - if encoding is None: - fs.write_binary_file(file_path, contents) - return - fs.write_text_file(file_path, contents) - - def write_output_files(self, filename, file_type, - output, expected, encoding, - print_text_diffs=False): - """Writes the test output, the expected output and optionally the diff - between the two to files in the results directory. - - The full output filename of the actual, for example, will be - <filename>-actual<file_type> - For instance, - my_test-actual.txt - - Args: - filename: The test filename - file_type: A string describing the test output file type, e.g. ".txt" - output: A string containing the test output - expected: A string containing the expected test output - print_text_diffs: True for text diffs. (FIXME: We should be able to get this from the file type?) - """ - self._make_output_directory(filename) - actual_filename = self.output_filename(filename, self.FILENAME_SUFFIX_ACTUAL + file_type) - expected_filename = self.output_filename(filename, self.FILENAME_SUFFIX_EXPECTED + file_type) - # FIXME: This function is poorly designed. We should be passing in some sort of - # encoding information from the callers. - if output: - self._write_into_file_at_path(actual_filename, output, encoding) - if expected: - self._write_into_file_at_path(expected_filename, expected, encoding) - - if not output or not expected: - return - - if not print_text_diffs: - return - - # Note: We pass encoding=None for all diff writes, as we treat diff - # output as binary. Diff output may contain multiple files in - # conflicting encodings. - diff = self._port.diff_text(expected, output, expected_filename, actual_filename) - diff_filename = self.output_filename(filename, self.FILENAME_SUFFIX_DIFF + file_type) - self._write_into_file_at_path(diff_filename, diff, encoding=None) - - # Shell out to wdiff to get colored inline diffs. - wdiff = self._port.wdiff_text(expected_filename, actual_filename) - wdiff_filename = self.output_filename(filename, self.FILENAME_SUFFIX_WDIFF) - self._write_into_file_at_path(wdiff_filename, wdiff, encoding=None) - - # Use WebKit's PrettyPatch.rb to get an HTML diff. - pretty_patch = self._port.pretty_patch_text(diff_filename) - pretty_patch_filename = self.output_filename(filename, self.FILENAME_SUFFIX_PRETTY_PATCH) - self._write_into_file_at_path(pretty_patch_filename, pretty_patch, encoding=None) diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py b/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py deleted file mode 100644 index 7af4d9c..0000000 --- a/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -""""Tests stray tests not covered by regular code paths.""" - -import test_type_base -import unittest - -from webkitpy.thirdparty.mock import Mock - - -class Test(unittest.TestCase): - - def test_compare_output_notimplemented(self): - test_type = test_type_base.TestTypeBase(None, None) - self.assertRaises(NotImplementedError, test_type.compare_output, - None, "foo.txt", '', - {}, 'Debug') - - -if __name__ == '__main__': - unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/text_diff.py b/Tools/Scripts/webkitpy/layout_tests/test_types/text_diff.py deleted file mode 100644 index 07c3d03..0000000 --- a/Tools/Scripts/webkitpy/layout_tests/test_types/text_diff.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Compares the text output of a test to the expected text output. - -If the output doesn't match, returns FailureTextMismatch and outputs the diff -files into the layout test results directory. -""" - -import errno -import logging - -from webkitpy.layout_tests.layout_package import test_failures -from webkitpy.layout_tests.test_types import test_type_base - -_log = logging.getLogger("webkitpy.layout_tests.test_types.text_diff") - - -class TestTextDiff(test_type_base.TestTypeBase): - - def _get_normalized_output_text(self, output): - """Returns the normalized text output, i.e. the output in which - the end-of-line characters are normalized to "\n".""" - # Running tests on Windows produces "\r\n". The "\n" part is helpfully - # changed to "\r\n" by our system (Python/Cygwin), resulting in - # "\r\r\n", when, in fact, we wanted to compare the text output with - # the normalized text expectation files. - return output.replace("\r\r\n", "\r\n").replace("\r\n", "\n") - - def compare_output(self, port, filename, options, actual_driver_output, - expected_driver_output): - """Implementation of CompareOutput that checks the output text against - the expected text from the LayoutTest directory.""" - failures = [] - - # Normalize text to diff - actual_text = self._get_normalized_output_text(actual_driver_output.text) - # Assuming expected_text is already normalized. - expected_text = expected_driver_output.text - - # Write output files for new tests, too. - if port.compare_text(actual_text, expected_text): - # Text doesn't match, write output files. - self.write_output_files(filename, ".txt", actual_text, - expected_text, encoding=None, - print_text_diffs=True) - - if expected_text == '': - failures.append(test_failures.FailureMissingResult()) - else: - failures.append(test_failures.FailureTextMismatch()) - - return failures diff --git a/Tools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests.py b/Tools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests.py index 7267aa6..256d081 100755 --- a/Tools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests.py +++ b/Tools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests.py @@ -53,7 +53,9 @@ def translate_includes(text): for filename, path in include_mapping.items(): search = r'(?:[^"\'= ]*/)?' + re.escape(filename) - replace = os.path.join(path, filename) + # We use '/' instead of os.path.join in order to produce consistent + # output cross-platform. + replace = path + '/' + filename text = re.sub(search, replace, text) return text |