summaryrefslogtreecommitdiffstats
path: root/Tools/Scripts/webkitpy/layout_tests
diff options
context:
space:
mode:
authorBen Murdoch <benm@google.com>2011-05-05 14:36:32 +0100
committerBen Murdoch <benm@google.com>2011-05-10 15:38:30 +0100
commitf05b935882198ccf7d81675736e3aeb089c5113a (patch)
tree4ea0ca838d9ef1b15cf17ddb3928efb427c7e5a1 /Tools/Scripts/webkitpy/layout_tests
parent60fbdcc62bced8db2cb1fd233cc4d1e4ea17db1b (diff)
downloadexternal_webkit-f05b935882198ccf7d81675736e3aeb089c5113a.zip
external_webkit-f05b935882198ccf7d81675736e3aeb089c5113a.tar.gz
external_webkit-f05b935882198ccf7d81675736e3aeb089c5113a.tar.bz2
Merge WebKit at r74534: Initial merge by git.
Change-Id: I6ccd1154fa1b19c2ec2a66878eb675738735f1eb
Diffstat (limited to 'Tools/Scripts/webkitpy/layout_tests')
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/__init__.py1
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py231
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py210
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py569
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py212
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py598
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py220
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py197
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py183
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/metered_stream.py146
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/metered_stream_unittest.py115
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/printing.py553
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py608
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/result_summary.py89
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py868
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py350
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py282
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures_unittest.py84
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_input.py47
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_output.py56
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py61
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_results_unittest.py52
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_results_uploader.py107
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner.py1218
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner_unittest.py102
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/__init__.py32
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/apache_http_server.py230
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/base.py862
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py315
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium.py546
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py152
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py73
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py190
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py182
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py40
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py193
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py173
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/chromium_win_unittest.py74
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/config.py169
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/config_mock.py50
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/config_standalone.py70
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/config_unittest.py202
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/dryrun.py132
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/factory.py113
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py188
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/google_chrome.py122
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/google_chrome_unittest.py103
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/gtk.py116
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/http_lock.py133
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/http_lock_unittest.py111
-rwxr-xr-xTools/Scripts/webkitpy/layout_tests/port/http_server.py233
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/http_server_base.py83
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/httpd2.pem41
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/lighttpd.conf90
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/mac.py152
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py81
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py97
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/qt.py119
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/server_process.py225
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/test.py343
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/test_files.py128
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py75
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/webkit.py497
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py68
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/websocket_server.py257
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/port/win.py75
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py966
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py157
-rwxr-xr-xTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py434
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py545
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/test_types/__init__.py0
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/test_types/image_diff.py146
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py223
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py47
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/test_types/text_diff.py93
-rwxr-xr-xTools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests.py160
-rw-r--r--Tools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests_unittest.py102
77 files changed, 17167 insertions, 0 deletions
diff --git a/Tools/Scripts/webkitpy/layout_tests/__init__.py b/Tools/Scripts/webkitpy/layout_tests/__init__.py
new file mode 100644
index 0000000..ef65bee
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/__init__.py
@@ -0,0 +1 @@
+# Required for Python to search this directory for module files
diff --git a/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py b/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py
new file mode 100644
index 0000000..51dcac8
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests.py
@@ -0,0 +1,231 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""deduplicate_tests -- lists duplicated between platforms.
+
+If platform/mac-leopard is missing an expected test output, we fall back on
+platform/mac. This means it's possible to grow redundant test outputs,
+where we have the same expected data in both a platform directory and another
+platform it falls back on.
+"""
+
+import collections
+import fnmatch
+import os
+import subprocess
+import sys
+import re
+import webkitpy.common.checkout.scm as scm
+import webkitpy.common.system.executive as executive
+import webkitpy.common.system.logutils as logutils
+import webkitpy.common.system.ospath as ospath
+import webkitpy.layout_tests.port.factory as port_factory
+
+_log = logutils.get_logger(__file__)
+
+_BASE_PLATFORM = 'base'
+
+
+def port_fallbacks():
+ """Get the port fallback information.
+ Returns:
+ A dictionary mapping platform name to a list of other platforms to fall
+ back on. All platforms fall back on 'base'.
+ """
+ fallbacks = {_BASE_PLATFORM: []}
+ platform_dir = os.path.join(scm.find_checkout_root(), 'LayoutTests',
+ 'platform')
+ for port_name in os.listdir(platform_dir):
+ try:
+ platforms = port_factory.get(port_name).baseline_search_path()
+ except NotImplementedError:
+ _log.error("'%s' lacks baseline_search_path(), please fix."
+ % port_name)
+ fallbacks[port_name] = [_BASE_PLATFORM]
+ continue
+ fallbacks[port_name] = [os.path.basename(p) for p in platforms][1:]
+ fallbacks[port_name].append(_BASE_PLATFORM)
+ return fallbacks
+
+
+def parse_git_output(git_output, glob_pattern):
+ """Parses the output of git ls-tree and filters based on glob_pattern.
+ Args:
+ git_output: result of git ls-tree -r HEAD LayoutTests.
+ glob_pattern: a pattern to filter the files.
+ Returns:
+ A dictionary mapping (test name, hash of content) => [paths]
+ """
+ hashes = collections.defaultdict(set)
+ for line in git_output.split('\n'):
+ if not line:
+ break
+ attrs, path = line.strip().split('\t')
+ if not fnmatch.fnmatch(path, glob_pattern):
+ continue
+ path = path[len('LayoutTests/'):]
+ match = re.match(r'^(platform/.*?/)?(.*)', path)
+ test = match.group(2)
+ _, _, hash = attrs.split(' ')
+ hashes[(test, hash)].add(path)
+ return hashes
+
+
+def cluster_file_hashes(glob_pattern):
+ """Get the hashes of all the test expectations in the tree.
+ We cheat and use git's hashes.
+ Args:
+ glob_pattern: a pattern to filter the files.
+ Returns:
+ A dictionary mapping (test name, hash of content) => [paths]
+ """
+
+ # A map of file hash => set of all files with that hash.
+ hashes = collections.defaultdict(set)
+
+ # Fill in the map.
+ cmd = ('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests')
+ try:
+ git_output = executive.Executive().run_command(cmd,
+ cwd=scm.find_checkout_root())
+ except OSError, e:
+ if e.errno == 2: # No such file or directory.
+ _log.error("Error: 'No such file' when running git.")
+ _log.error("This script requires git.")
+ sys.exit(1)
+ raise e
+ return parse_git_output(git_output, glob_pattern)
+
+
+def extract_platforms(paths):
+ """Extracts the platforms from a list of paths matching ^platform/(.*?)/.
+ Args:
+ paths: a list of paths.
+ Returns:
+ A dictionary containing all platforms from paths.
+ """
+ platforms = {}
+ for path in paths:
+ match = re.match(r'^platform/(.*?)/', path)
+ if match:
+ platform = match.group(1)
+ else:
+ platform = _BASE_PLATFORM
+ platforms[platform] = path
+ return platforms
+
+
+def has_intermediate_results(test, fallbacks, matching_platform,
+ path_exists=os.path.exists):
+ """Returns True if there is a test result that causes us to not delete
+ this duplicate.
+
+ For example, chromium-linux may be a duplicate of the checked in result,
+ but chromium-win may have a different result checked in. In this case,
+ we need to keep the duplicate results.
+
+ Args:
+ test: The test name.
+ fallbacks: A list of platforms we fall back on.
+ matching_platform: The platform that we found the duplicate test
+ result. We can stop checking here.
+ path_exists: Optional parameter that allows us to stub out
+ os.path.exists for testing.
+ """
+ for platform in fallbacks:
+ if platform == matching_platform:
+ return False
+ test_path = os.path.join('LayoutTests', 'platform', platform, test)
+ if path_exists(test_path):
+ return True
+ return False
+
+
+def get_relative_test_path(filename, relative_to,
+ checkout_root=scm.find_checkout_root()):
+ """Constructs a relative path to |filename| from |relative_to|.
+ Args:
+ filename: The test file we're trying to get a relative path to.
+ relative_to: The absolute path we're relative to.
+ Returns:
+ A relative path to filename or None if |filename| is not below
+ |relative_to|.
+ """
+ layout_test_dir = os.path.join(checkout_root, 'LayoutTests')
+ abs_path = os.path.join(layout_test_dir, filename)
+ return ospath.relpath(abs_path, relative_to)
+
+
+def find_dups(hashes, port_fallbacks, relative_to):
+ """Yields info about redundant test expectations.
+ Args:
+ hashes: a list of hashes as returned by cluster_file_hashes.
+ port_fallbacks: a list of fallback information as returned by
+ get_port_fallbacks.
+ relative_to: the directory that we want the results relative to
+ Returns:
+ a tuple containing (test, platform, fallback, platforms)
+ """
+ for (test, hash), cluster in hashes.items():
+ if len(cluster) < 2:
+ continue # Common case: only one file with that hash.
+
+ # Compute the list of platforms we have this particular hash for.
+ platforms = extract_platforms(cluster)
+ if len(platforms) == 1:
+ continue
+
+ # See if any of the platforms are redundant with each other.
+ for platform in platforms.keys():
+ for fallback in port_fallbacks[platform]:
+ if fallback not in platforms.keys():
+ continue
+ # We have to verify that there isn't an intermediate result
+ # that causes this duplicate hash to exist.
+ if has_intermediate_results(test, port_fallbacks[platform],
+ fallback):
+ continue
+ # We print the relative path so it's easy to pipe the results
+ # to xargs rm.
+ path = get_relative_test_path(platforms[platform], relative_to)
+ if not path:
+ continue
+ yield {
+ 'test': test,
+ 'platform': platform,
+ 'fallback': fallback,
+ 'path': path,
+ }
+
+
+def deduplicate(glob_pattern):
+ """Traverses LayoutTests and returns information about duplicated files.
+ Args:
+ glob pattern to filter the files in LayoutTests.
+ Returns:
+ a dictionary containing test, path, platform and fallback.
+ """
+ fallbacks = port_fallbacks()
+ hashes = cluster_file_hashes(glob_pattern)
+ return list(find_dups(hashes, fallbacks, os.getcwd()))
diff --git a/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py b/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py
new file mode 100644
index 0000000..309bf8d
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/deduplicate_tests_unittest.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit tests for deduplicate_tests.py."""
+
+import deduplicate_tests
+import os
+import unittest
+import webkitpy.common.checkout.scm as scm
+
+
+class MockExecutive(object):
+ last_run_command = []
+ response = ''
+
+ class Executive(object):
+ def run_command(self,
+ args,
+ cwd=None,
+ input=None,
+ error_handler=None,
+ return_exit_code=False,
+ return_stderr=True,
+ decode_output=True):
+ MockExecutive.last_run_command += [args]
+ return MockExecutive.response
+
+
+class ListDuplicatesTest(unittest.TestCase):
+ def setUp(self):
+ MockExecutive.last_run_command = []
+ MockExecutive.response = ''
+ deduplicate_tests.executive = MockExecutive
+ self._original_cwd = os.getcwd()
+ checkout_root = scm.find_checkout_root()
+ self.assertNotEqual(checkout_root, None)
+ os.chdir(checkout_root)
+
+ def tearDown(self):
+ os.chdir(self._original_cwd)
+
+ def test_parse_git_output(self):
+ git_output = (
+ '100644 blob 5053240b3353f6eb39f7cb00259785f16d121df2\tLayoutTests/mac/foo-expected.txt\n'
+ '100644 blob a004548d107ecc4e1ea08019daf0a14e8634a1ff\tLayoutTests/platform/chromium/foo-expected.txt\n'
+ '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/foo-expected.txt\n'
+ '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/animage.png\n'
+ '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/foo-expected.txt\n'
+ '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/animage.png\n'
+ '100644 blob 4303df5389ca87cae83dd3236b8dd84e16606517\tLayoutTests/platform/mac/foo-expected.txt\n')
+ hashes = deduplicate_tests.parse_git_output(git_output, '*')
+ expected = {('mac/foo-expected.txt', '5053240b3353f6eb39f7cb00259785f16d121df2'): set(['mac/foo-expected.txt']),
+ ('animage.png', 'abcdebc762e3aec5df03b5c04485b2cb3b65ffb1'): set(['platform/chromium-linux/animage.png', 'platform/chromium-win/animage.png']),
+ ('foo-expected.txt', '4303df5389ca87cae83dd3236b8dd84e16606517'): set(['platform/mac/foo-expected.txt']),
+ ('foo-expected.txt', 'd6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1'): set(['platform/chromium-linux/foo-expected.txt', 'platform/chromium-win/foo-expected.txt']),
+ ('foo-expected.txt', 'a004548d107ecc4e1ea08019daf0a14e8634a1ff'): set(['platform/chromium/foo-expected.txt'])}
+ self.assertEquals(expected, hashes)
+
+ hashes = deduplicate_tests.parse_git_output(git_output, '*.png')
+ expected = {('animage.png', 'abcdebc762e3aec5df03b5c04485b2cb3b65ffb1'): set(['platform/chromium-linux/animage.png', 'platform/chromium-win/animage.png'])}
+ self.assertEquals(expected, hashes)
+
+ def test_extract_platforms(self):
+ self.assertEquals({'foo': 'platform/foo/bar',
+ 'zoo': 'platform/zoo/com'},
+ deduplicate_tests.extract_platforms(['platform/foo/bar', 'platform/zoo/com']))
+ self.assertEquals({'foo': 'platform/foo/bar',
+ deduplicate_tests._BASE_PLATFORM: 'what/'},
+ deduplicate_tests.extract_platforms(['platform/foo/bar', 'what/']))
+
+ def test_has_intermediate_results(self):
+ test_cases = (
+ # If we found a duplicate in our first fallback, we have no
+ # intermediate results.
+ (False, ('fast/foo-expected.txt',
+ ['chromium-win', 'chromium', 'base'],
+ 'chromium-win',
+ lambda path: True)),
+ # Since chromium-win has a result, we have an intermediate result.
+ (True, ('fast/foo-expected.txt',
+ ['chromium-win', 'chromium', 'base'],
+ 'chromium',
+ lambda path: True)),
+ # There are no intermediate results.
+ (False, ('fast/foo-expected.txt',
+ ['chromium-win', 'chromium', 'base'],
+ 'chromium',
+ lambda path: False)),
+ # There are no intermediate results since a result for chromium is
+ # our duplicate file.
+ (False, ('fast/foo-expected.txt',
+ ['chromium-win', 'chromium', 'base'],
+ 'chromium',
+ lambda path: path == 'LayoutTests/platform/chromium/fast/foo-expected.txt')),
+ # We have an intermediate result in 'chromium' even though our
+ # duplicate is with the file in 'base'.
+ (True, ('fast/foo-expected.txt',
+ ['chromium-win', 'chromium', 'base'],
+ 'base',
+ lambda path: path == 'LayoutTests/platform/chromium/fast/foo-expected.txt')),
+ # We have an intermediate result in 'chromium-win' even though our
+ # duplicate is in 'base'.
+ (True, ('fast/foo-expected.txt',
+ ['chromium-win', 'chromium', 'base'],
+ 'base',
+ lambda path: path == 'LayoutTests/platform/chromium-win/fast/foo-expected.txt')),
+ )
+ for expected, inputs in test_cases:
+ self.assertEquals(expected,
+ deduplicate_tests.has_intermediate_results(*inputs))
+
+ def test_unique(self):
+ MockExecutive.response = (
+ '100644 blob 5053240b3353f6eb39f7cb00259785f16d121df2\tLayoutTests/mac/foo-expected.txt\n'
+ '100644 blob a004548d107ecc4e1ea08019daf0a14e8634a1ff\tLayoutTests/platform/chromium/foo-expected.txt\n'
+ '100644 blob abcd0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/foo-expected.txt\n'
+ '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/foo-expected.txt\n'
+ '100644 blob 4303df5389ca87cae83dd3236b8dd84e16606517\tLayoutTests/platform/mac/foo-expected.txt\n')
+ result = deduplicate_tests.deduplicate('*')
+ self.assertEquals(1, len(MockExecutive.last_run_command))
+ self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
+ self.assertEquals(0, len(result))
+
+ def test_duplicates(self):
+ MockExecutive.response = (
+ '100644 blob 5053240b3353f6eb39f7cb00259785f16d121df2\tLayoutTests/mac/foo-expected.txt\n'
+ '100644 blob a004548d107ecc4e1ea08019daf0a14e8634a1ff\tLayoutTests/platform/chromium/foo-expected.txt\n'
+ '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/foo-expected.txt\n'
+ '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-linux/animage.png\n'
+ '100644 blob d6bb0bc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/foo-expected.txt\n'
+ '100644 blob abcdebc762e3aec5df03b5c04485b2cb3b65ffb1\tLayoutTests/platform/chromium-win/animage.png\n'
+ '100644 blob 4303df5389ca87cae83dd3236b8dd84e16606517\tLayoutTests/platform/mac/foo-expected.txt\n')
+
+ result = deduplicate_tests.deduplicate('*')
+ self.assertEquals(1, len(MockExecutive.last_run_command))
+ self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
+ self.assertEquals(2, len(result))
+ self.assertEquals({'test': 'animage.png',
+ 'path': 'LayoutTests/platform/chromium-linux/animage.png',
+ 'fallback': 'chromium-win',
+ 'platform': 'chromium-linux'},
+ result[0])
+ self.assertEquals({'test': 'foo-expected.txt',
+ 'path': 'LayoutTests/platform/chromium-linux/foo-expected.txt',
+ 'fallback': 'chromium-win',
+ 'platform': 'chromium-linux'},
+ result[1])
+
+ result = deduplicate_tests.deduplicate('*.txt')
+ self.assertEquals(2, len(MockExecutive.last_run_command))
+ self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
+ self.assertEquals(1, len(result))
+ self.assertEquals({'test': 'foo-expected.txt',
+ 'path': 'LayoutTests/platform/chromium-linux/foo-expected.txt',
+ 'fallback': 'chromium-win',
+ 'platform': 'chromium-linux'},
+ result[0])
+
+ result = deduplicate_tests.deduplicate('*.png')
+ self.assertEquals(3, len(MockExecutive.last_run_command))
+ self.assertEquals(('git', 'ls-tree', '-r', 'HEAD', 'LayoutTests'), MockExecutive.last_run_command[-1])
+ self.assertEquals(1, len(result))
+ self.assertEquals({'test': 'animage.png',
+ 'path': 'LayoutTests/platform/chromium-linux/animage.png',
+ 'fallback': 'chromium-win',
+ 'platform': 'chromium-linux'},
+ result[0])
+
+ def test_get_relative_test_path(self):
+ checkout_root = scm.find_checkout_root()
+ layout_test_dir = os.path.join(checkout_root, 'LayoutTests')
+ test_cases = (
+ ('platform/mac/test.html',
+ ('platform/mac/test.html', layout_test_dir)),
+ ('LayoutTests/platform/mac/test.html',
+ ('platform/mac/test.html', checkout_root)),
+ (None,
+ ('platform/mac/test.html', os.path.join(checkout_root, 'WebCore'))),
+ ('test.html',
+ ('platform/mac/test.html', os.path.join(layout_test_dir, 'platform/mac'))),
+ (None,
+ ('platform/mac/test.html', os.path.join(layout_test_dir, 'platform/win'))),
+ )
+ for expected, inputs in test_cases:
+ self.assertEquals(expected,
+ deduplicate_tests.get_relative_test_path(*inputs))
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py
new file mode 100644
index 0000000..fdb8da6
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py
@@ -0,0 +1,569 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""A Thread object for running DumpRenderTree and processing URLs from a
+shared queue.
+
+Each thread runs a separate instance of the DumpRenderTree binary and validates
+the output. When there are no more URLs to process in the shared queue, the
+thread exits.
+"""
+
+from __future__ import with_statement
+
+import codecs
+import copy
+import logging
+import os
+import Queue
+import signal
+import sys
+import thread
+import threading
+import time
+
+
+from webkitpy.layout_tests.test_types import image_diff
+from webkitpy.layout_tests.test_types import test_type_base
+from webkitpy.layout_tests.test_types import text_diff
+
+import test_failures
+import test_output
+import test_results
+
+_log = logging.getLogger("webkitpy.layout_tests.layout_package."
+ "dump_render_tree_thread")
+
+
+def _expected_test_output(port, filename):
+ """Returns an expected TestOutput object."""
+ return test_output.TestOutput(port.expected_text(filename),
+ port.expected_image(filename),
+ port.expected_checksum(filename))
+
+def _process_output(port, options, test_input, test_types, test_args,
+ test_output, worker_name):
+ """Receives the output from a DumpRenderTree process, subjects it to a
+ number of tests, and returns a list of failure types the test produced.
+
+ Args:
+ port: port-specific hooks
+ options: command line options argument from optparse
+ proc: an active DumpRenderTree process
+ test_input: Object containing the test filename and timeout
+ test_types: list of test types to subject the output to
+ test_args: arguments to be passed to each test
+ test_output: a TestOutput object containing the output of the test
+ worker_name: worker name for logging
+
+ Returns: a TestResult object
+ """
+ failures = []
+
+ if test_output.crash:
+ failures.append(test_failures.FailureCrash())
+ if test_output.timeout:
+ failures.append(test_failures.FailureTimeout())
+
+ test_name = port.relative_test_filename(test_input.filename)
+ if test_output.crash:
+ _log.debug("%s Stacktrace for %s:\n%s" % (worker_name, test_name,
+ test_output.error))
+ filename = os.path.join(options.results_directory, test_name)
+ filename = os.path.splitext(filename)[0] + "-stack.txt"
+ port.maybe_make_directory(os.path.split(filename)[0])
+ with codecs.open(filename, "wb", "utf-8") as file:
+ file.write(test_output.error)
+ elif test_output.error:
+ _log.debug("%s %s output stderr lines:\n%s" % (worker_name, test_name,
+ test_output.error))
+
+ expected_test_output = _expected_test_output(port, test_input.filename)
+
+ # Check the output and save the results.
+ start_time = time.time()
+ time_for_diffs = {}
+ for test_type in test_types:
+ start_diff_time = time.time()
+ new_failures = test_type.compare_output(port, test_input.filename,
+ test_args, test_output,
+ expected_test_output)
+ # Don't add any more failures if we already have a crash, so we don't
+ # double-report those tests. We do double-report for timeouts since
+ # we still want to see the text and image output.
+ if not test_output.crash:
+ failures.extend(new_failures)
+ time_for_diffs[test_type.__class__.__name__] = (
+ time.time() - start_diff_time)
+
+ total_time_for_all_diffs = time.time() - start_diff_time
+ return test_results.TestResult(test_input.filename, failures, test_output.test_time,
+ total_time_for_all_diffs, time_for_diffs)
+
+
+def _pad_timeout(timeout):
+ """Returns a safe multiple of the per-test timeout value to use
+ to detect hung test threads.
+
+ """
+ # When we're running one test per DumpRenderTree process, we can
+ # enforce a hard timeout. The DumpRenderTree watchdog uses 2.5x
+ # the timeout; we want to be larger than that.
+ return timeout * 3
+
+
+def _milliseconds_to_seconds(msecs):
+ return float(msecs) / 1000.0
+
+
+def _should_fetch_expected_checksum(options):
+ return options.pixel_tests and not (options.new_baseline or options.reset_results)
+
+
+def _run_single_test(port, options, test_input, test_types, test_args, driver, worker_name):
+ # FIXME: Pull this into TestShellThread._run().
+
+ # The image hash is used to avoid doing an image dump if the
+ # checksums match, so it should be set to a blank value if we
+ # are generating a new baseline. (Otherwise, an image from a
+ # previous run will be copied into the baseline."""
+ if _should_fetch_expected_checksum(options):
+ test_input.image_hash = port.expected_checksum(test_input.filename)
+ test_output = driver.run_test(test_input)
+ return _process_output(port, options, test_input, test_types, test_args,
+ test_output, worker_name)
+
+
+class SingleTestThread(threading.Thread):
+ """Thread wrapper for running a single test file."""
+
+ def __init__(self, port, options, worker_number, worker_name,
+ test_input, test_types, test_args):
+ """
+ Args:
+ port: object implementing port-specific hooks
+ options: command line argument object from optparse
+ worker_number: worker number for tests
+ worker_name: for logging
+ test_input: Object containing the test filename and timeout
+ test_types: A list of TestType objects to run the test output
+ against.
+ test_args: A TestArguments object to pass to each TestType.
+ """
+
+ threading.Thread.__init__(self)
+ self._port = port
+ self._options = options
+ self._test_input = test_input
+ self._test_types = test_types
+ self._test_args = test_args
+ self._driver = None
+ self._worker_number = worker_number
+ self._name = worker_name
+
+ def run(self):
+ self._covered_run()
+
+ def _covered_run(self):
+ # FIXME: this is a separate routine to work around a bug
+ # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
+ self._driver = self._port.create_driver(self._worker_number)
+ self._driver.start()
+ self._test_result = _run_single_test(self._port, self._options,
+ self._test_input, self._test_types,
+ self._test_args, self._driver,
+ self._name)
+ self._driver.stop()
+
+ def get_test_result(self):
+ return self._test_result
+
+
+class WatchableThread(threading.Thread):
+ """This class abstracts an interface used by
+ run_webkit_tests.TestRunner._wait_for_threads_to_finish for thread
+ management."""
+ def __init__(self):
+ threading.Thread.__init__(self)
+ self._canceled = False
+ self._exception_info = None
+ self._next_timeout = None
+ self._thread_id = None
+
+ def cancel(self):
+ """Set a flag telling this thread to quit."""
+ self._canceled = True
+
+ def clear_next_timeout(self):
+ """Mark a flag telling this thread to stop setting timeouts."""
+ self._timeout = 0
+
+ def exception_info(self):
+ """If run() terminated on an uncaught exception, return it here
+ ((type, value, traceback) tuple).
+ Returns None if run() terminated normally. Meant to be called after
+ joining this thread."""
+ return self._exception_info
+
+ def id(self):
+ """Return a thread identifier."""
+ return self._thread_id
+
+ def next_timeout(self):
+ """Return the time the test is supposed to finish by."""
+ return self._next_timeout
+
+
+class TestShellThread(WatchableThread):
+ def __init__(self, port, options, worker_number, worker_name,
+ filename_list_queue, result_queue):
+ """Initialize all the local state for this DumpRenderTree thread.
+
+ Args:
+ port: interface to port-specific hooks
+ options: command line options argument from optparse
+ worker_number: identifier for a particular worker thread.
+ worker_name: for logging.
+ filename_list_queue: A thread safe Queue class that contains lists
+ of tuples of (filename, uri) pairs.
+ result_queue: A thread safe Queue class that will contain
+ serialized TestResult objects.
+ """
+ WatchableThread.__init__(self)
+ self._port = port
+ self._options = options
+ self._worker_number = worker_number
+ self._name = worker_name
+ self._filename_list_queue = filename_list_queue
+ self._result_queue = result_queue
+ self._filename_list = []
+ self._driver = None
+ self._test_group_timing_stats = {}
+ self._test_results = []
+ self._num_tests = 0
+ self._start_time = 0
+ self._stop_time = 0
+ self._have_http_lock = False
+ self._http_lock_wait_begin = 0
+ self._http_lock_wait_end = 0
+
+ self._test_types = []
+ for cls in self._get_test_type_classes():
+ self._test_types.append(cls(self._port,
+ self._options.results_directory))
+ self._test_args = self._get_test_args(worker_number)
+
+ # Current group of tests we're running.
+ self._current_group = None
+ # Number of tests in self._current_group.
+ self._num_tests_in_current_group = None
+ # Time at which we started running tests from self._current_group.
+ self._current_group_start_time = None
+
+ def _get_test_args(self, worker_number):
+ """Returns the tuple of arguments for tests and for DumpRenderTree."""
+ test_args = test_type_base.TestArguments()
+ test_args.new_baseline = self._options.new_baseline
+ test_args.reset_results = self._options.reset_results
+
+ return test_args
+
+ def _get_test_type_classes(self):
+ classes = [text_diff.TestTextDiff]
+ if self._options.pixel_tests:
+ classes.append(image_diff.ImageDiff)
+ return classes
+
+ def get_test_group_timing_stats(self):
+ """Returns a dictionary mapping test group to a tuple of
+ (number of tests in that group, time to run the tests)"""
+ return self._test_group_timing_stats
+
+ def get_test_results(self):
+ """Return the list of all tests run on this thread.
+
+ This is used to calculate per-thread statistics.
+
+ """
+ return self._test_results
+
+ def get_total_time(self):
+ return max(self._stop_time - self._start_time -
+ self._http_lock_wait_time(), 0.0)
+
+ def get_num_tests(self):
+ return self._num_tests
+
+ def run(self):
+ """Delegate main work to a helper method and watch for uncaught
+ exceptions."""
+ self._covered_run()
+
+ def _covered_run(self):
+ # FIXME: this is a separate routine to work around a bug
+ # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
+ self._thread_id = thread.get_ident()
+ self._start_time = time.time()
+ self._num_tests = 0
+ try:
+ _log.debug('%s starting' % (self.getName()))
+ self._run(test_runner=None, result_summary=None)
+ _log.debug('%s done (%d tests)' % (self.getName(),
+ self.get_num_tests()))
+ except KeyboardInterrupt:
+ self._exception_info = sys.exc_info()
+ _log.debug("%s interrupted" % self.getName())
+ except:
+ # Save the exception for our caller to see.
+ self._exception_info = sys.exc_info()
+ self._stop_time = time.time()
+ _log.error('%s dying, exception raised' % self.getName())
+
+ self._stop_time = time.time()
+
+ def run_in_main_thread(self, test_runner, result_summary):
+ """This hook allows us to run the tests from the main thread if
+ --num-test-shells==1, instead of having to always run two or more
+ threads. This allows us to debug the test harness without having to
+ do multi-threaded debugging."""
+ self._run(test_runner, result_summary)
+
+ def cancel(self):
+ """Clean up http lock and set a flag telling this thread to quit."""
+ self._stop_servers_with_lock()
+ WatchableThread.cancel(self)
+
+ def next_timeout(self):
+ """Return the time the test is supposed to finish by."""
+ if self._next_timeout:
+ return self._next_timeout + self._http_lock_wait_time()
+ return self._next_timeout
+
+ def _http_lock_wait_time(self):
+ """Return the time what http locking takes."""
+ if self._http_lock_wait_begin == 0:
+ return 0
+ if self._http_lock_wait_end == 0:
+ return time.time() - self._http_lock_wait_begin
+ return self._http_lock_wait_end - self._http_lock_wait_begin
+
+ def _run(self, test_runner, result_summary):
+ """Main work entry point of the thread. Basically we pull urls from the
+ filename queue and run the tests until we run out of urls.
+
+ If test_runner is not None, then we call test_runner.UpdateSummary()
+ with the results of each test."""
+ batch_size = self._options.batch_size
+ batch_count = 0
+
+ # Append tests we're running to the existing tests_run.txt file.
+ # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput.
+ tests_run_filename = os.path.join(self._options.results_directory,
+ "tests_run.txt")
+ tests_run_file = codecs.open(tests_run_filename, "a", "utf-8")
+
+ while True:
+ if self._canceled:
+ _log.debug('Testing cancelled')
+ tests_run_file.close()
+ return
+
+ if len(self._filename_list) is 0:
+ if self._current_group is not None:
+ self._test_group_timing_stats[self._current_group] = \
+ (self._num_tests_in_current_group,
+ time.time() - self._current_group_start_time)
+
+ try:
+ self._current_group, self._filename_list = \
+ self._filename_list_queue.get_nowait()
+ except Queue.Empty:
+ self._stop_servers_with_lock()
+ self._kill_dump_render_tree()
+ tests_run_file.close()
+ return
+
+ if self._current_group == "tests_to_http_lock":
+ self._start_servers_with_lock()
+ elif self._have_http_lock:
+ self._stop_servers_with_lock()
+
+ self._num_tests_in_current_group = len(self._filename_list)
+ self._current_group_start_time = time.time()
+
+ test_input = self._filename_list.pop()
+
+ # We have a url, run tests.
+ batch_count += 1
+ self._num_tests += 1
+ if self._options.run_singly:
+ result = self._run_test_in_another_thread(test_input)
+ else:
+ result = self._run_test_in_this_thread(test_input)
+
+ filename = test_input.filename
+ tests_run_file.write(filename + "\n")
+ if result.failures:
+ # Check and kill DumpRenderTree if we need to.
+ if len([1 for f in result.failures
+ if f.should_kill_dump_render_tree()]):
+ self._kill_dump_render_tree()
+ # Reset the batch count since the shell just bounced.
+ batch_count = 0
+ # Print the error message(s).
+ error_str = '\n'.join([' ' + f.message() for
+ f in result.failures])
+ _log.debug("%s %s failed:\n%s" % (self.getName(),
+ self._port.relative_test_filename(filename),
+ error_str))
+ else:
+ _log.debug("%s %s passed" % (self.getName(),
+ self._port.relative_test_filename(filename)))
+ self._result_queue.put(result.dumps())
+
+ if batch_size > 0 and batch_count >= batch_size:
+ # Bounce the shell and reset count.
+ self._kill_dump_render_tree()
+ batch_count = 0
+
+ if test_runner:
+ test_runner.update_summary(result_summary)
+
+ def _run_test_in_another_thread(self, test_input):
+ """Run a test in a separate thread, enforcing a hard time limit.
+
+ Since we can only detect the termination of a thread, not any internal
+ state or progress, we can only run per-test timeouts when running test
+ files singly.
+
+ Args:
+ test_input: Object containing the test filename and timeout
+
+ Returns:
+ A TestResult
+ """
+ worker = SingleTestThread(self._port,
+ self._options,
+ self._worker_number,
+ self._name,
+ test_input,
+ self._test_types,
+ self._test_args)
+
+ worker.start()
+
+ thread_timeout = _milliseconds_to_seconds(
+ _pad_timeout(int(test_input.timeout)))
+ thread._next_timeout = time.time() + thread_timeout
+ worker.join(thread_timeout)
+ if worker.isAlive():
+ # If join() returned with the thread still running, the
+ # DumpRenderTree is completely hung and there's nothing
+ # more we can do with it. We have to kill all the
+ # DumpRenderTrees to free it up. If we're running more than
+ # one DumpRenderTree thread, we'll end up killing the other
+ # DumpRenderTrees too, introducing spurious crashes. We accept
+ # that tradeoff in order to avoid losing the rest of this
+ # thread's results.
+ _log.error('Test thread hung: killing all DumpRenderTrees')
+ if worker._driver:
+ worker._driver.stop()
+
+ try:
+ result = worker.get_test_result()
+ except AttributeError, e:
+ # This gets raised if the worker thread has already exited.
+ failures = []
+ _log.error('Cannot get results of test: %s' %
+ test_input.filename)
+ result = test_results.TestResult(test_input.filename, failures=[],
+ test_run_time=0, total_time_for_all_diffs=0, time_for_diffs={})
+
+ return result
+
+ def _run_test_in_this_thread(self, test_input):
+ """Run a single test file using a shared DumpRenderTree process.
+
+ Args:
+ test_input: Object containing the test filename, uri and timeout
+
+ Returns: a TestResult object.
+ """
+ self._ensure_dump_render_tree_is_running()
+ thread_timeout = _milliseconds_to_seconds(
+ _pad_timeout(int(test_input.timeout)))
+ self._next_timeout = time.time() + thread_timeout
+ test_result = _run_single_test(self._port, self._options, test_input,
+ self._test_types, self._test_args,
+ self._driver, self._name)
+ self._test_results.append(test_result)
+ return test_result
+
+ def _ensure_dump_render_tree_is_running(self):
+ """Start the shared DumpRenderTree, if it's not running.
+
+ This is not for use when running tests singly, since those each start
+ a separate DumpRenderTree in their own thread.
+
+ """
+ # poll() is not threadsafe and can throw OSError due to:
+ # http://bugs.python.org/issue1731717
+ if not self._driver or self._driver.poll() is not None:
+ self._driver = self._port.create_driver(self._worker_number)
+ self._driver.start()
+
+ def _start_servers_with_lock(self):
+ """Acquire http lock and start the servers."""
+ self._http_lock_wait_begin = time.time()
+ _log.debug('Acquire http lock ...')
+ self._port.acquire_http_lock()
+ _log.debug('Starting HTTP server ...')
+ self._port.start_http_server()
+ _log.debug('Starting WebSocket server ...')
+ self._port.start_websocket_server()
+ self._http_lock_wait_end = time.time()
+ self._have_http_lock = True
+
+ def _stop_servers_with_lock(self):
+ """Stop the servers and release http lock."""
+ if self._have_http_lock:
+ _log.debug('Stopping HTTP server ...')
+ self._port.stop_http_server()
+ _log.debug('Stopping WebSocket server ...')
+ self._port.stop_websocket_server()
+ _log.debug('Release http lock ...')
+ self._port.release_http_lock()
+ self._have_http_lock = False
+
+ def _kill_dump_render_tree(self):
+ """Kill the DumpRenderTree process if it's running."""
+ if self._driver:
+ self._driver.stop()
+ self._driver = None
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py
new file mode 100644
index 0000000..b054c5b
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py
@@ -0,0 +1,212 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import logging
+import os
+
+from webkitpy.layout_tests.layout_package import json_results_generator
+from webkitpy.layout_tests.layout_package import test_expectations
+from webkitpy.layout_tests.layout_package import test_failures
+import webkitpy.thirdparty.simplejson as simplejson
+
+
+class JSONLayoutResultsGenerator(json_results_generator.JSONResultsGeneratorBase):
+ """A JSON results generator for layout tests."""
+
+ LAYOUT_TESTS_PATH = "LayoutTests"
+
+ # Additional JSON fields.
+ WONTFIX = "wontfixCounts"
+
+ # Note that we omit test_expectations.FAIL from this list because
+ # it should never show up (it's a legacy input expectation, never
+ # an output expectation).
+ FAILURE_TO_CHAR = {test_expectations.CRASH: "C",
+ test_expectations.TIMEOUT: "T",
+ test_expectations.IMAGE: "I",
+ test_expectations.TEXT: "F",
+ test_expectations.MISSING: "O",
+ test_expectations.IMAGE_PLUS_TEXT: "Z"}
+
+ def __init__(self, port, builder_name, build_name, build_number,
+ results_file_base_path, builder_base_url,
+ test_timings, expectations, result_summary, all_tests,
+ generate_incremental_results=False, test_results_server=None,
+ test_type="", master_name=""):
+ """Modifies the results.json file. Grabs it off the archive directory
+ if it is not found locally.
+
+ Args:
+ result_summary: ResultsSummary object storing the summary of the test
+ results.
+ """
+ super(JSONLayoutResultsGenerator, self).__init__(
+ builder_name, build_name, build_number, results_file_base_path,
+ builder_base_url, {}, port.test_repository_paths(),
+ generate_incremental_results, test_results_server,
+ test_type, master_name)
+
+ self._port = port
+ self._expectations = expectations
+
+ # We want relative paths to LayoutTest root for JSON output.
+ path_to_name = self._get_path_relative_to_layout_test_root
+ self._result_summary = result_summary
+ self._failures = dict(
+ (path_to_name(test), test_failures.determine_result_type(failures))
+ for (test, failures) in result_summary.failures.iteritems())
+ self._all_tests = [path_to_name(test) for test in all_tests]
+ self._test_timings = dict(
+ (path_to_name(test_tuple.filename), test_tuple.test_run_time)
+ for test_tuple in test_timings)
+
+ self.generate_json_output()
+
+ def _get_path_relative_to_layout_test_root(self, test):
+ """Returns the path of the test relative to the layout test root.
+ For example, for:
+ src/third_party/WebKit/LayoutTests/fast/forms/foo.html
+ We would return
+ fast/forms/foo.html
+ """
+ index = test.find(self.LAYOUT_TESTS_PATH)
+ if index is not -1:
+ index += len(self.LAYOUT_TESTS_PATH)
+
+ if index is -1:
+ # Already a relative path.
+ relativePath = test
+ else:
+ relativePath = test[index + 1:]
+
+ # Make sure all paths are unix-style.
+ return relativePath.replace('\\', '/')
+
+ # override
+ def _get_test_timing(self, test_name):
+ if test_name in self._test_timings:
+ # Floor for now to get time in seconds.
+ return int(self._test_timings[test_name])
+ return 0
+
+ # override
+ def _get_failed_test_names(self):
+ return set(self._failures.keys())
+
+ # override
+ def _get_modifier_char(self, test_name):
+ if test_name not in self._all_tests:
+ return self.NO_DATA_RESULT
+
+ if test_name in self._failures:
+ return self.FAILURE_TO_CHAR[self._failures[test_name]]
+
+ return self.PASS_RESULT
+
+ # override
+ def _get_result_char(self, test_name):
+ return self._get_modifier_char(test_name)
+
+ # override
+ def _convert_json_to_current_version(self, results_json):
+ archive_version = None
+ if self.VERSION_KEY in results_json:
+ archive_version = results_json[self.VERSION_KEY]
+
+ super(JSONLayoutResultsGenerator,
+ self)._convert_json_to_current_version(results_json)
+
+ # version 2->3
+ if archive_version == 2:
+ for results_for_builder in results_json.itervalues():
+ try:
+ test_results = results_for_builder[self.TESTS]
+ except:
+ continue
+
+ for test in test_results:
+ # Make sure all paths are relative
+ test_path = self._get_path_relative_to_layout_test_root(test)
+ if test_path != test:
+ test_results[test_path] = test_results[test]
+ del test_results[test]
+
+ # override
+ def _insert_failure_summaries(self, results_for_builder):
+ summary = self._result_summary
+
+ self._insert_item_into_raw_list(results_for_builder,
+ len((set(summary.failures.keys()) |
+ summary.tests_by_expectation[test_expectations.SKIP]) &
+ summary.tests_by_timeline[test_expectations.NOW]),
+ self.FIXABLE_COUNT)
+ self._insert_item_into_raw_list(results_for_builder,
+ self._get_failure_summary_entry(test_expectations.NOW),
+ self.FIXABLE)
+ self._insert_item_into_raw_list(results_for_builder,
+ len(self._expectations.get_tests_with_timeline(
+ test_expectations.NOW)), self.ALL_FIXABLE_COUNT)
+ self._insert_item_into_raw_list(results_for_builder,
+ self._get_failure_summary_entry(test_expectations.WONTFIX),
+ self.WONTFIX)
+
+ # override
+ def _normalize_results_json(self, test, test_name, tests):
+ super(JSONLayoutResultsGenerator, self)._normalize_results_json(
+ test, test_name, tests)
+
+ # Remove tests that don't exist anymore.
+ full_path = os.path.join(self._port.layout_tests_dir(), test_name)
+ full_path = os.path.normpath(full_path)
+ if not os.path.exists(full_path):
+ del tests[test_name]
+
+ def _get_failure_summary_entry(self, timeline):
+ """Creates a summary object to insert into the JSON.
+
+ Args:
+ summary ResultSummary object with test results
+ timeline current test_expectations timeline to build entry for
+ (e.g., test_expectations.NOW, etc.)
+ """
+ entry = {}
+ summary = self._result_summary
+ timeline_tests = summary.tests_by_timeline[timeline]
+ entry[self.SKIP_RESULT] = len(
+ summary.tests_by_expectation[test_expectations.SKIP] &
+ timeline_tests)
+ entry[self.PASS_RESULT] = len(
+ summary.tests_by_expectation[test_expectations.PASS] &
+ timeline_tests)
+ for failure_type in summary.tests_by_expectation.keys():
+ if failure_type not in self.FAILURE_TO_CHAR:
+ continue
+ count = len(summary.tests_by_expectation[failure_type] &
+ timeline_tests)
+ entry[self.FAILURE_TO_CHAR[failure_type]] = count
+ return entry
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py
new file mode 100644
index 0000000..54d129b
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py
@@ -0,0 +1,598 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import with_statement
+
+import codecs
+import logging
+import os
+import subprocess
+import sys
+import time
+import urllib2
+import xml.dom.minidom
+
+from webkitpy.layout_tests.layout_package import test_results_uploader
+
+import webkitpy.thirdparty.simplejson as simplejson
+
+# A JSON results generator for generic tests.
+# FIXME: move this code out of the layout_package directory.
+
+_log = logging.getLogger("webkitpy.layout_tests.layout_package.json_results_generator")
+
+class TestResult(object):
+ """A simple class that represents a single test result."""
+
+ # Test modifier constants.
+ (NONE, FAILS, FLAKY, DISABLED) = range(4)
+
+ def __init__(self, name, failed=False, elapsed_time=0):
+ self.name = name
+ self.failed = failed
+ self.time = elapsed_time
+
+ test_name = name
+ try:
+ test_name = name.split('.')[1]
+ except IndexError:
+ _log.warn("Invalid test name: %s.", name)
+ pass
+
+ if test_name.startswith('FAILS_'):
+ self.modifier = self.FAILS
+ elif test_name.startswith('FLAKY_'):
+ self.modifier = self.FLAKY
+ elif test_name.startswith('DISABLED_'):
+ self.modifier = self.DISABLED
+ else:
+ self.modifier = self.NONE
+
+ def fixable(self):
+ return self.failed or self.modifier == self.DISABLED
+
+
+class JSONResultsGeneratorBase(object):
+ """A JSON results generator for generic tests."""
+
+ MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG = 750
+ # Min time (seconds) that will be added to the JSON.
+ MIN_TIME = 1
+ JSON_PREFIX = "ADD_RESULTS("
+ JSON_SUFFIX = ");"
+
+ # Note that in non-chromium tests those chars are used to indicate
+ # test modifiers (FAILS, FLAKY, etc) but not actual test results.
+ PASS_RESULT = "P"
+ SKIP_RESULT = "X"
+ FAIL_RESULT = "F"
+ FLAKY_RESULT = "L"
+ NO_DATA_RESULT = "N"
+
+ MODIFIER_TO_CHAR = {TestResult.NONE: PASS_RESULT,
+ TestResult.DISABLED: SKIP_RESULT,
+ TestResult.FAILS: FAIL_RESULT,
+ TestResult.FLAKY: FLAKY_RESULT}
+
+ VERSION = 3
+ VERSION_KEY = "version"
+ RESULTS = "results"
+ TIMES = "times"
+ BUILD_NUMBERS = "buildNumbers"
+ TIME = "secondsSinceEpoch"
+ TESTS = "tests"
+
+ FIXABLE_COUNT = "fixableCount"
+ FIXABLE = "fixableCounts"
+ ALL_FIXABLE_COUNT = "allFixableCount"
+
+ RESULTS_FILENAME = "results.json"
+ INCREMENTAL_RESULTS_FILENAME = "incremental_results.json"
+
+ URL_FOR_TEST_LIST_JSON = \
+ "http://%s/testfile?builder=%s&name=%s&testlistjson=1&testtype=%s"
+
+ def __init__(self, builder_name, build_name, build_number,
+ results_file_base_path, builder_base_url,
+ test_results_map, svn_repositories=None,
+ generate_incremental_results=False,
+ test_results_server=None,
+ test_type="",
+ master_name=""):
+ """Modifies the results.json file. Grabs it off the archive directory
+ if it is not found locally.
+
+ Args
+ builder_name: the builder name (e.g. Webkit).
+ build_name: the build name (e.g. webkit-rel).
+ build_number: the build number.
+ results_file_base_path: Absolute path to the directory containing the
+ results json file.
+ builder_base_url: the URL where we have the archived test results.
+ If this is None no archived results will be retrieved.
+ test_results_map: A dictionary that maps test_name to TestResult.
+ svn_repositories: A (json_field_name, svn_path) pair for SVN
+ repositories that tests rely on. The SVN revision will be
+ included in the JSON with the given json_field_name.
+ generate_incremental_results: If true, generate incremental json file
+ from current run results.
+ test_results_server: server that hosts test results json.
+ test_type: test type string (e.g. 'layout-tests').
+ master_name: the name of the buildbot master.
+ """
+ self._builder_name = builder_name
+ self._build_name = build_name
+ self._build_number = build_number
+ self._builder_base_url = builder_base_url
+ self._results_directory = results_file_base_path
+ self._results_file_path = os.path.join(results_file_base_path,
+ self.RESULTS_FILENAME)
+ self._incremental_results_file_path = os.path.join(
+ results_file_base_path, self.INCREMENTAL_RESULTS_FILENAME)
+
+ self._test_results_map = test_results_map
+ self._test_results = test_results_map.values()
+ self._generate_incremental_results = generate_incremental_results
+
+ self._svn_repositories = svn_repositories
+ if not self._svn_repositories:
+ self._svn_repositories = {}
+
+ self._test_results_server = test_results_server
+ self._test_type = test_type
+ self._master_name = master_name
+
+ self._json = None
+ self._archived_results = None
+
+ def generate_json_output(self):
+ """Generates the JSON output file."""
+
+ # Generate the JSON output file that has full results.
+ # FIXME: stop writing out the full results file once all bots use
+ # incremental results.
+ if not self._json:
+ self._json = self.get_json()
+ if self._json:
+ self._generate_json_file(self._json, self._results_file_path)
+
+ # Generate the JSON output file that only has incremental results.
+ if self._generate_incremental_results:
+ json = self.get_json(incremental=True)
+ if json:
+ self._generate_json_file(
+ json, self._incremental_results_file_path)
+
+ def get_json(self, incremental=False):
+ """Gets the results for the results.json file."""
+ results_json = {}
+ if not incremental:
+ if self._json:
+ return self._json
+
+ if self._archived_results:
+ results_json = self._archived_results
+
+ if not results_json:
+ results_json, error = self._get_archived_json_results(incremental)
+ if error:
+ # If there was an error don't write a results.json
+ # file at all as it would lose all the information on the
+ # bot.
+ _log.error("Archive directory is inaccessible. Not "
+ "modifying or clobbering the results.json "
+ "file: " + str(error))
+ return None
+
+ builder_name = self._builder_name
+ if results_json and builder_name not in results_json:
+ _log.debug("Builder name (%s) is not in the results.json file."
+ % builder_name)
+
+ self._convert_json_to_current_version(results_json)
+
+ if builder_name not in results_json:
+ results_json[builder_name] = (
+ self._create_results_for_builder_json())
+
+ results_for_builder = results_json[builder_name]
+
+ self._insert_generic_metadata(results_for_builder)
+
+ self._insert_failure_summaries(results_for_builder)
+
+ # Update the all failing tests with result type and time.
+ tests = results_for_builder[self.TESTS]
+ all_failing_tests = self._get_failed_test_names()
+ all_failing_tests.update(tests.iterkeys())
+ for test in all_failing_tests:
+ self._insert_test_time_and_result(test, tests, incremental)
+
+ return results_json
+
+ def set_archived_results(self, archived_results):
+ self._archived_results = archived_results
+
+ def upload_json_files(self, json_files):
+ """Uploads the given json_files to the test_results_server (if the
+ test_results_server is given)."""
+ if not self._test_results_server:
+ return
+
+ if not self._master_name:
+ _log.error("--test-results-server was set, but --master-name was not. Not uploading JSON files.")
+ return
+
+ _log.info("Uploading JSON files for builder: %s", self._builder_name)
+ attrs = [("builder", self._builder_name),
+ ("testtype", self._test_type),
+ ("master", self._master_name)]
+
+ files = [(file, os.path.join(self._results_directory, file))
+ for file in json_files]
+
+ uploader = test_results_uploader.TestResultsUploader(
+ self._test_results_server)
+ try:
+ # Set uploading timeout in case appengine server is having problem.
+ # 120 seconds are more than enough to upload test results.
+ uploader.upload(attrs, files, 120)
+ except Exception, err:
+ _log.error("Upload failed: %s" % err)
+ return
+
+ _log.info("JSON files uploaded.")
+
+ def _generate_json_file(self, json, file_path):
+ # Specify separators in order to get compact encoding.
+ json_data = simplejson.dumps(json, separators=(',', ':'))
+ json_string = self.JSON_PREFIX + json_data + self.JSON_SUFFIX
+
+ results_file = codecs.open(file_path, "w", "utf-8")
+ results_file.write(json_string)
+ results_file.close()
+
+ def _get_test_timing(self, test_name):
+ """Returns test timing data (elapsed time) in second
+ for the given test_name."""
+ if test_name in self._test_results_map:
+ # Floor for now to get time in seconds.
+ return int(self._test_results_map[test_name].time)
+ return 0
+
+ def _get_failed_test_names(self):
+ """Returns a set of failed test names."""
+ return set([r.name for r in self._test_results if r.failed])
+
+ def _get_modifier_char(self, test_name):
+ """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT,
+ PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test modifier
+ for the given test_name.
+ """
+ if test_name not in self._test_results_map:
+ return self.__class__.NO_DATA_RESULT
+
+ test_result = self._test_results_map[test_name]
+ if test_result.modifier in self.MODIFIER_TO_CHAR.keys():
+ return self.MODIFIER_TO_CHAR[test_result.modifier]
+
+ return self.__class__.PASS_RESULT
+
+ def _get_result_char(self, test_name):
+ """Returns a single char (e.g. SKIP_RESULT, FAIL_RESULT,
+ PASS_RESULT, NO_DATA_RESULT, etc) that indicates the test result
+ for the given test_name.
+ """
+ if test_name not in self._test_results_map:
+ return self.__class__.NO_DATA_RESULT
+
+ test_result = self._test_results_map[test_name]
+ if test_result.modifier == TestResult.DISABLED:
+ return self.__class__.SKIP_RESULT
+
+ if test_result.failed:
+ return self.__class__.FAIL_RESULT
+
+ return self.__class__.PASS_RESULT
+
+ # FIXME: Callers should use scm.py instead.
+ # FIXME: Identify and fix the run-time errors that were observed on Windows
+ # chromium buildbot when we had updated this code to use scm.py once before.
+ def _get_svn_revision(self, in_directory):
+ """Returns the svn revision for the given directory.
+
+ Args:
+ in_directory: The directory where svn is to be run.
+ """
+ if os.path.exists(os.path.join(in_directory, '.svn')):
+ # Note: Not thread safe: http://bugs.python.org/issue2320
+ output = subprocess.Popen(["svn", "info", "--xml"],
+ cwd=in_directory,
+ shell=(sys.platform == 'win32'),
+ stdout=subprocess.PIPE).communicate()[0]
+ try:
+ dom = xml.dom.minidom.parseString(output)
+ return dom.getElementsByTagName('entry')[0].getAttribute(
+ 'revision')
+ except xml.parsers.expat.ExpatError:
+ return ""
+ return ""
+
+ def _get_archived_json_results(self, for_incremental=False):
+ """Reads old results JSON file if it exists.
+ Returns (archived_results, error) tuple where error is None if results
+ were successfully read.
+
+ if for_incremental is True, download JSON file that only contains test
+ name list from test-results server. This is for generating incremental
+ JSON so the file generated has info for tests that failed before but
+ pass or are skipped from current run.
+ """
+ results_json = {}
+ old_results = None
+ error = None
+
+ if os.path.exists(self._results_file_path) and not for_incremental:
+ with codecs.open(self._results_file_path, "r", "utf-8") as file:
+ old_results = file.read()
+ elif self._builder_base_url or for_incremental:
+ if for_incremental:
+ if not self._test_results_server:
+ # starting from fresh if no test results server specified.
+ return {}, None
+
+ results_file_url = (self.URL_FOR_TEST_LIST_JSON %
+ (urllib2.quote(self._test_results_server),
+ urllib2.quote(self._builder_name),
+ self.RESULTS_FILENAME,
+ urllib2.quote(self._test_type)))
+ else:
+ # Check if we have the archived JSON file on the buildbot
+ # server.
+ results_file_url = (self._builder_base_url +
+ self._build_name + "/" + self.RESULTS_FILENAME)
+ _log.error("Local results.json file does not exist. Grabbing "
+ "it off the archive at " + results_file_url)
+
+ try:
+ results_file = urllib2.urlopen(results_file_url)
+ info = results_file.info()
+ old_results = results_file.read()
+ except urllib2.HTTPError, http_error:
+ # A non-4xx status code means the bot is hosed for some reason
+ # and we can't grab the results.json file off of it.
+ if (http_error.code < 400 and http_error.code >= 500):
+ error = http_error
+ except urllib2.URLError, url_error:
+ error = url_error
+
+ if old_results:
+ # Strip the prefix and suffix so we can get the actual JSON object.
+ old_results = old_results[len(self.JSON_PREFIX):
+ len(old_results) - len(self.JSON_SUFFIX)]
+
+ try:
+ results_json = simplejson.loads(old_results)
+ except:
+ _log.debug("results.json was not valid JSON. Clobbering.")
+ # The JSON file is not valid JSON. Just clobber the results.
+ results_json = {}
+ else:
+ _log.debug('Old JSON results do not exist. Starting fresh.')
+ results_json = {}
+
+ return results_json, error
+
+ def _insert_failure_summaries(self, results_for_builder):
+ """Inserts aggregate pass/failure statistics into the JSON.
+ This method reads self._test_results and generates
+ FIXABLE, FIXABLE_COUNT and ALL_FIXABLE_COUNT entries.
+
+ Args:
+ results_for_builder: Dictionary containing the test results for a
+ single builder.
+ """
+ # Insert the number of tests that failed or skipped.
+ fixable_count = len([r for r in self._test_results if r.fixable()])
+ self._insert_item_into_raw_list(results_for_builder,
+ fixable_count, self.FIXABLE_COUNT)
+
+ # Create a test modifiers (FAILS, FLAKY etc) summary dictionary.
+ entry = {}
+ for test_name in self._test_results_map.iterkeys():
+ result_char = self._get_modifier_char(test_name)
+ entry[result_char] = entry.get(result_char, 0) + 1
+
+ # Insert the pass/skip/failure summary dictionary.
+ self._insert_item_into_raw_list(results_for_builder, entry,
+ self.FIXABLE)
+
+ # Insert the number of all the tests that are supposed to pass.
+ all_test_count = len(self._test_results)
+ self._insert_item_into_raw_list(results_for_builder,
+ all_test_count, self.ALL_FIXABLE_COUNT)
+
+ def _insert_item_into_raw_list(self, results_for_builder, item, key):
+ """Inserts the item into the list with the given key in the results for
+ this builder. Creates the list if no such list exists.
+
+ Args:
+ results_for_builder: Dictionary containing the test results for a
+ single builder.
+ item: Number or string to insert into the list.
+ key: Key in results_for_builder for the list to insert into.
+ """
+ if key in results_for_builder:
+ raw_list = results_for_builder[key]
+ else:
+ raw_list = []
+
+ raw_list.insert(0, item)
+ raw_list = raw_list[:self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG]
+ results_for_builder[key] = raw_list
+
+ def _insert_item_run_length_encoded(self, item, encoded_results):
+ """Inserts the item into the run-length encoded results.
+
+ Args:
+ item: String or number to insert.
+ encoded_results: run-length encoded results. An array of arrays, e.g.
+ [[3,'A'],[1,'Q']] encodes AAAQ.
+ """
+ if len(encoded_results) and item == encoded_results[0][1]:
+ num_results = encoded_results[0][0]
+ if num_results <= self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG:
+ encoded_results[0][0] = num_results + 1
+ else:
+ # Use a list instead of a class for the run-length encoding since
+ # we want the serialized form to be concise.
+ encoded_results.insert(0, [1, item])
+
+ def _insert_generic_metadata(self, results_for_builder):
+ """ Inserts generic metadata (such as version number, current time etc)
+ into the JSON.
+
+ Args:
+ results_for_builder: Dictionary containing the test results for
+ a single builder.
+ """
+ self._insert_item_into_raw_list(results_for_builder,
+ self._build_number, self.BUILD_NUMBERS)
+
+ # Include SVN revisions for the given repositories.
+ for (name, path) in self._svn_repositories:
+ self._insert_item_into_raw_list(results_for_builder,
+ self._get_svn_revision(path),
+ name + 'Revision')
+
+ self._insert_item_into_raw_list(results_for_builder,
+ int(time.time()),
+ self.TIME)
+
+ def _insert_test_time_and_result(self, test_name, tests, incremental=False):
+ """ Insert a test item with its results to the given tests dictionary.
+
+ Args:
+ tests: Dictionary containing test result entries.
+ """
+
+ result = self._get_result_char(test_name)
+ time = self._get_test_timing(test_name)
+
+ if test_name not in tests:
+ tests[test_name] = self._create_results_and_times_json()
+
+ thisTest = tests[test_name]
+ if self.RESULTS in thisTest:
+ self._insert_item_run_length_encoded(result, thisTest[self.RESULTS])
+ else:
+ thisTest[self.RESULTS] = [[1, result]]
+
+ if self.TIMES in thisTest:
+ self._insert_item_run_length_encoded(time, thisTest[self.TIMES])
+ else:
+ thisTest[self.TIMES] = [[1, time]]
+
+ # Don't normalize the incremental results json because we need results
+ # for tests that pass or have no data from current run.
+ if not incremental:
+ self._normalize_results_json(thisTest, test_name, tests)
+
+ def _convert_json_to_current_version(self, results_json):
+ """If the JSON does not match the current version, converts it to the
+ current version and adds in the new version number.
+ """
+ if (self.VERSION_KEY in results_json and
+ results_json[self.VERSION_KEY] == self.VERSION):
+ return
+
+ results_json[self.VERSION_KEY] = self.VERSION
+
+ def _create_results_and_times_json(self):
+ results_and_times = {}
+ results_and_times[self.RESULTS] = []
+ results_and_times[self.TIMES] = []
+ return results_and_times
+
+ def _create_results_for_builder_json(self):
+ results_for_builder = {}
+ results_for_builder[self.TESTS] = {}
+ return results_for_builder
+
+ def _remove_items_over_max_number_of_builds(self, encoded_list):
+ """Removes items from the run-length encoded list after the final
+ item that exceeds the max number of builds to track.
+
+ Args:
+ encoded_results: run-length encoded results. An array of arrays, e.g.
+ [[3,'A'],[1,'Q']] encodes AAAQ.
+ """
+ num_builds = 0
+ index = 0
+ for result in encoded_list:
+ num_builds = num_builds + result[0]
+ index = index + 1
+ if num_builds > self.MAX_NUMBER_OF_BUILD_RESULTS_TO_LOG:
+ return encoded_list[:index]
+ return encoded_list
+
+ def _normalize_results_json(self, test, test_name, tests):
+ """ Prune tests where all runs pass or tests that no longer exist and
+ truncate all results to maxNumberOfBuilds.
+
+ Args:
+ test: ResultsAndTimes object for this test.
+ test_name: Name of the test.
+ tests: The JSON object with all the test results for this builder.
+ """
+ test[self.RESULTS] = self._remove_items_over_max_number_of_builds(
+ test[self.RESULTS])
+ test[self.TIMES] = self._remove_items_over_max_number_of_builds(
+ test[self.TIMES])
+
+ is_all_pass = self._is_results_all_of_type(test[self.RESULTS],
+ self.PASS_RESULT)
+ is_all_no_data = self._is_results_all_of_type(test[self.RESULTS],
+ self.NO_DATA_RESULT)
+ max_time = max([time[1] for time in test[self.TIMES]])
+
+ # Remove all passes/no-data from the results to reduce noise and
+ # filesize. If a test passes every run, but takes > MIN_TIME to run,
+ # don't throw away the data.
+ if is_all_no_data or (is_all_pass and max_time <= self.MIN_TIME):
+ del tests[test_name]
+
+ def _is_results_all_of_type(self, results, type):
+ """Returns whether all the results are of the given type
+ (e.g. all passes)."""
+ return len(results) == 1 and results[0][1] == type
+
+
+# Left here not to break anything.
+class JSONResultsGenerator(JSONResultsGeneratorBase):
+ pass
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py
new file mode 100644
index 0000000..dad549a
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py
@@ -0,0 +1,220 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit tests for json_results_generator.py."""
+
+import unittest
+import optparse
+import random
+import shutil
+import tempfile
+
+from webkitpy.layout_tests.layout_package import json_results_generator
+from webkitpy.layout_tests.layout_package import test_expectations
+
+
+class JSONGeneratorTest(unittest.TestCase):
+ def setUp(self):
+ self.builder_name = 'DUMMY_BUILDER_NAME'
+ self.build_name = 'DUMMY_BUILD_NAME'
+ self.build_number = 'DUMMY_BUILDER_NUMBER'
+
+ # For archived results.
+ self._json = None
+ self._num_runs = 0
+ self._tests_set = set([])
+ self._test_timings = {}
+ self._failed_count_map = {}
+
+ self._PASS_count = 0
+ self._DISABLED_count = 0
+ self._FLAKY_count = 0
+ self._FAILS_count = 0
+ self._fixable_count = 0
+
+ def _test_json_generation(self, passed_tests_list, failed_tests_list):
+ tests_set = set(passed_tests_list) | set(failed_tests_list)
+
+ DISABLED_tests = set([t for t in tests_set
+ if t.startswith('DISABLED_')])
+ FLAKY_tests = set([t for t in tests_set
+ if t.startswith('FLAKY_')])
+ FAILS_tests = set([t for t in tests_set
+ if t.startswith('FAILS_')])
+ PASS_tests = tests_set - (DISABLED_tests | FLAKY_tests | FAILS_tests)
+
+ failed_tests = set(failed_tests_list) - DISABLED_tests
+ failed_count_map = dict([(t, 1) for t in failed_tests])
+
+ test_timings = {}
+ i = 0
+ for test in tests_set:
+ test_timings[test] = float(self._num_runs * 100 + i)
+ i += 1
+
+ test_results_map = dict()
+ for test in tests_set:
+ test_results_map[test] = json_results_generator.TestResult(test,
+ failed=(test in failed_tests),
+ elapsed_time=test_timings[test])
+
+ generator = json_results_generator.JSONResultsGeneratorBase(
+ self.builder_name, self.build_name, self.build_number,
+ '',
+ None, # don't fetch past json results archive
+ test_results_map)
+
+ failed_count_map = dict([(t, 1) for t in failed_tests])
+
+ # Test incremental json results
+ incremental_json = generator.get_json(incremental=True)
+ self._verify_json_results(
+ tests_set,
+ test_timings,
+ failed_count_map,
+ len(PASS_tests),
+ len(DISABLED_tests),
+ len(FLAKY_tests),
+ len(DISABLED_tests | failed_tests),
+ incremental_json,
+ 1)
+
+ # Test aggregated json results
+ generator.set_archived_results(self._json)
+ json = generator.get_json(incremental=False)
+ self._json = json
+ self._num_runs += 1
+ self._tests_set |= tests_set
+ self._test_timings.update(test_timings)
+ self._PASS_count += len(PASS_tests)
+ self._DISABLED_count += len(DISABLED_tests)
+ self._FLAKY_count += len(FLAKY_tests)
+ self._fixable_count += len(DISABLED_tests | failed_tests)
+
+ get = self._failed_count_map.get
+ for test in failed_count_map.iterkeys():
+ self._failed_count_map[test] = get(test, 0) + 1
+
+ self._verify_json_results(
+ self._tests_set,
+ self._test_timings,
+ self._failed_count_map,
+ self._PASS_count,
+ self._DISABLED_count,
+ self._FLAKY_count,
+ self._fixable_count,
+ self._json,
+ self._num_runs)
+
+ def _verify_json_results(self, tests_set, test_timings, failed_count_map,
+ PASS_count, DISABLED_count, FLAKY_count,
+ fixable_count,
+ json, num_runs):
+ # Aliasing to a short name for better access to its constants.
+ JRG = json_results_generator.JSONResultsGeneratorBase
+
+ self.assertTrue(JRG.VERSION_KEY in json)
+ self.assertTrue(self.builder_name in json)
+
+ buildinfo = json[self.builder_name]
+ self.assertTrue(JRG.FIXABLE in buildinfo)
+ self.assertTrue(JRG.TESTS in buildinfo)
+ self.assertEqual(len(buildinfo[JRG.BUILD_NUMBERS]), num_runs)
+ self.assertEqual(buildinfo[JRG.BUILD_NUMBERS][0], self.build_number)
+
+ if tests_set or DISABLED_count:
+ fixable = {}
+ for fixable_items in buildinfo[JRG.FIXABLE]:
+ for (type, count) in fixable_items.iteritems():
+ if type in fixable:
+ fixable[type] = fixable[type] + count
+ else:
+ fixable[type] = count
+
+ if PASS_count:
+ self.assertEqual(fixable[JRG.PASS_RESULT], PASS_count)
+ else:
+ self.assertTrue(JRG.PASS_RESULT not in fixable or
+ fixable[JRG.PASS_RESULT] == 0)
+ if DISABLED_count:
+ self.assertEqual(fixable[JRG.SKIP_RESULT], DISABLED_count)
+ else:
+ self.assertTrue(JRG.SKIP_RESULT not in fixable or
+ fixable[JRG.SKIP_RESULT] == 0)
+ if FLAKY_count:
+ self.assertEqual(fixable[JRG.FLAKY_RESULT], FLAKY_count)
+ else:
+ self.assertTrue(JRG.FLAKY_RESULT not in fixable or
+ fixable[JRG.FLAKY_RESULT] == 0)
+
+ if failed_count_map:
+ tests = buildinfo[JRG.TESTS]
+ for test_name in failed_count_map.iterkeys():
+ self.assertTrue(test_name in tests)
+ test = tests[test_name]
+
+ failed = 0
+ for result in test[JRG.RESULTS]:
+ if result[1] == JRG.FAIL_RESULT:
+ failed += result[0]
+ self.assertEqual(failed_count_map[test_name], failed)
+
+ timing_count = 0
+ for timings in test[JRG.TIMES]:
+ if timings[1] == test_timings[test_name]:
+ timing_count = timings[0]
+ self.assertEqual(1, timing_count)
+
+ if fixable_count:
+ self.assertEqual(sum(buildinfo[JRG.FIXABLE_COUNT]), fixable_count)
+
+ def test_json_generation(self):
+ self._test_json_generation([], [])
+ self._test_json_generation(['A1', 'B1'], [])
+ self._test_json_generation([], ['FAILS_A2', 'FAILS_B2'])
+ self._test_json_generation(['DISABLED_A3', 'DISABLED_B3'], [])
+ self._test_json_generation(['A4'], ['B4', 'FAILS_C4'])
+ self._test_json_generation(['DISABLED_C5', 'DISABLED_D5'], ['A5', 'B5'])
+ self._test_json_generation(
+ ['A6', 'B6', 'FAILS_C6', 'DISABLED_E6', 'DISABLED_F6'],
+ ['FAILS_D6'])
+
+ # Generate JSON with the same test sets. (Both incremental results and
+ # archived results must be updated appropriately.)
+ self._test_json_generation(
+ ['A', 'FLAKY_B', 'DISABLED_C'],
+ ['FAILS_D', 'FLAKY_E'])
+ self._test_json_generation(
+ ['A', 'DISABLED_C', 'FLAKY_E'],
+ ['FLAKY_B', 'FAILS_D'])
+ self._test_json_generation(
+ ['FLAKY_B', 'DISABLED_C', 'FAILS_D'],
+ ['A', 'FLAKY_E'])
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py
new file mode 100644
index 0000000..e0ca8db
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py
@@ -0,0 +1,197 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Module for handling messages, threads, processes, and concurrency for run-webkit-tests.
+
+Testing is accomplished by having a manager (TestRunner) gather all of the
+tests to be run, and sending messages to a pool of workers (TestShellThreads)
+to run each test. Each worker communicates with one driver (usually
+DumpRenderTree) to run one test at a time and then compare the output against
+what we expected to get.
+
+This modules provides a message broker that connects the manager to the
+workers: it provides a messaging abstraction and message loops, and
+handles launching threads and/or processes depending on the
+requested configuration.
+"""
+
+import logging
+import sys
+import time
+import traceback
+
+import dump_render_tree_thread
+
+_log = logging.getLogger(__name__)
+
+
+def get(port, options):
+ """Return an instance of a WorkerMessageBroker."""
+ worker_model = options.worker_model
+ if worker_model == 'old-inline':
+ return InlineBroker(port, options)
+ if worker_model == 'old-threads':
+ return MultiThreadedBroker(port, options)
+ raise ValueError('unsupported value for --worker-model: %s' % worker_model)
+
+
+class _WorkerState(object):
+ def __init__(self, name):
+ self.name = name
+ self.thread = None
+
+
+class WorkerMessageBroker(object):
+ def __init__(self, port, options):
+ self._port = port
+ self._options = options
+ self._num_workers = int(self._options.child_processes)
+
+ # This maps worker names to their _WorkerState values.
+ self._workers = {}
+
+ def _threads(self):
+ return tuple([w.thread for w in self._workers.values()])
+
+ def start_workers(self, test_runner):
+ """Starts up the pool of workers for running the tests.
+
+ Args:
+ test_runner: a handle to the manager/TestRunner object
+ """
+ self._test_runner = test_runner
+ for worker_number in xrange(self._num_workers):
+ worker = _WorkerState('worker-%d' % worker_number)
+ worker.thread = self._start_worker(worker_number, worker.name)
+ self._workers[worker.name] = worker
+ return self._threads()
+
+ def _start_worker(self, worker_number, worker_name):
+ raise NotImplementedError
+
+ def run_message_loop(self):
+ """Loop processing messages until done."""
+ raise NotImplementedError
+
+ def cancel_workers(self):
+ """Cancel/interrupt any workers that are still alive."""
+ pass
+
+ def cleanup(self):
+ """Perform any necessary cleanup on shutdown."""
+ pass
+
+
+class InlineBroker(WorkerMessageBroker):
+ def _start_worker(self, worker_number, worker_name):
+ # FIXME: Replace with something that isn't a thread.
+ thread = dump_render_tree_thread.TestShellThread(self._port,
+ self._options, worker_number, worker_name,
+ self._test_runner._current_filename_queue,
+ self._test_runner._result_queue)
+ # Note: Don't start() the thread! If we did, it would actually
+ # create another thread and start executing it, and we'd no longer
+ # be single-threaded.
+ return thread
+
+ def run_message_loop(self):
+ thread = self._threads()[0]
+ thread.run_in_main_thread(self._test_runner,
+ self._test_runner._current_result_summary)
+ self._test_runner.update()
+
+
+class MultiThreadedBroker(WorkerMessageBroker):
+ def _start_worker(self, worker_number, worker_name):
+ thread = dump_render_tree_thread.TestShellThread(self._port,
+ self._options, worker_number, worker_name,
+ self._test_runner._current_filename_queue,
+ self._test_runner._result_queue)
+ thread.start()
+ return thread
+
+ def run_message_loop(self):
+ threads = self._threads()
+
+ # Loop through all the threads waiting for them to finish.
+ some_thread_is_alive = True
+ while some_thread_is_alive:
+ some_thread_is_alive = False
+ t = time.time()
+ for thread in threads:
+ exception_info = thread.exception_info()
+ if exception_info is not None:
+ # Re-raise the thread's exception here to make it
+ # clear that testing was aborted. Otherwise,
+ # the tests that did not run would be assumed
+ # to have passed.
+ raise exception_info[0], exception_info[1], exception_info[2]
+
+ if thread.isAlive():
+ some_thread_is_alive = True
+ next_timeout = thread.next_timeout()
+ if next_timeout and t > next_timeout:
+ log_wedged_worker(thread.getName(), thread.id())
+ thread.clear_next_timeout()
+
+ self._test_runner.update()
+
+ if some_thread_is_alive:
+ time.sleep(0.01)
+
+ def cancel_workers(self):
+ threads = self._threads()
+ for thread in threads:
+ thread.cancel()
+
+
+def log_wedged_worker(name, id):
+ """Log information about the given worker state."""
+ stack = _find_thread_stack(id)
+ assert(stack is not None)
+ _log.error("")
+ _log.error("%s (tid %d) is wedged" % (name, id))
+ _log_stack(stack)
+ _log.error("")
+
+
+def _find_thread_stack(id):
+ """Returns a stack object that can be used to dump a stack trace for
+ the given thread id (or None if the id is not found)."""
+ for thread_id, stack in sys._current_frames().items():
+ if thread_id == id:
+ return stack
+ return None
+
+
+def _log_stack(stack):
+ """Log a stack trace to log.error()."""
+ for filename, lineno, name, line in traceback.extract_stack(stack):
+ _log.error('File: "%s", line %d, in %s' % (filename, lineno, name))
+ if line:
+ _log.error(' %s' % line.strip())
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py
new file mode 100644
index 0000000..6f04fd3
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py
@@ -0,0 +1,183 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import logging
+import Queue
+import sys
+import thread
+import threading
+import time
+import unittest
+
+from webkitpy.common import array_stream
+from webkitpy.common.system import outputcapture
+from webkitpy.tool import mocktool
+
+from webkitpy.layout_tests import run_webkit_tests
+
+import message_broker
+
+
+class TestThread(threading.Thread):
+ def __init__(self, started_queue, stopping_queue):
+ threading.Thread.__init__(self)
+ self._thread_id = None
+ self._started_queue = started_queue
+ self._stopping_queue = stopping_queue
+ self._timeout = False
+ self._timeout_queue = Queue.Queue()
+ self._exception_info = None
+
+ def id(self):
+ return self._thread_id
+
+ def getName(self):
+ return "worker-0"
+
+ def run(self):
+ self._covered_run()
+
+ def _covered_run(self):
+ # FIXME: this is a separate routine to work around a bug
+ # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
+ self._thread_id = thread.get_ident()
+ try:
+ self._started_queue.put('')
+ msg = self._stopping_queue.get()
+ if msg == 'KeyboardInterrupt':
+ raise KeyboardInterrupt
+ elif msg == 'Exception':
+ raise ValueError()
+ elif msg == 'Timeout':
+ self._timeout = True
+ self._timeout_queue.get()
+ except:
+ self._exception_info = sys.exc_info()
+
+ def exception_info(self):
+ return self._exception_info
+
+ def next_timeout(self):
+ if self._timeout:
+ self._timeout_queue.put('done')
+ return time.time() - 10
+ return time.time()
+
+ def clear_next_timeout(self):
+ self._next_timeout = None
+
+class TestHandler(logging.Handler):
+ def __init__(self, astream):
+ logging.Handler.__init__(self)
+ self._stream = astream
+
+ def emit(self, record):
+ self._stream.write(self.format(record))
+
+
+class MultiThreadedBrokerTest(unittest.TestCase):
+ class MockTestRunner(object):
+ def __init__(self):
+ pass
+
+ def __del__(self):
+ pass
+
+ def update(self):
+ pass
+
+ def run_one_thread(self, msg):
+ runner = self.MockTestRunner()
+ port = None
+ options = mocktool.MockOptions(child_processes='1')
+ starting_queue = Queue.Queue()
+ stopping_queue = Queue.Queue()
+ broker = message_broker.MultiThreadedBroker(port, options)
+ broker._test_runner = runner
+ child_thread = TestThread(starting_queue, stopping_queue)
+ broker._workers['worker-0'] = message_broker._WorkerState('worker-0')
+ broker._workers['worker-0'].thread = child_thread
+ child_thread.start()
+ started_msg = starting_queue.get()
+ stopping_queue.put(msg)
+ return broker.run_message_loop()
+
+ def test_basic(self):
+ interrupted = self.run_one_thread('')
+ self.assertFalse(interrupted)
+
+ def test_interrupt(self):
+ self.assertRaises(KeyboardInterrupt, self.run_one_thread, 'KeyboardInterrupt')
+
+ def test_timeout(self):
+ oc = outputcapture.OutputCapture()
+ oc.capture_output()
+ interrupted = self.run_one_thread('Timeout')
+ self.assertFalse(interrupted)
+ oc.restore_output()
+
+ def test_exception(self):
+ self.assertRaises(ValueError, self.run_one_thread, 'Exception')
+
+
+class Test(unittest.TestCase):
+ def test_find_thread_stack_found(self):
+ id, stack = sys._current_frames().items()[0]
+ found_stack = message_broker._find_thread_stack(id)
+ self.assertNotEqual(found_stack, None)
+
+ def test_find_thread_stack_not_found(self):
+ found_stack = message_broker._find_thread_stack(0)
+ self.assertEqual(found_stack, None)
+
+ def test_log_wedged_worker(self):
+ oc = outputcapture.OutputCapture()
+ oc.capture_output()
+ logger = message_broker._log
+ astream = array_stream.ArrayStream()
+ handler = TestHandler(astream)
+ logger.addHandler(handler)
+
+ starting_queue = Queue.Queue()
+ stopping_queue = Queue.Queue()
+ child_thread = TestThread(starting_queue, stopping_queue)
+ child_thread.start()
+ msg = starting_queue.get()
+
+ message_broker.log_wedged_worker(child_thread.getName(),
+ child_thread.id())
+ stopping_queue.put('')
+ child_thread.join(timeout=1.0)
+
+ self.assertFalse(astream.empty())
+ self.assertFalse(child_thread.isAlive())
+ oc.restore_output()
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/metered_stream.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/metered_stream.py
new file mode 100644
index 0000000..20646a1
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/metered_stream.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+Package that implements a stream wrapper that has 'meters' as well as
+regular output. A 'meter' is a single line of text that can be erased
+and rewritten repeatedly, without producing multiple lines of output. It
+can be used to produce effects like progress bars.
+
+This package should only be called by the printing module in the layout_tests
+package.
+"""
+
+import logging
+
+_log = logging.getLogger("webkitpy.layout_tests.metered_stream")
+
+
+class MeteredStream:
+ """This class is a wrapper around a stream that allows you to implement
+ meters (progress bars, etc.).
+
+ It can be used directly as a stream, by calling write(), but provides
+ two other methods for output, update(), and progress().
+
+ In normal usage, update() will overwrite the output of the immediately
+ preceding update() (write() also will overwrite update()). So, calling
+ multiple update()s in a row can provide an updating status bar (note that
+ if an update string contains newlines, only the text following the last
+ newline will be overwritten/erased).
+
+ If the MeteredStream is constructed in "verbose" mode (i.e., by passing
+ verbose=true), then update() no longer overwrite a previous update(), and
+ instead the call is equivalent to write(), although the text is
+ actually sent to the logger rather than to the stream passed
+ to the constructor.
+
+ progress() is just like update(), except that if you are in verbose mode,
+ progress messages are not output at all (they are dropped). This is
+ used for things like progress bars which are presumed to be unwanted in
+ verbose mode.
+
+ Note that the usual usage for this class is as a destination for
+ a logger that can also be written to directly (i.e., some messages go
+ through the logger, some don't). We thus have to dance around a
+ layering inversion in update() for things to work correctly.
+ """
+
+ def __init__(self, verbose, stream):
+ """
+ Args:
+ verbose: whether progress is a no-op and updates() aren't overwritten
+ stream: output stream to write to
+ """
+ self._dirty = False
+ self._verbose = verbose
+ self._stream = stream
+ self._last_update = ""
+
+ def write(self, txt):
+ """Write to the stream, overwriting and resetting the meter."""
+ if self._dirty:
+ self._write(txt)
+ self._dirty = False
+ self._last_update = ''
+ else:
+ self._stream.write(txt)
+
+ def flush(self):
+ """Flush any buffered output."""
+ self._stream.flush()
+
+ def progress(self, str):
+ """
+ Write a message to the stream that will get overwritten.
+
+ This is used for progress updates that don't need to be preserved in
+ the log. If the MeteredStream was initialized with verbose==True,
+ then this output is discarded. We have this in case we are logging
+ lots of output and the update()s will get lost or won't work
+ properly (typically because verbose streams are redirected to files).
+
+ """
+ if self._verbose:
+ return
+ self._write(str)
+
+ def update(self, str):
+ """
+ Write a message that is also included when logging verbosely.
+
+ This routine preserves the same console logging behavior as progress(),
+ but will also log the message if verbose() was true.
+
+ """
+ # Note this is a separate routine that calls either into the logger
+ # or the metering stream. We have to be careful to avoid a layering
+ # inversion (stream calling back into the logger).
+ if self._verbose:
+ _log.info(str)
+ else:
+ self._write(str)
+
+ def _write(self, str):
+ """Actually write the message to the stream."""
+
+ # FIXME: Figure out if there is a way to detect if we're writing
+ # to a stream that handles CRs correctly (e.g., terminals). That might
+ # be a cleaner way of handling this.
+
+ # Print the necessary number of backspaces to erase the previous
+ # message.
+ if len(self._last_update):
+ self._stream.write("\b" * len(self._last_update) +
+ " " * len(self._last_update) +
+ "\b" * len(self._last_update))
+ self._stream.write(str)
+ last_newline = str.rfind("\n")
+ self._last_update = str[(last_newline + 1):]
+ self._dirty = True
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/metered_stream_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/metered_stream_unittest.py
new file mode 100644
index 0000000..9421ff8
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/metered_stream_unittest.py
@@ -0,0 +1,115 @@
+#!/usr/bin/python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit tests for metered_stream.py."""
+
+import os
+import optparse
+import pdb
+import sys
+import unittest
+
+from webkitpy.common.array_stream import ArrayStream
+from webkitpy.layout_tests.layout_package import metered_stream
+
+
+class TestMeteredStream(unittest.TestCase):
+ def test_regular(self):
+ a = ArrayStream()
+ m = metered_stream.MeteredStream(verbose=False, stream=a)
+ self.assertTrue(a.empty())
+
+ # basic test - note that the flush() is a no-op, but we include it
+ # for coverage.
+ m.write("foo")
+ m.flush()
+ exp = ['foo']
+ self.assertEquals(a.get(), exp)
+
+ # now check that a second write() does not overwrite the first.
+ m.write("bar")
+ exp.append('bar')
+ self.assertEquals(a.get(), exp)
+
+ m.update("batter")
+ exp.append('batter')
+ self.assertEquals(a.get(), exp)
+
+ # The next update() should overwrite the laste update() but not the
+ # other text. Note that the cursor is effectively positioned at the
+ # end of 'foo', even though we had to erase three more characters.
+ m.update("foo")
+ exp.append('\b\b\b\b\b\b \b\b\b\b\b\b')
+ exp.append('foo')
+ self.assertEquals(a.get(), exp)
+
+ m.progress("progress")
+ exp.append('\b\b\b \b\b\b')
+ exp.append('progress')
+ self.assertEquals(a.get(), exp)
+
+ # now check that a write() does overwrite the progress bar
+ m.write("foo")
+ exp.append('\b\b\b\b\b\b\b\b \b\b\b\b\b\b\b\b')
+ exp.append('foo')
+ self.assertEquals(a.get(), exp)
+
+ # Now test that we only back up to the most recent newline.
+
+ # Note also that we do not back up to erase the most recent write(),
+ # i.e., write()s do not get erased.
+ a.reset()
+ m.update("foo\nbar")
+ m.update("baz")
+ self.assertEquals(a.get(), ['foo\nbar', '\b\b\b \b\b\b', 'baz'])
+
+ def test_verbose(self):
+ a = ArrayStream()
+ m = metered_stream.MeteredStream(verbose=True, stream=a)
+ self.assertTrue(a.empty())
+ m.write("foo")
+ self.assertEquals(a.get(), ['foo'])
+
+ import logging
+ b = ArrayStream()
+ logger = logging.getLogger()
+ handler = logging.StreamHandler(b)
+ logger.addHandler(handler)
+ m.update("bar")
+ logger.handlers.remove(handler)
+ self.assertEquals(a.get(), ['foo'])
+ self.assertEquals(b.get(), ['bar\n'])
+
+ m.progress("dropped")
+ self.assertEquals(a.get(), ['foo'])
+ self.assertEquals(b.get(), ['bar\n'])
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/printing.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/printing.py
new file mode 100644
index 0000000..7a6aad1
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/printing.py
@@ -0,0 +1,553 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Package that handles non-debug, non-file output for run-webkit-tests."""
+
+import logging
+import optparse
+import os
+import pdb
+
+from webkitpy.layout_tests.layout_package import metered_stream
+from webkitpy.layout_tests.layout_package import test_expectations
+
+_log = logging.getLogger("webkitpy.layout_tests.printer")
+
+TestExpectationsFile = test_expectations.TestExpectationsFile
+
+NUM_SLOW_TESTS_TO_LOG = 10
+
+PRINT_DEFAULT = ("misc,one-line-progress,one-line-summary,unexpected,"
+ "unexpected-results,updates")
+PRINT_EVERYTHING = ("actual,config,expected,misc,one-line-progress,"
+ "one-line-summary,slowest,timing,unexpected,"
+ "unexpected-results,updates")
+
+HELP_PRINTING = """
+Output for run-webkit-tests is controlled by a comma-separated list of
+values passed to --print. Values either influence the overall output, or
+the output at the beginning of the run, during the run, or at the end:
+
+Overall options:
+ nothing don't print anything. This overrides every other option
+ default include the default options. This is useful for logging
+ the default options plus additional settings.
+ everything print everything (except the trace-* options and the
+ detailed-progress option, see below for the full list )
+ misc print miscellaneous things like blank lines
+
+At the beginning of the run:
+ config print the test run configuration
+ expected print a summary of what is expected to happen
+ (# passes, # failures, etc.)
+
+During the run:
+ detailed-progress print one dot per test completed
+ one-line-progress print a one-line progress bar
+ unexpected print any unexpected results as they occur
+ updates print updates on which stage is executing
+ trace-everything print detailed info on every test's results
+ (baselines, expectation, time it took to run). If
+ this is specified it will override the '*-progress'
+ options, the 'trace-unexpected' option, and the
+ 'unexpected' option.
+ trace-unexpected like 'trace-everything', but only for tests with
+ unexpected results. If this option is specified,
+ it will override the 'unexpected' option.
+
+At the end of the run:
+ actual print a summary of the actual results
+ slowest print %(slowest)d slowest tests and the time they took
+ timing print timing statistics
+ unexpected-results print a list of the tests with unexpected results
+ one-line-summary print a one-line summary of the run
+
+Notes:
+ - 'detailed-progress' can only be used if running in a single thread
+ (using --child-processes=1) or a single queue of tests (using
+ --experimental-fully-parallel). If these conditions aren't true,
+ 'one-line-progress' will be used instead.
+ - If both 'detailed-progress' and 'one-line-progress' are specified (and
+ both are possible), 'detailed-progress' will be used.
+ - If 'nothing' is specified, it overrides all of the other options.
+ - Specifying --verbose is equivalent to --print everything plus it
+ changes the format of the log messages to add timestamps and other
+ information. If you specify --verbose and --print X, then X overrides
+ the --print everything implied by --verbose.
+
+--print 'everything' is equivalent to --print '%(everything)s'.
+
+The default (--print default) is equivalent to --print '%(default)s'.
+""" % {'slowest': NUM_SLOW_TESTS_TO_LOG, 'everything': PRINT_EVERYTHING,
+ 'default': PRINT_DEFAULT}
+
+
+def print_options():
+ return [
+ # Note: We use print_options rather than just 'print' because print
+ # is a reserved word.
+ # Note: Also, we don't specify a default value so we can detect when
+ # no flag is specified on the command line and use different defaults
+ # based on whether or not --verbose is specified (since --print
+ # overrides --verbose).
+ optparse.make_option("--print", dest="print_options",
+ help=("controls print output of test run. "
+ "Use --help-printing for more.")),
+ optparse.make_option("--help-printing", action="store_true",
+ help="show detailed help on controlling print output"),
+ optparse.make_option("-v", "--verbose", action="store_true",
+ default=False, help="include debug-level logging"),
+ ]
+
+
+def parse_print_options(print_options, verbose, child_processes,
+ is_fully_parallel):
+ """Parse the options provided to --print and dedup and rank them.
+
+ Returns
+ a set() of switches that govern how logging is done
+
+ """
+ if print_options:
+ switches = set(print_options.split(','))
+ elif verbose:
+ switches = set(PRINT_EVERYTHING.split(','))
+ else:
+ switches = set(PRINT_DEFAULT.split(','))
+
+ if 'nothing' in switches:
+ return set()
+
+ if (child_processes != 1 and not is_fully_parallel and
+ 'detailed-progress' in switches):
+ _log.warn("Can only print 'detailed-progress' if running "
+ "with --child-processes=1 or "
+ "with --experimental-fully-parallel. "
+ "Using 'one-line-progress' instead.")
+ switches.discard('detailed-progress')
+ switches.add('one-line-progress')
+
+ if 'everything' in switches:
+ switches.discard('everything')
+ switches.update(set(PRINT_EVERYTHING.split(',')))
+
+ if 'default' in switches:
+ switches.discard('default')
+ switches.update(set(PRINT_DEFAULT.split(',')))
+
+ if 'detailed-progress' in switches:
+ switches.discard('one-line-progress')
+
+ if 'trace-everything' in switches:
+ switches.discard('detailed-progress')
+ switches.discard('one-line-progress')
+ switches.discard('trace-unexpected')
+ switches.discard('unexpected')
+
+ if 'trace-unexpected' in switches:
+ switches.discard('unexpected')
+
+ return switches
+
+
+def _configure_logging(stream, verbose):
+ log_fmt = '%(message)s'
+ log_datefmt = '%y%m%d %H:%M:%S'
+ log_level = logging.INFO
+ if verbose:
+ log_fmt = ('%(asctime)s %(process)d %(filename)s:%(lineno)d '
+ '%(levelname)s %(message)s')
+ log_level = logging.DEBUG
+
+ root = logging.getLogger()
+ handler = logging.StreamHandler(stream)
+ handler.setFormatter(logging.Formatter(log_fmt, None))
+ root.addHandler(handler)
+ root.setLevel(log_level)
+ return handler
+
+
+def _restore_logging(handler_to_remove):
+ root = logging.getLogger()
+ root.handlers.remove(handler_to_remove)
+
+
+class Printer(object):
+ """Class handling all non-debug-logging printing done by run-webkit-tests.
+
+ Printing from run-webkit-tests falls into two buckets: general or
+ regular output that is read only by humans and can be changed at any
+ time, and output that is parsed by buildbots (and humans) and hence
+ must be changed more carefully and in coordination with the buildbot
+ parsing code (in chromium.org's buildbot/master.chromium/scripts/master/
+ log_parser/webkit_test_command.py script).
+
+ By default the buildbot-parsed code gets logged to stdout, and regular
+ output gets logged to stderr."""
+ def __init__(self, port, options, regular_output, buildbot_output,
+ child_processes, is_fully_parallel):
+ """
+ Args
+ port interface to port-specific routines
+ options OptionParser object with command line settings
+ regular_output stream to which output intended only for humans
+ should be written
+ buildbot_output stream to which output intended to be read by
+ the buildbots (and humans) should be written
+ child_processes number of parallel threads running (usually
+ controlled by --child-processes)
+ is_fully_parallel are the tests running in a single queue, or
+ in shards (usually controlled by
+ --experimental-fully-parallel)
+
+ Note that the last two args are separate rather than bundled into
+ the options structure so that this object does not assume any flags
+ set in options that weren't returned from logging_options(), above.
+ The two are used to determine whether or not we can sensibly use
+ the 'detailed-progress' option, or can only use 'one-line-progress'.
+ """
+ self._buildbot_stream = buildbot_output
+ self._options = options
+ self._port = port
+ self._stream = regular_output
+
+ # These are used for --print detailed-progress to track status by
+ # directory.
+ self._current_dir = None
+ self._current_progress_str = ""
+ self._current_test_number = 0
+
+ self._meter = metered_stream.MeteredStream(options.verbose,
+ regular_output)
+ self._logging_handler = _configure_logging(self._meter,
+ options.verbose)
+
+ self.switches = parse_print_options(options.print_options,
+ options.verbose, child_processes, is_fully_parallel)
+
+ def cleanup(self):
+ """Restore logging configuration to its initial settings."""
+ if self._logging_handler:
+ _restore_logging(self._logging_handler)
+ self._logging_handler = None
+
+ def __del__(self):
+ self.cleanup()
+
+ # These two routines just hide the implementation of the switches.
+ def disabled(self, option):
+ return not option in self.switches
+
+ def enabled(self, option):
+ return option in self.switches
+
+ def help_printing(self):
+ self._write(HELP_PRINTING)
+
+ def print_actual(self, msg):
+ if self.disabled('actual'):
+ return
+ self._buildbot_stream.write("%s\n" % msg)
+
+ def print_config(self, msg):
+ self.write(msg, 'config')
+
+ def print_expected(self, msg):
+ self.write(msg, 'expected')
+
+ def print_timing(self, msg):
+ self.write(msg, 'timing')
+
+ def print_one_line_summary(self, total, expected, unexpected):
+ """Print a one-line summary of the test run to stdout.
+
+ Args:
+ total: total number of tests run
+ expected: number of expected results
+ unexpected: number of unexpected results
+ """
+ if self.disabled('one-line-summary'):
+ return
+
+ incomplete = total - expected - unexpected
+ if incomplete:
+ self._write("")
+ incomplete_str = " (%d didn't run)" % incomplete
+ expected_str = str(expected)
+ else:
+ incomplete_str = ""
+ expected_str = "All %d" % expected
+
+ if unexpected == 0:
+ self._write("%s tests ran as expected%s." %
+ (expected_str, incomplete_str))
+ elif expected == 1:
+ self._write("1 test ran as expected, %d didn't%s:" %
+ (unexpected, incomplete_str))
+ else:
+ self._write("%d tests ran as expected, %d didn't%s:" %
+ (expected, unexpected, incomplete_str))
+ self._write("")
+
+ def print_test_result(self, result, expected, exp_str, got_str):
+ """Print the result of the test as determined by --print.
+
+ This routine is used to print the details of each test as it completes.
+
+ Args:
+ result - The actual TestResult object
+ expected - Whether the result we got was an expected result
+ exp_str - What we expected to get (used for tracing)
+ got_str - What we actually got (used for tracing)
+
+ Note that we need all of these arguments even though they seem
+ somewhat redundant, in order to keep this routine from having to
+ known anything about the set of expectations.
+ """
+ if (self.enabled('trace-everything') or
+ self.enabled('trace-unexpected') and not expected):
+ self._print_test_trace(result, exp_str, got_str)
+ elif (not expected and self.enabled('unexpected') and
+ self.disabled('detailed-progress')):
+ # Note: 'detailed-progress' handles unexpected results internally,
+ # so we skip it here.
+ self._print_unexpected_test_result(result)
+
+ def _print_test_trace(self, result, exp_str, got_str):
+ """Print detailed results of a test (triggered by --print trace-*).
+ For each test, print:
+ - location of the expected baselines
+ - expected results
+ - actual result
+ - timing info
+ """
+ filename = result.filename
+ test_name = self._port.relative_test_filename(filename)
+ self._write('trace: %s' % test_name)
+ txt_file = self._port.expected_filename(filename, '.txt')
+ if self._port.path_exists(txt_file):
+ self._write(' txt: %s' %
+ self._port.relative_test_filename(txt_file))
+ else:
+ self._write(' txt: <none>')
+ checksum_file = self._port.expected_filename(filename, '.checksum')
+ if self._port.path_exists(checksum_file):
+ self._write(' sum: %s' %
+ self._port.relative_test_filename(checksum_file))
+ else:
+ self._write(' sum: <none>')
+ png_file = self._port.expected_filename(filename, '.png')
+ if self._port.path_exists(png_file):
+ self._write(' png: %s' %
+ self._port.relative_test_filename(png_file))
+ else:
+ self._write(' png: <none>')
+ self._write(' exp: %s' % exp_str)
+ self._write(' got: %s' % got_str)
+ self._write(' took: %-.3f' % result.test_run_time)
+ self._write('')
+
+ def _print_unexpected_test_result(self, result):
+ """Prints one unexpected test result line."""
+ desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result.type][0]
+ self.write(" %s -> unexpected %s" %
+ (self._port.relative_test_filename(result.filename),
+ desc), "unexpected")
+
+ def print_progress(self, result_summary, retrying, test_list):
+ """Print progress through the tests as determined by --print."""
+ if self.enabled('detailed-progress'):
+ self._print_detailed_progress(result_summary, test_list)
+ elif self.enabled('one-line-progress'):
+ self._print_one_line_progress(result_summary, retrying)
+ else:
+ return
+
+ if result_summary.remaining == 0:
+ self._meter.update('')
+
+ def _print_one_line_progress(self, result_summary, retrying):
+ """Displays the progress through the test run."""
+ percent_complete = 100 * (result_summary.expected +
+ result_summary.unexpected) / result_summary.total
+ action = "Testing"
+ if retrying:
+ action = "Retrying"
+ self._meter.progress("%s (%d%%): %d ran as expected, %d didn't,"
+ " %d left" % (action, percent_complete, result_summary.expected,
+ result_summary.unexpected, result_summary.remaining))
+
+ def _print_detailed_progress(self, result_summary, test_list):
+ """Display detailed progress output where we print the directory name
+ and one dot for each completed test. This is triggered by
+ "--log detailed-progress"."""
+ if self._current_test_number == len(test_list):
+ return
+
+ next_test = test_list[self._current_test_number]
+ next_dir = os.path.dirname(
+ self._port.relative_test_filename(next_test))
+ if self._current_progress_str == "":
+ self._current_progress_str = "%s: " % (next_dir)
+ self._current_dir = next_dir
+
+ while next_test in result_summary.results:
+ if next_dir != self._current_dir:
+ self._meter.write("%s\n" % (self._current_progress_str))
+ self._current_progress_str = "%s: ." % (next_dir)
+ self._current_dir = next_dir
+ else:
+ self._current_progress_str += "."
+
+ if (next_test in result_summary.unexpected_results and
+ self.enabled('unexpected')):
+ self._meter.write("%s\n" % self._current_progress_str)
+ test_result = result_summary.results[next_test]
+ self._print_unexpected_test_result(test_result)
+ self._current_progress_str = "%s: " % self._current_dir
+
+ self._current_test_number += 1
+ if self._current_test_number == len(test_list):
+ break
+
+ next_test = test_list[self._current_test_number]
+ next_dir = os.path.dirname(
+ self._port.relative_test_filename(next_test))
+
+ if result_summary.remaining:
+ remain_str = " (%d)" % (result_summary.remaining)
+ self._meter.progress("%s%s" % (self._current_progress_str,
+ remain_str))
+ else:
+ self._meter.progress("%s" % (self._current_progress_str))
+
+ def print_unexpected_results(self, unexpected_results):
+ """Prints a list of the unexpected results to the buildbot stream."""
+ if self.disabled('unexpected-results'):
+ return
+
+ passes = {}
+ flaky = {}
+ regressions = {}
+
+ for test, results in unexpected_results['tests'].iteritems():
+ actual = results['actual'].split(" ")
+ expected = results['expected'].split(" ")
+ if actual == ['PASS']:
+ if 'CRASH' in expected:
+ _add_to_dict_of_lists(passes,
+ 'Expected to crash, but passed',
+ test)
+ elif 'TIMEOUT' in expected:
+ _add_to_dict_of_lists(passes,
+ 'Expected to timeout, but passed',
+ test)
+ else:
+ _add_to_dict_of_lists(passes,
+ 'Expected to fail, but passed',
+ test)
+ elif len(actual) > 1:
+ # We group flaky tests by the first actual result we got.
+ _add_to_dict_of_lists(flaky, actual[0], test)
+ else:
+ _add_to_dict_of_lists(regressions, results['actual'], test)
+
+ if len(passes) or len(flaky) or len(regressions):
+ self._buildbot_stream.write("\n")
+
+ if len(passes):
+ for key, tests in passes.iteritems():
+ self._buildbot_stream.write("%s: (%d)\n" % (key, len(tests)))
+ tests.sort()
+ for test in tests:
+ self._buildbot_stream.write(" %s\n" % test)
+ self._buildbot_stream.write("\n")
+ self._buildbot_stream.write("\n")
+
+ if len(flaky):
+ descriptions = TestExpectationsFile.EXPECTATION_DESCRIPTIONS
+ for key, tests in flaky.iteritems():
+ result = TestExpectationsFile.EXPECTATIONS[key.lower()]
+ self._buildbot_stream.write("Unexpected flakiness: %s (%d)\n"
+ % (descriptions[result][1], len(tests)))
+ tests.sort()
+
+ for test in tests:
+ result = unexpected_results['tests'][test]
+ actual = result['actual'].split(" ")
+ expected = result['expected'].split(" ")
+ result = TestExpectationsFile.EXPECTATIONS[key.lower()]
+ new_expectations_list = list(set(actual) | set(expected))
+ self._buildbot_stream.write(" %s = %s\n" %
+ (test, " ".join(new_expectations_list)))
+ self._buildbot_stream.write("\n")
+ self._buildbot_stream.write("\n")
+
+ if len(regressions):
+ descriptions = TestExpectationsFile.EXPECTATION_DESCRIPTIONS
+ for key, tests in regressions.iteritems():
+ result = TestExpectationsFile.EXPECTATIONS[key.lower()]
+ self._buildbot_stream.write(
+ "Regressions: Unexpected %s : (%d)\n" % (
+ descriptions[result][1], len(tests)))
+ tests.sort()
+ for test in tests:
+ self._buildbot_stream.write(" %s = %s\n" % (test, key))
+ self._buildbot_stream.write("\n")
+ self._buildbot_stream.write("\n")
+
+ if len(unexpected_results['tests']) and self._options.verbose:
+ self._buildbot_stream.write("%s\n" % ("-" * 78))
+
+ def print_update(self, msg):
+ if self.disabled('updates'):
+ return
+ self._meter.update(msg)
+
+ def write(self, msg, option="misc"):
+ if self.disabled(option):
+ return
+ self._write(msg)
+
+ def _write(self, msg):
+ # FIXME: we could probably get away with calling _log.info() all of
+ # the time, but there doesn't seem to be a good way to test the output
+ # from the logger :(.
+ if self._options.verbose:
+ _log.info(msg)
+ else:
+ self._meter.write("%s\n" % msg)
+
+#
+# Utility routines used by the Controller class
+#
+
+
+def _add_to_dict_of_lists(dict, key, value):
+ dict.setdefault(key, []).append(value)
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py
new file mode 100644
index 0000000..0e478c8
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py
@@ -0,0 +1,608 @@
+#!/usr/bin/python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit tests for printing.py."""
+
+import os
+import optparse
+import pdb
+import sys
+import unittest
+import logging
+
+from webkitpy.common import array_stream
+from webkitpy.common.system import logtesting
+from webkitpy.layout_tests import port
+
+from webkitpy.layout_tests.layout_package import printing
+from webkitpy.layout_tests.layout_package import result_summary
+from webkitpy.layout_tests.layout_package import test_expectations
+from webkitpy.layout_tests.layout_package import test_failures
+from webkitpy.layout_tests.layout_package import test_results
+from webkitpy.layout_tests.layout_package import test_runner
+
+
+def get_options(args):
+ print_options = printing.print_options()
+ option_parser = optparse.OptionParser(option_list=print_options)
+ return option_parser.parse_args(args)
+
+
+class TestUtilityFunctions(unittest.TestCase):
+ def test_configure_logging(self):
+ options, args = get_options([])
+ stream = array_stream.ArrayStream()
+ handler = printing._configure_logging(stream, options.verbose)
+ logging.info("this should be logged")
+ self.assertFalse(stream.empty())
+
+ stream.reset()
+ logging.debug("this should not be logged")
+ self.assertTrue(stream.empty())
+
+ printing._restore_logging(handler)
+
+ stream.reset()
+ options, args = get_options(['--verbose'])
+ handler = printing._configure_logging(stream, options.verbose)
+ logging.debug("this should be logged")
+ self.assertFalse(stream.empty())
+ printing._restore_logging(handler)
+
+ def test_print_options(self):
+ options, args = get_options([])
+ self.assertTrue(options is not None)
+
+ def test_parse_print_options(self):
+ def test_switches(args, expected_switches_str,
+ verbose=False, child_processes=1,
+ is_fully_parallel=False):
+ options, args = get_options(args)
+ if expected_switches_str:
+ expected_switches = set(expected_switches_str.split(','))
+ else:
+ expected_switches = set()
+ switches = printing.parse_print_options(options.print_options,
+ verbose,
+ child_processes,
+ is_fully_parallel)
+ self.assertEqual(expected_switches, switches)
+
+ # test that we default to the default set of switches
+ test_switches([], printing.PRINT_DEFAULT)
+
+ # test that verbose defaults to everything
+ test_switches([], printing.PRINT_EVERYTHING, verbose=True)
+
+ # test that --print default does what it's supposed to
+ test_switches(['--print', 'default'], printing.PRINT_DEFAULT)
+
+ # test that --print nothing does what it's supposed to
+ test_switches(['--print', 'nothing'], None)
+
+ # test that --print everything does what it's supposed to
+ test_switches(['--print', 'everything'], printing.PRINT_EVERYTHING)
+
+ # this tests that '--print X' overrides '--verbose'
+ test_switches(['--print', 'actual'], 'actual', verbose=True)
+
+
+
+class Testprinter(unittest.TestCase):
+ def get_printer(self, args=None, single_threaded=False,
+ is_fully_parallel=False):
+ printing_options = printing.print_options()
+ option_parser = optparse.OptionParser(option_list=printing_options)
+ options, args = option_parser.parse_args(args)
+ self._port = port.get('test', options)
+ nproc = 2
+ if single_threaded:
+ nproc = 1
+
+ regular_output = array_stream.ArrayStream()
+ buildbot_output = array_stream.ArrayStream()
+ printer = printing.Printer(self._port, options, regular_output,
+ buildbot_output, single_threaded,
+ is_fully_parallel)
+ return printer, regular_output, buildbot_output
+
+ def get_result(self, test, result_type=test_expectations.PASS, run_time=0):
+ failures = []
+ if result_type == test_expectations.TIMEOUT:
+ failures = [test_failures.FailureTimeout()]
+ elif result_type == test_expectations.CRASH:
+ failures = [test_failures.FailureCrash()]
+ path = os.path.join(self._port.layout_tests_dir(), test)
+ return test_results.TestResult(path, failures, run_time,
+ total_time_for_all_diffs=0,
+ time_for_diffs=0)
+
+ def get_result_summary(self, tests, expectations_str):
+ test_paths = [os.path.join(self._port.layout_tests_dir(), test) for
+ test in tests]
+ expectations = test_expectations.TestExpectations(
+ self._port, test_paths, expectations_str,
+ self._port.test_platform_name(), is_debug_mode=False,
+ is_lint_mode=False)
+
+ rs = result_summary.ResultSummary(expectations, test_paths)
+ return test_paths, rs, expectations
+
+ def test_help_printer(self):
+ # Here and below we'll call the "regular" printer err and the
+ # buildbot printer out; this corresponds to how things run on the
+ # bots with stderr and stdout.
+ printer, err, out = self.get_printer()
+
+ # This routine should print something to stdout. testing what it is
+ # is kind of pointless.
+ printer.help_printing()
+ self.assertFalse(err.empty())
+ self.assertTrue(out.empty())
+
+ def do_switch_tests(self, method_name, switch, to_buildbot,
+ message='hello', exp_err=None, exp_bot=None):
+ def do_helper(method_name, switch, message, exp_err, exp_bot):
+ printer, err, bot = self.get_printer(['--print', switch])
+ getattr(printer, method_name)(message)
+ self.assertEqual(err.get(), exp_err)
+ self.assertEqual(bot.get(), exp_bot)
+
+ if to_buildbot:
+ if exp_err is None:
+ exp_err = []
+ if exp_bot is None:
+ exp_bot = [message + "\n"]
+ else:
+ if exp_err is None:
+ exp_err = [message + "\n"]
+ if exp_bot is None:
+ exp_bot = []
+ do_helper(method_name, 'nothing', 'hello', [], [])
+ do_helper(method_name, switch, 'hello', exp_err, exp_bot)
+ do_helper(method_name, 'everything', 'hello', exp_err, exp_bot)
+
+ def test_configure_and_cleanup(self):
+ # This test verifies that calling cleanup repeatedly and deleting
+ # the object is safe.
+ printer, err, out = self.get_printer(['--print', 'everything'])
+ printer.cleanup()
+ printer.cleanup()
+ printer = None
+
+ def test_print_actual(self):
+ # Actual results need to be logged to the buildbot's stream.
+ self.do_switch_tests('print_actual', 'actual', to_buildbot=True)
+
+ def test_print_actual_buildbot(self):
+ # FIXME: Test that the format of the actual results matches what the
+ # buildbot is expecting.
+ pass
+
+ def test_print_config(self):
+ self.do_switch_tests('print_config', 'config', to_buildbot=False)
+
+ def test_print_expected(self):
+ self.do_switch_tests('print_expected', 'expected', to_buildbot=False)
+
+ def test_print_timing(self):
+ self.do_switch_tests('print_timing', 'timing', to_buildbot=False)
+
+ def test_print_update(self):
+ # Note that there shouldn't be a carriage return here; updates()
+ # are meant to be overwritten.
+ self.do_switch_tests('print_update', 'updates', to_buildbot=False,
+ message='hello', exp_err=['hello'])
+
+ def test_print_one_line_summary(self):
+ printer, err, out = self.get_printer(['--print', 'nothing'])
+ printer.print_one_line_summary(1, 1, 0)
+ self.assertTrue(err.empty())
+
+ printer, err, out = self.get_printer(['--print', 'one-line-summary'])
+ printer.print_one_line_summary(1, 1, 0)
+ self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])
+
+ printer, err, out = self.get_printer(['--print', 'everything'])
+ printer.print_one_line_summary(1, 1, 0)
+ self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])
+
+ err.reset()
+ printer.print_one_line_summary(2, 1, 1)
+ self.assertEquals(err.get(),
+ ["1 test ran as expected, 1 didn't:\n", "\n"])
+
+ err.reset()
+ printer.print_one_line_summary(3, 2, 1)
+ self.assertEquals(err.get(),
+ ["2 tests ran as expected, 1 didn't:\n", "\n"])
+
+ err.reset()
+ printer.print_one_line_summary(3, 2, 0)
+ self.assertEquals(err.get(),
+ ['\n', "2 tests ran as expected (1 didn't run).\n",
+ '\n'])
+
+
+ def test_print_test_result(self):
+ # Note here that we don't use meaningful exp_str and got_str values;
+ # the actual contents of the string are treated opaquely by
+ # print_test_result() when tracing, and usually we don't want
+ # to test what exactly is printed, just that something
+ # was printed (or that nothing was printed).
+ #
+ # FIXME: this is actually some goofy layering; it would be nice
+ # we could refactor it so that the args weren't redundant. Maybe
+ # the TestResult should contain what was expected, and the
+ # strings could be derived from the TestResult?
+ printer, err, out = self.get_printer(['--print', 'nothing'])
+ result = self.get_result('passes/image.html')
+ printer.print_test_result(result, expected=False, exp_str='',
+ got_str='')
+ self.assertTrue(err.empty())
+
+ printer, err, out = self.get_printer(['--print', 'unexpected'])
+ printer.print_test_result(result, expected=True, exp_str='',
+ got_str='')
+ self.assertTrue(err.empty())
+ printer.print_test_result(result, expected=False, exp_str='',
+ got_str='')
+ self.assertEquals(err.get(),
+ [' passes/image.html -> unexpected pass\n'])
+
+ printer, err, out = self.get_printer(['--print', 'everything'])
+ printer.print_test_result(result, expected=True, exp_str='',
+ got_str='')
+ self.assertTrue(err.empty())
+
+ printer.print_test_result(result, expected=False, exp_str='',
+ got_str='')
+ self.assertEquals(err.get(),
+ [' passes/image.html -> unexpected pass\n'])
+
+ printer, err, out = self.get_printer(['--print', 'nothing'])
+ printer.print_test_result(result, expected=False, exp_str='',
+ got_str='')
+ self.assertTrue(err.empty())
+
+ printer, err, out = self.get_printer(['--print',
+ 'trace-unexpected'])
+ printer.print_test_result(result, expected=True, exp_str='',
+ got_str='')
+ self.assertTrue(err.empty())
+
+ printer, err, out = self.get_printer(['--print',
+ 'trace-unexpected'])
+ printer.print_test_result(result, expected=False, exp_str='',
+ got_str='')
+ self.assertFalse(err.empty())
+
+ printer, err, out = self.get_printer(['--print',
+ 'trace-unexpected'])
+ result = self.get_result("passes/text.html")
+ printer.print_test_result(result, expected=False, exp_str='',
+ got_str='')
+ self.assertFalse(err.empty())
+
+ err.reset()
+ printer.print_test_result(result, expected=False, exp_str='',
+ got_str='')
+ self.assertFalse(err.empty())
+
+ printer, err, out = self.get_printer(['--print', 'trace-everything'])
+ result = self.get_result('passes/image.html')
+ printer.print_test_result(result, expected=True, exp_str='',
+ got_str='')
+ result = self.get_result('failures/expected/missing_text.html')
+ printer.print_test_result(result, expected=True, exp_str='',
+ got_str='')
+ result = self.get_result('failures/expected/missing_check.html')
+ printer.print_test_result(result, expected=True, exp_str='',
+ got_str='')
+ result = self.get_result('failures/expected/missing_image.html')
+ printer.print_test_result(result, expected=True, exp_str='',
+ got_str='')
+ self.assertFalse(err.empty())
+
+ err.reset()
+ printer.print_test_result(result, expected=False, exp_str='',
+ got_str='')
+
+ def test_print_progress(self):
+ expectations = ''
+
+ # test that we print nothing
+ printer, err, out = self.get_printer(['--print', 'nothing'])
+ tests = ['passes/text.html', 'failures/expected/timeout.html',
+ 'failures/expected/crash.html']
+ paths, rs, exp = self.get_result_summary(tests, expectations)
+
+ printer.print_progress(rs, False, paths)
+ self.assertTrue(out.empty())
+ self.assertTrue(err.empty())
+
+ printer.print_progress(rs, True, paths)
+ self.assertTrue(out.empty())
+ self.assertTrue(err.empty())
+
+ # test regular functionality
+ printer, err, out = self.get_printer(['--print',
+ 'one-line-progress'])
+ printer.print_progress(rs, False, paths)
+ self.assertTrue(out.empty())
+ self.assertFalse(err.empty())
+
+ err.reset()
+ out.reset()
+ printer.print_progress(rs, True, paths)
+ self.assertFalse(err.empty())
+ self.assertTrue(out.empty())
+
+ def test_print_progress__detailed(self):
+ tests = ['passes/text.html', 'failures/expected/timeout.html',
+ 'failures/expected/crash.html']
+ expectations = 'failures/expected/timeout.html = TIMEOUT'
+
+ # first, test that it is disabled properly
+ # should still print one-line-progress
+ printer, err, out = self.get_printer(
+ ['--print', 'detailed-progress'], single_threaded=False)
+ paths, rs, exp = self.get_result_summary(tests, expectations)
+ printer.print_progress(rs, False, paths)
+ self.assertFalse(err.empty())
+ self.assertTrue(out.empty())
+
+ # now test the enabled paths
+ printer, err, out = self.get_printer(
+ ['--print', 'detailed-progress'], single_threaded=True)
+ paths, rs, exp = self.get_result_summary(tests, expectations)
+ printer.print_progress(rs, False, paths)
+ self.assertFalse(err.empty())
+ self.assertTrue(out.empty())
+
+ err.reset()
+ out.reset()
+ printer.print_progress(rs, True, paths)
+ self.assertFalse(err.empty())
+ self.assertTrue(out.empty())
+
+ rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False)
+ rs.add(self.get_result('failures/expected/timeout.html'), True)
+ rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True)
+ err.reset()
+ out.reset()
+ printer.print_progress(rs, False, paths)
+ self.assertFalse(err.empty())
+ self.assertTrue(out.empty())
+
+ # We only clear the meter when retrying w/ detailed-progress.
+ err.reset()
+ out.reset()
+ printer.print_progress(rs, True, paths)
+ self.assertFalse(err.empty())
+ self.assertTrue(out.empty())
+
+ printer, err, out = self.get_printer(
+ ['--print', 'detailed-progress,unexpected'], single_threaded=True)
+ paths, rs, exp = self.get_result_summary(tests, expectations)
+ printer.print_progress(rs, False, paths)
+ self.assertFalse(err.empty())
+ self.assertTrue(out.empty())
+
+ err.reset()
+ out.reset()
+ printer.print_progress(rs, True, paths)
+ self.assertFalse(err.empty())
+ self.assertTrue(out.empty())
+
+ rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False)
+ rs.add(self.get_result('failures/expected/timeout.html'), True)
+ rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True)
+ err.reset()
+ out.reset()
+ printer.print_progress(rs, False, paths)
+ self.assertFalse(err.empty())
+ self.assertTrue(out.empty())
+
+ # We only clear the meter when retrying w/ detailed-progress.
+ err.reset()
+ out.reset()
+ printer.print_progress(rs, True, paths)
+ self.assertFalse(err.empty())
+ self.assertTrue(out.empty())
+
+ def test_write_nothing(self):
+ printer, err, out = self.get_printer(['--print', 'nothing'])
+ printer.write("foo")
+ self.assertTrue(err.empty())
+
+ def test_write_misc(self):
+ printer, err, out = self.get_printer(['--print', 'misc'])
+ printer.write("foo")
+ self.assertFalse(err.empty())
+ err.reset()
+ printer.write("foo", "config")
+ self.assertTrue(err.empty())
+
+ def test_write_everything(self):
+ printer, err, out = self.get_printer(['--print', 'everything'])
+ printer.write("foo")
+ self.assertFalse(err.empty())
+ err.reset()
+ printer.write("foo", "config")
+ self.assertFalse(err.empty())
+
+ def test_write_verbose(self):
+ printer, err, out = self.get_printer(['--verbose'])
+ printer.write("foo")
+ self.assertTrue(not err.empty() and "foo" in err.get()[0])
+ self.assertTrue(out.empty())
+
+ def test_print_unexpected_results(self):
+ # This routine is the only one that prints stuff that the bots
+ # care about.
+ #
+ # FIXME: there's some weird layering going on here. It seems
+ # like we shouldn't be both using an expectations string and
+ # having to specify whether or not the result was expected.
+ # This whole set of tests should probably be rewritten.
+ #
+ # FIXME: Plus, the fact that we're having to call into
+ # run_webkit_tests is clearly a layering inversion.
+ def get_unexpected_results(expected, passing, flaky):
+ """Return an unexpected results summary matching the input description.
+
+ There are a lot of different combinations of test results that
+ can be tested; this routine produces various combinations based
+ on the values of the input flags.
+
+ Args
+ expected: whether the tests ran as expected
+ passing: whether the tests should all pass
+ flaky: whether the tests should be flaky (if False, they
+ produce the same results on both runs; if True, they
+ all pass on the second run).
+
+ """
+ paths, rs, exp = self.get_result_summary(tests, expectations)
+ if expected:
+ rs.add(self.get_result('passes/text.html', test_expectations.PASS),
+ expected)
+ rs.add(self.get_result('failures/expected/timeout.html',
+ test_expectations.TIMEOUT), expected)
+ rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH),
+ expected)
+ elif passing:
+ rs.add(self.get_result('passes/text.html'), expected)
+ rs.add(self.get_result('failures/expected/timeout.html'), expected)
+ rs.add(self.get_result('failures/expected/crash.html'), expected)
+ else:
+ rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT),
+ expected)
+ rs.add(self.get_result('failures/expected/timeout.html',
+ test_expectations.CRASH), expected)
+ rs.add(self.get_result('failures/expected/crash.html',
+ test_expectations.TIMEOUT),
+ expected)
+ retry = rs
+ if flaky:
+ paths, retry, exp = self.get_result_summary(tests,
+ expectations)
+ retry.add(self.get_result('passes/text.html'), True)
+ retry.add(self.get_result('failures/expected/timeout.html'), True)
+ retry.add(self.get_result('failures/expected/crash.html'), True)
+ unexpected_results = test_runner.summarize_unexpected_results(
+ self._port, exp, rs, retry)
+ return unexpected_results
+
+ tests = ['passes/text.html', 'failures/expected/timeout.html',
+ 'failures/expected/crash.html']
+ expectations = ''
+
+ printer, err, out = self.get_printer(['--print', 'nothing'])
+ ur = get_unexpected_results(expected=False, passing=False, flaky=False)
+ printer.print_unexpected_results(ur)
+ self.assertTrue(err.empty())
+ self.assertTrue(out.empty())
+
+ printer, err, out = self.get_printer(['--print',
+ 'unexpected-results'])
+
+ # test everything running as expected
+ ur = get_unexpected_results(expected=True, passing=False, flaky=False)
+ printer.print_unexpected_results(ur)
+ self.assertTrue(err.empty())
+ self.assertTrue(out.empty())
+
+ # test failures
+ err.reset()
+ out.reset()
+ ur = get_unexpected_results(expected=False, passing=False, flaky=False)
+ printer.print_unexpected_results(ur)
+ self.assertTrue(err.empty())
+ self.assertFalse(out.empty())
+
+ # test unexpected flaky results
+ err.reset()
+ out.reset()
+ ur = get_unexpected_results(expected=False, passing=True, flaky=False)
+ printer.print_unexpected_results(ur)
+ self.assertTrue(err.empty())
+ self.assertFalse(out.empty())
+
+ # test unexpected passes
+ err.reset()
+ out.reset()
+ ur = get_unexpected_results(expected=False, passing=False, flaky=True)
+ printer.print_unexpected_results(ur)
+ self.assertTrue(err.empty())
+ self.assertFalse(out.empty())
+
+ err.reset()
+ out.reset()
+ printer, err, out = self.get_printer(['--print', 'everything'])
+ ur = get_unexpected_results(expected=False, passing=False, flaky=False)
+ printer.print_unexpected_results(ur)
+ self.assertTrue(err.empty())
+ self.assertFalse(out.empty())
+
+ expectations = """
+failures/expected/crash.html = CRASH
+failures/expected/timeout.html = TIMEOUT
+"""
+ err.reset()
+ out.reset()
+ ur = get_unexpected_results(expected=False, passing=False, flaky=False)
+ printer.print_unexpected_results(ur)
+ self.assertTrue(err.empty())
+ self.assertFalse(out.empty())
+
+ err.reset()
+ out.reset()
+ ur = get_unexpected_results(expected=False, passing=True, flaky=False)
+ printer.print_unexpected_results(ur)
+ self.assertTrue(err.empty())
+ self.assertFalse(out.empty())
+
+ # Test handling of --verbose as well.
+ err.reset()
+ out.reset()
+ printer, err, out = self.get_printer(['--verbose'])
+ ur = get_unexpected_results(expected=False, passing=False, flaky=False)
+ printer.print_unexpected_results(ur)
+ self.assertTrue(err.empty())
+ self.assertFalse(out.empty())
+
+ def test_print_unexpected_results_buildbot(self):
+ # FIXME: Test that print_unexpected_results() produces the printer the
+ # buildbot is expecting.
+ pass
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/result_summary.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/result_summary.py
new file mode 100644
index 0000000..80fd6ac
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/result_summary.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Run layout tests."""
+
+import logging
+
+import test_expectations
+
+_log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
+
+TestExpectationsFile = test_expectations.TestExpectationsFile
+
+
+class ResultSummary(object):
+ """A class for partitioning the test results we get into buckets.
+
+ This class is basically a glorified struct and it's private to this file
+ so we don't bother with any information hiding."""
+
+ def __init__(self, expectations, test_files):
+ self.total = len(test_files)
+ self.remaining = self.total
+ self.expectations = expectations
+ self.expected = 0
+ self.unexpected = 0
+ self.unexpected_failures = 0
+ self.unexpected_crashes_or_timeouts = 0
+ self.tests_by_expectation = {}
+ self.tests_by_timeline = {}
+ self.results = {}
+ self.unexpected_results = {}
+ self.failures = {}
+ self.tests_by_expectation[test_expectations.SKIP] = set()
+ for expectation in TestExpectationsFile.EXPECTATIONS.values():
+ self.tests_by_expectation[expectation] = set()
+ for timeline in TestExpectationsFile.TIMELINES.values():
+ self.tests_by_timeline[timeline] = (
+ expectations.get_tests_with_timeline(timeline))
+
+ def add(self, result, expected):
+ """Add a TestResult into the appropriate bin.
+
+ Args:
+ result: TestResult
+ expected: whether the result was what we expected it to be.
+ """
+
+ self.tests_by_expectation[result.type].add(result.filename)
+ self.results[result.filename] = result
+ self.remaining -= 1
+ if len(result.failures):
+ self.failures[result.filename] = result.failures
+ if expected:
+ self.expected += 1
+ else:
+ self.unexpected_results[result.filename] = result.type
+ self.unexpected += 1
+ if len(result.failures):
+ self.unexpected_failures += 1
+ if result.type == test_expectations.CRASH or result.type == test_expectations.TIMEOUT:
+ self.unexpected_crashes_or_timeouts += 1
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py
new file mode 100644
index 0000000..8645fc1
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations.py
@@ -0,0 +1,868 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""A helper class for reading in and dealing with tests expectations
+for layout tests.
+"""
+
+import logging
+import os
+import re
+import sys
+
+import webkitpy.thirdparty.simplejson as simplejson
+
+_log = logging.getLogger("webkitpy.layout_tests.layout_package."
+ "test_expectations")
+
+# Test expectation and modifier constants.
+(PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, TIMEOUT, CRASH, SKIP, WONTFIX,
+ SLOW, REBASELINE, MISSING, FLAKY, NOW, NONE) = range(15)
+
+# Test expectation file update action constants
+(NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM, ADD_PLATFORMS_EXCEPT_THIS) = range(4)
+
+
+def result_was_expected(result, expected_results, test_needs_rebaselining,
+ test_is_skipped):
+ """Returns whether we got a result we were expecting.
+ Args:
+ result: actual result of a test execution
+ expected_results: set of results listed in test_expectations
+ test_needs_rebaselining: whether test was marked as REBASELINE
+ test_is_skipped: whether test was marked as SKIP"""
+ if result in expected_results:
+ return True
+ if result in (IMAGE, TEXT, IMAGE_PLUS_TEXT) and FAIL in expected_results:
+ return True
+ if result == MISSING and test_needs_rebaselining:
+ return True
+ if result == SKIP and test_is_skipped:
+ return True
+ return False
+
+
+def remove_pixel_failures(expected_results):
+ """Returns a copy of the expected results for a test, except that we
+ drop any pixel failures and return the remaining expectations. For example,
+ if we're not running pixel tests, then tests expected to fail as IMAGE
+ will PASS."""
+ expected_results = expected_results.copy()
+ if IMAGE in expected_results:
+ expected_results.remove(IMAGE)
+ expected_results.add(PASS)
+ if IMAGE_PLUS_TEXT in expected_results:
+ expected_results.remove(IMAGE_PLUS_TEXT)
+ expected_results.add(TEXT)
+ return expected_results
+
+
+class TestExpectations:
+ TEST_LIST = "test_expectations.txt"
+
+ def __init__(self, port, tests, expectations, test_platform_name,
+ is_debug_mode, is_lint_mode, overrides=None):
+ """Loads and parses the test expectations given in the string.
+ Args:
+ port: handle to object containing platform-specific functionality
+ test: list of all of the test files
+ expectations: test expectations as a string
+ test_platform_name: name of the platform to match expectations
+ against. Note that this may be different than
+ port.test_platform_name() when is_lint_mode is True.
+ is_debug_mode: whether to use the DEBUG or RELEASE modifiers
+ in the expectations
+ is_lint_mode: If True, just parse the expectations string
+ looking for errors.
+ overrides: test expectations that are allowed to override any
+ entries in |expectations|. This is used by callers
+ that need to manage two sets of expectations (e.g., upstream
+ and downstream expectations).
+ """
+ self._expected_failures = TestExpectationsFile(port, expectations,
+ tests, test_platform_name, is_debug_mode, is_lint_mode,
+ overrides=overrides)
+
+ # TODO(ojan): Allow for removing skipped tests when getting the list of
+ # tests to run, but not when getting metrics.
+ # TODO(ojan): Replace the Get* calls here with the more sane API exposed
+ # by TestExpectationsFile below. Maybe merge the two classes entirely?
+
+ def get_expectations_json_for_all_platforms(self):
+ return (
+ self._expected_failures.get_expectations_json_for_all_platforms())
+
+ def get_rebaselining_failures(self):
+ return (self._expected_failures.get_test_set(REBASELINE, FAIL) |
+ self._expected_failures.get_test_set(REBASELINE, IMAGE) |
+ self._expected_failures.get_test_set(REBASELINE, TEXT) |
+ self._expected_failures.get_test_set(REBASELINE,
+ IMAGE_PLUS_TEXT))
+
+ def get_options(self, test):
+ return self._expected_failures.get_options(test)
+
+ def get_expectations(self, test):
+ return self._expected_failures.get_expectations(test)
+
+ def get_expectations_string(self, test):
+ """Returns the expectatons for the given test as an uppercase string.
+ If there are no expectations for the test, then "PASS" is returned."""
+ expectations = self.get_expectations(test)
+ retval = []
+
+ for expectation in expectations:
+ retval.append(self.expectation_to_string(expectation))
+
+ return " ".join(retval)
+
+ def expectation_to_string(self, expectation):
+ """Return the uppercased string equivalent of a given expectation."""
+ for item in TestExpectationsFile.EXPECTATIONS.items():
+ if item[1] == expectation:
+ return item[0].upper()
+ raise ValueError(expectation)
+
+ def get_tests_with_result_type(self, result_type):
+ return self._expected_failures.get_tests_with_result_type(result_type)
+
+ def get_tests_with_timeline(self, timeline):
+ return self._expected_failures.get_tests_with_timeline(timeline)
+
+ def matches_an_expected_result(self, test, result,
+ pixel_tests_are_enabled):
+ expected_results = self._expected_failures.get_expectations(test)
+ if not pixel_tests_are_enabled:
+ expected_results = remove_pixel_failures(expected_results)
+ return result_was_expected(result, expected_results,
+ self.is_rebaselining(test), self.has_modifier(test, SKIP))
+
+ def is_rebaselining(self, test):
+ return self._expected_failures.has_modifier(test, REBASELINE)
+
+ def has_modifier(self, test, modifier):
+ return self._expected_failures.has_modifier(test, modifier)
+
+ def remove_platform_from_expectations(self, tests, platform):
+ return self._expected_failures.remove_platform_from_expectations(
+ tests, platform)
+
+
+def strip_comments(line):
+ """Strips comments from a line and return None if the line is empty
+ or else the contents of line with leading and trailing spaces removed
+ and all other whitespace collapsed"""
+
+ commentIndex = line.find('//')
+ if commentIndex is -1:
+ commentIndex = len(line)
+
+ line = re.sub(r'\s+', ' ', line[:commentIndex].strip())
+ if line == '':
+ return None
+ else:
+ return line
+
+
+class ParseError(Exception):
+ def __init__(self, fatal, errors):
+ self.fatal = fatal
+ self.errors = errors
+
+ def __str__(self):
+ return '\n'.join(map(str, self.errors))
+
+ def __repr__(self):
+ return 'ParseError(fatal=%s, errors=%s)' % (fatal, errors)
+
+
+class ModifiersAndExpectations:
+ """A holder for modifiers and expectations on a test that serializes to
+ JSON."""
+
+ def __init__(self, modifiers, expectations):
+ self.modifiers = modifiers
+ self.expectations = expectations
+
+
+class ExpectationsJsonEncoder(simplejson.JSONEncoder):
+ """JSON encoder that can handle ModifiersAndExpectations objects."""
+ def default(self, obj):
+ # A ModifiersAndExpectations object has two fields, each of which
+ # is a dict. Since JSONEncoders handle all the builtin types directly,
+ # the only time this routine should be called is on the top level
+ # object (i.e., the encoder shouldn't recurse).
+ assert isinstance(obj, ModifiersAndExpectations)
+ return {"modifiers": obj.modifiers,
+ "expectations": obj.expectations}
+
+
+class TestExpectationsFile:
+ """Test expectation files consist of lines with specifications of what
+ to expect from layout test cases. The test cases can be directories
+ in which case the expectations apply to all test cases in that
+ directory and any subdirectory. The format of the file is along the
+ lines of:
+
+ LayoutTests/fast/js/fixme.js = FAIL
+ LayoutTests/fast/js/flaky.js = FAIL PASS
+ LayoutTests/fast/js/crash.js = CRASH TIMEOUT FAIL PASS
+ ...
+
+ To add other options:
+ SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS
+ DEBUG : LayoutTests/fast/js/no-good.js = TIMEOUT PASS
+ DEBUG SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS
+ LINUX DEBUG SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS
+ LINUX WIN : LayoutTests/fast/js/no-good.js = TIMEOUT PASS
+
+ SKIP: Doesn't run the test.
+ SLOW: The test takes a long time to run, but does not timeout indefinitely.
+ WONTFIX: For tests that we never intend to pass on a given platform.
+ DEBUG: Expectations apply only to the debug build.
+ RELEASE: Expectations apply only to release build.
+ LINUX/WIN/WIN-XP/WIN-VISTA/WIN-7/MAC: Expectations apply only to these
+ platforms.
+
+ Notes:
+ -A test cannot be both SLOW and TIMEOUT
+ -A test should only be one of IMAGE, TEXT, IMAGE+TEXT, or FAIL. FAIL is
+ a migratory state that currently means either IMAGE, TEXT, or
+ IMAGE+TEXT. Once we have finished migrating the expectations, we will
+ change FAIL to have the meaning of IMAGE+TEXT and remove the IMAGE+TEXT
+ identifier.
+ -A test can be included twice, but not via the same path.
+ -If a test is included twice, then the more precise path wins.
+ -CRASH tests cannot be WONTFIX
+ """
+
+ EXPECTATIONS = {'pass': PASS,
+ 'fail': FAIL,
+ 'text': TEXT,
+ 'image': IMAGE,
+ 'image+text': IMAGE_PLUS_TEXT,
+ 'timeout': TIMEOUT,
+ 'crash': CRASH,
+ 'missing': MISSING}
+
+ EXPECTATION_DESCRIPTIONS = {SKIP: ('skipped', 'skipped'),
+ PASS: ('pass', 'passes'),
+ FAIL: ('failure', 'failures'),
+ TEXT: ('text diff mismatch',
+ 'text diff mismatch'),
+ IMAGE: ('image mismatch', 'image mismatch'),
+ IMAGE_PLUS_TEXT: ('image and text mismatch',
+ 'image and text mismatch'),
+ CRASH: ('DumpRenderTree crash',
+ 'DumpRenderTree crashes'),
+ TIMEOUT: ('test timed out', 'tests timed out'),
+ MISSING: ('no expected result found',
+ 'no expected results found')}
+
+ EXPECTATION_ORDER = (PASS, CRASH, TIMEOUT, MISSING, IMAGE_PLUS_TEXT,
+ TEXT, IMAGE, FAIL, SKIP)
+
+ BUILD_TYPES = ('debug', 'release')
+
+ MODIFIERS = {'skip': SKIP,
+ 'wontfix': WONTFIX,
+ 'slow': SLOW,
+ 'rebaseline': REBASELINE,
+ 'none': NONE}
+
+ TIMELINES = {'wontfix': WONTFIX,
+ 'now': NOW}
+
+ RESULT_TYPES = {'skip': SKIP,
+ 'pass': PASS,
+ 'fail': FAIL,
+ 'flaky': FLAKY}
+
+ def __init__(self, port, expectations, full_test_list, test_platform_name,
+ is_debug_mode, is_lint_mode, overrides=None):
+ """
+ expectations: Contents of the expectations file
+ full_test_list: The list of all tests to be run pending processing of
+ the expections for those tests.
+ test_platform_name: name of the platform to match expectations
+ against. Note that this may be different than
+ port.test_platform_name() when is_lint_mode is True.
+ is_debug_mode: Whether we testing a test_shell built debug mode.
+ is_lint_mode: Whether this is just linting test_expecatations.txt.
+ overrides: test expectations that are allowed to override any
+ entries in |expectations|. This is used by callers
+ that need to manage two sets of expectations (e.g., upstream
+ and downstream expectations).
+ """
+
+ self._port = port
+ self._expectations = expectations
+ self._full_test_list = full_test_list
+ self._test_platform_name = test_platform_name
+ self._is_debug_mode = is_debug_mode
+ self._is_lint_mode = is_lint_mode
+ self._overrides = overrides
+ self._errors = []
+ self._non_fatal_errors = []
+
+ # Maps relative test paths as listed in the expectations file to a
+ # list of maps containing modifiers and expectations for each time
+ # the test is listed in the expectations file.
+ self._all_expectations = {}
+
+ # Maps a test to its list of expectations.
+ self._test_to_expectations = {}
+
+ # Maps a test to its list of options (string values)
+ self._test_to_options = {}
+
+ # Maps a test to its list of modifiers: the constants associated with
+ # the options minus any bug or platform strings
+ self._test_to_modifiers = {}
+
+ # Maps a test to the base path that it was listed with in the list.
+ self._test_list_paths = {}
+
+ self._modifier_to_tests = self._dict_of_sets(self.MODIFIERS)
+ self._expectation_to_tests = self._dict_of_sets(self.EXPECTATIONS)
+ self._timeline_to_tests = self._dict_of_sets(self.TIMELINES)
+ self._result_type_to_tests = self._dict_of_sets(self.RESULT_TYPES)
+
+ self._read(self._get_iterable_expectations(self._expectations),
+ overrides_allowed=False)
+
+ # List of tests that are in the overrides file (used for checking for
+ # duplicates inside the overrides file itself). Note that just because
+ # a test is in this set doesn't mean it's necessarily overridding a
+ # expectation in the regular expectations; the test might not be
+ # mentioned in the regular expectations file at all.
+ self._overridding_tests = set()
+
+ if overrides:
+ self._read(self._get_iterable_expectations(self._overrides),
+ overrides_allowed=True)
+
+ self._handle_any_read_errors()
+ self._process_tests_without_expectations()
+
+ def _handle_any_read_errors(self):
+ if len(self._errors) or len(self._non_fatal_errors):
+ if self._is_debug_mode:
+ build_type = 'DEBUG'
+ else:
+ build_type = 'RELEASE'
+ _log.error('')
+ _log.error("FAILURES FOR PLATFORM: %s, BUILD_TYPE: %s" %
+ (self._test_platform_name.upper(), build_type))
+
+ for error in self._errors:
+ _log.error(error)
+ for error in self._non_fatal_errors:
+ _log.error(error)
+
+ if len(self._errors):
+ raise ParseError(fatal=True, errors=self._errors)
+ if len(self._non_fatal_errors) and self._is_lint_mode:
+ raise ParseError(fatal=False, errors=self._non_fatal_errors)
+
+ def _process_tests_without_expectations(self):
+ expectations = set([PASS])
+ options = []
+ modifiers = []
+ if self._full_test_list:
+ for test in self._full_test_list:
+ if not test in self._test_list_paths:
+ self._add_test(test, modifiers, expectations, options,
+ overrides_allowed=False)
+
+ def _dict_of_sets(self, strings_to_constants):
+ """Takes a dict of strings->constants and returns a dict mapping
+ each constant to an empty set."""
+ d = {}
+ for c in strings_to_constants.values():
+ d[c] = set()
+ return d
+
+ def _get_iterable_expectations(self, expectations_str):
+ """Returns an object that can be iterated over. Allows for not caring
+ about whether we're iterating over a file or a new-line separated
+ string."""
+ iterable = [x + "\n" for x in expectations_str.split("\n")]
+ # Strip final entry if it's empty to avoid added in an extra
+ # newline.
+ if iterable[-1] == "\n":
+ return iterable[:-1]
+ return iterable
+
+ def get_test_set(self, modifier, expectation=None, include_skips=True):
+ if expectation is None:
+ tests = self._modifier_to_tests[modifier]
+ else:
+ tests = (self._expectation_to_tests[expectation] &
+ self._modifier_to_tests[modifier])
+
+ if not include_skips:
+ tests = tests - self.get_test_set(SKIP, expectation)
+
+ return tests
+
+ def get_tests_with_result_type(self, result_type):
+ return self._result_type_to_tests[result_type]
+
+ def get_tests_with_timeline(self, timeline):
+ return self._timeline_to_tests[timeline]
+
+ def get_options(self, test):
+ """This returns the entire set of options for the given test
+ (the modifiers plus the BUGXXXX identifier). This is used by the
+ LTTF dashboard."""
+ return self._test_to_options[test]
+
+ def has_modifier(self, test, modifier):
+ return test in self._modifier_to_tests[modifier]
+
+ def get_expectations(self, test):
+ return self._test_to_expectations[test]
+
+ def get_expectations_json_for_all_platforms(self):
+ # Specify separators in order to get compact encoding.
+ return ExpectationsJsonEncoder(separators=(',', ':')).encode(
+ self._all_expectations)
+
+ def get_non_fatal_errors(self):
+ return self._non_fatal_errors
+
+ def remove_platform_from_expectations(self, tests, platform):
+ """Returns a copy of the expectations with the tests matching the
+ platform removed.
+
+ If a test is in the test list and has an option that matches the given
+ platform, remove the matching platform and save the updated test back
+ to the file. If no other platforms remaining after removal, delete the
+ test from the file.
+
+ Args:
+ tests: list of tests that need to update..
+ platform: which platform option to remove.
+
+ Returns:
+ the updated string.
+ """
+
+ assert(platform)
+ f_orig = self._get_iterable_expectations(self._expectations)
+ f_new = []
+
+ tests_removed = 0
+ tests_updated = 0
+ lineno = 0
+ for line in f_orig:
+ lineno += 1
+ action = self._get_platform_update_action(line, lineno, tests,
+ platform)
+ assert(action in (NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM,
+ ADD_PLATFORMS_EXCEPT_THIS))
+ if action == NO_CHANGE:
+ # Save the original line back to the file
+ _log.debug('No change to test: %s', line)
+ f_new.append(line)
+ elif action == REMOVE_TEST:
+ tests_removed += 1
+ _log.info('Test removed: %s', line)
+ elif action == REMOVE_PLATFORM:
+ parts = line.split(':')
+ new_options = parts[0].replace(platform.upper() + ' ', '', 1)
+ new_line = ('%s:%s' % (new_options, parts[1]))
+ f_new.append(new_line)
+ tests_updated += 1
+ _log.info('Test updated: ')
+ _log.info(' old: %s', line)
+ _log.info(' new: %s', new_line)
+ elif action == ADD_PLATFORMS_EXCEPT_THIS:
+ parts = line.split(':')
+ new_options = parts[0]
+ for p in self._port.test_platform_names():
+ p = p.upper()
+ # This is a temp solution for rebaselining tool.
+ # Do not add tags WIN-7 and WIN-VISTA to test expectations
+ # if the original line does not specify the platform
+ # option.
+ # TODO(victorw): Remove WIN-VISTA and WIN-7 once we have
+ # reliable Win 7 and Win Vista buildbots setup.
+ if not p in (platform.upper(), 'WIN-VISTA', 'WIN-7'):
+ new_options += p + ' '
+ new_line = ('%s:%s' % (new_options, parts[1]))
+ f_new.append(new_line)
+ tests_updated += 1
+ _log.info('Test updated: ')
+ _log.info(' old: %s', line)
+ _log.info(' new: %s', new_line)
+
+ _log.info('Total tests removed: %d', tests_removed)
+ _log.info('Total tests updated: %d', tests_updated)
+
+ return "".join(f_new)
+
+ def parse_expectations_line(self, line, lineno):
+ """Parses a line from test_expectations.txt and returns a tuple
+ with the test path, options as a list, expectations as a list."""
+ line = strip_comments(line)
+ if not line:
+ return (None, None, None)
+
+ options = []
+ if line.find(":") is -1:
+ test_and_expectation = line.split("=")
+ else:
+ parts = line.split(":")
+ options = self._get_options_list(parts[0])
+ test_and_expectation = parts[1].split('=')
+
+ test = test_and_expectation[0].strip()
+ if (len(test_and_expectation) is not 2):
+ self._add_error(lineno, "Missing expectations.",
+ test_and_expectation)
+ expectations = None
+ else:
+ expectations = self._get_options_list(test_and_expectation[1])
+
+ return (test, options, expectations)
+
+ def _get_platform_update_action(self, line, lineno, tests, platform):
+ """Check the platform option and return the action needs to be taken.
+
+ Args:
+ line: current line in test expectations file.
+ lineno: current line number of line
+ tests: list of tests that need to update..
+ platform: which platform option to remove.
+
+ Returns:
+ NO_CHANGE: no change to the line (comments, test not in the list etc)
+ REMOVE_TEST: remove the test from file.
+ REMOVE_PLATFORM: remove this platform option from the test.
+ ADD_PLATFORMS_EXCEPT_THIS: add all the platforms except this one.
+ """
+ test, options, expectations = self.parse_expectations_line(line,
+ lineno)
+ if not test or test not in tests:
+ return NO_CHANGE
+
+ has_any_platform = False
+ for option in options:
+ if option in self._port.test_platform_names():
+ has_any_platform = True
+ if not option == platform:
+ return REMOVE_PLATFORM
+
+ # If there is no platform specified, then it means apply to all
+ # platforms. Return the action to add all the platforms except this
+ # one.
+ if not has_any_platform:
+ return ADD_PLATFORMS_EXCEPT_THIS
+
+ return REMOVE_TEST
+
+ def _has_valid_modifiers_for_current_platform(self, options, lineno,
+ test_and_expectations, modifiers):
+ """Returns true if the current platform is in the options list or if
+ no platforms are listed and if there are no fatal errors in the
+ options list.
+
+ Args:
+ options: List of lowercase options.
+ lineno: The line in the file where the test is listed.
+ test_and_expectations: The path and expectations for the test.
+ modifiers: The set to populate with modifiers.
+ """
+ has_any_platform = False
+ has_bug_id = False
+ for option in options:
+ if option in self.MODIFIERS:
+ modifiers.add(option)
+ elif option in self._port.test_platform_names():
+ has_any_platform = True
+ elif re.match(r'bug\d', option) != None:
+ self._add_error(lineno, 'Bug must be either BUGCR, BUGWK, or BUGV8_ for test: %s' %
+ option, test_and_expectations)
+ elif option.startswith('bug'):
+ has_bug_id = True
+ elif option not in self.BUILD_TYPES:
+ self._add_error(lineno, 'Invalid modifier for test: %s' %
+ option, test_and_expectations)
+
+ if has_any_platform and not self._match_platform(options):
+ return False
+
+ if not has_bug_id and 'wontfix' not in options:
+ # TODO(ojan): Turn this into an AddError call once all the
+ # tests have BUG identifiers.
+ self._log_non_fatal_error(lineno, 'Test lacks BUG modifier.',
+ test_and_expectations)
+
+ if 'release' in options or 'debug' in options:
+ if self._is_debug_mode and 'debug' not in options:
+ return False
+ if not self._is_debug_mode and 'release' not in options:
+ return False
+
+ if self._is_lint_mode and 'rebaseline' in options:
+ self._add_error(lineno,
+ 'REBASELINE should only be used for running rebaseline.py. '
+ 'Cannot be checked in.', test_and_expectations)
+
+ return True
+
+ def _match_platform(self, options):
+ """Match the list of options against our specified platform. If any
+ of the options prefix-match self._platform, return True. This handles
+ the case where a test is marked WIN and the platform is WIN-VISTA.
+
+ Args:
+ options: list of options
+ """
+ for opt in options:
+ if self._test_platform_name.startswith(opt):
+ return True
+ return False
+
+ def _add_to_all_expectations(self, test, options, expectations):
+ # Make all paths unix-style so the dashboard doesn't need to.
+ test = test.replace('\\', '/')
+ if not test in self._all_expectations:
+ self._all_expectations[test] = []
+ self._all_expectations[test].append(
+ ModifiersAndExpectations(options, expectations))
+
+ def _read(self, expectations, overrides_allowed):
+ """For each test in an expectations iterable, generate the
+ expectations for it."""
+ lineno = 0
+ for line in expectations:
+ lineno += 1
+
+ test_list_path, options, expectations = \
+ self.parse_expectations_line(line, lineno)
+ if not expectations:
+ continue
+
+ self._add_to_all_expectations(test_list_path,
+ " ".join(options).upper(),
+ " ".join(expectations).upper())
+
+ modifiers = set()
+ if options and not self._has_valid_modifiers_for_current_platform(
+ options, lineno, test_list_path, modifiers):
+ continue
+
+ expectations = self._parse_expectations(expectations, lineno,
+ test_list_path)
+
+ if 'slow' in options and TIMEOUT in expectations:
+ self._add_error(lineno,
+ 'A test can not be both slow and timeout. If it times out '
+ 'indefinitely, then it should be just timeout.',
+ test_list_path)
+
+ full_path = os.path.join(self._port.layout_tests_dir(),
+ test_list_path)
+ full_path = os.path.normpath(full_path)
+ # WebKit's way of skipping tests is to add a -disabled suffix.
+ # So we should consider the path existing if the path or the
+ # -disabled version exists.
+ if (not self._port.path_exists(full_path)
+ and not self._port.path_exists(full_path + '-disabled')):
+ # Log a non fatal error here since you hit this case any
+ # time you update test_expectations.txt without syncing
+ # the LayoutTests directory
+ self._log_non_fatal_error(lineno, 'Path does not exist.',
+ test_list_path)
+ continue
+
+ if not self._full_test_list:
+ tests = [test_list_path]
+ else:
+ tests = self._expand_tests(test_list_path)
+
+ self._add_tests(tests, expectations, test_list_path, lineno,
+ modifiers, options, overrides_allowed)
+
+ def _get_options_list(self, listString):
+ return [part.strip().lower() for part in listString.strip().split(' ')]
+
+ def _parse_expectations(self, expectations, lineno, test_list_path):
+ result = set()
+ for part in expectations:
+ if not part in self.EXPECTATIONS:
+ self._add_error(lineno, 'Unsupported expectation: %s' % part,
+ test_list_path)
+ continue
+ expectation = self.EXPECTATIONS[part]
+ result.add(expectation)
+ return result
+
+ def _expand_tests(self, test_list_path):
+ """Convert the test specification to an absolute, normalized
+ path and make sure directories end with the OS path separator."""
+ # FIXME: full_test_list can quickly contain a big amount of
+ # elements. We should consider at some point to use a more
+ # efficient structure instead of a list. Maybe a dictionary of
+ # lists to represent the tree of tests, leaves being test
+ # files and nodes being categories.
+
+ path = os.path.join(self._port.layout_tests_dir(), test_list_path)
+ path = os.path.normpath(path)
+ if self._port.path_isdir(path):
+ # this is a test category, return all the tests of the category.
+ path = os.path.join(path, '')
+
+ return [test for test in self._full_test_list if test.startswith(path)]
+
+ # this is a test file, do a quick check if it's in the
+ # full test suite.
+ result = []
+ if path in self._full_test_list:
+ result = [path, ]
+ return result
+
+ def _add_tests(self, tests, expectations, test_list_path, lineno,
+ modifiers, options, overrides_allowed):
+ for test in tests:
+ if self._already_seen_test(test, test_list_path, lineno,
+ overrides_allowed):
+ continue
+
+ self._clear_expectations_for_test(test, test_list_path)
+ self._add_test(test, modifiers, expectations, options,
+ overrides_allowed)
+
+ def _add_test(self, test, modifiers, expectations, options,
+ overrides_allowed):
+ """Sets the expected state for a given test.
+
+ This routine assumes the test has not been added before. If it has,
+ use _ClearExpectationsForTest() to reset the state prior to
+ calling this.
+
+ Args:
+ test: test to add
+ modifiers: sequence of modifier keywords ('wontfix', 'slow', etc.)
+ expectations: sequence of expectations (PASS, IMAGE, etc.)
+ options: sequence of keywords and bug identifiers.
+ overrides_allowed: whether we're parsing the regular expectations
+ or the overridding expectations"""
+ self._test_to_expectations[test] = expectations
+ for expectation in expectations:
+ self._expectation_to_tests[expectation].add(test)
+
+ self._test_to_options[test] = options
+ self._test_to_modifiers[test] = set()
+ for modifier in modifiers:
+ mod_value = self.MODIFIERS[modifier]
+ self._modifier_to_tests[mod_value].add(test)
+ self._test_to_modifiers[test].add(mod_value)
+
+ if 'wontfix' in modifiers:
+ self._timeline_to_tests[WONTFIX].add(test)
+ else:
+ self._timeline_to_tests[NOW].add(test)
+
+ if 'skip' in modifiers:
+ self._result_type_to_tests[SKIP].add(test)
+ elif expectations == set([PASS]):
+ self._result_type_to_tests[PASS].add(test)
+ elif len(expectations) > 1:
+ self._result_type_to_tests[FLAKY].add(test)
+ else:
+ self._result_type_to_tests[FAIL].add(test)
+
+ if overrides_allowed:
+ self._overridding_tests.add(test)
+
+ def _clear_expectations_for_test(self, test, test_list_path):
+ """Remove prexisting expectations for this test.
+ This happens if we are seeing a more precise path
+ than a previous listing.
+ """
+ if test in self._test_list_paths:
+ self._test_to_expectations.pop(test, '')
+ self._remove_from_sets(test, self._expectation_to_tests)
+ self._remove_from_sets(test, self._modifier_to_tests)
+ self._remove_from_sets(test, self._timeline_to_tests)
+ self._remove_from_sets(test, self._result_type_to_tests)
+
+ self._test_list_paths[test] = os.path.normpath(test_list_path)
+
+ def _remove_from_sets(self, test, dict):
+ """Removes the given test from the sets in the dictionary.
+
+ Args:
+ test: test to look for
+ dict: dict of sets of files"""
+ for set_of_tests in dict.itervalues():
+ if test in set_of_tests:
+ set_of_tests.remove(test)
+
+ def _already_seen_test(self, test, test_list_path, lineno,
+ allow_overrides):
+ """Returns true if we've already seen a more precise path for this test
+ than the test_list_path.
+ """
+ if not test in self._test_list_paths:
+ return False
+
+ prev_base_path = self._test_list_paths[test]
+ if (prev_base_path == os.path.normpath(test_list_path)):
+ if (not allow_overrides or test in self._overridding_tests):
+ if allow_overrides:
+ expectation_source = "override"
+ else:
+ expectation_source = "expectation"
+ self._add_error(lineno, 'Duplicate %s.' % expectation_source,
+ test)
+ return True
+ else:
+ # We have seen this path, but that's okay because its
+ # in the overrides and the earlier path was in the
+ # expectations.
+ return False
+
+ # Check if we've already seen a more precise path.
+ return prev_base_path.startswith(os.path.normpath(test_list_path))
+
+ def _add_error(self, lineno, msg, path):
+ """Reports an error that will prevent running the tests. Does not
+ immediately raise an exception because we'd like to aggregate all the
+ errors so they can all be printed out."""
+ self._errors.append('Line:%s %s %s' % (lineno, msg, path))
+
+ def _log_non_fatal_error(self, lineno, msg, path):
+ """Reports an error that will not prevent running the tests. These are
+ still errors, but not bad enough to warrant breaking test running."""
+ self._non_fatal_errors.append('Line:%s %s %s' % (lineno, msg, path))
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py
new file mode 100644
index 0000000..34771f3
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py
@@ -0,0 +1,350 @@
+#!/usr/bin/python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit tests for test_expectations.py."""
+
+import os
+import sys
+import unittest
+
+from webkitpy.layout_tests import port
+from webkitpy.layout_tests.layout_package.test_expectations import *
+
+class FunctionsTest(unittest.TestCase):
+ def test_result_was_expected(self):
+ # test basics
+ self.assertEquals(result_was_expected(PASS, set([PASS]),
+ False, False), True)
+ self.assertEquals(result_was_expected(TEXT, set([PASS]),
+ False, False), False)
+
+ # test handling of FAIL expectations
+ self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]),
+ False, False), True)
+ self.assertEquals(result_was_expected(IMAGE, set([FAIL]),
+ False, False), True)
+ self.assertEquals(result_was_expected(TEXT, set([FAIL]),
+ False, False), True)
+ self.assertEquals(result_was_expected(CRASH, set([FAIL]),
+ False, False), False)
+
+ # test handling of SKIPped tests and results
+ self.assertEquals(result_was_expected(SKIP, set([CRASH]),
+ False, True), True)
+ self.assertEquals(result_was_expected(SKIP, set([CRASH]),
+ False, False), False)
+
+ # test handling of MISSING results and the REBASELINE modifier
+ self.assertEquals(result_was_expected(MISSING, set([PASS]),
+ True, False), True)
+ self.assertEquals(result_was_expected(MISSING, set([PASS]),
+ False, False), False)
+
+ def test_remove_pixel_failures(self):
+ self.assertEquals(remove_pixel_failures(set([TEXT])),
+ set([TEXT]))
+ self.assertEquals(remove_pixel_failures(set([PASS])),
+ set([PASS]))
+ self.assertEquals(remove_pixel_failures(set([IMAGE])),
+ set([PASS]))
+ self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])),
+ set([TEXT]))
+ self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
+ set([PASS, CRASH]))
+
+
+class Base(unittest.TestCase):
+ def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
+ self._port = port.get('test', None)
+ self._exp = None
+ unittest.TestCase.__init__(self, testFunc)
+
+ def get_test(self, test_name):
+ return os.path.join(self._port.layout_tests_dir(), test_name)
+
+ def get_basic_tests(self):
+ return [self.get_test('failures/expected/text.html'),
+ self.get_test('failures/expected/image_checksum.html'),
+ self.get_test('failures/expected/crash.html'),
+ self.get_test('failures/expected/missing_text.html'),
+ self.get_test('failures/expected/image.html'),
+ self.get_test('passes/text.html')]
+
+ def get_basic_expectations(self):
+ return """
+BUG_TEST : failures/expected/text.html = TEXT
+BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
+BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
+BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
+BUG_TEST WONTFIX WIN : failures/expected/image.html = IMAGE
+"""
+
+ def parse_exp(self, expectations, overrides=None, is_lint_mode=False,
+ is_debug_mode=False):
+ self._exp = TestExpectations(self._port,
+ tests=self.get_basic_tests(),
+ expectations=expectations,
+ test_platform_name=self._port.test_platform_name(),
+ is_debug_mode=is_debug_mode,
+ is_lint_mode=is_lint_mode,
+ overrides=overrides)
+
+ def assert_exp(self, test, result):
+ self.assertEquals(self._exp.get_expectations(self.get_test(test)),
+ set([result]))
+
+
+class TestExpectationsTest(Base):
+ def test_basic(self):
+ self.parse_exp(self.get_basic_expectations())
+ self.assert_exp('failures/expected/text.html', TEXT)
+ self.assert_exp('failures/expected/image_checksum.html', IMAGE)
+ self.assert_exp('passes/text.html', PASS)
+ self.assert_exp('failures/expected/image.html', PASS)
+
+ def test_multiple_results(self):
+ self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
+ self.assertEqual(self._exp.get_expectations(
+ self.get_test('failures/expected/text.html')),
+ set([TEXT, CRASH]))
+
+ def test_precedence(self):
+ # This tests handling precedence of specific lines over directories
+ # and tests expectations covering entire directories.
+ exp_str = """
+BUGX : failures/expected/text.html = TEXT
+BUGX WONTFIX : failures/expected = IMAGE
+"""
+ self.parse_exp(exp_str)
+ self.assert_exp('failures/expected/text.html', TEXT)
+ self.assert_exp('failures/expected/crash.html', IMAGE)
+
+ def test_category_expectations(self):
+ # This test checks unknown tests are not present in the
+ # expectations and that known test part of a test category is
+ # present in the expectations.
+ exp_str = """
+BUGX WONTFIX : failures/expected = IMAGE
+"""
+ self.parse_exp(exp_str)
+ test_name = 'failures/expected/unknown-test.html'
+ unknown_test = self.get_test(test_name)
+ self.assertRaises(KeyError, self._exp.get_expectations,
+ unknown_test)
+ self.assert_exp('failures/expected/crash.html', IMAGE)
+
+ def test_release_mode(self):
+ self.parse_exp('BUGX DEBUG : failures/expected/text.html = TEXT',
+ is_debug_mode=True)
+ self.assert_exp('failures/expected/text.html', TEXT)
+ self.parse_exp('BUGX RELEASE : failures/expected/text.html = TEXT',
+ is_debug_mode=True)
+ self.assert_exp('failures/expected/text.html', PASS)
+ self.parse_exp('BUGX DEBUG : failures/expected/text.html = TEXT',
+ is_debug_mode=False)
+ self.assert_exp('failures/expected/text.html', PASS)
+ self.parse_exp('BUGX RELEASE : failures/expected/text.html = TEXT',
+ is_debug_mode=False)
+ self.assert_exp('failures/expected/text.html', TEXT)
+
+ def test_get_options(self):
+ self.parse_exp(self.get_basic_expectations())
+ self.assertEqual(self._exp.get_options(
+ self.get_test('passes/text.html')), [])
+
+ def test_expectations_json_for_all_platforms(self):
+ self.parse_exp(self.get_basic_expectations())
+ json_str = self._exp.get_expectations_json_for_all_platforms()
+ # FIXME: test actual content?
+ self.assertTrue(json_str)
+
+ def test_get_expectations_string(self):
+ self.parse_exp(self.get_basic_expectations())
+ self.assertEquals(self._exp.get_expectations_string(
+ self.get_test('failures/expected/text.html')),
+ 'TEXT')
+
+ def test_expectation_to_string(self):
+ # Normal cases are handled by other tests.
+ self.parse_exp(self.get_basic_expectations())
+ self.assertRaises(ValueError, self._exp.expectation_to_string,
+ -1)
+
+ def test_get_test_set(self):
+ # Handle some corner cases for this routine not covered by other tests.
+ self.parse_exp(self.get_basic_expectations())
+ s = self._exp._expected_failures.get_test_set(WONTFIX)
+ self.assertEqual(s,
+ set([self.get_test('failures/expected/crash.html'),
+ self.get_test('failures/expected/image_checksum.html')]))
+ s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH)
+ self.assertEqual(s,
+ set([self.get_test('failures/expected/crash.html')]))
+ s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH,
+ include_skips=False)
+ self.assertEqual(s, set([]))
+
+ def test_parse_error_fatal(self):
+ try:
+ self.parse_exp("""FOO : failures/expected/text.html = TEXT
+SKIP : failures/expected/image.html""")
+ self.assertFalse(True, "ParseError wasn't raised")
+ except ParseError, e:
+ self.assertTrue(e.fatal)
+ exp_errors = [u'Line:1 Invalid modifier for test: foo failures/expected/text.html',
+ u"Line:2 Missing expectations. [' failures/expected/image.html']"]
+ self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
+ self.assertEqual(e.errors, exp_errors)
+
+ def test_parse_error_nonfatal(self):
+ try:
+ self.parse_exp('SKIP : failures/expected/text.html = TEXT',
+ is_lint_mode=True)
+ self.assertFalse(True, "ParseError wasn't raised")
+ except ParseError, e:
+ self.assertFalse(e.fatal)
+ exp_errors = [u'Line:1 Test lacks BUG modifier. failures/expected/text.html']
+ self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
+ self.assertEqual(e.errors, exp_errors)
+
+ def test_syntax_missing_expectation(self):
+ # This is missing the expectation.
+ self.assertRaises(ParseError, self.parse_exp,
+ 'BUG_TEST: failures/expected/text.html',
+ is_debug_mode=True)
+
+ def test_syntax_invalid_option(self):
+ self.assertRaises(ParseError, self.parse_exp,
+ 'BUG_TEST FOO: failures/expected/text.html = PASS')
+
+ def test_syntax_invalid_expectation(self):
+ # This is missing the expectation.
+ self.assertRaises(ParseError, self.parse_exp,
+ 'BUG_TEST: failures/expected/text.html = FOO')
+
+ def test_syntax_missing_bugid(self):
+ # This should log a non-fatal error.
+ self.parse_exp('SLOW : failures/expected/text.html = TEXT')
+ self.assertEqual(
+ len(self._exp._expected_failures.get_non_fatal_errors()), 1)
+
+ def test_semantic_slow_and_timeout(self):
+ # A test cannot be SLOW and expected to TIMEOUT.
+ self.assertRaises(ParseError, self.parse_exp,
+ 'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT')
+
+ def test_semantic_rebaseline(self):
+ # Can't lint a file w/ 'REBASELINE' in it.
+ self.assertRaises(ParseError, self.parse_exp,
+ 'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
+ is_lint_mode=True)
+
+ def test_semantic_duplicates(self):
+ self.assertRaises(ParseError, self.parse_exp, """
+BUG_TEST : failures/expected/text.html = TEXT
+BUG_TEST : failures/expected/text.html = IMAGE""")
+
+ self.assertRaises(ParseError, self.parse_exp,
+ self.get_basic_expectations(), """
+BUG_TEST : failures/expected/text.html = TEXT
+BUG_TEST : failures/expected/text.html = IMAGE""")
+
+ def test_semantic_missing_file(self):
+ # This should log a non-fatal error.
+ self.parse_exp('BUG_TEST : missing_file.html = TEXT')
+ self.assertEqual(
+ len(self._exp._expected_failures.get_non_fatal_errors()), 1)
+
+
+ def test_overrides(self):
+ self.parse_exp(self.get_basic_expectations(), """
+BUG_OVERRIDE : failures/expected/text.html = IMAGE""")
+ self.assert_exp('failures/expected/text.html', IMAGE)
+
+ def test_matches_an_expected_result(self):
+
+ def match(test, result, pixel_tests_enabled):
+ return self._exp.matches_an_expected_result(
+ self.get_test(test), result, pixel_tests_enabled)
+
+ self.parse_exp(self.get_basic_expectations())
+ self.assertTrue(match('failures/expected/text.html', TEXT, True))
+ self.assertTrue(match('failures/expected/text.html', TEXT, False))
+ self.assertFalse(match('failures/expected/text.html', CRASH, True))
+ self.assertFalse(match('failures/expected/text.html', CRASH, False))
+ self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
+ True))
+ self.assertTrue(match('failures/expected/image_checksum.html', PASS,
+ False))
+ self.assertTrue(match('failures/expected/crash.html', SKIP, False))
+ self.assertTrue(match('passes/text.html', PASS, False))
+
+
+class RebaseliningTest(Base):
+ """Test rebaselining-specific functionality."""
+ def assertRemove(self, platform, input_expectations, expected_expectations):
+ self.parse_exp(input_expectations)
+ test = self.get_test('failures/expected/text.html')
+ actual_expectations = self._exp.remove_platform_from_expectations(
+ test, platform)
+ self.assertEqual(expected_expectations, actual_expectations)
+
+ def test_no_get_rebaselining_failures(self):
+ self.parse_exp(self.get_basic_expectations())
+ self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
+
+ def test_get_rebaselining_failures_expand(self):
+ self.parse_exp("""
+BUG_TEST REBASELINE : failures/expected/text.html = TEXT
+""")
+ self.assertEqual(len(self._exp.get_rebaselining_failures()), 1)
+
+ def test_remove_expand(self):
+ self.assertRemove('mac',
+ 'BUGX REBASELINE : failures/expected/text.html = TEXT\n',
+ 'BUGX REBASELINE WIN : failures/expected/text.html = TEXT\n')
+
+ def test_remove_mac_win(self):
+ self.assertRemove('mac',
+ 'BUGX REBASELINE MAC WIN : failures/expected/text.html = TEXT\n',
+ 'BUGX REBASELINE WIN : failures/expected/text.html = TEXT\n')
+
+ def test_remove_mac_mac(self):
+ self.assertRemove('mac',
+ 'BUGX REBASELINE MAC : failures/expected/text.html = TEXT\n',
+ '')
+
+ def test_remove_nothing(self):
+ self.assertRemove('mac',
+ '\n\n',
+ '\n\n')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py
new file mode 100644
index 0000000..6d55761
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py
@@ -0,0 +1,282 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Classes for failures that occur during tests."""
+
+import os
+import test_expectations
+
+import cPickle
+
+
+def determine_result_type(failure_list):
+ """Takes a set of test_failures and returns which result type best fits
+ the list of failures. "Best fits" means we use the worst type of failure.
+
+ Returns:
+ one of the test_expectations result types - PASS, TEXT, CRASH, etc."""
+
+ if not failure_list or len(failure_list) == 0:
+ return test_expectations.PASS
+
+ failure_types = [type(f) for f in failure_list]
+ if FailureCrash in failure_types:
+ return test_expectations.CRASH
+ elif FailureTimeout in failure_types:
+ return test_expectations.TIMEOUT
+ elif (FailureMissingResult in failure_types or
+ FailureMissingImage in failure_types or
+ FailureMissingImageHash in failure_types):
+ return test_expectations.MISSING
+ else:
+ is_text_failure = FailureTextMismatch in failure_types
+ is_image_failure = (FailureImageHashIncorrect in failure_types or
+ FailureImageHashMismatch in failure_types)
+ if is_text_failure and is_image_failure:
+ return test_expectations.IMAGE_PLUS_TEXT
+ elif is_text_failure:
+ return test_expectations.TEXT
+ elif is_image_failure:
+ return test_expectations.IMAGE
+ else:
+ raise ValueError("unclassifiable set of failures: "
+ + str(failure_types))
+
+
+class TestFailure(object):
+ """Abstract base class that defines the failure interface."""
+
+ @staticmethod
+ def loads(s):
+ """Creates a TestFailure object from the specified string."""
+ return cPickle.loads(s)
+
+ @staticmethod
+ def message():
+ """Returns a string describing the failure in more detail."""
+ raise NotImplementedError
+
+ def __eq__(self, other):
+ return self.__class__.__name__ == other.__class__.__name__
+
+ def __ne__(self, other):
+ return self.__class__.__name__ != other.__class__.__name__
+
+ def dumps(self):
+ """Returns the string/JSON representation of a TestFailure."""
+ return cPickle.dumps(self)
+
+ def result_html_output(self, filename):
+ """Returns an HTML string to be included on the results.html page."""
+ raise NotImplementedError
+
+ def should_kill_dump_render_tree(self):
+ """Returns True if we should kill DumpRenderTree before the next
+ test."""
+ return False
+
+ def relative_output_filename(self, filename, modifier):
+ """Returns a relative filename inside the output dir that contains
+ modifier.
+
+ For example, if filename is fast\dom\foo.html and modifier is
+ "-expected.txt", the return value is fast\dom\foo-expected.txt
+
+ Args:
+ filename: relative filename to test file
+ modifier: a string to replace the extension of filename with
+
+ Return:
+ The relative windows path to the output filename
+ """
+ return os.path.splitext(filename)[0] + modifier
+
+
+class FailureWithType(TestFailure):
+ """Base class that produces standard HTML output based on the test type.
+
+ Subclasses may commonly choose to override the ResultHtmlOutput, but still
+ use the standard OutputLinks.
+ """
+
+ def __init__(self):
+ TestFailure.__init__(self)
+
+ # Filename suffixes used by ResultHtmlOutput.
+ OUT_FILENAMES = ()
+
+ def output_links(self, filename, out_names):
+ """Returns a string holding all applicable output file links.
+
+ Args:
+ filename: the test filename, used to construct the result file names
+ out_names: list of filename suffixes for the files. If three or more
+ suffixes are in the list, they should be [actual, expected, diff,
+ wdiff]. Two suffixes should be [actual, expected], and a
+ single item is the [actual] filename suffix.
+ If out_names is empty, returns the empty string.
+ """
+ # FIXME: Seems like a bad idea to separate the display name data
+ # from the path data by hard-coding the display name here
+ # and passing in the path information via out_names.
+ #
+ # FIXME: Also, we don't know for sure that these files exist,
+ # and we shouldn't be creating links to files that don't exist
+ # (for example, if we don't actually have wdiff output).
+ links = ['']
+ uris = [self.relative_output_filename(filename, fn) for
+ fn in out_names]
+ if len(uris) > 1:
+ links.append("<a href='%s'>expected</a>" % uris[1])
+ if len(uris) > 0:
+ links.append("<a href='%s'>actual</a>" % uris[0])
+ if len(uris) > 2:
+ links.append("<a href='%s'>diff</a>" % uris[2])
+ if len(uris) > 3:
+ links.append("<a href='%s'>wdiff</a>" % uris[3])
+ if len(uris) > 4:
+ links.append("<a href='%s'>pretty diff</a>" % uris[4])
+ return ' '.join(links)
+
+ def result_html_output(self, filename):
+ return self.message() + self.output_links(filename, self.OUT_FILENAMES)
+
+
+class FailureTimeout(TestFailure):
+ """Test timed out. We also want to restart DumpRenderTree if this
+ happens."""
+
+ @staticmethod
+ def message():
+ return "Test timed out"
+
+ def result_html_output(self, filename):
+ return "<strong>%s</strong>" % self.message()
+
+ def should_kill_dump_render_tree(self):
+ return True
+
+
+class FailureCrash(TestFailure):
+ """Test shell crashed."""
+
+ @staticmethod
+ def message():
+ return "Test shell crashed"
+
+ def result_html_output(self, filename):
+ # FIXME: create a link to the minidump file
+ stack = self.relative_output_filename(filename, "-stack.txt")
+ return "<strong>%s</strong> <a href=%s>stack</a>" % (self.message(),
+ stack)
+
+ def should_kill_dump_render_tree(self):
+ return True
+
+
+class FailureMissingResult(FailureWithType):
+ """Expected result was missing."""
+ OUT_FILENAMES = ("-actual.txt",)
+
+ @staticmethod
+ def message():
+ return "No expected results found"
+
+ def result_html_output(self, filename):
+ return ("<strong>%s</strong>" % self.message() +
+ self.output_links(filename, self.OUT_FILENAMES))
+
+
+class FailureTextMismatch(FailureWithType):
+ """Text diff output failed."""
+ # Filename suffixes used by ResultHtmlOutput.
+ # FIXME: Why don't we use the constants from TestTypeBase here?
+ OUT_FILENAMES = ("-actual.txt", "-expected.txt", "-diff.txt",
+ "-wdiff.html", "-pretty-diff.html")
+
+ @staticmethod
+ def message():
+ return "Text diff mismatch"
+
+
+class FailureMissingImageHash(FailureWithType):
+ """Actual result hash was missing."""
+ # Chrome doesn't know to display a .checksum file as text, so don't bother
+ # putting in a link to the actual result.
+
+ @staticmethod
+ def message():
+ return "No expected image hash found"
+
+ def result_html_output(self, filename):
+ return "<strong>%s</strong>" % self.message()
+
+
+class FailureMissingImage(FailureWithType):
+ """Actual result image was missing."""
+ OUT_FILENAMES = ("-actual.png",)
+
+ @staticmethod
+ def message():
+ return "No expected image found"
+
+ def result_html_output(self, filename):
+ return ("<strong>%s</strong>" % self.message() +
+ self.output_links(filename, self.OUT_FILENAMES))
+
+
+class FailureImageHashMismatch(FailureWithType):
+ """Image hashes didn't match."""
+ OUT_FILENAMES = ("-actual.png", "-expected.png", "-diff.png")
+
+ @staticmethod
+ def message():
+ # We call this a simple image mismatch to avoid confusion, since
+ # we link to the PNGs rather than the checksums.
+ return "Image mismatch"
+
+
+class FailureImageHashIncorrect(FailureWithType):
+ """Actual result hash is incorrect."""
+ # Chrome doesn't know to display a .checksum file as text, so don't bother
+ # putting in a link to the actual result.
+
+ @staticmethod
+ def message():
+ return "Images match, expected image hash incorrect. "
+
+ def result_html_output(self, filename):
+ return "<strong>%s</strong>" % self.message()
+
+# Convenient collection of all failure classes for anything that might
+# need to enumerate over them all.
+ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult,
+ FailureTextMismatch, FailureMissingImageHash,
+ FailureMissingImage, FailureImageHashMismatch,
+ FailureImageHashIncorrect)
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures_unittest.py
new file mode 100644
index 0000000..3e3528d
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures_unittest.py
@@ -0,0 +1,84 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+""""Tests code paths not covered by the regular unit tests."""
+
+import unittest
+
+from webkitpy.layout_tests.layout_package.test_failures import *
+
+
+class Test(unittest.TestCase):
+ def assertResultHtml(self, failure_obj):
+ self.assertNotEqual(failure_obj.result_html_output('foo'), None)
+
+ def assert_loads(self, cls):
+ failure_obj = cls()
+ s = failure_obj.dumps()
+ new_failure_obj = TestFailure.loads(s)
+ self.assertTrue(isinstance(new_failure_obj, cls))
+
+ self.assertEqual(failure_obj, new_failure_obj)
+
+ # Also test that != is implemented.
+ self.assertFalse(failure_obj != new_failure_obj)
+
+ def test_crash(self):
+ self.assertResultHtml(FailureCrash())
+
+ def test_hash_incorrect(self):
+ self.assertResultHtml(FailureImageHashIncorrect())
+
+ def test_missing(self):
+ self.assertResultHtml(FailureMissingResult())
+
+ def test_missing_image(self):
+ self.assertResultHtml(FailureMissingImage())
+
+ def test_missing_image_hash(self):
+ self.assertResultHtml(FailureMissingImageHash())
+
+ def test_timeout(self):
+ self.assertResultHtml(FailureTimeout())
+
+ def test_unknown_failure_type(self):
+ class UnknownFailure(TestFailure):
+ pass
+
+ failure_obj = UnknownFailure()
+ self.assertRaises(ValueError, determine_result_type, [failure_obj])
+ self.assertRaises(NotImplementedError, failure_obj.message)
+ self.assertRaises(NotImplementedError, failure_obj.result_html_output,
+ "foo.txt")
+
+ def test_loads(self):
+ for c in ALL_FAILURE_CLASSES:
+ self.assert_loads(c)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_input.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_input.py
new file mode 100644
index 0000000..4b027c0
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_input.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+class TestInput:
+ """Groups information about a test for easy passing of data."""
+
+ def __init__(self, filename, timeout):
+ """Holds the input parameters for a test.
+ Args:
+ filename: Full path to the test.
+ timeout: Timeout in msecs the driver should use while running the test
+ """
+ # FIXME: filename should really be test_name as a relative path.
+ self.filename = filename
+ self.timeout = timeout
+ # The image_hash is used to avoid doing an image dump if the
+ # checksums match. The image_hash is set later, and only if it is needed
+ # for the test.
+ self.image_hash = None
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_output.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_output.py
new file mode 100644
index 0000000..e809be6
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_output.py
@@ -0,0 +1,56 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+class TestOutput(object):
+ """Groups information about a test output for easy passing of data.
+
+ This is used not only for a actual test output, but also for grouping
+ expected test output.
+ """
+
+ def __init__(self, text, image, image_hash,
+ crash=None, test_time=None, timeout=None, error=None):
+ """Initializes a TestOutput object.
+
+ Args:
+ text: a text output
+ image: an image output
+ image_hash: a string containing the checksum of the image
+ crash: a boolean indicating whether the driver crashed on the test
+ test_time: a time which the test has taken
+ timeout: a boolean indicating whehter the test timed out
+ error: any unexpected or additional (or error) text output
+ """
+ self.text = text
+ self.image = image
+ self.image_hash = image_hash
+ self.crash = crash
+ self.test_time = test_time
+ self.timeout = timeout
+ self.error = error
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py
new file mode 100644
index 0000000..2417fb7
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py
@@ -0,0 +1,61 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import cPickle
+
+import test_failures
+
+
+class TestResult(object):
+ """Data object containing the results of a single test."""
+
+ @staticmethod
+ def loads(str):
+ return cPickle.loads(str)
+
+ def __init__(self, filename, failures, test_run_time,
+ total_time_for_all_diffs, time_for_diffs):
+ self.failures = failures
+ self.filename = filename
+ self.test_run_time = test_run_time
+ self.time_for_diffs = time_for_diffs
+ self.total_time_for_all_diffs = total_time_for_all_diffs
+ self.type = test_failures.determine_result_type(failures)
+
+ def __eq__(self, other):
+ return (self.filename == other.filename and
+ self.failures == other.failures and
+ self.test_run_time == other.test_run_time and
+ self.time_for_diffs == other.time_for_diffs and
+ self.total_time_for_all_diffs == other.total_time_for_all_diffs)
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ def dumps(self):
+ return cPickle.dumps(self)
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results_unittest.py
new file mode 100644
index 0000000..5921666
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results_unittest.py
@@ -0,0 +1,52 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import unittest
+
+from test_results import TestResult
+
+
+class Test(unittest.TestCase):
+ def test_loads(self):
+ result = TestResult(filename='foo',
+ failures=[],
+ test_run_time=1.1,
+ total_time_for_all_diffs=0.5,
+ time_for_diffs=0.5)
+ s = result.dumps()
+ new_result = TestResult.loads(s)
+ self.assertTrue(isinstance(new_result, TestResult))
+
+ self.assertEqual(new_result, result)
+
+ # Also check that != is implemented.
+ self.assertFalse(new_result != result)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results_uploader.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results_uploader.py
new file mode 100644
index 0000000..033c8c6
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results_uploader.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import with_statement
+
+import codecs
+import mimetypes
+import socket
+import urllib2
+
+from webkitpy.common.net.networktransaction import NetworkTransaction
+
+def get_mime_type(filename):
+ return mimetypes.guess_type(filename)[0] or 'application/octet-stream'
+
+
+def _encode_multipart_form_data(fields, files):
+ """Encode form fields for multipart/form-data.
+
+ Args:
+ fields: A sequence of (name, value) elements for regular form fields.
+ files: A sequence of (name, filename, value) elements for data to be
+ uploaded as files.
+ Returns:
+ (content_type, body) ready for httplib.HTTP instance.
+
+ Source:
+ http://code.google.com/p/rietveld/source/browse/trunk/upload.py
+ """
+ BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
+ CRLF = '\r\n'
+ lines = []
+
+ for key, value in fields:
+ lines.append('--' + BOUNDARY)
+ lines.append('Content-Disposition: form-data; name="%s"' % key)
+ lines.append('')
+ if isinstance(value, unicode):
+ value = value.encode('utf-8')
+ lines.append(value)
+
+ for key, filename, value in files:
+ lines.append('--' + BOUNDARY)
+ lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))
+ lines.append('Content-Type: %s' % get_mime_type(filename))
+ lines.append('')
+ if isinstance(value, unicode):
+ value = value.encode('utf-8')
+ lines.append(value)
+
+ lines.append('--' + BOUNDARY + '--')
+ lines.append('')
+ body = CRLF.join(lines)
+ content_type = 'multipart/form-data; boundary=%s' % BOUNDARY
+ return content_type, body
+
+
+class TestResultsUploader:
+ def __init__(self, host):
+ self._host = host
+
+ def _upload_files(self, attrs, file_objs):
+ url = "http://%s/testfile/upload" % self._host
+ content_type, data = _encode_multipart_form_data(attrs, file_objs)
+ headers = {"Content-Type": content_type}
+ request = urllib2.Request(url, data, headers)
+ urllib2.urlopen(request)
+
+ def upload(self, params, files, timeout_seconds):
+ file_objs = []
+ for filename, path in files:
+ with codecs.open(path, "rb") as file:
+ file_objs.append(('file', filename, file.read()))
+
+ orig_timeout = socket.getdefaulttimeout()
+ try:
+ socket.setdefaulttimeout(timeout_seconds)
+ NetworkTransaction(timeout_seconds=timeout_seconds).run(
+ lambda: self._upload_files(params, file_objs))
+ finally:
+ socket.setdefaulttimeout(orig_timeout)
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner.py
new file mode 100644
index 0000000..24d04ca
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner.py
@@ -0,0 +1,1218 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+The TestRunner class runs a series of tests (TestType interface) against a set
+of test files. If a test file fails a TestType, it returns a list TestFailure
+objects to the TestRunner. The TestRunner then aggregates the TestFailures to
+create a final report.
+"""
+
+from __future__ import with_statement
+
+import codecs
+import errno
+import logging
+import math
+import os
+import Queue
+import random
+import shutil
+import sys
+import time
+
+from result_summary import ResultSummary
+from test_input import TestInput
+
+import dump_render_tree_thread
+import json_layout_results_generator
+import message_broker
+import printing
+import test_expectations
+import test_failures
+import test_results
+import test_results_uploader
+
+from webkitpy.thirdparty import simplejson
+from webkitpy.tool import grammar
+
+_log = logging.getLogger("webkitpy.layout_tests.run_webkit_tests")
+
+# Builder base URL where we have the archived test results.
+BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
+
+LAYOUT_TESTS_DIRECTORY = "LayoutTests" + os.sep
+
+TestExpectationsFile = test_expectations.TestExpectationsFile
+
+
+def summarize_unexpected_results(port_obj, expectations, result_summary,
+ retry_summary):
+ """Summarize any unexpected results as a dict.
+
+ FIXME: split this data structure into a separate class?
+
+ Args:
+ port_obj: interface to port-specific hooks
+ expectations: test_expectations.TestExpectations object
+ result_summary: summary object from initial test runs
+ retry_summary: summary object from final test run of retried tests
+ Returns:
+ A dictionary containing a summary of the unexpected results from the
+ run, with the following fields:
+ 'version': a version indicator (1 in this version)
+ 'fixable': # of fixable tests (NOW - PASS)
+ 'skipped': # of skipped tests (NOW & SKIPPED)
+ 'num_regressions': # of non-flaky failures
+ 'num_flaky': # of flaky failures
+ 'num_passes': # of unexpected passes
+ 'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
+ """
+ results = {}
+ results['version'] = 1
+
+ tbe = result_summary.tests_by_expectation
+ tbt = result_summary.tests_by_timeline
+ results['fixable'] = len(tbt[test_expectations.NOW] -
+ tbe[test_expectations.PASS])
+ results['skipped'] = len(tbt[test_expectations.NOW] &
+ tbe[test_expectations.SKIP])
+
+ num_passes = 0
+ num_flaky = 0
+ num_regressions = 0
+ keywords = {}
+ for k, v in TestExpectationsFile.EXPECTATIONS.iteritems():
+ keywords[v] = k.upper()
+
+ tests = {}
+ for filename, result in result_summary.unexpected_results.iteritems():
+ # Note that if a test crashed in the original run, we ignore
+ # whether or not it crashed when we retried it (if we retried it),
+ # and always consider the result not flaky.
+ test = port_obj.relative_test_filename(filename)
+ expected = expectations.get_expectations_string(filename)
+ actual = [keywords[result]]
+
+ if result == test_expectations.PASS:
+ num_passes += 1
+ elif result == test_expectations.CRASH:
+ num_regressions += 1
+ else:
+ if filename not in retry_summary.unexpected_results:
+ actual.extend(expectations.get_expectations_string(
+ filename).split(" "))
+ num_flaky += 1
+ else:
+ retry_result = retry_summary.unexpected_results[filename]
+ if result != retry_result:
+ actual.append(keywords[retry_result])
+ num_flaky += 1
+ else:
+ num_regressions += 1
+
+ tests[test] = {}
+ tests[test]['expected'] = expected
+ tests[test]['actual'] = " ".join(actual)
+
+ results['tests'] = tests
+ results['num_passes'] = num_passes
+ results['num_flaky'] = num_flaky
+ results['num_regressions'] = num_regressions
+
+ return results
+
+
+class TestRunInterruptedException(Exception):
+ """Raised when a test run should be stopped immediately."""
+ def __init__(self, reason):
+ self.reason = reason
+
+
+class TestRunner:
+ """A class for managing running a series of tests on a series of layout
+ test files."""
+
+ HTTP_SUBDIR = os.sep.join(['', 'http', ''])
+ WEBSOCKET_SUBDIR = os.sep.join(['', 'websocket', ''])
+
+ # The per-test timeout in milliseconds, if no --time-out-ms option was
+ # given to run_webkit_tests. This should correspond to the default timeout
+ # in DumpRenderTree.
+ DEFAULT_TEST_TIMEOUT_MS = 6 * 1000
+
+ def __init__(self, port, options, printer):
+ """Initialize test runner data structures.
+
+ Args:
+ port: an object implementing port-specific
+ options: a dictionary of command line options
+ printer: a Printer object to record updates to.
+ """
+ self._port = port
+ self._options = options
+ self._printer = printer
+ self._message_broker = None
+
+ # disable wss server. need to install pyOpenSSL on buildbots.
+ # self._websocket_secure_server = websocket_server.PyWebSocket(
+ # options.results_directory, use_tls=True, port=9323)
+
+ # a set of test files, and the same tests as a list
+ self._test_files = set()
+ self._test_files_list = None
+ self._result_queue = Queue.Queue()
+ self._retrying = False
+
+ def collect_tests(self, args, last_unexpected_results):
+ """Find all the files to test.
+
+ Args:
+ args: list of test arguments from the command line
+ last_unexpected_results: list of unexpected results to retest, if any
+
+ """
+ paths = [self._strip_test_dir_prefix(arg) for arg in args if arg and arg != '']
+ paths += last_unexpected_results
+ if self._options.test_list:
+ paths += read_test_files(self._options.test_list)
+ self._test_files = self._port.tests(paths)
+
+ def _strip_test_dir_prefix(self, path):
+ if path.startswith(LAYOUT_TESTS_DIRECTORY):
+ return path[len(LAYOUT_TESTS_DIRECTORY):]
+ return path
+
+ def lint(self):
+ lint_failed = False
+
+ # Creating the expecations for each platform/configuration pair does
+ # all the test list parsing and ensures it's correct syntax (e.g. no
+ # dupes).
+ for platform_name in self._port.test_platform_names():
+ try:
+ self.parse_expectations(platform_name, is_debug_mode=True)
+ except test_expectations.ParseError:
+ lint_failed = True
+ try:
+ self.parse_expectations(platform_name, is_debug_mode=False)
+ except test_expectations.ParseError:
+ lint_failed = True
+
+ self._printer.write("")
+ if lint_failed:
+ _log.error("Lint failed.")
+ return -1
+
+ _log.info("Lint succeeded.")
+ return 0
+
+ def parse_expectations(self, test_platform_name, is_debug_mode):
+ """Parse the expectations from the test_list files and return a data
+ structure holding them. Throws an error if the test_list files have
+ invalid syntax."""
+ if self._options.lint_test_files:
+ test_files = None
+ else:
+ test_files = self._test_files
+
+ expectations_str = self._port.test_expectations()
+ overrides_str = self._port.test_expectations_overrides()
+ self._expectations = test_expectations.TestExpectations(
+ self._port, test_files, expectations_str, test_platform_name,
+ is_debug_mode, self._options.lint_test_files,
+ overrides=overrides_str)
+ return self._expectations
+
+ def prepare_lists_and_print_output(self):
+ """Create appropriate subsets of test lists and returns a
+ ResultSummary object. Also prints expected test counts.
+ """
+
+ # Remove skipped - both fixable and ignored - files from the
+ # top-level list of files to test.
+ num_all_test_files = len(self._test_files)
+ self._printer.print_expected("Found: %d tests" %
+ (len(self._test_files)))
+ if not num_all_test_files:
+ _log.critical('No tests to run.')
+ return None
+
+ skipped = set()
+ if num_all_test_files > 1 and not self._options.force:
+ skipped = self._expectations.get_tests_with_result_type(
+ test_expectations.SKIP)
+ self._test_files -= skipped
+
+ # Create a sorted list of test files so the subset chunk,
+ # if used, contains alphabetically consecutive tests.
+ self._test_files_list = list(self._test_files)
+ if self._options.randomize_order:
+ random.shuffle(self._test_files_list)
+ else:
+ self._test_files_list.sort()
+
+ # If the user specifies they just want to run a subset of the tests,
+ # just grab a subset of the non-skipped tests.
+ if self._options.run_chunk or self._options.run_part:
+ chunk_value = self._options.run_chunk or self._options.run_part
+ test_files = self._test_files_list
+ try:
+ (chunk_num, chunk_len) = chunk_value.split(":")
+ chunk_num = int(chunk_num)
+ assert(chunk_num >= 0)
+ test_size = int(chunk_len)
+ assert(test_size > 0)
+ except:
+ _log.critical("invalid chunk '%s'" % chunk_value)
+ return None
+
+ # Get the number of tests
+ num_tests = len(test_files)
+
+ # Get the start offset of the slice.
+ if self._options.run_chunk:
+ chunk_len = test_size
+ # In this case chunk_num can be really large. We need
+ # to make the slave fit in the current number of tests.
+ slice_start = (chunk_num * chunk_len) % num_tests
+ else:
+ # Validate the data.
+ assert(test_size <= num_tests)
+ assert(chunk_num <= test_size)
+
+ # To count the chunk_len, and make sure we don't skip
+ # some tests, we round to the next value that fits exactly
+ # all the parts.
+ rounded_tests = num_tests
+ if rounded_tests % test_size != 0:
+ rounded_tests = (num_tests + test_size -
+ (num_tests % test_size))
+
+ chunk_len = rounded_tests / test_size
+ slice_start = chunk_len * (chunk_num - 1)
+ # It does not mind if we go over test_size.
+
+ # Get the end offset of the slice.
+ slice_end = min(num_tests, slice_start + chunk_len)
+
+ files = test_files[slice_start:slice_end]
+
+ tests_run_msg = 'Running: %d tests (chunk slice [%d:%d] of %d)' % (
+ (slice_end - slice_start), slice_start, slice_end, num_tests)
+ self._printer.print_expected(tests_run_msg)
+
+ # If we reached the end and we don't have enough tests, we run some
+ # from the beginning.
+ if slice_end - slice_start < chunk_len:
+ extra = chunk_len - (slice_end - slice_start)
+ extra_msg = (' last chunk is partial, appending [0:%d]' %
+ extra)
+ self._printer.print_expected(extra_msg)
+ tests_run_msg += "\n" + extra_msg
+ files.extend(test_files[0:extra])
+ tests_run_filename = os.path.join(self._options.results_directory,
+ "tests_run.txt")
+ with codecs.open(tests_run_filename, "w", "utf-8") as file:
+ file.write(tests_run_msg + "\n")
+
+ len_skip_chunk = int(len(files) * len(skipped) /
+ float(len(self._test_files)))
+ skip_chunk_list = list(skipped)[0:len_skip_chunk]
+ skip_chunk = set(skip_chunk_list)
+
+ # Update expectations so that the stats are calculated correctly.
+ # We need to pass a list that includes the right # of skipped files
+ # to ParseExpectations so that ResultSummary() will get the correct
+ # stats. So, we add in the subset of skipped files, and then
+ # subtract them back out.
+ self._test_files_list = files + skip_chunk_list
+ self._test_files = set(self._test_files_list)
+
+ self._expectations = self.parse_expectations(
+ self._port.test_platform_name(),
+ self._options.configuration == 'Debug')
+
+ self._test_files = set(files)
+ self._test_files_list = files
+ else:
+ skip_chunk = skipped
+
+ result_summary = ResultSummary(self._expectations,
+ self._test_files | skip_chunk)
+ self._print_expected_results_of_type(result_summary,
+ test_expectations.PASS, "passes")
+ self._print_expected_results_of_type(result_summary,
+ test_expectations.FAIL, "failures")
+ self._print_expected_results_of_type(result_summary,
+ test_expectations.FLAKY, "flaky")
+ self._print_expected_results_of_type(result_summary,
+ test_expectations.SKIP, "skipped")
+
+ if self._options.force:
+ self._printer.print_expected('Running all tests, including '
+ 'skips (--force)')
+ else:
+ # Note that we don't actually run the skipped tests (they were
+ # subtracted out of self._test_files, above), but we stub out the
+ # results here so the statistics can remain accurate.
+ for test in skip_chunk:
+ result = test_results.TestResult(test,
+ failures=[], test_run_time=0, total_time_for_all_diffs=0,
+ time_for_diffs=0)
+ result.type = test_expectations.SKIP
+ result_summary.add(result, expected=True)
+ self._printer.print_expected('')
+
+ return result_summary
+
+ def _get_dir_for_test_file(self, test_file):
+ """Returns the highest-level directory by which to shard the given
+ test file."""
+ index = test_file.rfind(os.sep + LAYOUT_TESTS_DIRECTORY)
+
+ test_file = test_file[index + len(LAYOUT_TESTS_DIRECTORY):]
+ test_file_parts = test_file.split(os.sep, 1)
+ directory = test_file_parts[0]
+ test_file = test_file_parts[1]
+
+ # The http tests are very stable on mac/linux.
+ # TODO(ojan): Make the http server on Windows be apache so we can
+ # turn shard the http tests there as well. Switching to apache is
+ # what made them stable on linux/mac.
+ return_value = directory
+ while ((directory != 'http' or sys.platform in ('darwin', 'linux2'))
+ and test_file.find(os.sep) >= 0):
+ test_file_parts = test_file.split(os.sep, 1)
+ directory = test_file_parts[0]
+ return_value = os.path.join(return_value, directory)
+ test_file = test_file_parts[1]
+
+ return return_value
+
+ def _get_test_input_for_file(self, test_file):
+ """Returns the appropriate TestInput object for the file. Mostly this
+ is used for looking up the timeout value (in ms) to use for the given
+ test."""
+ if self._test_is_slow(test_file):
+ return TestInput(test_file, self._options.slow_time_out_ms)
+ return TestInput(test_file, self._options.time_out_ms)
+
+ def _test_requires_lock(self, test_file):
+ """Return True if the test needs to be locked when
+ running multiple copies of NRWTs."""
+ split_path = test_file.split(os.sep)
+ return 'http' in split_path or 'websocket' in split_path
+
+ def _test_is_slow(self, test_file):
+ return self._expectations.has_modifier(test_file,
+ test_expectations.SLOW)
+
+ def _shard_tests(self, test_files, use_real_shards):
+ """Groups tests into batches.
+ This helps ensure that tests that depend on each other (aka bad tests!)
+ continue to run together as most cross-tests dependencies tend to
+ occur within the same directory. If use_real_shards is False, we
+ put each (non-HTTP/websocket) test into its own shard for maximum
+ concurrency instead of trying to do any sort of real sharding.
+
+ Return:
+ A list of lists of TestInput objects.
+ """
+ # FIXME: when we added http locking, we changed how this works such
+ # that we always lump all of the HTTP threads into a single shard.
+ # That will slow down experimental-fully-parallel, but it's unclear
+ # what the best alternative is completely revamping how we track
+ # when to grab the lock.
+
+ test_lists = []
+ tests_to_http_lock = []
+ if not use_real_shards:
+ for test_file in test_files:
+ test_input = self._get_test_input_for_file(test_file)
+ if self._test_requires_lock(test_file):
+ tests_to_http_lock.append(test_input)
+ else:
+ test_lists.append((".", [test_input]))
+ else:
+ tests_by_dir = {}
+ for test_file in test_files:
+ directory = self._get_dir_for_test_file(test_file)
+ test_input = self._get_test_input_for_file(test_file)
+ if self._test_requires_lock(test_file):
+ tests_to_http_lock.append(test_input)
+ else:
+ tests_by_dir.setdefault(directory, [])
+ tests_by_dir[directory].append(test_input)
+ # Sort by the number of tests in the dir so that the ones with the
+ # most tests get run first in order to maximize parallelization.
+ # Number of tests is a good enough, but not perfect, approximation
+ # of how long that set of tests will take to run. We can't just use
+ # a PriorityQueue until we move to Python 2.6.
+ for directory in tests_by_dir:
+ test_list = tests_by_dir[directory]
+ # Keep the tests in alphabetical order.
+ # FIXME: Remove once tests are fixed so they can be run in any
+ # order.
+ test_list.reverse()
+ test_list_tuple = (directory, test_list)
+ test_lists.append(test_list_tuple)
+ test_lists.sort(lambda a, b: cmp(len(b[1]), len(a[1])))
+
+ # Put the http tests first. There are only a couple hundred of them,
+ # but each http test takes a very long time to run, so sorting by the
+ # number of tests doesn't accurately capture how long they take to run.
+ if tests_to_http_lock:
+ tests_to_http_lock.reverse()
+ test_lists.insert(0, ("tests_to_http_lock", tests_to_http_lock))
+
+ return test_lists
+
+ def _contains_tests(self, subdir):
+ for test_file in self._test_files:
+ if test_file.find(subdir) >= 0:
+ return True
+ return False
+
+ def _num_workers(self):
+ return int(self._options.child_processes)
+
+ def _run_tests(self, file_list, result_summary):
+ """Runs the tests in the file_list.
+
+ Return: A tuple (interrupted, keyboard_interrupted, thread_timings,
+ test_timings, individual_test_timings)
+ interrupted is whether the run was interrupted
+ keyboard_interrupted is whether the interruption was because someone
+ typed Ctrl^C
+ thread_timings is a list of dicts with the total runtime
+ of each thread with 'name', 'num_tests', 'total_time' properties
+ test_timings is a list of timings for each sharded subdirectory
+ of the form [time, directory_name, num_tests]
+ individual_test_timings is a list of run times for each test
+ in the form {filename:filename, test_run_time:test_run_time}
+ result_summary: summary object to populate with the results
+ """
+
+ self._printer.print_update('Sharding tests ...')
+ num_workers = self._num_workers()
+ test_lists = self._shard_tests(file_list,
+ num_workers > 1 and not self._options.experimental_fully_parallel)
+ filename_queue = Queue.Queue()
+ for item in test_lists:
+ filename_queue.put(item)
+
+ self._printer.print_update('Starting %s ...' %
+ grammar.pluralize('worker', num_workers))
+ self._message_broker = message_broker.get(self._port, self._options)
+ broker = self._message_broker
+ self._current_filename_queue = filename_queue
+ self._current_result_summary = result_summary
+
+ if not self._options.dry_run:
+ threads = broker.start_workers(self)
+ else:
+ threads = {}
+
+ self._printer.print_update("Starting testing ...")
+ keyboard_interrupted = False
+ interrupted = False
+ if not self._options.dry_run:
+ try:
+ broker.run_message_loop()
+ except KeyboardInterrupt:
+ _log.info("Interrupted, exiting")
+ broker.cancel_workers()
+ keyboard_interrupted = True
+ interrupted = True
+ except TestRunInterruptedException, e:
+ _log.info(e.reason)
+ broker.cancel_workers()
+ interrupted = True
+ except:
+ # Unexpected exception; don't try to clean up workers.
+ _log.info("Exception raised, exiting")
+ raise
+
+ thread_timings, test_timings, individual_test_timings = \
+ self._collect_timing_info(threads)
+
+ broker.cleanup()
+ self._message_broker = None
+ return (interrupted, keyboard_interrupted, thread_timings, test_timings,
+ individual_test_timings)
+
+ def update(self):
+ self.update_summary(self._current_result_summary)
+
+ def _collect_timing_info(self, threads):
+ test_timings = {}
+ individual_test_timings = []
+ thread_timings = []
+
+ for thread in threads:
+ thread_timings.append({'name': thread.getName(),
+ 'num_tests': thread.get_num_tests(),
+ 'total_time': thread.get_total_time()})
+ test_timings.update(thread.get_test_group_timing_stats())
+ individual_test_timings.extend(thread.get_test_results())
+
+ return (thread_timings, test_timings, individual_test_timings)
+
+ def needs_http(self):
+ """Returns whether the test runner needs an HTTP server."""
+ return self._contains_tests(self.HTTP_SUBDIR)
+
+ def needs_websocket(self):
+ """Returns whether the test runner needs a WEBSOCKET server."""
+ return self._contains_tests(self.WEBSOCKET_SUBDIR)
+
+ def set_up_run(self):
+ """Configures the system to be ready to run tests.
+
+ Returns a ResultSummary object if we should continue to run tests,
+ or None if we should abort.
+
+ """
+ # This must be started before we check the system dependencies,
+ # since the helper may do things to make the setup correct.
+ self._printer.print_update("Starting helper ...")
+ self._port.start_helper()
+
+ # Check that the system dependencies (themes, fonts, ...) are correct.
+ if not self._options.nocheck_sys_deps:
+ self._printer.print_update("Checking system dependencies ...")
+ if not self._port.check_sys_deps(self.needs_http()):
+ self._port.stop_helper()
+ return None
+
+ if self._options.clobber_old_results:
+ self._clobber_old_results()
+
+ # Create the output directory if it doesn't already exist.
+ self._port.maybe_make_directory(self._options.results_directory)
+
+ self._port.setup_test_run()
+
+ self._printer.print_update("Preparing tests ...")
+ result_summary = self.prepare_lists_and_print_output()
+ if not result_summary:
+ return None
+
+ return result_summary
+
+ def run(self, result_summary):
+ """Run all our tests on all our test files.
+
+ For each test file, we run each test type. If there are any failures,
+ we collect them for reporting.
+
+ Args:
+ result_summary: a summary object tracking the test results.
+
+ Return:
+ The number of unexpected results (0 == success)
+ """
+ # gather_test_files() must have been called first to initialize us.
+ # If we didn't find any files to test, we've errored out already in
+ # prepare_lists_and_print_output().
+ assert(len(self._test_files))
+
+ start_time = time.time()
+
+ interrupted, keyboard_interrupted, thread_timings, test_timings, \
+ individual_test_timings = (
+ self._run_tests(self._test_files_list, result_summary))
+
+ # We exclude the crashes from the list of results to retry, because
+ # we want to treat even a potentially flaky crash as an error.
+ failures = self._get_failures(result_summary, include_crashes=False)
+ retry_summary = result_summary
+ while (len(failures) and self._options.retry_failures and
+ not self._retrying and not interrupted):
+ _log.info('')
+ _log.info("Retrying %d unexpected failure(s) ..." % len(failures))
+ _log.info('')
+ self._retrying = True
+ retry_summary = ResultSummary(self._expectations, failures.keys())
+ # Note that we intentionally ignore the return value here.
+ self._run_tests(failures.keys(), retry_summary)
+ failures = self._get_failures(retry_summary, include_crashes=True)
+
+ end_time = time.time()
+
+ self._print_timing_statistics(end_time - start_time,
+ thread_timings, test_timings,
+ individual_test_timings,
+ result_summary)
+
+ self._print_result_summary(result_summary)
+
+ sys.stdout.flush()
+ sys.stderr.flush()
+
+ self._printer.print_one_line_summary(result_summary.total,
+ result_summary.expected,
+ result_summary.unexpected)
+
+ unexpected_results = summarize_unexpected_results(self._port,
+ self._expectations, result_summary, retry_summary)
+ self._printer.print_unexpected_results(unexpected_results)
+
+ if (self._options.record_results and not self._options.dry_run and
+ not interrupted):
+ # Write the same data to log files and upload generated JSON files
+ # to appengine server.
+ self._upload_json_files(unexpected_results, result_summary,
+ individual_test_timings)
+
+ # Write the summary to disk (results.html) and display it if requested.
+ if not self._options.dry_run:
+ wrote_results = self._write_results_html_file(result_summary)
+ if self._options.show_results and wrote_results:
+ self._show_results_html_file()
+
+ # Now that we've completed all the processing we can, we re-raise
+ # a KeyboardInterrupt if necessary so the caller can handle it.
+ if keyboard_interrupted:
+ raise KeyboardInterrupt
+
+ # Ignore flaky failures and unexpected passes so we don't turn the
+ # bot red for those.
+ return unexpected_results['num_regressions']
+
+ def clean_up_run(self):
+ """Restores the system after we're done running tests."""
+
+ _log.debug("flushing stdout")
+ sys.stdout.flush()
+ _log.debug("flushing stderr")
+ sys.stderr.flush()
+ _log.debug("stopping helper")
+ self._port.stop_helper()
+
+ def update_summary(self, result_summary):
+ """Update the summary and print results with any completed tests."""
+ while True:
+ try:
+ result = test_results.TestResult.loads(self._result_queue.get_nowait())
+ except Queue.Empty:
+ return
+
+ expected = self._expectations.matches_an_expected_result(
+ result.filename, result.type, self._options.pixel_tests)
+ result_summary.add(result, expected)
+ exp_str = self._expectations.get_expectations_string(
+ result.filename)
+ got_str = self._expectations.expectation_to_string(result.type)
+ self._printer.print_test_result(result, expected, exp_str, got_str)
+ self._printer.print_progress(result_summary, self._retrying,
+ self._test_files_list)
+
+ def interrupt_if_at_failure_limit(limit, count, message):
+ if limit and count >= limit:
+ raise TestRunInterruptedException(message % count)
+
+ interrupt_if_at_failure_limit(
+ self._options.exit_after_n_failures,
+ result_summary.unexpected_failures,
+ "Aborting run since %d failures were reached")
+ interrupt_if_at_failure_limit(
+ self._options.exit_after_n_crashes_or_timeouts,
+ result_summary.unexpected_crashes_or_timeouts,
+ "Aborting run since %d crashes or timeouts were reached")
+
+ def _clobber_old_results(self):
+ # Just clobber the actual test results directories since the other
+ # files in the results directory are explicitly used for cross-run
+ # tracking.
+ self._printer.print_update("Clobbering old results in %s" %
+ self._options.results_directory)
+ layout_tests_dir = self._port.layout_tests_dir()
+ possible_dirs = self._port.test_dirs()
+ for dirname in possible_dirs:
+ if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
+ shutil.rmtree(os.path.join(self._options.results_directory,
+ dirname),
+ ignore_errors=True)
+
+ def _get_failures(self, result_summary, include_crashes):
+ """Filters a dict of results and returns only the failures.
+
+ Args:
+ result_summary: the results of the test run
+ include_crashes: whether crashes are included in the output.
+ We use False when finding the list of failures to retry
+ to see if the results were flaky. Although the crashes may also be
+ flaky, we treat them as if they aren't so that they're not ignored.
+ Returns:
+ a dict of files -> results
+ """
+ failed_results = {}
+ for test, result in result_summary.unexpected_results.iteritems():
+ if (result == test_expectations.PASS or
+ result == test_expectations.CRASH and not include_crashes):
+ continue
+ failed_results[test] = result
+
+ return failed_results
+
+ def _upload_json_files(self, unexpected_results, result_summary,
+ individual_test_timings):
+ """Writes the results of the test run as JSON files into the results
+ dir and upload the files to the appengine server.
+
+ There are three different files written into the results dir:
+ unexpected_results.json: A short list of any unexpected results.
+ This is used by the buildbots to display results.
+ expectations.json: This is used by the flakiness dashboard.
+ results.json: A full list of the results - used by the flakiness
+ dashboard and the aggregate results dashboard.
+
+ Args:
+ unexpected_results: dict of unexpected results
+ result_summary: full summary object
+ individual_test_timings: list of test times (used by the flakiness
+ dashboard).
+ """
+ results_directory = self._options.results_directory
+ _log.debug("Writing JSON files in %s." % results_directory)
+ unexpected_json_path = os.path.join(results_directory, "unexpected_results.json")
+ with codecs.open(unexpected_json_path, "w", "utf-8") as file:
+ simplejson.dump(unexpected_results, file, sort_keys=True, indent=2)
+
+ # Write a json file of the test_expectations.txt file for the layout
+ # tests dashboard.
+ expectations_path = os.path.join(results_directory, "expectations.json")
+ expectations_json = \
+ self._expectations.get_expectations_json_for_all_platforms()
+ with codecs.open(expectations_path, "w", "utf-8") as file:
+ file.write(u"ADD_EXPECTATIONS(%s);" % expectations_json)
+
+ generator = json_layout_results_generator.JSONLayoutResultsGenerator(
+ self._port, self._options.builder_name, self._options.build_name,
+ self._options.build_number, self._options.results_directory,
+ BUILDER_BASE_URL, individual_test_timings,
+ self._expectations, result_summary, self._test_files_list,
+ not self._options.upload_full_results,
+ self._options.test_results_server,
+ "layout-tests",
+ self._options.master_name)
+
+ _log.debug("Finished writing JSON files.")
+
+ json_files = ["expectations.json"]
+ if self._options.upload_full_results:
+ json_files.append("results.json")
+ else:
+ json_files.append("incremental_results.json")
+
+ generator.upload_json_files(json_files)
+
+ def _print_config(self):
+ """Prints the configuration for the test run."""
+ p = self._printer
+ p.print_config("Using port '%s'" % self._port.name())
+ p.print_config("Placing test results in %s" %
+ self._options.results_directory)
+ if self._options.new_baseline:
+ p.print_config("Placing new baselines in %s" %
+ self._port.baseline_path())
+ p.print_config("Using %s build" % self._options.configuration)
+ if self._options.pixel_tests:
+ p.print_config("Pixel tests enabled")
+ else:
+ p.print_config("Pixel tests disabled")
+
+ p.print_config("Regular timeout: %s, slow test timeout: %s" %
+ (self._options.time_out_ms,
+ self._options.slow_time_out_ms))
+
+ if self._num_workers() == 1:
+ p.print_config("Running one %s" % self._port.driver_name())
+ else:
+ p.print_config("Running %s %ss in parallel" %
+ (self._options.child_processes,
+ self._port.driver_name()))
+ p.print_config('Command line: ' +
+ ' '.join(self._port.driver_cmd_line()))
+ p.print_config("Worker model: %s" % self._options.worker_model)
+ p.print_config("")
+
+ def _print_expected_results_of_type(self, result_summary,
+ result_type, result_type_str):
+ """Print the number of the tests in a given result class.
+
+ Args:
+ result_summary - the object containing all the results to report on
+ result_type - the particular result type to report in the summary.
+ result_type_str - a string description of the result_type.
+ """
+ tests = self._expectations.get_tests_with_result_type(result_type)
+ now = result_summary.tests_by_timeline[test_expectations.NOW]
+ wontfix = result_summary.tests_by_timeline[test_expectations.WONTFIX]
+
+ # We use a fancy format string in order to print the data out in a
+ # nicely-aligned table.
+ fmtstr = ("Expect: %%5d %%-8s (%%%dd now, %%%dd wontfix)"
+ % (self._num_digits(now), self._num_digits(wontfix)))
+ self._printer.print_expected(fmtstr %
+ (len(tests), result_type_str, len(tests & now), len(tests & wontfix)))
+
+ def _num_digits(self, num):
+ """Returns the number of digits needed to represent the length of a
+ sequence."""
+ ndigits = 1
+ if len(num):
+ ndigits = int(math.log10(len(num))) + 1
+ return ndigits
+
+ def _print_timing_statistics(self, total_time, thread_timings,
+ directory_test_timings, individual_test_timings,
+ result_summary):
+ """Record timing-specific information for the test run.
+
+ Args:
+ total_time: total elapsed time (in seconds) for the test run
+ thread_timings: wall clock time each thread ran for
+ directory_test_timings: timing by directory
+ individual_test_timings: timing by file
+ result_summary: summary object for the test run
+ """
+ self._printer.print_timing("Test timing:")
+ self._printer.print_timing(" %6.2f total testing time" % total_time)
+ self._printer.print_timing("")
+ self._printer.print_timing("Thread timing:")
+ cuml_time = 0
+ for t in thread_timings:
+ self._printer.print_timing(" %10s: %5d tests, %6.2f secs" %
+ (t['name'], t['num_tests'], t['total_time']))
+ cuml_time += t['total_time']
+ self._printer.print_timing(" %6.2f cumulative, %6.2f optimal" %
+ (cuml_time, cuml_time / int(self._options.child_processes)))
+ self._printer.print_timing("")
+
+ self._print_aggregate_test_statistics(individual_test_timings)
+ self._print_individual_test_times(individual_test_timings,
+ result_summary)
+ self._print_directory_timings(directory_test_timings)
+
+ def _print_aggregate_test_statistics(self, individual_test_timings):
+ """Prints aggregate statistics (e.g. median, mean, etc.) for all tests.
+ Args:
+ individual_test_timings: List of TestResults for all tests.
+ """
+ test_types = [] # Unit tests don't actually produce any timings.
+ if individual_test_timings:
+ test_types = individual_test_timings[0].time_for_diffs.keys()
+ times_for_dump_render_tree = []
+ times_for_diff_processing = []
+ times_per_test_type = {}
+ for test_type in test_types:
+ times_per_test_type[test_type] = []
+
+ for test_stats in individual_test_timings:
+ times_for_dump_render_tree.append(test_stats.test_run_time)
+ times_for_diff_processing.append(
+ test_stats.total_time_for_all_diffs)
+ time_for_diffs = test_stats.time_for_diffs
+ for test_type in test_types:
+ times_per_test_type[test_type].append(
+ time_for_diffs[test_type])
+
+ self._print_statistics_for_test_timings(
+ "PER TEST TIME IN TESTSHELL (seconds):",
+ times_for_dump_render_tree)
+ self._print_statistics_for_test_timings(
+ "PER TEST DIFF PROCESSING TIMES (seconds):",
+ times_for_diff_processing)
+ for test_type in test_types:
+ self._print_statistics_for_test_timings(
+ "PER TEST TIMES BY TEST TYPE: %s" % test_type,
+ times_per_test_type[test_type])
+
+ def _print_individual_test_times(self, individual_test_timings,
+ result_summary):
+ """Prints the run times for slow, timeout and crash tests.
+ Args:
+ individual_test_timings: List of TestStats for all tests.
+ result_summary: summary object for test run
+ """
+ # Reverse-sort by the time spent in DumpRenderTree.
+ individual_test_timings.sort(lambda a, b:
+ cmp(b.test_run_time, a.test_run_time))
+
+ num_printed = 0
+ slow_tests = []
+ timeout_or_crash_tests = []
+ unexpected_slow_tests = []
+ for test_tuple in individual_test_timings:
+ filename = test_tuple.filename
+ is_timeout_crash_or_slow = False
+ if self._test_is_slow(filename):
+ is_timeout_crash_or_slow = True
+ slow_tests.append(test_tuple)
+
+ if filename in result_summary.failures:
+ result = result_summary.results[filename].type
+ if (result == test_expectations.TIMEOUT or
+ result == test_expectations.CRASH):
+ is_timeout_crash_or_slow = True
+ timeout_or_crash_tests.append(test_tuple)
+
+ if (not is_timeout_crash_or_slow and
+ num_printed < printing.NUM_SLOW_TESTS_TO_LOG):
+ num_printed = num_printed + 1
+ unexpected_slow_tests.append(test_tuple)
+
+ self._printer.print_timing("")
+ self._print_test_list_timing("%s slowest tests that are not "
+ "marked as SLOW and did not timeout/crash:" %
+ printing.NUM_SLOW_TESTS_TO_LOG, unexpected_slow_tests)
+ self._printer.print_timing("")
+ self._print_test_list_timing("Tests marked as SLOW:", slow_tests)
+ self._printer.print_timing("")
+ self._print_test_list_timing("Tests that timed out or crashed:",
+ timeout_or_crash_tests)
+ self._printer.print_timing("")
+
+ def _print_test_list_timing(self, title, test_list):
+ """Print timing info for each test.
+
+ Args:
+ title: section heading
+ test_list: tests that fall in this section
+ """
+ if self._printer.disabled('slowest'):
+ return
+
+ self._printer.print_timing(title)
+ for test_tuple in test_list:
+ filename = test_tuple.filename[len(
+ self._port.layout_tests_dir()) + 1:]
+ filename = filename.replace('\\', '/')
+ test_run_time = round(test_tuple.test_run_time, 1)
+ self._printer.print_timing(" %s took %s seconds" %
+ (filename, test_run_time))
+
+ def _print_directory_timings(self, directory_test_timings):
+ """Print timing info by directory for any directories that
+ take > 10 seconds to run.
+
+ Args:
+ directory_test_timing: time info for each directory
+ """
+ timings = []
+ for directory in directory_test_timings:
+ num_tests, time_for_directory = directory_test_timings[directory]
+ timings.append((round(time_for_directory, 1), directory,
+ num_tests))
+ timings.sort()
+
+ self._printer.print_timing("Time to process slowest subdirectories:")
+ min_seconds_to_print = 10
+ for timing in timings:
+ if timing[0] > min_seconds_to_print:
+ self._printer.print_timing(
+ " %s took %s seconds to run %s tests." % (timing[1],
+ timing[0], timing[2]))
+ self._printer.print_timing("")
+
+ def _print_statistics_for_test_timings(self, title, timings):
+ """Prints the median, mean and standard deviation of the values in
+ timings.
+
+ Args:
+ title: Title for these timings.
+ timings: A list of floats representing times.
+ """
+ self._printer.print_timing(title)
+ timings.sort()
+
+ num_tests = len(timings)
+ if not num_tests:
+ return
+ percentile90 = timings[int(.9 * num_tests)]
+ percentile99 = timings[int(.99 * num_tests)]
+
+ if num_tests % 2 == 1:
+ median = timings[((num_tests - 1) / 2) - 1]
+ else:
+ lower = timings[num_tests / 2 - 1]
+ upper = timings[num_tests / 2]
+ median = (float(lower + upper)) / 2
+
+ mean = sum(timings) / num_tests
+
+ for time in timings:
+ sum_of_deviations = math.pow(time - mean, 2)
+
+ std_deviation = math.sqrt(sum_of_deviations / num_tests)
+ self._printer.print_timing(" Median: %6.3f" % median)
+ self._printer.print_timing(" Mean: %6.3f" % mean)
+ self._printer.print_timing(" 90th percentile: %6.3f" % percentile90)
+ self._printer.print_timing(" 99th percentile: %6.3f" % percentile99)
+ self._printer.print_timing(" Standard dev: %6.3f" % std_deviation)
+ self._printer.print_timing("")
+
+ def _print_result_summary(self, result_summary):
+ """Print a short summary about how many tests passed.
+
+ Args:
+ result_summary: information to log
+ """
+ failed = len(result_summary.failures)
+ skipped = len(
+ result_summary.tests_by_expectation[test_expectations.SKIP])
+ total = result_summary.total
+ passed = total - failed - skipped
+ pct_passed = 0.0
+ if total > 0:
+ pct_passed = float(passed) * 100 / total
+
+ self._printer.print_actual("")
+ self._printer.print_actual("=> Results: %d/%d tests passed (%.1f%%)" %
+ (passed, total, pct_passed))
+ self._printer.print_actual("")
+ self._print_result_summary_entry(result_summary,
+ test_expectations.NOW, "Tests to be fixed")
+
+ self._printer.print_actual("")
+ self._print_result_summary_entry(result_summary,
+ test_expectations.WONTFIX,
+ "Tests that will only be fixed if they crash (WONTFIX)")
+ self._printer.print_actual("")
+
+ def _print_result_summary_entry(self, result_summary, timeline,
+ heading):
+ """Print a summary block of results for a particular timeline of test.
+
+ Args:
+ result_summary: summary to print results for
+ timeline: the timeline to print results for (NOT, WONTFIX, etc.)
+ heading: a textual description of the timeline
+ """
+ total = len(result_summary.tests_by_timeline[timeline])
+ not_passing = (total -
+ len(result_summary.tests_by_expectation[test_expectations.PASS] &
+ result_summary.tests_by_timeline[timeline]))
+ self._printer.print_actual("=> %s (%d):" % (heading, not_passing))
+
+ for result in TestExpectationsFile.EXPECTATION_ORDER:
+ if result == test_expectations.PASS:
+ continue
+ results = (result_summary.tests_by_expectation[result] &
+ result_summary.tests_by_timeline[timeline])
+ desc = TestExpectationsFile.EXPECTATION_DESCRIPTIONS[result]
+ if not_passing and len(results):
+ pct = len(results) * 100.0 / not_passing
+ self._printer.print_actual(" %5d %-24s (%4.1f%%)" %
+ (len(results), desc[len(results) != 1], pct))
+
+ def _results_html(self, test_files, failures, title="Test Failures", override_time=None):
+ """
+ test_files = a list of file paths
+ failures = dictionary mapping test paths to failure objects
+ title = title printed at top of test
+ override_time = current time (used by unit tests)
+ """
+ page = """<html>
+ <head>
+ <title>Layout Test Results (%(time)s)</title>
+ </head>
+ <body>
+ <h2>%(title)s (%(time)s)</h2>
+ """ % {'title': title, 'time': override_time or time.asctime()}
+
+ for test_file in sorted(test_files):
+ test_name = self._port.relative_test_filename(test_file)
+ test_url = self._port.filename_to_uri(test_file)
+ page += u"<p><a href='%s'>%s</a><br />\n" % (test_url, test_name)
+ test_failures = failures.get(test_file, [])
+ for failure in test_failures:
+ page += (u"&nbsp;&nbsp;%s<br/>" %
+ failure.result_html_output(test_name))
+ page += "</p>\n"
+ page += "</body></html>\n"
+ return page
+
+ def _write_results_html_file(self, result_summary):
+ """Write results.html which is a summary of tests that failed.
+
+ Args:
+ result_summary: a summary of the results :)
+
+ Returns:
+ True if any results were written (since expected failures may be
+ omitted)
+ """
+ # test failures
+ if self._options.full_results_html:
+ results_title = "Test Failures"
+ test_files = result_summary.failures.keys()
+ else:
+ results_title = "Unexpected Test Failures"
+ unexpected_failures = self._get_failures(result_summary,
+ include_crashes=True)
+ test_files = unexpected_failures.keys()
+ if not len(test_files):
+ return False
+
+ out_filename = os.path.join(self._options.results_directory,
+ "results.html")
+ with codecs.open(out_filename, "w", "utf-8") as results_file:
+ html = self._results_html(test_files, result_summary.failures, results_title)
+ results_file.write(html)
+
+ return True
+
+ def _show_results_html_file(self):
+ """Shows the results.html page."""
+ results_filename = os.path.join(self._options.results_directory,
+ "results.html")
+ self._port.show_results_html_file(results_filename)
+
+
+def read_test_files(files):
+ tests = []
+ for file in files:
+ try:
+ with codecs.open(file, 'r', 'utf-8') as file_contents:
+ # FIXME: This could be cleaner using a list comprehension.
+ for line in file_contents:
+ line = test_expectations.strip_comments(line)
+ if line:
+ tests.append(line)
+ except IOError, e:
+ if e.errno == errno.ENOENT:
+ _log.critical('')
+ _log.critical('--test-list file "%s" not found' % file)
+ raise
+ return tests
diff --git a/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner_unittest.py b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner_unittest.py
new file mode 100644
index 0000000..3c564ae
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/layout_package/test_runner_unittest.py
@@ -0,0 +1,102 @@
+#!/usr/bin/python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit tests for TestRunner()."""
+
+import unittest
+
+from webkitpy.thirdparty.mock import Mock
+
+import test_runner
+
+
+class TestRunnerWrapper(test_runner.TestRunner):
+ def _get_test_input_for_file(self, test_file):
+ return test_file
+
+
+class TestRunnerTest(unittest.TestCase):
+ def test_results_html(self):
+ mock_port = Mock()
+ mock_port.relative_test_filename = lambda name: name
+ mock_port.filename_to_uri = lambda name: name
+
+ runner = test_runner.TestRunner(port=mock_port, options=Mock(),
+ printer=Mock())
+ expected_html = u"""<html>
+ <head>
+ <title>Layout Test Results (time)</title>
+ </head>
+ <body>
+ <h2>Title (time)</h2>
+ <p><a href='test_path'>test_path</a><br />
+</p>
+</body></html>
+"""
+ html = runner._results_html(["test_path"], {}, "Title", override_time="time")
+ self.assertEqual(html, expected_html)
+
+ def test_shard_tests(self):
+ # Test that _shard_tests in test_runner.TestRunner really
+ # put the http tests first in the queue.
+ runner = TestRunnerWrapper(port=Mock(), options=Mock(),
+ printer=Mock())
+
+ test_list = [
+ "LayoutTests/websocket/tests/unicode.htm",
+ "LayoutTests/animations/keyframes.html",
+ "LayoutTests/http/tests/security/view-source-no-refresh.html",
+ "LayoutTests/websocket/tests/websocket-protocol-ignored.html",
+ "LayoutTests/fast/css/display-none-inline-style-change-crash.html",
+ "LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html",
+ "LayoutTests/dom/html/level2/html/HTMLAnchorElement03.html",
+ "LayoutTests/ietestcenter/Javascript/11.1.5_4-4-c-1.html",
+ "LayoutTests/dom/html/level2/html/HTMLAnchorElement06.html",
+ ]
+
+ expected_tests_to_http_lock = set([
+ 'LayoutTests/websocket/tests/unicode.htm',
+ 'LayoutTests/http/tests/security/view-source-no-refresh.html',
+ 'LayoutTests/websocket/tests/websocket-protocol-ignored.html',
+ 'LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html',
+ ])
+
+ # FIXME: Ideally the HTTP tests don't have to all be in one shard.
+ single_thread_results = runner._shard_tests(test_list, False)
+ multi_thread_results = runner._shard_tests(test_list, True)
+
+ self.assertEqual("tests_to_http_lock", single_thread_results[0][0])
+ self.assertEqual(expected_tests_to_http_lock, set(single_thread_results[0][1]))
+ self.assertEqual("tests_to_http_lock", multi_thread_results[0][0])
+ self.assertEqual(expected_tests_to_http_lock, set(multi_thread_results[0][1]))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/__init__.py b/Tools/Scripts/webkitpy/layout_tests/port/__init__.py
new file mode 100644
index 0000000..e3ad6f4
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/__init__.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Port-specific entrypoints for the layout tests test infrastructure."""
+
+from factory import get
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/apache_http_server.py b/Tools/Scripts/webkitpy/layout_tests/port/apache_http_server.py
new file mode 100644
index 0000000..46617f6
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/apache_http_server.py
@@ -0,0 +1,230 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""A class to start/stop the apache http server used by layout tests."""
+
+
+from __future__ import with_statement
+
+import codecs
+import logging
+import optparse
+import os
+import re
+import subprocess
+import sys
+
+import http_server_base
+
+_log = logging.getLogger("webkitpy.layout_tests.port.apache_http_server")
+
+
+class LayoutTestApacheHttpd(http_server_base.HttpServerBase):
+
+ def __init__(self, port_obj, output_dir):
+ """Args:
+ port_obj: handle to the platform-specific routines
+ output_dir: the absolute path to the layout test result directory
+ """
+ http_server_base.HttpServerBase.__init__(self, port_obj)
+ self._output_dir = output_dir
+ self._httpd_proc = None
+ port_obj.maybe_make_directory(output_dir)
+
+ self.mappings = [{'port': 8000},
+ {'port': 8080},
+ {'port': 8081},
+ {'port': 8443, 'sslcert': True}]
+
+ # The upstream .conf file assumed the existence of /tmp/WebKit for
+ # placing apache files like the lock file there.
+ self._runtime_path = os.path.join("/tmp", "WebKit")
+ port_obj.maybe_make_directory(self._runtime_path)
+
+ # The PID returned when Apache is started goes away (due to dropping
+ # privileges?). The proper controlling PID is written to a file in the
+ # apache runtime directory.
+ self._pid_file = os.path.join(self._runtime_path, 'httpd.pid')
+
+ test_dir = self._port_obj.layout_tests_dir()
+ js_test_resources_dir = self._cygwin_safe_join(test_dir, "fast", "js",
+ "resources")
+ mime_types_path = self._cygwin_safe_join(test_dir, "http", "conf",
+ "mime.types")
+ cert_file = self._cygwin_safe_join(test_dir, "http", "conf",
+ "webkit-httpd.pem")
+ access_log = self._cygwin_safe_join(output_dir, "access_log.txt")
+ error_log = self._cygwin_safe_join(output_dir, "error_log.txt")
+ document_root = self._cygwin_safe_join(test_dir, "http", "tests")
+
+ # FIXME: We shouldn't be calling a protected method of _port_obj!
+ executable = self._port_obj._path_to_apache()
+ if self._is_cygwin():
+ executable = self._get_cygwin_path(executable)
+
+ cmd = [executable,
+ '-f', "\"%s\"" % self._get_apache_config_file_path(test_dir, output_dir),
+ '-C', "\'DocumentRoot \"%s\"\'" % document_root,
+ '-c', "\'Alias /js-test-resources \"%s\"'" % js_test_resources_dir,
+ '-C', "\'Listen %s\'" % "127.0.0.1:8000",
+ '-C', "\'Listen %s\'" % "127.0.0.1:8081",
+ '-c', "\'TypesConfig \"%s\"\'" % mime_types_path,
+ '-c', "\'CustomLog \"%s\" common\'" % access_log,
+ '-c', "\'ErrorLog \"%s\"\'" % error_log,
+ '-C', "\'User \"%s\"\'" % os.environ.get("USERNAME",
+ os.environ.get("USER", ""))]
+
+ if self._is_cygwin():
+ cygbin = self._port_obj._path_from_base('third_party', 'cygwin',
+ 'bin')
+ # Not entirely sure why, but from cygwin we need to run the
+ # httpd command through bash.
+ self._start_cmd = [
+ os.path.join(cygbin, 'bash.exe'),
+ '-c',
+ 'PATH=%s %s' % (self._get_cygwin_path(cygbin), " ".join(cmd)),
+ ]
+ else:
+ # TODO(ojan): When we get cygwin using Apache 2, use set the
+ # cert file for cygwin as well.
+ cmd.extend(['-c', "\'SSLCertificateFile %s\'" % cert_file])
+ # Join the string here so that Cygwin/Windows and Mac/Linux
+ # can use the same code. Otherwise, we could remove the single
+ # quotes above and keep cmd as a sequence.
+ self._start_cmd = " ".join(cmd)
+
+ def _is_cygwin(self):
+ return sys.platform in ("win32", "cygwin")
+
+ def _cygwin_safe_join(self, *parts):
+ """Returns a platform appropriate path."""
+ path = os.path.join(*parts)
+ if self._is_cygwin():
+ return self._get_cygwin_path(path)
+ return path
+
+ def _get_cygwin_path(self, path):
+ """Convert a Windows path to a cygwin path.
+
+ The cygpath utility insists on converting paths that it thinks are
+ Cygwin root paths to what it thinks the correct roots are. So paths
+ such as "C:\b\slave\webkit-release\build\third_party\cygwin\bin"
+ are converted to plain "/usr/bin". To avoid this, we
+ do the conversion manually.
+
+ The path is expected to be an absolute path, on any drive.
+ """
+ drive_regexp = re.compile(r'([a-z]):[/\\]', re.IGNORECASE)
+
+ def lower_drive(matchobj):
+ return '/cygdrive/%s/' % matchobj.group(1).lower()
+ path = drive_regexp.sub(lower_drive, path)
+ return path.replace('\\', '/')
+
+ def _get_apache_config_file_path(self, test_dir, output_dir):
+ """Returns the path to the apache config file to use.
+ Args:
+ test_dir: absolute path to the LayoutTests directory.
+ output_dir: absolute path to the layout test results directory.
+ """
+ httpd_config = self._port_obj._path_to_apache_config_file()
+ httpd_config_copy = os.path.join(output_dir, "httpd.conf")
+ # httpd.conf is always utf-8 according to http://archive.apache.org/gnats/10125
+ with codecs.open(httpd_config, "r", "utf-8") as httpd_config_file:
+ httpd_conf = httpd_config_file.read()
+ if self._is_cygwin():
+ # This is a gross hack, but it lets us use the upstream .conf file
+ # and our checked in cygwin. This tells the server the root
+ # directory to look in for .so modules. It will use this path
+ # plus the relative paths to the .so files listed in the .conf
+ # file. We have apache/cygwin checked into our tree so
+ # people don't have to install it into their cygwin.
+ cygusr = self._port_obj._path_from_base('third_party', 'cygwin',
+ 'usr')
+ httpd_conf = httpd_conf.replace('ServerRoot "/usr"',
+ 'ServerRoot "%s"' % self._get_cygwin_path(cygusr))
+
+ with codecs.open(httpd_config_copy, "w", "utf-8") as file:
+ file.write(httpd_conf)
+
+ if self._is_cygwin():
+ return self._get_cygwin_path(httpd_config_copy)
+ return httpd_config_copy
+
+ def _get_virtual_host_config(self, document_root, port, ssl=False):
+ """Returns a <VirtualHost> directive block for an httpd.conf file.
+ It will listen to 127.0.0.1 on each of the given port.
+ """
+ return '\n'.join(('<VirtualHost 127.0.0.1:%s>' % port,
+ 'DocumentRoot "%s"' % document_root,
+ ssl and 'SSLEngine On' or '',
+ '</VirtualHost>', ''))
+
+ def _start_httpd_process(self):
+ """Starts the httpd process and returns whether there were errors."""
+ # Use shell=True because we join the arguments into a string for
+ # the sake of Window/Cygwin and it needs quoting that breaks
+ # shell=False.
+ # FIXME: We should not need to be joining shell arguments into strings.
+ # shell=True is a trail of tears.
+ # Note: Not thread safe: http://bugs.python.org/issue2320
+ self._httpd_proc = subprocess.Popen(self._start_cmd,
+ stderr=subprocess.PIPE,
+ shell=True)
+ err = self._httpd_proc.stderr.read()
+ if len(err):
+ _log.debug(err)
+ return False
+ return True
+
+ def start(self):
+ """Starts the apache http server."""
+ # Stop any currently running servers.
+ self.stop()
+
+ _log.debug("Starting apache http server")
+ server_started = self.wait_for_action(self._start_httpd_process)
+ if server_started:
+ _log.debug("Apache started. Testing ports")
+ server_started = self.wait_for_action(
+ self.is_server_running_on_all_ports)
+
+ if server_started:
+ _log.debug("Server successfully started")
+ else:
+ raise Exception('Failed to start http server')
+
+ def stop(self):
+ """Stops the apache http server."""
+ _log.debug("Shutting down any running http servers")
+ httpd_pid = None
+ if os.path.exists(self._pid_file):
+ httpd_pid = int(open(self._pid_file).readline())
+ # FIXME: We shouldn't be calling a protected method of _port_obj!
+ self._port_obj._shut_down_http_server(httpd_pid)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/base.py b/Tools/Scripts/webkitpy/layout_tests/port/base.py
new file mode 100644
index 0000000..97b54c9
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/base.py
@@ -0,0 +1,862 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the Google name nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Abstract base class of Port-specific entrypoints for the layout tests
+test infrastructure (the Port and Driver classes)."""
+
+import cgi
+import difflib
+import errno
+import os
+import shlex
+import sys
+import time
+
+import apache_http_server
+import config as port_config
+import http_lock
+import http_server
+import test_files
+import websocket_server
+
+from webkitpy.common import system
+from webkitpy.common.system import filesystem
+from webkitpy.common.system import logutils
+from webkitpy.common.system import path
+from webkitpy.common.system.executive import Executive, ScriptError
+from webkitpy.common.system.user import User
+
+
+_log = logutils.get_logger(__file__)
+
+
+class DummyOptions(object):
+ """Fake implementation of optparse.Values. Cloned from
+ webkitpy.tool.mocktool.MockOptions.
+
+ """
+
+ def __init__(self, **kwargs):
+ # The caller can set option values using keyword arguments. We don't
+ # set any values by default because we don't know how this
+ # object will be used. Generally speaking unit tests should
+ # subclass this or provider wrapper functions that set a common
+ # set of options.
+ for key, value in kwargs.items():
+ self.__dict__[key] = value
+
+
+# FIXME: This class should merge with webkitpy.webkit_port at some point.
+class Port(object):
+ """Abstract class for Port-specific hooks for the layout_test package."""
+
+ def __init__(self, port_name=None, options=None,
+ executive=None,
+ user=None,
+ filesystem=None,
+ config=None,
+ **kwargs):
+ self._name = port_name
+ self._options = options
+ if self._options is None:
+ # FIXME: Ideally we'd have a package-wide way to get a
+ # well-formed options object that had all of the necessary
+ # options defined on it.
+ self._options = DummyOptions()
+ self._executive = executive or Executive()
+ self._user = user or User()
+ self._filesystem = filesystem or system.filesystem.FileSystem()
+ self._config = config or port_config.Config(self._executive,
+ self._filesystem)
+ self._helper = None
+ self._http_server = None
+ self._webkit_base_dir = None
+ self._websocket_server = None
+ self._http_lock = None
+
+ # Python's Popen has a bug that causes any pipes opened to a
+ # process that can't be executed to be leaked. Since this
+ # code is specifically designed to tolerate exec failures
+ # to gracefully handle cases where wdiff is not installed,
+ # the bug results in a massive file descriptor leak. As a
+ # workaround, if an exec failure is ever experienced for
+ # wdiff, assume it's not available. This will leak one
+ # file descriptor but that's better than leaking each time
+ # wdiff would be run.
+ #
+ # http://mail.python.org/pipermail/python-list/
+ # 2008-August/505753.html
+ # http://bugs.python.org/issue3210
+ self._wdiff_available = True
+
+ self._pretty_patch_path = self.path_from_webkit_base("Websites",
+ "bugs.webkit.org", "PrettyPatch", "prettify.rb")
+ self._pretty_patch_available = True
+ self.set_option_default('configuration', None)
+ if self._options.configuration is None:
+ self._options.configuration = self.default_configuration()
+
+ def default_child_processes(self):
+ """Return the number of DumpRenderTree instances to use for this
+ port."""
+ return self._executive.cpu_count()
+
+ def baseline_path(self):
+ """Return the absolute path to the directory to store new baselines
+ in for this port."""
+ raise NotImplementedError('Port.baseline_path')
+
+ def baseline_search_path(self):
+ """Return a list of absolute paths to directories to search under for
+ baselines. The directories are searched in order."""
+ raise NotImplementedError('Port.baseline_search_path')
+
+ def check_build(self, needs_http):
+ """This routine is used to ensure that the build is up to date
+ and all the needed binaries are present."""
+ raise NotImplementedError('Port.check_build')
+
+ def check_sys_deps(self, needs_http):
+ """If the port needs to do some runtime checks to ensure that the
+ tests can be run successfully, it should override this routine.
+ This step can be skipped with --nocheck-sys-deps.
+
+ Returns whether the system is properly configured."""
+ return True
+
+ def check_image_diff(self, override_step=None, logging=True):
+ """This routine is used to check whether image_diff binary exists."""
+ raise NotImplementedError('Port.check_image_diff')
+
+ def check_pretty_patch(self):
+ """Checks whether we can use the PrettyPatch ruby script."""
+
+ # check if Ruby is installed
+ try:
+ result = self._executive.run_command(['ruby', '--version'])
+ except OSError, e:
+ if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
+ _log.error("Ruby is not installed; "
+ "can't generate pretty patches.")
+ _log.error('')
+ return False
+
+ if not self.path_exists(self._pretty_patch_path):
+ _log.error('Unable to find %s .' % self._pretty_patch_path)
+ _log.error("Can't generate pretty patches.")
+ _log.error('')
+ return False
+
+ return True
+
+ def compare_text(self, expected_text, actual_text):
+ """Return whether or not the two strings are *not* equal. This
+ routine is used to diff text output.
+
+ While this is a generic routine, we include it in the Port
+ interface so that it can be overriden for testing purposes."""
+ return expected_text != actual_text
+
+ def diff_image(self, expected_contents, actual_contents,
+ diff_filename=None, tolerance=0):
+ """Compare two images and produce a delta image file.
+
+ Return True if the two images are different, False if they are the same.
+ Also produce a delta image of the two images and write that into
+ |diff_filename| if it is not None.
+
+ |tolerance| should be a percentage value (0.0 - 100.0).
+ If it is omitted, the port default tolerance value is used.
+
+ """
+ raise NotImplementedError('Port.diff_image')
+
+
+ def diff_text(self, expected_text, actual_text,
+ expected_filename, actual_filename):
+ """Returns a string containing the diff of the two text strings
+ in 'unified diff' format.
+
+ While this is a generic routine, we include it in the Port
+ interface so that it can be overriden for testing purposes."""
+
+ # The filenames show up in the diff output, make sure they're
+ # raw bytes and not unicode, so that they don't trigger join()
+ # trying to decode the input.
+ def to_raw_bytes(str):
+ if isinstance(str, unicode):
+ return str.encode('utf-8')
+ return str
+ expected_filename = to_raw_bytes(expected_filename)
+ actual_filename = to_raw_bytes(actual_filename)
+ diff = difflib.unified_diff(expected_text.splitlines(True),
+ actual_text.splitlines(True),
+ expected_filename,
+ actual_filename)
+ return ''.join(diff)
+
+ def driver_name(self):
+ """Returns the name of the actual binary that is performing the test,
+ so that it can be referred to in log messages. In most cases this
+ will be DumpRenderTree, but if a port uses a binary with a different
+ name, it can be overridden here."""
+ return "DumpRenderTree"
+
+ def expected_baselines(self, filename, suffix, all_baselines=False):
+ """Given a test name, finds where the baseline results are located.
+
+ Args:
+ filename: absolute filename to test file
+ suffix: file suffix of the expected results, including dot; e.g.
+ '.txt' or '.png'. This should not be None, but may be an empty
+ string.
+ all_baselines: If True, return an ordered list of all baseline paths
+ for the given platform. If False, return only the first one.
+ Returns
+ a list of ( platform_dir, results_filename ), where
+ platform_dir - abs path to the top of the results tree (or test
+ tree)
+ results_filename - relative path from top of tree to the results
+ file
+ (os.path.join of the two gives you the full path to the file,
+ unless None was returned.)
+ Return values will be in the format appropriate for the current
+ platform (e.g., "\\" for path separators on Windows). If the results
+ file is not found, then None will be returned for the directory,
+ but the expected relative pathname will still be returned.
+
+ This routine is generic but lives here since it is used in
+ conjunction with the other baseline and filename routines that are
+ platform specific.
+ """
+ testname = os.path.splitext(self.relative_test_filename(filename))[0]
+
+ baseline_filename = testname + '-expected' + suffix
+
+ baseline_search_path = self.baseline_search_path()
+
+ baselines = []
+ for platform_dir in baseline_search_path:
+ if self.path_exists(self._filesystem.join(platform_dir,
+ baseline_filename)):
+ baselines.append((platform_dir, baseline_filename))
+
+ if not all_baselines and baselines:
+ return baselines
+
+ # If it wasn't found in a platform directory, return the expected
+ # result in the test directory, even if no such file actually exists.
+ platform_dir = self.layout_tests_dir()
+ if self.path_exists(self._filesystem.join(platform_dir,
+ baseline_filename)):
+ baselines.append((platform_dir, baseline_filename))
+
+ if baselines:
+ return baselines
+
+ return [(None, baseline_filename)]
+
+ def expected_filename(self, filename, suffix):
+ """Given a test name, returns an absolute path to its expected results.
+
+ If no expected results are found in any of the searched directories,
+ the directory in which the test itself is located will be returned.
+ The return value is in the format appropriate for the platform
+ (e.g., "\\" for path separators on windows).
+
+ Args:
+ filename: absolute filename to test file
+ suffix: file suffix of the expected results, including dot; e.g. '.txt'
+ or '.png'. This should not be None, but may be an empty string.
+ platform: the most-specific directory name to use to build the
+ search list of directories, e.g., 'chromium-win', or
+ 'chromium-mac-leopard' (we follow the WebKit format)
+
+ This routine is generic but is implemented here to live alongside
+ the other baseline and filename manipulation routines.
+ """
+ platform_dir, baseline_filename = self.expected_baselines(
+ filename, suffix)[0]
+ if platform_dir:
+ return self._filesystem.join(platform_dir, baseline_filename)
+ return self._filesystem.join(self.layout_tests_dir(), baseline_filename)
+
+ def expected_checksum(self, test):
+ """Returns the checksum of the image we expect the test to produce, or None if it is a text-only test."""
+ path = self.expected_filename(test, '.checksum')
+ if not self.path_exists(path):
+ return None
+ return self._filesystem.read_text_file(path)
+
+ def expected_image(self, test):
+ """Returns the image we expect the test to produce."""
+ path = self.expected_filename(test, '.png')
+ if not self.path_exists(path):
+ return None
+ return self._filesystem.read_binary_file(path)
+
+ def expected_text(self, test):
+ """Returns the text output we expect the test to produce.
+ End-of-line characters are normalized to '\n'."""
+ # FIXME: DRT output is actually utf-8, but since we don't decode the
+ # output from DRT (instead treating it as a binary string), we read the
+ # baselines as a binary string, too.
+ path = self.expected_filename(test, '.txt')
+ if not self.path_exists(path):
+ return ''
+ text = self._filesystem.read_binary_file(path)
+ return text.replace("\r\n", "\n")
+
+ def filename_to_uri(self, filename):
+ """Convert a test file (which is an absolute path) to a URI."""
+ LAYOUTTEST_HTTP_DIR = "http/tests/"
+ LAYOUTTEST_WEBSOCKET_DIR = "http/tests/websocket/tests/"
+
+ relative_path = self.relative_test_filename(filename)
+ port = None
+ use_ssl = False
+
+ if (relative_path.startswith(LAYOUTTEST_WEBSOCKET_DIR)
+ or relative_path.startswith(LAYOUTTEST_HTTP_DIR)):
+ relative_path = relative_path[len(LAYOUTTEST_HTTP_DIR):]
+ port = 8000
+
+ # Make http/tests/local run as local files. This is to mimic the
+ # logic in run-webkit-tests.
+ #
+ # TODO(dpranke): remove the media reference and the SSL reference?
+ if (port and not relative_path.startswith("local/") and
+ not relative_path.startswith("media/")):
+ if relative_path.startswith("ssl/"):
+ port += 443
+ protocol = "https"
+ else:
+ protocol = "http"
+ return "%s://127.0.0.1:%u/%s" % (protocol, port, relative_path)
+
+ return path.abspath_to_uri(os.path.abspath(filename))
+
+ def tests(self, paths):
+ """Return the list of tests found (relative to layout_tests_dir()."""
+ return test_files.find(self, paths)
+
+ def test_dirs(self):
+ """Returns the list of top-level test directories.
+
+ Used by --clobber-old-results."""
+ layout_tests_dir = self.layout_tests_dir()
+ return filter(lambda x: self._filesystem.isdir(self._filesystem.join(layout_tests_dir, x)),
+ self._filesystem.listdir(layout_tests_dir))
+
+ def path_isdir(self, path):
+ """Return True if the path refers to a directory of tests."""
+ # Used by test_expectations.py to apply rules to whole directories.
+ return self._filesystem.isdir(path)
+
+ def path_exists(self, path):
+ """Return True if the path refers to an existing test or baseline."""
+ # Used by test_expectations.py to determine if an entry refers to a
+ # valid test and by printing.py to determine if baselines exist.
+ return self._filesystem.exists(path)
+
+ def driver_cmd_line(self):
+ """Prints the DRT command line that will be used."""
+ driver = self.create_driver(0)
+ return driver.cmd_line()
+
+ def update_baseline(self, path, data, encoding):
+ """Updates the baseline for a test.
+
+ Args:
+ path: the actual path to use for baseline, not the path to
+ the test. This function is used to update either generic or
+ platform-specific baselines, but we can't infer which here.
+ data: contents of the baseline.
+ encoding: file encoding to use for the baseline.
+ """
+ # FIXME: remove the encoding parameter in favor of text/binary
+ # functions.
+ if encoding is None:
+ self._filesystem.write_binary_file(path, data)
+ else:
+ self._filesystem.write_text_file(path, data)
+
+ def uri_to_test_name(self, uri):
+ """Return the base layout test name for a given URI.
+
+ This returns the test name for a given URI, e.g., if you passed in
+ "file:///src/LayoutTests/fast/html/keygen.html" it would return
+ "fast/html/keygen.html".
+
+ """
+ test = uri
+ if uri.startswith("file:///"):
+ prefix = path.abspath_to_uri(self.layout_tests_dir()) + "/"
+ return test[len(prefix):]
+
+ if uri.startswith("http://127.0.0.1:8880/"):
+ # websocket tests
+ return test.replace('http://127.0.0.1:8880/', '')
+
+ if uri.startswith("http://"):
+ # regular HTTP test
+ return test.replace('http://127.0.0.1:8000/', 'http/tests/')
+
+ if uri.startswith("https://"):
+ return test.replace('https://127.0.0.1:8443/', 'http/tests/')
+
+ raise NotImplementedError('unknown url type: %s' % uri)
+
+ def layout_tests_dir(self):
+ """Return the absolute path to the top of the LayoutTests directory."""
+ return self.path_from_webkit_base('LayoutTests')
+
+ def skips_layout_test(self, test_name):
+ """Figures out if the givent test is being skipped or not.
+
+ Test categories are handled as well."""
+ for test_or_category in self.skipped_layout_tests():
+ if test_or_category == test_name:
+ return True
+ category = self._filesystem.join(self.layout_tests_dir(),
+ test_or_category)
+ if (self._filesystem.isdir(category) and
+ test_name.startswith(test_or_category)):
+ return True
+ return False
+
+ def maybe_make_directory(self, *path):
+ """Creates the specified directory if it doesn't already exist."""
+ self._filesystem.maybe_make_directory(*path)
+
+ def name(self):
+ """Return the name of the port (e.g., 'mac', 'chromium-win-xp').
+
+ Note that this is different from the test_platform_name(), which
+ may be different (e.g., 'win-xp' instead of 'chromium-win-xp'."""
+ return self._name
+
+ def get_option(self, name, default_value=None):
+ # FIXME: Eventually we should not have to do a test for
+ # hasattr(), and we should be able to just do
+ # self.options.value. See additional FIXME in the constructor.
+ if hasattr(self._options, name):
+ return getattr(self._options, name)
+ return default_value
+
+ def set_option_default(self, name, default_value):
+ if not hasattr(self._options, name):
+ return setattr(self._options, name, default_value)
+
+ def path_from_webkit_base(self, *comps):
+ """Returns the full path to path made by joining the top of the
+ WebKit source tree and the list of path components in |*comps|."""
+ return self._config.path_from_webkit_base(*comps)
+
+ def script_path(self, script_name):
+ return self._config.script_path(script_name)
+
+ def path_to_test_expectations_file(self):
+ """Update the test expectations to the passed-in string.
+
+ This is used by the rebaselining tool. Raises NotImplementedError
+ if the port does not use expectations files."""
+ raise NotImplementedError('Port.path_to_test_expectations_file')
+
+ def relative_test_filename(self, filename):
+ """Relative unix-style path for a filename under the LayoutTests
+ directory. Filenames outside the LayoutTests directory should raise
+ an error."""
+ assert filename.startswith(self.layout_tests_dir()), "%s did not start with %s" % (filename, self.layout_tests_dir())
+ return filename[len(self.layout_tests_dir()) + 1:]
+
+ def results_directory(self):
+ """Absolute path to the place to store the test results."""
+ raise NotImplementedError('Port.results_directory')
+
+ def setup_test_run(self):
+ """Perform port-specific work at the beginning of a test run."""
+ pass
+
+ def setup_environ_for_server(self):
+ """Perform port-specific work at the beginning of a server launch.
+
+ Returns:
+ Operating-system's environment.
+ """
+ return os.environ.copy()
+
+ def show_results_html_file(self, results_filename):
+ """This routine should display the HTML file pointed at by
+ results_filename in a users' browser."""
+ return self._user.open_url(results_filename)
+
+ def create_driver(self, worker_number):
+ """Return a newly created base.Driver subclass for starting/stopping
+ the test driver."""
+ raise NotImplementedError('Port.create_driver')
+
+ def start_helper(self):
+ """If a port needs to reconfigure graphics settings or do other
+ things to ensure a known test configuration, it should override this
+ method."""
+ pass
+
+ def start_http_server(self):
+ """Start a web server if it is available. Do nothing if
+ it isn't. This routine is allowed to (and may) fail if a server
+ is already running."""
+ if self.get_option('use_apache'):
+ self._http_server = apache_http_server.LayoutTestApacheHttpd(self,
+ self.get_option('results_directory'))
+ else:
+ self._http_server = http_server.Lighttpd(self,
+ self.get_option('results_directory'))
+ self._http_server.start()
+
+ def start_websocket_server(self):
+ """Start a websocket server if it is available. Do nothing if
+ it isn't. This routine is allowed to (and may) fail if a server
+ is already running."""
+ self._websocket_server = websocket_server.PyWebSocket(self,
+ self.get_option('results_directory'))
+ self._websocket_server.start()
+
+ def acquire_http_lock(self):
+ self._http_lock = http_lock.HttpLock(None)
+ self._http_lock.wait_for_httpd_lock()
+
+ def stop_helper(self):
+ """Shut down the test helper if it is running. Do nothing if
+ it isn't, or it isn't available. If a port overrides start_helper()
+ it must override this routine as well."""
+ pass
+
+ def stop_http_server(self):
+ """Shut down the http server if it is running. Do nothing if
+ it isn't, or it isn't available."""
+ if self._http_server:
+ self._http_server.stop()
+
+ def stop_websocket_server(self):
+ """Shut down the websocket server if it is running. Do nothing if
+ it isn't, or it isn't available."""
+ if self._websocket_server:
+ self._websocket_server.stop()
+
+ def release_http_lock(self):
+ if self._http_lock:
+ self._http_lock.cleanup_http_lock()
+
+ def test_expectations(self):
+ """Returns the test expectations for this port.
+
+ Basically this string should contain the equivalent of a
+ test_expectations file. See test_expectations.py for more details."""
+ raise NotImplementedError('Port.test_expectations')
+
+ def test_expectations_overrides(self):
+ """Returns an optional set of overrides for the test_expectations.
+
+ This is used by ports that have code in two repositories, and where
+ it is possible that you might need "downstream" expectations that
+ temporarily override the "upstream" expectations until the port can
+ sync up the two repos."""
+ return None
+
+ def test_base_platform_names(self):
+ """Return a list of the 'base' platforms on your port. The base
+ platforms represent different architectures, operating systems,
+ or implementations (as opposed to different versions of a single
+ platform). For example, 'mac' and 'win' might be different base
+ platforms, wherease 'mac-tiger' and 'mac-leopard' might be
+ different platforms. This routine is used by the rebaselining tool
+ and the dashboards, and the strings correspond to the identifiers
+ in your test expectations (*not* necessarily the platform names
+ themselves)."""
+ raise NotImplementedError('Port.base_test_platforms')
+
+ def test_platform_name(self):
+ """Returns the string that corresponds to the given platform name
+ in the test expectations. This may be the same as name(), or it
+ may be different. For example, chromium returns 'mac' for
+ 'chromium-mac'."""
+ raise NotImplementedError('Port.test_platform_name')
+
+ def test_platforms(self):
+ """Returns the list of test platform identifiers as used in the
+ test_expectations and on dashboards, the rebaselining tool, etc.
+
+ Note that this is not necessarily the same as the list of ports,
+ which must be globally unique (e.g., both 'chromium-mac' and 'mac'
+ might return 'mac' as a test_platform name'."""
+ raise NotImplementedError('Port.platforms')
+
+ def test_platform_name_to_name(self, test_platform_name):
+ """Returns the Port platform name that corresponds to the name as
+ referenced in the expectations file. E.g., "mac" returns
+ "chromium-mac" on the Chromium ports."""
+ raise NotImplementedError('Port.test_platform_name_to_name')
+
+ def version(self):
+ """Returns a string indicating the version of a given platform, e.g.
+ '-leopard' or '-xp'.
+
+ This is used to help identify the exact port when parsing test
+ expectations, determining search paths, and logging information."""
+ raise NotImplementedError('Port.version')
+
+ def test_repository_paths(self):
+ """Returns a list of (repository_name, repository_path) tuples
+ of its depending code base. By default it returns a list that only
+ contains a ('webkit', <webkitRepossitoryPath>) tuple.
+ """
+ return [('webkit', self.layout_tests_dir())]
+
+
+ _WDIFF_DEL = '##WDIFF_DEL##'
+ _WDIFF_ADD = '##WDIFF_ADD##'
+ _WDIFF_END = '##WDIFF_END##'
+
+ def _format_wdiff_output_as_html(self, wdiff):
+ wdiff = cgi.escape(wdiff)
+ wdiff = wdiff.replace(self._WDIFF_DEL, "<span class=del>")
+ wdiff = wdiff.replace(self._WDIFF_ADD, "<span class=add>")
+ wdiff = wdiff.replace(self._WDIFF_END, "</span>")
+ html = "<head><style>.del { background: #faa; } "
+ html += ".add { background: #afa; }</style></head>"
+ html += "<pre>%s</pre>" % wdiff
+ return html
+
+ def _wdiff_command(self, actual_filename, expected_filename):
+ executable = self._path_to_wdiff()
+ return [executable,
+ "--start-delete=%s" % self._WDIFF_DEL,
+ "--end-delete=%s" % self._WDIFF_END,
+ "--start-insert=%s" % self._WDIFF_ADD,
+ "--end-insert=%s" % self._WDIFF_END,
+ actual_filename,
+ expected_filename]
+
+ @staticmethod
+ def _handle_wdiff_error(script_error):
+ # Exit 1 means the files differed, any other exit code is an error.
+ if script_error.exit_code != 1:
+ raise script_error
+
+ def _run_wdiff(self, actual_filename, expected_filename):
+ """Runs wdiff and may throw exceptions.
+ This is mostly a hook for unit testing."""
+ # Diffs are treated as binary as they may include multiple files
+ # with conflicting encodings. Thus we do not decode the output.
+ command = self._wdiff_command(actual_filename, expected_filename)
+ wdiff = self._executive.run_command(command, decode_output=False,
+ error_handler=self._handle_wdiff_error)
+ return self._format_wdiff_output_as_html(wdiff)
+
+ def wdiff_text(self, actual_filename, expected_filename):
+ """Returns a string of HTML indicating the word-level diff of the
+ contents of the two filenames. Returns an empty string if word-level
+ diffing isn't available."""
+ if not self._wdiff_available:
+ return ""
+ try:
+ # It's possible to raise a ScriptError we pass wdiff invalid paths.
+ return self._run_wdiff(actual_filename, expected_filename)
+ except OSError, e:
+ if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
+ # Silently ignore cases where wdiff is missing.
+ self._wdiff_available = False
+ return ""
+ raise
+
+ # This is a class variable so we can test error output easily.
+ _pretty_patch_error_html = "Failed to run PrettyPatch, see error log."
+
+ def pretty_patch_text(self, diff_path):
+ if not self._pretty_patch_available:
+ return self._pretty_patch_error_html
+ command = ("ruby", "-I", os.path.dirname(self._pretty_patch_path),
+ self._pretty_patch_path, diff_path)
+ try:
+ # Diffs are treated as binary (we pass decode_output=False) as they
+ # may contain multiple files of conflicting encodings.
+ return self._executive.run_command(command, decode_output=False)
+ except OSError, e:
+ # If the system is missing ruby log the error and stop trying.
+ self._pretty_patch_available = False
+ _log.error("Failed to run PrettyPatch (%s): %s" % (command, e))
+ return self._pretty_patch_error_html
+ except ScriptError, e:
+ # If ruby failed to run for some reason, log the command
+ # output and stop trying.
+ self._pretty_patch_available = False
+ _log.error("Failed to run PrettyPatch (%s):\n%s" % (command,
+ e.message_with_output()))
+ return self._pretty_patch_error_html
+
+ def default_configuration(self):
+ return self._config.default_configuration()
+
+ #
+ # PROTECTED ROUTINES
+ #
+ # The routines below should only be called by routines in this class
+ # or any of its subclasses.
+ #
+ def _webkit_build_directory(self, args):
+ return self._config.build_directory(args[0])
+
+ def _path_to_apache(self):
+ """Returns the full path to the apache binary.
+
+ This is needed only by ports that use the apache_http_server module."""
+ raise NotImplementedError('Port.path_to_apache')
+
+ def _path_to_apache_config_file(self):
+ """Returns the full path to the apache binary.
+
+ This is needed only by ports that use the apache_http_server module."""
+ raise NotImplementedError('Port.path_to_apache_config_file')
+
+ def _path_to_driver(self, configuration=None):
+ """Returns the full path to the test driver (DumpRenderTree)."""
+ raise NotImplementedError('Port._path_to_driver')
+
+ def _path_to_webcore_library(self):
+ """Returns the full path to a built copy of WebCore."""
+ raise NotImplementedError('Port.path_to_webcore_library')
+
+ def _path_to_helper(self):
+ """Returns the full path to the layout_test_helper binary, which
+ is used to help configure the system for the test run, or None
+ if no helper is needed.
+
+ This is likely only used by start/stop_helper()."""
+ raise NotImplementedError('Port._path_to_helper')
+
+ def _path_to_image_diff(self):
+ """Returns the full path to the image_diff binary, or None if it
+ is not available.
+
+ This is likely used only by diff_image()"""
+ raise NotImplementedError('Port.path_to_image_diff')
+
+ def _path_to_lighttpd(self):
+ """Returns the path to the LigHTTPd binary.
+
+ This is needed only by ports that use the http_server.py module."""
+ raise NotImplementedError('Port._path_to_lighttpd')
+
+ def _path_to_lighttpd_modules(self):
+ """Returns the path to the LigHTTPd modules directory.
+
+ This is needed only by ports that use the http_server.py module."""
+ raise NotImplementedError('Port._path_to_lighttpd_modules')
+
+ def _path_to_lighttpd_php(self):
+ """Returns the path to the LigHTTPd PHP executable.
+
+ This is needed only by ports that use the http_server.py module."""
+ raise NotImplementedError('Port._path_to_lighttpd_php')
+
+ def _path_to_wdiff(self):
+ """Returns the full path to the wdiff binary, or None if it is
+ not available.
+
+ This is likely used only by wdiff_text()"""
+ raise NotImplementedError('Port._path_to_wdiff')
+
+ def _shut_down_http_server(self, pid):
+ """Forcefully and synchronously kills the web server.
+
+ This routine should only be called from http_server.py or its
+ subclasses."""
+ raise NotImplementedError('Port._shut_down_http_server')
+
+ def _webkit_baseline_path(self, platform):
+ """Return the full path to the top of the baseline tree for a
+ given platform."""
+ return self._filesystem.join(self.layout_tests_dir(), 'platform',
+ platform)
+
+
+class Driver:
+ """Abstract interface for the DumpRenderTree interface."""
+
+ def __init__(self, port, worker_number):
+ """Initialize a Driver to subsequently run tests.
+
+ Typically this routine will spawn DumpRenderTree in a config
+ ready for subsequent input.
+
+ port - reference back to the port object.
+ worker_number - identifier for a particular worker/driver instance
+ """
+ raise NotImplementedError('Driver.__init__')
+
+ def run_test(self, test_input):
+ """Run a single test and return the results.
+
+ Note that it is okay if a test times out or crashes and leaves
+ the driver in an indeterminate state. The upper layers of the program
+ are responsible for cleaning up and ensuring things are okay.
+
+ Args:
+ test_input: a TestInput object
+
+ Returns a TestOutput object.
+ """
+ raise NotImplementedError('Driver.run_test')
+
+ # FIXME: This is static so we can test it w/o creating a Base instance.
+ @classmethod
+ def _command_wrapper(cls, wrapper_option):
+ # Hook for injecting valgrind or other runtime instrumentation,
+ # used by e.g. tools/valgrind/valgrind_tests.py.
+ wrapper = []
+ browser_wrapper = os.environ.get("BROWSER_WRAPPER", None)
+ if browser_wrapper:
+ # FIXME: There seems to be no reason to use BROWSER_WRAPPER over --wrapper.
+ # Remove this code any time after the date listed below.
+ _log.error("BROWSER_WRAPPER is deprecated, please use --wrapper instead.")
+ _log.error("BROWSER_WRAPPER will be removed any time after June 1st 2010 and your scripts will break.")
+ wrapper += [browser_wrapper]
+
+ if wrapper_option:
+ wrapper += shlex.split(wrapper_option)
+ return wrapper
+
+ def poll(self):
+ """Returns None if the Driver is still running. Returns the returncode
+ if it has exited."""
+ raise NotImplementedError('Driver.poll')
+
+ def stop(self):
+ raise NotImplementedError('Driver.stop')
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py
new file mode 100644
index 0000000..8d586e3
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py
@@ -0,0 +1,315 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import optparse
+import os
+import sys
+import tempfile
+import unittest
+
+from webkitpy.common.system.executive import Executive, ScriptError
+from webkitpy.common.system import executive_mock
+from webkitpy.common.system import filesystem
+from webkitpy.common.system import outputcapture
+from webkitpy.common.system.path import abspath_to_uri
+from webkitpy.thirdparty.mock import Mock
+from webkitpy.tool import mocktool
+
+import base
+import config
+import config_mock
+
+
+class PortTest(unittest.TestCase):
+ def test_format_wdiff_output_as_html(self):
+ output = "OUTPUT %s %s %s" % (base.Port._WDIFF_DEL, base.Port._WDIFF_ADD, base.Port._WDIFF_END)
+ html = base.Port()._format_wdiff_output_as_html(output)
+ expected_html = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre>OUTPUT <span class=del> <span class=add> </span></pre>"
+ self.assertEqual(html, expected_html)
+
+ def test_wdiff_command(self):
+ port = base.Port()
+ port._path_to_wdiff = lambda: "/path/to/wdiff"
+ command = port._wdiff_command("/actual/path", "/expected/path")
+ expected_command = [
+ "/path/to/wdiff",
+ "--start-delete=##WDIFF_DEL##",
+ "--end-delete=##WDIFF_END##",
+ "--start-insert=##WDIFF_ADD##",
+ "--end-insert=##WDIFF_END##",
+ "/actual/path",
+ "/expected/path",
+ ]
+ self.assertEqual(command, expected_command)
+
+ def _file_with_contents(self, contents, encoding="utf-8"):
+ new_file = tempfile.NamedTemporaryFile()
+ new_file.write(contents.encode(encoding))
+ new_file.flush()
+ return new_file
+
+ def test_pretty_patch_os_error(self):
+ port = base.Port(executive=executive_mock.MockExecutive2(exception=OSError))
+ oc = outputcapture.OutputCapture()
+ oc.capture_output()
+ self.assertEqual(port.pretty_patch_text("patch.txt"),
+ port._pretty_patch_error_html)
+
+ # This tests repeated calls to make sure we cache the result.
+ self.assertEqual(port.pretty_patch_text("patch.txt"),
+ port._pretty_patch_error_html)
+ oc.restore_output()
+
+ def test_pretty_patch_script_error(self):
+ # FIXME: This is some ugly white-box test hacking ...
+ base._pretty_patch_available = True
+ port = base.Port(executive=executive_mock.MockExecutive2(exception=ScriptError))
+ self.assertEqual(port.pretty_patch_text("patch.txt"),
+ port._pretty_patch_error_html)
+
+ # This tests repeated calls to make sure we cache the result.
+ self.assertEqual(port.pretty_patch_text("patch.txt"),
+ port._pretty_patch_error_html)
+
+ def test_run_wdiff(self):
+ executive = Executive()
+ # This may fail on some systems. We could ask the port
+ # object for the wdiff path, but since we don't know what
+ # port object to use, this is sufficient for now.
+ try:
+ wdiff_path = executive.run_command(["which", "wdiff"]).rstrip()
+ except Exception, e:
+ wdiff_path = None
+
+ port = base.Port()
+ port._path_to_wdiff = lambda: wdiff_path
+
+ if wdiff_path:
+ # "with tempfile.NamedTemporaryFile() as actual" does not seem to work in Python 2.5
+ actual = self._file_with_contents(u"foo")
+ expected = self._file_with_contents(u"bar")
+ wdiff = port._run_wdiff(actual.name, expected.name)
+ expected_wdiff = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre><span class=del>foo</span><span class=add>bar</span></pre>"
+ self.assertEqual(wdiff, expected_wdiff)
+ # Running the full wdiff_text method should give the same result.
+ port._wdiff_available = True # In case it's somehow already disabled.
+ wdiff = port.wdiff_text(actual.name, expected.name)
+ self.assertEqual(wdiff, expected_wdiff)
+ # wdiff should still be available after running wdiff_text with a valid diff.
+ self.assertTrue(port._wdiff_available)
+ actual.close()
+ expected.close()
+
+ # Bogus paths should raise a script error.
+ self.assertRaises(ScriptError, port._run_wdiff, "/does/not/exist", "/does/not/exist2")
+ self.assertRaises(ScriptError, port.wdiff_text, "/does/not/exist", "/does/not/exist2")
+ # wdiff will still be available after running wdiff_text with invalid paths.
+ self.assertTrue(port._wdiff_available)
+ base._wdiff_available = True
+
+ # If wdiff does not exist _run_wdiff should throw an OSError.
+ port._path_to_wdiff = lambda: "/invalid/path/to/wdiff"
+ self.assertRaises(OSError, port._run_wdiff, "foo", "bar")
+
+ # wdiff_text should not throw an error if wdiff does not exist.
+ self.assertEqual(port.wdiff_text("foo", "bar"), "")
+ # However wdiff should not be available after running wdiff_text if wdiff is missing.
+ self.assertFalse(port._wdiff_available)
+
+ def test_diff_text(self):
+ port = base.Port()
+ # Make sure that we don't run into decoding exceptions when the
+ # filenames are unicode, with regular or malformed input (expected or
+ # actual input is always raw bytes, not unicode).
+ port.diff_text('exp', 'act', 'exp.txt', 'act.txt')
+ port.diff_text('exp', 'act', u'exp.txt', 'act.txt')
+ port.diff_text('exp', 'act', u'a\xac\u1234\u20ac\U00008000', 'act.txt')
+
+ port.diff_text('exp' + chr(255), 'act', 'exp.txt', 'act.txt')
+ port.diff_text('exp' + chr(255), 'act', u'exp.txt', 'act.txt')
+
+ # Though expected and actual files should always be read in with no
+ # encoding (and be stored as str objects), test unicode inputs just to
+ # be safe.
+ port.diff_text(u'exp', 'act', 'exp.txt', 'act.txt')
+ port.diff_text(
+ u'a\xac\u1234\u20ac\U00008000', 'act', 'exp.txt', 'act.txt')
+
+ # And make sure we actually get diff output.
+ diff = port.diff_text('foo', 'bar', 'exp.txt', 'act.txt')
+ self.assertTrue('foo' in diff)
+ self.assertTrue('bar' in diff)
+ self.assertTrue('exp.txt' in diff)
+ self.assertTrue('act.txt' in diff)
+ self.assertFalse('nosuchthing' in diff)
+
+ def test_default_configuration_notfound(self):
+ # Test that we delegate to the config object properly.
+ port = base.Port(config=config_mock.MockConfig(default_configuration='default'))
+ self.assertEqual(port.default_configuration(), 'default')
+
+ def test_layout_tests_skipping(self):
+ port = base.Port()
+ port.skipped_layout_tests = lambda: ['foo/bar.html', 'media']
+ self.assertTrue(port.skips_layout_test('foo/bar.html'))
+ self.assertTrue(port.skips_layout_test('media/video-zoom.html'))
+ self.assertFalse(port.skips_layout_test('foo/foo.html'))
+
+ def test_setup_test_run(self):
+ port = base.Port()
+ # This routine is a no-op. We just test it for coverage.
+ port.setup_test_run()
+
+ def test_test_dirs(self):
+ port = base.Port()
+ dirs = port.test_dirs()
+ self.assertTrue('canvas' in dirs)
+ self.assertTrue('css2.1' in dirs)
+
+ def test_filename_to_uri(self):
+ port = base.Port()
+ layout_test_dir = port.layout_tests_dir()
+ test_file = os.path.join(layout_test_dir, "foo", "bar.html")
+
+ # On Windows, absolute paths are of the form "c:\foo.txt". However,
+ # all current browsers (except for Opera) normalize file URLs by
+ # prepending an additional "/" as if the absolute path was
+ # "/c:/foo.txt". This means that all file URLs end up with "file:///"
+ # at the beginning.
+ if sys.platform == 'win32':
+ prefix = "file:///"
+ path = test_file.replace("\\", "/")
+ else:
+ prefix = "file://"
+ path = test_file
+
+ self.assertEqual(port.filename_to_uri(test_file),
+ abspath_to_uri(test_file))
+
+ def test_get_option__set(self):
+ options, args = optparse.OptionParser().parse_args([])
+ options.foo = 'bar'
+ port = base.Port(options=options)
+ self.assertEqual(port.get_option('foo'), 'bar')
+
+ def test_get_option__unset(self):
+ port = base.Port()
+ self.assertEqual(port.get_option('foo'), None)
+
+ def test_get_option__default(self):
+ port = base.Port()
+ self.assertEqual(port.get_option('foo', 'bar'), 'bar')
+
+ def test_set_option_default__unset(self):
+ port = base.Port()
+ port.set_option_default('foo', 'bar')
+ self.assertEqual(port.get_option('foo'), 'bar')
+
+ def test_set_option_default__set(self):
+ options, args = optparse.OptionParser().parse_args([])
+ options.foo = 'bar'
+ port = base.Port(options=options)
+ # This call should have no effect.
+ port.set_option_default('foo', 'new_bar')
+ self.assertEqual(port.get_option('foo'), 'bar')
+
+ def test_name__unset(self):
+ port = base.Port()
+ self.assertEqual(port.name(), None)
+
+ def test_name__set(self):
+ port = base.Port(port_name='foo')
+ self.assertEqual(port.name(), 'foo')
+
+
+class VirtualTest(unittest.TestCase):
+ """Tests that various methods expected to be virtual are."""
+ def assertVirtual(self, method, *args, **kwargs):
+ self.assertRaises(NotImplementedError, method, *args, **kwargs)
+
+ def test_virtual_methods(self):
+ port = base.Port()
+ self.assertVirtual(port.baseline_path)
+ self.assertVirtual(port.baseline_search_path)
+ self.assertVirtual(port.check_build, None)
+ self.assertVirtual(port.check_image_diff)
+ self.assertVirtual(port.create_driver, 0)
+ self.assertVirtual(port.diff_image, None, None)
+ self.assertVirtual(port.path_to_test_expectations_file)
+ self.assertVirtual(port.test_platform_name)
+ self.assertVirtual(port.results_directory)
+ self.assertVirtual(port.test_expectations)
+ self.assertVirtual(port.test_base_platform_names)
+ self.assertVirtual(port.test_platform_name)
+ self.assertVirtual(port.test_platforms)
+ self.assertVirtual(port.test_platform_name_to_name, None)
+ self.assertVirtual(port.version)
+ self.assertVirtual(port._path_to_apache)
+ self.assertVirtual(port._path_to_apache_config_file)
+ self.assertVirtual(port._path_to_driver)
+ self.assertVirtual(port._path_to_helper)
+ self.assertVirtual(port._path_to_image_diff)
+ self.assertVirtual(port._path_to_lighttpd)
+ self.assertVirtual(port._path_to_lighttpd_modules)
+ self.assertVirtual(port._path_to_lighttpd_php)
+ self.assertVirtual(port._path_to_wdiff)
+ self.assertVirtual(port._shut_down_http_server, None)
+
+ def test_virtual_driver_method(self):
+ self.assertRaises(NotImplementedError, base.Driver, base.Port(),
+ 0)
+
+ def test_virtual_driver_methods(self):
+ class VirtualDriver(base.Driver):
+ def __init__(self):
+ pass
+
+ driver = VirtualDriver()
+ self.assertVirtual(driver.run_test, None)
+ self.assertVirtual(driver.poll)
+ self.assertVirtual(driver.stop)
+
+
+class DriverTest(unittest.TestCase):
+
+ def _assert_wrapper(self, wrapper_string, expected_wrapper):
+ wrapper = base.Driver._command_wrapper(wrapper_string)
+ self.assertEqual(wrapper, expected_wrapper)
+
+ def test_command_wrapper(self):
+ self._assert_wrapper(None, [])
+ self._assert_wrapper("valgrind", ["valgrind"])
+
+ # Validate that shlex works as expected.
+ command_with_spaces = "valgrind --smc-check=\"check with spaces!\" --foo"
+ expected_parse = ["valgrind", "--smc-check=check with spaces!", "--foo"]
+ self._assert_wrapper(command_with_spaces, expected_parse)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py
new file mode 100644
index 0000000..012e9cc
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium.py
@@ -0,0 +1,546 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Chromium implementations of the Port interface."""
+
+from __future__ import with_statement
+
+import codecs
+import errno
+import logging
+import os
+import re
+import shutil
+import signal
+import subprocess
+import sys
+import tempfile
+import time
+import webbrowser
+
+from webkitpy.common.system.path import cygpath
+from webkitpy.layout_tests.layout_package import test_expectations
+from webkitpy.layout_tests.layout_package import test_output
+
+import base
+import http_server
+
+# Chromium DRT on OSX uses WebKitDriver.
+if sys.platform == 'darwin':
+ import webkit
+
+import websocket_server
+
+_log = logging.getLogger("webkitpy.layout_tests.port.chromium")
+
+
+# FIXME: This function doesn't belong in this package.
+def check_file_exists(path_to_file, file_description, override_step=None,
+ logging=True):
+ """Verify the file is present where expected or log an error.
+
+ Args:
+ file_name: The (human friendly) name or description of the file
+ you're looking for (e.g., "HTTP Server"). Used for error logging.
+ override_step: An optional string to be logged if the check fails.
+ logging: Whether or not log the error messages."""
+ if not os.path.exists(path_to_file):
+ if logging:
+ _log.error('Unable to find %s' % file_description)
+ _log.error(' at %s' % path_to_file)
+ if override_step:
+ _log.error(' %s' % override_step)
+ _log.error('')
+ return False
+ return True
+
+
+class ChromiumPort(base.Port):
+ """Abstract base class for Chromium implementations of the Port class."""
+
+ def __init__(self, **kwargs):
+ base.Port.__init__(self, **kwargs)
+ self._chromium_base_dir = None
+
+ def baseline_path(self):
+ return self._webkit_baseline_path(self._name)
+
+ def check_build(self, needs_http):
+ result = True
+
+ dump_render_tree_binary_path = self._path_to_driver()
+ result = check_file_exists(dump_render_tree_binary_path,
+ 'test driver') and result
+ if result and self.get_option('build'):
+ result = self._check_driver_build_up_to_date(
+ self.get_option('configuration'))
+ else:
+ _log.error('')
+
+ helper_path = self._path_to_helper()
+ if helper_path:
+ result = check_file_exists(helper_path,
+ 'layout test helper') and result
+
+ if self.get_option('pixel_tests'):
+ result = self.check_image_diff(
+ 'To override, invoke with --no-pixel-tests') and result
+
+ # It's okay if pretty patch isn't available, but we will at
+ # least log a message.
+ self.check_pretty_patch()
+
+ return result
+
+ def check_sys_deps(self, needs_http):
+ cmd = [self._path_to_driver(), '--check-layout-test-sys-deps']
+ if self._executive.run_command(cmd, return_exit_code=True):
+ _log.error('System dependencies check failed.')
+ _log.error('To override, invoke with --nocheck-sys-deps')
+ _log.error('')
+ return False
+ return True
+
+ def check_image_diff(self, override_step=None, logging=True):
+ image_diff_path = self._path_to_image_diff()
+ return check_file_exists(image_diff_path, 'image diff exe',
+ override_step, logging)
+
+ def diff_image(self, expected_contents, actual_contents,
+ diff_filename=None):
+ executable = self._path_to_image_diff()
+
+ tempdir = tempfile.mkdtemp()
+ expected_filename = os.path.join(tempdir, "expected.png")
+ with open(expected_filename, 'w+b') as file:
+ file.write(expected_contents)
+ actual_filename = os.path.join(tempdir, "actual.png")
+ with open(actual_filename, 'w+b') as file:
+ file.write(actual_contents)
+
+ if diff_filename:
+ cmd = [executable, '--diff', expected_filename,
+ actual_filename, diff_filename]
+ else:
+ cmd = [executable, expected_filename, actual_filename]
+
+ result = True
+ try:
+ exit_code = self._executive.run_command(cmd, return_exit_code=True)
+ if exit_code == 0:
+ # The images are the same.
+ result = False
+ elif exit_code != 1:
+ _log.error("image diff returned an exit code of "
+ + str(exit_code))
+ # Returning False here causes the script to think that we
+ # successfully created the diff even though we didn't. If
+ # we return True, we think that the images match but the hashes
+ # don't match.
+ # FIXME: Figure out why image_diff returns other values.
+ result = False
+ except OSError, e:
+ if e.errno == errno.ENOENT or e.errno == errno.EACCES:
+ _compare_available = False
+ else:
+ raise e
+ finally:
+ shutil.rmtree(tempdir, ignore_errors=True)
+ return result
+
+ def driver_name(self):
+ if self._options.use_test_shell:
+ return "test_shell"
+ return "DumpRenderTree"
+
+ def path_from_chromium_base(self, *comps):
+ """Returns the full path to path made by joining the top of the
+ Chromium source tree and the list of path components in |*comps|."""
+ if not self._chromium_base_dir:
+ abspath = os.path.abspath(__file__)
+ offset = abspath.find('third_party')
+ if offset == -1:
+ self._chromium_base_dir = os.path.join(
+ abspath[0:abspath.find('Tools')],
+ 'WebKit', 'chromium')
+ else:
+ self._chromium_base_dir = abspath[0:offset]
+ return os.path.join(self._chromium_base_dir, *comps)
+
+ def path_to_test_expectations_file(self):
+ return self.path_from_webkit_base('LayoutTests', 'platform',
+ 'chromium', 'test_expectations.txt')
+
+ def results_directory(self):
+ try:
+ return self.path_from_chromium_base('webkit',
+ self.get_option('configuration'),
+ self.get_option('results_directory'))
+ except AssertionError:
+ return self._build_path(self.get_option('configuration'),
+ self.get_option('results_directory'))
+
+ def setup_test_run(self):
+ # Delete the disk cache if any to ensure a clean test run.
+ dump_render_tree_binary_path = self._path_to_driver()
+ cachedir = os.path.split(dump_render_tree_binary_path)[0]
+ cachedir = os.path.join(cachedir, "cache")
+ if os.path.exists(cachedir):
+ shutil.rmtree(cachedir)
+
+ def create_driver(self, worker_number):
+ """Starts a new Driver and returns a handle to it."""
+ if not self.get_option('use_test_shell') and sys.platform == 'darwin':
+ return webkit.WebKitDriver(self, worker_number)
+ return ChromiumDriver(self, worker_number)
+
+ def start_helper(self):
+ helper_path = self._path_to_helper()
+ if helper_path:
+ _log.debug("Starting layout helper %s" % helper_path)
+ # Note: Not thread safe: http://bugs.python.org/issue2320
+ self._helper = subprocess.Popen([helper_path],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None)
+ is_ready = self._helper.stdout.readline()
+ if not is_ready.startswith('ready'):
+ _log.error("layout_test_helper failed to be ready")
+
+ def stop_helper(self):
+ if self._helper:
+ _log.debug("Stopping layout test helper")
+ self._helper.stdin.write("x\n")
+ self._helper.stdin.close()
+ # wait() is not threadsafe and can throw OSError due to:
+ # http://bugs.python.org/issue1731717
+ self._helper.wait()
+
+ def test_base_platform_names(self):
+ return ('linux', 'mac', 'win')
+
+ def test_expectations(self):
+ """Returns the test expectations for this port.
+
+ Basically this string should contain the equivalent of a
+ test_expectations file. See test_expectations.py for more details."""
+ expectations_path = self.path_to_test_expectations_file()
+ with codecs.open(expectations_path, "r", "utf-8") as file:
+ return file.read()
+
+ def test_expectations_overrides(self):
+ try:
+ overrides_path = self.path_from_chromium_base('webkit', 'tools',
+ 'layout_tests', 'test_expectations.txt')
+ except AssertionError:
+ return None
+ if not os.path.exists(overrides_path):
+ return None
+ with codecs.open(overrides_path, "r", "utf-8") as file:
+ return file.read()
+
+ def skipped_layout_tests(self, extra_test_files=None):
+ expectations_str = self.test_expectations()
+ overrides_str = self.test_expectations_overrides()
+ test_platform_name = self.test_platform_name()
+ is_debug_mode = False
+
+ all_test_files = self.tests([])
+ if extra_test_files:
+ all_test_files.update(extra_test_files)
+
+ expectations = test_expectations.TestExpectations(
+ self, all_test_files, expectations_str, test_platform_name,
+ is_debug_mode, is_lint_mode=False, overrides=overrides_str)
+ tests_dir = self.layout_tests_dir()
+ return [self.relative_test_filename(test)
+ for test in expectations.get_tests_with_result_type(test_expectations.SKIP)]
+
+ def test_platform_names(self):
+ return self.test_base_platform_names() + ('win-xp',
+ 'win-vista', 'win-7')
+
+ def test_platform_name_to_name(self, test_platform_name):
+ if test_platform_name in self.test_platform_names():
+ return 'chromium-' + test_platform_name
+ raise ValueError('Unsupported test_platform_name: %s' %
+ test_platform_name)
+
+ def test_repository_paths(self):
+ # Note: for JSON file's backward-compatibility we use 'chrome' rather
+ # than 'chromium' here.
+ repos = super(ChromiumPort, self).test_repository_paths()
+ repos.append(('chrome', self.path_from_chromium_base()))
+ return repos
+
+ #
+ # PROTECTED METHODS
+ #
+ # These routines should only be called by other methods in this file
+ # or any subclasses.
+ #
+
+ def _check_driver_build_up_to_date(self, configuration):
+ if configuration in ('Debug', 'Release'):
+ try:
+ debug_path = self._path_to_driver('Debug')
+ release_path = self._path_to_driver('Release')
+
+ debug_mtime = os.stat(debug_path).st_mtime
+ release_mtime = os.stat(release_path).st_mtime
+
+ if (debug_mtime > release_mtime and configuration == 'Release' or
+ release_mtime > debug_mtime and configuration == 'Debug'):
+ _log.warning('You are not running the most '
+ 'recent DumpRenderTree binary. You need to '
+ 'pass --debug or not to select between '
+ 'Debug and Release.')
+ _log.warning('')
+ # This will fail if we don't have both a debug and release binary.
+ # That's fine because, in this case, we must already be running the
+ # most up-to-date one.
+ except OSError:
+ pass
+ return True
+
+ def _chromium_baseline_path(self, platform):
+ if platform is None:
+ platform = self.name()
+ return self.path_from_webkit_base('LayoutTests', 'platform', platform)
+
+ def _convert_path(self, path):
+ """Handles filename conversion for subprocess command line args."""
+ # See note above in diff_image() for why we need this.
+ if sys.platform == 'cygwin':
+ return cygpath(path)
+ return path
+
+ def _path_to_image_diff(self):
+ binary_name = 'ImageDiff'
+ if self.get_option('use_test_shell'):
+ binary_name = 'image_diff'
+ return self._build_path(self.get_option('configuration'), binary_name)
+
+
+class ChromiumDriver(base.Driver):
+ """Abstract interface for test_shell."""
+
+ def __init__(self, port, worker_number):
+ self._port = port
+ self._worker_number = worker_number
+ self._image_path = None
+ if self._port.get_option('pixel_tests'):
+ self._image_path = os.path.join(
+ self._port.get_option('results_directory'),
+ 'png_result%s.png' % self._worker_number)
+
+ def cmd_line(self):
+ cmd = self._command_wrapper(self._port.get_option('wrapper'))
+ cmd.append(self._port._path_to_driver())
+ if self._port.get_option('pixel_tests'):
+ # See note above in diff_image() for why we need _convert_path().
+ cmd.append("--pixel-tests=" +
+ self._port._convert_path(self._image_path))
+
+ if self._port.get_option('use_test_shell'):
+ cmd.append('--layout-tests')
+ else:
+ cmd.append('--test-shell')
+
+ if self._port.get_option('startup_dialog'):
+ cmd.append('--testshell-startup-dialog')
+
+ if self._port.get_option('gp_fault_error_box'):
+ cmd.append('--gp-fault-error-box')
+
+ if self._port.get_option('js_flags') is not None:
+ cmd.append('--js-flags="' + self._port.get_option('js_flags') + '"')
+
+ if self._port.get_option('multiple_loads') > 0:
+ cmd.append('--multiple-loads=' + str(self._port.get_option('multiple_loads')))
+
+ # test_shell does not support accelerated compositing.
+ if not self._port.get_option("use_test_shell"):
+ if self._port.get_option('accelerated_compositing'):
+ cmd.append('--enable-accelerated-compositing')
+ if self._port.get_option('accelerated_2d_canvas'):
+ cmd.append('--enable-accelerated-2d-canvas')
+ return cmd
+
+ def start(self):
+ # FIXME: Should be an error to call this method twice.
+ cmd = self.cmd_line()
+
+ # We need to pass close_fds=True to work around Python bug #2320
+ # (otherwise we can hang when we kill DumpRenderTree when we are running
+ # multiple threads). See http://bugs.python.org/issue2320 .
+ # Note that close_fds isn't supported on Windows, but this bug only
+ # shows up on Mac and Linux.
+ close_flag = sys.platform not in ('win32', 'cygwin')
+ self._proc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ close_fds=close_flag)
+
+ def poll(self):
+ # poll() is not threadsafe and can throw OSError due to:
+ # http://bugs.python.org/issue1731717
+ return self._proc.poll()
+
+ def _write_command_and_read_line(self, input=None):
+ """Returns a tuple: (line, did_crash)"""
+ try:
+ if input:
+ if isinstance(input, unicode):
+ # TestShell expects utf-8
+ input = input.encode("utf-8")
+ self._proc.stdin.write(input)
+ # DumpRenderTree text output is always UTF-8. However some tests
+ # (e.g. webarchive) may spit out binary data instead of text so we
+ # don't bother to decode the output (for either DRT or test_shell).
+ line = self._proc.stdout.readline()
+ # We could assert() here that line correctly decodes as UTF-8.
+ return (line, False)
+ except IOError, e:
+ _log.error("IOError communicating w/ test_shell: " + str(e))
+ return (None, True)
+
+ def _test_shell_command(self, uri, timeoutms, checksum):
+ cmd = uri
+ if timeoutms:
+ cmd += ' ' + str(timeoutms)
+ if checksum:
+ cmd += ' ' + checksum
+ cmd += "\n"
+ return cmd
+
+ def _output_image(self):
+ """Returns the image output which driver generated."""
+ png_path = self._image_path
+ if png_path and os.path.isfile(png_path):
+ with open(png_path, 'rb') as image_file:
+ return image_file.read()
+ else:
+ return None
+
+ def _output_image_with_retry(self):
+ # Retry a few more times because open() sometimes fails on Windows,
+ # raising "IOError: [Errno 13] Permission denied:"
+ retry_num = 50
+ timeout_seconds = 5.0
+ for i in range(retry_num):
+ try:
+ return self._output_image()
+ except IOError, e:
+ if e.errno == errno.EACCES:
+ time.sleep(timeout_seconds / retry_num)
+ else:
+ raise e
+ return self._output_image()
+
+ def run_test(self, test_input):
+ output = []
+ error = []
+ crash = False
+ timeout = False
+ actual_uri = None
+ actual_checksum = None
+
+ start_time = time.time()
+
+ uri = self._port.filename_to_uri(test_input.filename)
+ cmd = self._test_shell_command(uri, test_input.timeout,
+ test_input.image_hash)
+ (line, crash) = self._write_command_and_read_line(input=cmd)
+
+ while not crash and line.rstrip() != "#EOF":
+ # Make sure we haven't crashed.
+ if line == '' and self.poll() is not None:
+ # This is hex code 0xc000001d, which is used for abrupt
+ # termination. This happens if we hit ctrl+c from the prompt
+ # and we happen to be waiting on test_shell.
+ # sdoyon: Not sure for which OS and in what circumstances the
+ # above code is valid. What works for me under Linux to detect
+ # ctrl+c is for the subprocess returncode to be negative
+ # SIGINT. And that agrees with the subprocess documentation.
+ if (-1073741510 == self._proc.returncode or
+ - signal.SIGINT == self._proc.returncode):
+ raise KeyboardInterrupt
+ crash = True
+ break
+
+ # Don't include #URL lines in our output
+ if line.startswith("#URL:"):
+ actual_uri = line.rstrip()[5:]
+ if uri != actual_uri:
+ # GURL capitalizes the drive letter of a file URL.
+ if (not re.search("^file:///[a-z]:", uri) or
+ uri.lower() != actual_uri.lower()):
+ _log.fatal("Test got out of sync:\n|%s|\n|%s|" %
+ (uri, actual_uri))
+ raise AssertionError("test out of sync")
+ elif line.startswith("#MD5:"):
+ actual_checksum = line.rstrip()[5:]
+ elif line.startswith("#TEST_TIMED_OUT"):
+ timeout = True
+ # Test timed out, but we still need to read until #EOF.
+ elif actual_uri:
+ output.append(line)
+ else:
+ error.append(line)
+
+ (line, crash) = self._write_command_and_read_line(input=None)
+
+ run_time = time.time() - start_time
+ return test_output.TestOutput(
+ ''.join(output), self._output_image_with_retry(), actual_checksum,
+ crash, run_time, timeout, ''.join(error))
+
+ def stop(self):
+ if self._proc:
+ self._proc.stdin.close()
+ self._proc.stdout.close()
+ if self._proc.stderr:
+ self._proc.stderr.close()
+ if sys.platform not in ('win32', 'cygwin'):
+ # Closing stdin/stdout/stderr hangs sometimes on OS X,
+ # (see __init__(), above), and anyway we don't want to hang
+ # the harness if test_shell is buggy, so we wait a couple
+ # seconds to give test_shell a chance to clean up, but then
+ # force-kill the process if necessary.
+ KILL_TIMEOUT = 3.0
+ timeout = time.time() + KILL_TIMEOUT
+ # poll() is not threadsafe and can throw OSError due to:
+ # http://bugs.python.org/issue1731717
+ while self._proc.poll() is None and time.time() < timeout:
+ time.sleep(0.1)
+ # poll() is not threadsafe and can throw OSError due to:
+ # http://bugs.python.org/issue1731717
+ if self._proc.poll() is None:
+ _log.warning('stopping test driver timed out, '
+ 'killing it')
+ self._port._executive.kill_process(self._proc.pid)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py
new file mode 100644
index 0000000..c1f5c8d
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu.py
@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import with_statement
+
+import codecs
+import os
+import sys
+
+import chromium_linux
+import chromium_mac
+import chromium_win
+
+
+def get(**kwargs):
+ """Some tests have slightly different results when run while using
+ hardware acceleration. In those cases, we prepend an additional directory
+ to the baseline paths."""
+ port_name = kwargs.get('port_name', None)
+ if port_name == 'chromium-gpu':
+ if sys.platform in ('cygwin', 'win32'):
+ port_name = 'chromium-gpu-win'
+ elif sys.platform == 'linux2':
+ port_name = 'chromium-gpu-linux'
+ elif sys.platform == 'darwin':
+ port_name = 'chromium-gpu-mac'
+ else:
+ raise NotImplementedError('unsupported platform: %s' %
+ sys.platform)
+
+ if port_name == 'chromium-gpu-linux':
+ return ChromiumGpuLinuxPort(**kwargs)
+
+ if port_name.startswith('chromium-gpu-mac'):
+ return ChromiumGpuMacPort(**kwargs)
+
+ if port_name.startswith('chromium-gpu-win'):
+ return ChromiumGpuWinPort(**kwargs)
+
+ raise NotImplementedError('unsupported port: %s' % port_name)
+
+
+def _set_gpu_options(options):
+ if options:
+ if options.accelerated_compositing is None:
+ options.accelerated_compositing = True
+ if options.accelerated_2d_canvas is None:
+ options.accelerated_2d_canvas = True
+
+ # FIXME: Remove this after http://codereview.chromium.org/5133001/ is enabled
+ # on the bots.
+ if options.builder_name is not None and not ' - GPU' in options.builder_name:
+ options.builder_name = options.builder_name + ' - GPU'
+
+
+def _gpu_overrides(port):
+ try:
+ overrides_path = port.path_from_chromium_base('webkit', 'tools',
+ 'layout_tests', 'test_expectations_gpu.txt')
+ except AssertionError:
+ return None
+ if not os.path.exists(overrides_path):
+ return None
+ with codecs.open(overrides_path, "r", "utf-8") as file:
+ return file.read()
+
+
+class ChromiumGpuLinuxPort(chromium_linux.ChromiumLinuxPort):
+ def __init__(self, **kwargs):
+ kwargs.setdefault('port_name', 'chromium-gpu-linux')
+ _set_gpu_options(kwargs.get('options'))
+ chromium_linux.ChromiumLinuxPort.__init__(self, **kwargs)
+
+ def baseline_search_path(self):
+ # Mimic the Linux -> Win expectations fallback in the ordinary Chromium port.
+ return (map(self._webkit_baseline_path, ['chromium-gpu-linux', 'chromium-gpu-win', 'chromium-gpu']) +
+ chromium_linux.ChromiumLinuxPort.baseline_search_path(self))
+
+ def default_child_processes(self):
+ return 1
+
+ def path_to_test_expectations_file(self):
+ return self.path_from_webkit_base('LayoutTests', 'platform',
+ 'chromium-gpu', 'test_expectations.txt')
+
+ def test_expectations_overrides(self):
+ return _gpu_overrides(self)
+
+
+class ChromiumGpuMacPort(chromium_mac.ChromiumMacPort):
+ def __init__(self, **kwargs):
+ kwargs.setdefault('port_name', 'chromium-gpu-mac')
+ _set_gpu_options(kwargs.get('options'))
+ chromium_mac.ChromiumMacPort.__init__(self, **kwargs)
+
+ def baseline_search_path(self):
+ return (map(self._webkit_baseline_path, ['chromium-gpu-mac', 'chromium-gpu']) +
+ chromium_mac.ChromiumMacPort.baseline_search_path(self))
+
+ def default_child_processes(self):
+ return 1
+
+ def path_to_test_expectations_file(self):
+ return self.path_from_webkit_base('LayoutTests', 'platform',
+ 'chromium-gpu', 'test_expectations.txt')
+
+ def test_expectations_overrides(self):
+ return _gpu_overrides(self)
+
+
+class ChromiumGpuWinPort(chromium_win.ChromiumWinPort):
+ def __init__(self, **kwargs):
+ kwargs.setdefault('port_name', 'chromium-gpu-win' + self.version())
+ _set_gpu_options(kwargs.get('options'))
+ chromium_win.ChromiumWinPort.__init__(self, **kwargs)
+
+ def baseline_search_path(self):
+ return (map(self._webkit_baseline_path, ['chromium-gpu-win', 'chromium-gpu']) +
+ chromium_win.ChromiumWinPort.baseline_search_path(self))
+
+ def default_child_processes(self):
+ return 1
+
+ def path_to_test_expectations_file(self):
+ return self.path_from_webkit_base('LayoutTests', 'platform',
+ 'chromium-gpu', 'test_expectations.txt')
+
+ def test_expectations_overrides(self):
+ return _gpu_overrides(self)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py
new file mode 100644
index 0000000..ad0404c
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_gpu_unittest.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import unittest
+
+from webkitpy.tool import mocktool
+import chromium_gpu
+
+
+class ChromiumGpuTest(unittest.TestCase):
+ def test_get_chromium_gpu_linux(self):
+ self.assertOverridesWorked('chromium-gpu-linux')
+
+ def test_get_chromium_gpu_mac(self):
+ self.assertOverridesWorked('chromium-gpu-mac')
+
+ def test_get_chromium_gpu_win(self):
+ self.assertOverridesWorked('chromium-gpu-win')
+
+ def assertOverridesWorked(self, port_name):
+ # test that we got the right port
+ mock_options = mocktool.MockOptions(accelerated_compositing=None,
+ accelerated_2d_canvas=None,
+ builder_name='foo',
+ child_processes=None)
+ port = chromium_gpu.get(port_name=port_name, options=mock_options)
+ self.assertTrue(port._options.accelerated_compositing)
+ self.assertTrue(port._options.accelerated_2d_canvas)
+ self.assertEqual(port.default_child_processes(), 1)
+ self.assertEqual(port._options.builder_name, 'foo - GPU')
+
+ # we use startswith() instead of Equal to gloss over platform versions.
+ self.assertTrue(port.name().startswith(port_name))
+
+ # test that it has the right directories in front of the search path.
+ paths = port.baseline_search_path()
+ self.assertEqual(port._webkit_baseline_path(port_name), paths[0])
+ if port_name == 'chromium-gpu-linux':
+ self.assertEqual(port._webkit_baseline_path('chromium-gpu-win'), paths[1])
+ self.assertEqual(port._webkit_baseline_path('chromium-gpu'), paths[2])
+ else:
+ self.assertEqual(port._webkit_baseline_path('chromium-gpu'), paths[1])
+
+ # Test that we have the right expectations file.
+ self.assertTrue('chromium-gpu' in
+ port.path_to_test_expectations_file())
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py
new file mode 100644
index 0000000..5d9dd87
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_linux.py
@@ -0,0 +1,190 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Chromium Linux implementation of the Port interface."""
+
+import logging
+import os
+import signal
+
+import chromium
+
+_log = logging.getLogger("webkitpy.layout_tests.port.chromium_linux")
+
+
+class ChromiumLinuxPort(chromium.ChromiumPort):
+ """Chromium Linux implementation of the Port class."""
+
+ def __init__(self, **kwargs):
+ kwargs.setdefault('port_name', 'chromium-linux')
+ chromium.ChromiumPort.__init__(self, **kwargs)
+
+ def baseline_search_path(self):
+ port_names = ["chromium-linux", "chromium-win", "chromium", "win", "mac"]
+ return map(self._webkit_baseline_path, port_names)
+
+ def check_build(self, needs_http):
+ result = chromium.ChromiumPort.check_build(self, needs_http)
+ if needs_http:
+ if self.get_option('use_apache'):
+ result = self._check_apache_install() and result
+ else:
+ result = self._check_lighttpd_install() and result
+ result = self._check_wdiff_install() and result
+
+ if not result:
+ _log.error('For complete Linux build requirements, please see:')
+ _log.error('')
+ _log.error(' http://code.google.com/p/chromium/wiki/'
+ 'LinuxBuildInstructions')
+ return result
+
+ def test_platform_name(self):
+ # We use 'linux' instead of 'chromium-linux' in test_expectations.txt.
+ return 'linux'
+
+ def version(self):
+ # We don't have different versions on linux.
+ return ''
+
+ #
+ # PROTECTED METHODS
+ #
+
+ def _build_path(self, *comps):
+ base = self.path_from_chromium_base()
+ if os.path.exists(os.path.join(base, 'sconsbuild')):
+ return os.path.join(base, 'sconsbuild', *comps)
+ if os.path.exists(os.path.join(base, 'out', *comps)) or self.get_option('use_test_shell'):
+ return os.path.join(base, 'out', *comps)
+ base = self.path_from_webkit_base()
+ if os.path.exists(os.path.join(base, 'sconsbuild')):
+ return os.path.join(base, 'sconsbuild', *comps)
+ return os.path.join(base, 'out', *comps)
+
+ def _check_apache_install(self):
+ result = chromium.check_file_exists(self._path_to_apache(),
+ "apache2")
+ result = chromium.check_file_exists(self._path_to_apache_config_file(),
+ "apache2 config file") and result
+ if not result:
+ _log.error(' Please install using: "sudo apt-get install '
+ 'apache2 libapache2-mod-php5"')
+ _log.error('')
+ return result
+
+ def _check_lighttpd_install(self):
+ result = chromium.check_file_exists(
+ self._path_to_lighttpd(), "LigHTTPd executable")
+ result = chromium.check_file_exists(self._path_to_lighttpd_php(),
+ "PHP CGI executable") and result
+ result = chromium.check_file_exists(self._path_to_lighttpd_modules(),
+ "LigHTTPd modules") and result
+ if not result:
+ _log.error(' Please install using: "sudo apt-get install '
+ 'lighttpd php5-cgi"')
+ _log.error('')
+ return result
+
+ def _check_wdiff_install(self):
+ result = chromium.check_file_exists(self._path_to_wdiff(), 'wdiff')
+ if not result:
+ _log.error(' Please install using: "sudo apt-get install '
+ 'wdiff"')
+ _log.error('')
+ # FIXME: The ChromiumMac port always returns True.
+ return result
+
+ def _path_to_apache(self):
+ if self._is_redhat_based():
+ return '/usr/sbin/httpd'
+ else:
+ return '/usr/sbin/apache2'
+
+ def _path_to_apache_config_file(self):
+ if self._is_redhat_based():
+ config_name = 'fedora-httpd.conf'
+ else:
+ config_name = 'apache2-debian-httpd.conf'
+
+ return os.path.join(self.layout_tests_dir(), 'http', 'conf',
+ config_name)
+
+ def _path_to_lighttpd(self):
+ return "/usr/sbin/lighttpd"
+
+ def _path_to_lighttpd_modules(self):
+ return "/usr/lib/lighttpd"
+
+ def _path_to_lighttpd_php(self):
+ return "/usr/bin/php-cgi"
+
+ def _path_to_driver(self, configuration=None):
+ if not configuration:
+ configuration = self.get_option('configuration')
+ binary_name = 'DumpRenderTree'
+ if self.get_option('use_test_shell'):
+ binary_name = 'test_shell'
+ return self._build_path(configuration, binary_name)
+
+ def _path_to_helper(self):
+ return None
+
+ def _path_to_wdiff(self):
+ if self._is_redhat_based():
+ return '/usr/bin/dwdiff'
+ else:
+ return '/usr/bin/wdiff'
+
+ def _is_redhat_based(self):
+ return os.path.exists(os.path.join('/etc', 'redhat-release'))
+
+ def _shut_down_http_server(self, server_pid):
+ """Shut down the lighttpd web server. Blocks until it's fully
+ shut down.
+
+ Args:
+ server_pid: The process ID of the running server.
+ """
+ # server_pid is not set when "http_server.py stop" is run manually.
+ if server_pid is None:
+ # TODO(mmoss) This isn't ideal, since it could conflict with
+ # lighttpd processes not started by http_server.py,
+ # but good enough for now.
+ self._executive.kill_all("lighttpd")
+ self._executive.kill_all("apache2")
+ else:
+ try:
+ os.kill(server_pid, signal.SIGTERM)
+ # TODO(mmoss) Maybe throw in a SIGKILL just to be sure?
+ except OSError:
+ # Sometimes we get a bad PID (e.g. from a stale httpd.pid
+ # file), so if kill fails on the given PID, just try to
+ # 'killall' web servers.
+ self._shut_down_http_server(None)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py
new file mode 100644
index 0000000..f638e01
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac.py
@@ -0,0 +1,182 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Chromium Mac implementation of the Port interface."""
+
+import logging
+import os
+import platform
+import signal
+
+import chromium
+
+from webkitpy.common.system.executive import Executive
+
+_log = logging.getLogger("webkitpy.layout_tests.port.chromium_mac")
+
+
+class ChromiumMacPort(chromium.ChromiumPort):
+ """Chromium Mac implementation of the Port class."""
+
+ def __init__(self, **kwargs):
+ kwargs.setdefault('port_name', 'chromium-mac')
+ chromium.ChromiumPort.__init__(self, **kwargs)
+
+ def baseline_search_path(self):
+ port_names = [
+ "chromium-mac" + self.version(),
+ "chromium-mac",
+ "chromium",
+ "mac" + self.version(),
+ "mac",
+ ]
+ return map(self._webkit_baseline_path, port_names)
+
+ def check_build(self, needs_http):
+ result = chromium.ChromiumPort.check_build(self, needs_http)
+ result = self._check_wdiff_install() and result
+ if not result:
+ _log.error('For complete Mac build requirements, please see:')
+ _log.error('')
+ _log.error(' http://code.google.com/p/chromium/wiki/'
+ 'MacBuildInstructions')
+ return result
+
+ def default_child_processes(self):
+ # FIXME: we need to run single-threaded for now. See
+ # https://bugs.webkit.org/show_bug.cgi?id=38553. Unfortunately this
+ # routine is called right before the logger is configured, so if we
+ # try to _log.warning(), it gets thrown away.
+ import sys
+ sys.stderr.write("Defaulting to one child - see https://bugs.webkit.org/show_bug.cgi?id=38553\n")
+ return 1
+
+ def driver_name(self):
+ """name for this port's equivalent of DumpRenderTree."""
+ if self.get_option('use_test_shell'):
+ return "TestShell"
+ return "DumpRenderTree"
+
+ def test_platform_name(self):
+ # We use 'mac' instead of 'chromium-mac'
+ return 'mac'
+
+ def version(self):
+ # FIXME: It's strange that this string is -version, not just version.
+ os_version_string = platform.mac_ver()[0] # e.g. "10.5.6"
+ if not os_version_string:
+ return '-leopard'
+ release_version = int(os_version_string.split('.')[1])
+ # we don't support 'tiger' or earlier releases
+ if release_version == 5:
+ return '-leopard'
+ elif release_version == 6:
+ return '-snowleopard'
+ return ''
+
+ #
+ # PROTECTED METHODS
+ #
+
+ def _build_path(self, *comps):
+ path = self.path_from_chromium_base('xcodebuild', *comps)
+ if os.path.exists(path) or self.get_option('use_test_shell'):
+ return path
+ return self.path_from_webkit_base('WebKit', 'chromium', 'xcodebuild',
+ *comps)
+
+ def _check_wdiff_install(self):
+ try:
+ # We're ignoring the return and always returning True
+ self._executive.run_command([self._path_to_wdiff()], error_handler=Executive.ignore_error)
+ except OSError:
+ _log.warning('wdiff not found. Install using MacPorts or some '
+ 'other means')
+ return True
+
+ def _lighttpd_path(self, *comps):
+ return self.path_from_chromium_base('third_party', 'lighttpd',
+ 'mac', *comps)
+
+ def _path_to_apache(self):
+ return '/usr/sbin/httpd'
+
+ def _path_to_apache_config_file(self):
+ return os.path.join(self.layout_tests_dir(), 'http', 'conf',
+ 'apache2-httpd.conf')
+
+ def _path_to_lighttpd(self):
+ return self._lighttpd_path('bin', 'lighttpd')
+
+ def _path_to_lighttpd_modules(self):
+ return self._lighttpd_path('lib')
+
+ def _path_to_lighttpd_php(self):
+ return self._lighttpd_path('bin', 'php-cgi')
+
+ def _path_to_driver(self, configuration=None):
+ # FIXME: make |configuration| happy with case-sensitive file
+ # systems.
+ if not configuration:
+ configuration = self.get_option('configuration')
+ return self._build_path(configuration, self.driver_name() + '.app',
+ 'Contents', 'MacOS', self.driver_name())
+
+ def _path_to_helper(self):
+ binary_name = 'LayoutTestHelper'
+ if self.get_option('use_test_shell'):
+ binary_name = 'layout_test_helper'
+ return self._build_path(self.get_option('configuration'), binary_name)
+
+ def _path_to_wdiff(self):
+ return 'wdiff'
+
+ def _shut_down_http_server(self, server_pid):
+ """Shut down the lighttpd web server. Blocks until it's fully
+ shut down.
+
+ Args:
+ server_pid: The process ID of the running server.
+ """
+ # server_pid is not set when "http_server.py stop" is run manually.
+ if server_pid is None:
+ # TODO(mmoss) This isn't ideal, since it could conflict with
+ # lighttpd processes not started by http_server.py,
+ # but good enough for now.
+ self._executive.kill_all('lighttpd')
+ self._executive.kill_all('httpd')
+ else:
+ try:
+ os.kill(server_pid, signal.SIGTERM)
+ # TODO(mmoss) Maybe throw in a SIGKILL just to be sure?
+ except OSError:
+ # Sometimes we get a bad PID (e.g. from a stale httpd.pid
+ # file), so if kill fails on the given PID, just try to
+ # 'killall' web servers.
+ self._shut_down_http_server(None)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py
new file mode 100644
index 0000000..d63faa0
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_mac_unittest.py
@@ -0,0 +1,40 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import chromium_mac
+import unittest
+
+from webkitpy.thirdparty.mock import Mock
+
+
+class ChromiumMacPortTest(unittest.TestCase):
+
+ def test_check_wdiff_install(self):
+ port = chromium_mac.ChromiumMacPort()
+ # Currently is always true, just logs if missing.
+ self.assertTrue(port._check_wdiff_install())
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py
new file mode 100644
index 0000000..c87984f
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py
@@ -0,0 +1,193 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import unittest
+import StringIO
+
+from webkitpy.tool import mocktool
+from webkitpy.thirdparty.mock import Mock
+
+import chromium
+import chromium_linux
+import chromium_mac
+import chromium_win
+
+class ChromiumDriverTest(unittest.TestCase):
+
+ def setUp(self):
+ mock_port = Mock()
+ mock_port.get_option = lambda option_name: ''
+ self.driver = chromium.ChromiumDriver(mock_port, worker_number=0)
+
+ def test_test_shell_command(self):
+ expected_command = "test.html 2 checksum\n"
+ self.assertEqual(self.driver._test_shell_command("test.html", 2, "checksum"), expected_command)
+
+ def _assert_write_command_and_read_line(self, input=None, expected_line=None, expected_stdin=None, expected_crash=False):
+ if not expected_stdin:
+ if input:
+ expected_stdin = input
+ else:
+ # We reset stdin, so we should expect stdin.getValue = ""
+ expected_stdin = ""
+ self.driver._proc.stdin = StringIO.StringIO()
+ line, did_crash = self.driver._write_command_and_read_line(input)
+ self.assertEqual(self.driver._proc.stdin.getvalue(), expected_stdin)
+ self.assertEqual(line, expected_line)
+ self.assertEqual(did_crash, expected_crash)
+
+ def test_write_command_and_read_line(self):
+ self.driver._proc = Mock()
+ # Set up to read 3 lines before we get an IOError
+ self.driver._proc.stdout = StringIO.StringIO("first\nsecond\nthird\n")
+
+ unicode_input = u"I \u2661 Unicode"
+ utf8_input = unicode_input.encode("utf-8")
+ # Test unicode input conversion to utf-8
+ self._assert_write_command_and_read_line(input=unicode_input, expected_stdin=utf8_input, expected_line="first\n")
+ # Test str() input.
+ self._assert_write_command_and_read_line(input="foo", expected_line="second\n")
+ # Test input=None
+ self._assert_write_command_and_read_line(expected_line="third\n")
+ # Test reading from a closed/empty stream.
+ # reading from a StringIO does not raise IOError like a real file would, so raise IOError manually.
+ def mock_readline():
+ raise IOError
+ self.driver._proc.stdout.readline = mock_readline
+ self._assert_write_command_and_read_line(expected_crash=True)
+
+
+class ChromiumPortTest(unittest.TestCase):
+ class TestMacPort(chromium_mac.ChromiumMacPort):
+ def __init__(self, options):
+ chromium_mac.ChromiumMacPort.__init__(self,
+ port_name='test-port',
+ options=options)
+
+ def default_configuration(self):
+ self.default_configuration_called = True
+ return 'default'
+
+ class TestLinuxPort(chromium_linux.ChromiumLinuxPort):
+ def __init__(self, options):
+ chromium_linux.ChromiumLinuxPort.__init__(self,
+ port_name='test-port',
+ options=options)
+
+ def default_configuration(self):
+ self.default_configuration_called = True
+ return 'default'
+
+ def test_path_to_image_diff(self):
+ mock_options = mocktool.MockOptions()
+ port = ChromiumPortTest.TestLinuxPort(options=mock_options)
+ self.assertTrue(port._path_to_image_diff().endswith(
+ '/out/default/ImageDiff'), msg=port._path_to_image_diff())
+ port = ChromiumPortTest.TestMacPort(options=mock_options)
+ self.assertTrue(port._path_to_image_diff().endswith(
+ '/xcodebuild/default/ImageDiff'))
+ mock_options = mocktool.MockOptions(use_test_shell=True)
+ port = ChromiumPortTest.TestLinuxPort(options=mock_options)
+ self.assertTrue(port._path_to_image_diff().endswith(
+ '/out/default/image_diff'), msg=port._path_to_image_diff())
+ port = ChromiumPortTest.TestMacPort(options=mock_options)
+ self.assertTrue(port._path_to_image_diff().endswith(
+ '/xcodebuild/default/image_diff'))
+ # FIXME: Figure out how this is going to work on Windows.
+ #port = chromium_win.ChromiumWinPort('test-port', options=MockOptions())
+
+ def test_skipped_layout_tests(self):
+ mock_options = mocktool.MockOptions()
+ port = ChromiumPortTest.TestLinuxPort(options=mock_options)
+
+ fake_test = os.path.join(port.layout_tests_dir(), "fast/js/not-good.js")
+
+ port.test_expectations = lambda: """BUG_TEST SKIP : fast/js/not-good.js = TEXT
+LINUX WIN : fast/js/very-good.js = TIMEOUT PASS"""
+ port.test_expectations_overrides = lambda: ''
+ port.tests = lambda paths: set()
+ port.path_exists = lambda test: True
+
+ skipped_tests = port.skipped_layout_tests(extra_test_files=[fake_test, ])
+ self.assertTrue("fast/js/not-good.js" in skipped_tests)
+
+ def test_default_configuration(self):
+ mock_options = mocktool.MockOptions()
+ port = ChromiumPortTest.TestLinuxPort(options=mock_options)
+ self.assertEquals(mock_options.configuration, 'default')
+ self.assertTrue(port.default_configuration_called)
+
+ mock_options = mocktool.MockOptions(configuration=None)
+ port = ChromiumPortTest.TestLinuxPort(mock_options)
+ self.assertEquals(mock_options.configuration, 'default')
+ self.assertTrue(port.default_configuration_called)
+
+ def test_diff_image(self):
+ class TestPort(ChromiumPortTest.TestLinuxPort):
+ def _path_to_image_diff(self):
+ return "/path/to/image_diff"
+
+ class MockExecute:
+ def __init__(self, result):
+ self._result = result
+
+ def run_command(self,
+ args,
+ cwd=None,
+ input=None,
+ error_handler=None,
+ return_exit_code=False,
+ return_stderr=True,
+ decode_output=False):
+ if return_exit_code:
+ return self._result
+ return ''
+
+ mock_options = mocktool.MockOptions()
+ port = ChromiumPortTest.TestLinuxPort(mock_options)
+
+ # Images are different.
+ port._executive = MockExecute(0)
+ self.assertEquals(False, port.diff_image("EXPECTED", "ACTUAL"))
+
+ # Images are the same.
+ port._executive = MockExecute(1)
+ self.assertEquals(True, port.diff_image("EXPECTED", "ACTUAL"))
+
+ # There was some error running image_diff.
+ port._executive = MockExecute(2)
+ exception_raised = False
+ try:
+ port.diff_image("EXPECTED", "ACTUAL")
+ except ValueError, e:
+ exception_raised = True
+ self.assertFalse(exception_raised)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py
new file mode 100644
index 0000000..d080f82
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Chromium Win implementation of the Port interface."""
+
+import logging
+import os
+import sys
+
+import chromium
+
+_log = logging.getLogger("webkitpy.layout_tests.port.chromium_win")
+
+
+class ChromiumWinPort(chromium.ChromiumPort):
+ """Chromium Win implementation of the Port class."""
+
+ def __init__(self, **kwargs):
+ kwargs.setdefault('port_name', 'chromium-win' + self.version())
+ chromium.ChromiumPort.__init__(self, **kwargs)
+
+ def setup_environ_for_server(self):
+ env = chromium.ChromiumPort.setup_environ_for_server(self)
+ # Put the cygwin directory first in the path to find cygwin1.dll.
+ env["PATH"] = "%s;%s" % (
+ self.path_from_chromium_base("third_party", "cygwin", "bin"),
+ env["PATH"])
+ # Configure the cygwin directory so that pywebsocket finds proper
+ # python executable to run cgi program.
+ env["CYGWIN_PATH"] = self.path_from_chromium_base(
+ "third_party", "cygwin", "bin")
+ if (sys.platform == "win32" and self.get_option('register_cygwin')):
+ setup_mount = self.path_from_chromium_base("third_party",
+ "cygwin",
+ "setup_mount.bat")
+ self._executive.run_command([setup_mount])
+ return env
+
+ def baseline_search_path(self):
+ port_names = []
+ if self._name.endswith('-win-xp'):
+ port_names.append("chromium-win-xp")
+ if self._name.endswith('-win-xp') or self._name.endswith('-win-vista'):
+ port_names.append("chromium-win-vista")
+ # FIXME: This may need to include mac-snowleopard like win.py.
+ port_names.extend(["chromium-win", "chromium", "win", "mac"])
+ return map(self._webkit_baseline_path, port_names)
+
+ def check_build(self, needs_http):
+ result = chromium.ChromiumPort.check_build(self, needs_http)
+ if not result:
+ _log.error('For complete Windows build requirements, please '
+ 'see:')
+ _log.error('')
+ _log.error(' http://dev.chromium.org/developers/how-tos/'
+ 'build-instructions-windows')
+ return result
+
+ def relative_test_filename(self, filename):
+ path = filename[len(self.layout_tests_dir()) + 1:]
+ return path.replace('\\', '/')
+
+ def test_platform_name(self):
+ # We return 'win-xp', not 'chromium-win-xp' here, for convenience.
+ return 'win' + self.version()
+
+ def version(self):
+ if not hasattr(sys, 'getwindowsversion'):
+ return ''
+ winver = sys.getwindowsversion()
+ if winver[0] == 6 and (winver[1] == 1):
+ return '-7'
+ if winver[0] == 6 and (winver[1] == 0):
+ return '-vista'
+ if winver[0] == 5 and (winver[1] == 1 or winver[1] == 2):
+ return '-xp'
+ return ''
+
+ #
+ # PROTECTED ROUTINES
+ #
+ def _build_path(self, *comps):
+ p = self.path_from_chromium_base('webkit', *comps)
+ if os.path.exists(p):
+ return p
+ p = self.path_from_chromium_base('chrome', *comps)
+ if os.path.exists(p) or self.get_option('use_test_shell'):
+ return p
+ return os.path.join(self.path_from_webkit_base(), 'WebKit', 'chromium',
+ *comps)
+
+ def _lighttpd_path(self, *comps):
+ return self.path_from_chromium_base('third_party', 'lighttpd', 'win',
+ *comps)
+
+ def _path_to_apache(self):
+ return self.path_from_chromium_base('third_party', 'cygwin', 'usr',
+ 'sbin', 'httpd')
+
+ def _path_to_apache_config_file(self):
+ return os.path.join(self.layout_tests_dir(), 'http', 'conf',
+ 'cygwin-httpd.conf')
+
+ def _path_to_lighttpd(self):
+ return self._lighttpd_path('LightTPD.exe')
+
+ def _path_to_lighttpd_modules(self):
+ return self._lighttpd_path('lib')
+
+ def _path_to_lighttpd_php(self):
+ return self._lighttpd_path('php5', 'php-cgi.exe')
+
+ def _path_to_driver(self, configuration=None):
+ if not configuration:
+ configuration = self.get_option('configuration')
+ binary_name = 'DumpRenderTree.exe'
+ if self.get_option('use_test_shell'):
+ binary_name = 'test_shell.exe'
+ return self._build_path(configuration, binary_name)
+
+ def _path_to_helper(self):
+ binary_name = 'LayoutTestHelper.exe'
+ if self.get_option('use_test_shell'):
+ binary_name = 'layout_test_helper.exe'
+ return self._build_path(self.get_option('configuration'), binary_name)
+
+ def _path_to_image_diff(self):
+ binary_name = 'ImageDiff.exe'
+ if self.get_option('use_test_shell'):
+ binary_name = 'image_diff.exe'
+ return self._build_path(self.get_option('configuration'), binary_name)
+
+ def _path_to_wdiff(self):
+ return self.path_from_chromium_base('third_party', 'cygwin', 'bin',
+ 'wdiff.exe')
+
+ def _shut_down_http_server(self, server_pid):
+ """Shut down the lighttpd web server. Blocks until it's fully
+ shut down.
+
+ Args:
+ server_pid: The process ID of the running server.
+ """
+ # FIXME: Why are we ignoring server_pid and calling
+ # _kill_all instead of Executive.kill_process(pid)?
+ self._executive.kill_all("LightTPD.exe")
+ self._executive.kill_all("httpd.exe")
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/chromium_win_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win_unittest.py
new file mode 100644
index 0000000..36f3c6b
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/chromium_win_unittest.py
@@ -0,0 +1,74 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import sys
+import unittest
+import chromium_win
+from webkitpy.common.system import outputcapture
+from webkitpy.tool import mocktool
+
+
+class ChromiumWinTest(unittest.TestCase):
+
+ class RegisterCygwinOption(object):
+ def __init__(self):
+ self.register_cygwin = True
+
+ def setUp(self):
+ self.orig_platform = sys.platform
+
+ def tearDown(self):
+ sys.platform = self.orig_platform
+
+ def _mock_path_from_chromium_base(self, *comps):
+ return os.path.join("/chromium/src", *comps)
+
+ def test_setup_environ_for_server(self):
+ port = chromium_win.ChromiumWinPort()
+ port._executive = mocktool.MockExecutive(should_log=True)
+ port.path_from_chromium_base = self._mock_path_from_chromium_base
+ output = outputcapture.OutputCapture()
+ orig_environ = os.environ.copy()
+ env = output.assert_outputs(self, port.setup_environ_for_server)
+ self.assertEqual(orig_environ["PATH"], os.environ["PATH"])
+ self.assertNotEqual(env["PATH"], os.environ["PATH"])
+
+ def test_setup_environ_for_server_register_cygwin(self):
+ sys.platform = "win32"
+ port = chromium_win.ChromiumWinPort(
+ options=ChromiumWinTest.RegisterCygwinOption())
+ port._executive = mocktool.MockExecutive(should_log=True)
+ port.path_from_chromium_base = self._mock_path_from_chromium_base
+ setup_mount = self._mock_path_from_chromium_base("third_party",
+ "cygwin",
+ "setup_mount.bat")
+ expected_stderr = "MOCK run_command: %s\n" % [setup_mount]
+ output = outputcapture.OutputCapture()
+ output.assert_outputs(self, port.setup_environ_for_server,
+ expected_stderr=expected_stderr)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/config.py b/Tools/Scripts/webkitpy/layout_tests/port/config.py
new file mode 100644
index 0000000..e08ed9d
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/config.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Wrapper objects for WebKit-specific utility routines."""
+
+# FIXME: This file needs to be unified with common/checkout/scm.py and
+# common/config/ports.py .
+
+import os
+
+from webkitpy.common.system import logutils
+from webkitpy.common.system import executive
+
+
+_log = logutils.get_logger(__file__)
+
+#
+# FIXME: This is used to record if we've already hit the filesystem to look
+# for a default configuration. We cache this to speed up the unit tests,
+# but this can be reset with clear_cached_configuration(). This should be
+# replaced with us consistently using MockConfigs() for tests that don't
+# hit the filesystem at all and provide a reliable value.
+#
+_have_determined_configuration = False
+_configuration = "Release"
+
+
+def clear_cached_configuration():
+ global _have_determined_configuration, _configuration
+ _have_determined_configuration = False
+ _configuration = "Release"
+
+
+class Config(object):
+ _FLAGS_FROM_CONFIGURATIONS = {
+ "Debug": "--debug",
+ "Release": "--release",
+ }
+
+ def __init__(self, executive, filesystem):
+ self._executive = executive
+ self._filesystem = filesystem
+ self._webkit_base_dir = None
+ self._default_configuration = None
+ self._build_directories = {}
+
+ def build_directory(self, configuration):
+ """Returns the path to the build directory for the configuration."""
+ if configuration:
+ flags = ["--configuration",
+ self._FLAGS_FROM_CONFIGURATIONS[configuration]]
+ else:
+ configuration = ""
+ flags = ["--top-level"]
+
+ if not self._build_directories.get(configuration):
+ args = ["perl", self._script_path("webkit-build-directory")] + flags
+ self._build_directories[configuration] = (
+ self._executive.run_command(args).rstrip())
+
+ return self._build_directories[configuration]
+
+ def build_dumprendertree(self, configuration):
+ """Builds DRT in the given configuration.
+
+ Returns True if the build was successful and up-to-date."""
+ flag = self._FLAGS_FROM_CONFIGURATIONS[configuration]
+ exit_code = self._executive.run_command([
+ self._script_path("build-dumprendertree"), flag],
+ return_exit_code=True)
+ if exit_code != 0:
+ _log.error("Failed to build DumpRenderTree")
+ return False
+ return True
+
+ def default_configuration(self):
+ """Returns the default configuration for the user.
+
+ Returns the value set by 'set-webkit-configuration', or "Release"
+ if that has not been set. This mirrors the logic in webkitdirs.pm."""
+ if not self._default_configuration:
+ self._default_configuration = self._determine_configuration()
+ if not self._default_configuration:
+ self._default_configuration = 'Release'
+ if self._default_configuration not in self._FLAGS_FROM_CONFIGURATIONS:
+ _log.warn("Configuration \"%s\" is not a recognized value.\n" %
+ self._default_configuration)
+ _log.warn("Scripts may fail. "
+ "See 'set-webkit-configuration --help'.")
+ return self._default_configuration
+
+ def path_from_webkit_base(self, *comps):
+ return self._filesystem.join(self.webkit_base_dir(), *comps)
+
+ def webkit_base_dir(self):
+ """Returns the absolute path to the top of the WebKit tree.
+
+ Raises an AssertionError if the top dir can't be determined."""
+ # Note: this code somewhat duplicates the code in
+ # scm.find_checkout_root(). However, that code only works if the top
+ # of the SCM repository also matches the top of the WebKit tree. The
+ # Chromium ports, for example, only check out subdirectories like
+ # Tools/Scripts, and so we still have to do additional work
+ # to find the top of the tree.
+ #
+ # This code will also work if there is no SCM system at all.
+ if not self._webkit_base_dir:
+ abspath = os.path.abspath(__file__)
+ self._webkit_base_dir = abspath[0:abspath.find('Tools') - 1]
+ return self._webkit_base_dir
+
+ def _script_path(self, script_name):
+ return self._filesystem.join(self.webkit_base_dir(), "Tools",
+ "Scripts", script_name)
+
+ def _determine_configuration(self):
+ # This mirrors the logic in webkitdirs.pm:determineConfiguration().
+ #
+ # FIXME: See the comment at the top of the file regarding unit tests
+ # and our use of global mutable static variables.
+ global _have_determined_configuration, _configuration
+ if not _have_determined_configuration:
+ contents = self._read_configuration()
+ if not contents:
+ contents = "Release"
+ if contents == "Deployment":
+ contents = "Release"
+ if contents == "Development":
+ contents = "Debug"
+ _configuration = contents
+ _have_determined_configuration = True
+ return _configuration
+
+ def _read_configuration(self):
+ try:
+ configuration_path = self._filesystem.join(self.build_directory(None),
+ "Configuration")
+ if not self._filesystem.exists(configuration_path):
+ return None
+ except (OSError, executive.ScriptError):
+ return None
+
+ return self._filesystem.read_text_file(configuration_path).rstrip()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/config_mock.py b/Tools/Scripts/webkitpy/layout_tests/port/config_mock.py
new file mode 100644
index 0000000..af71fa3
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/config_mock.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Wrapper objects for WebKit-specific utility routines."""
+
+
+class MockConfig(object):
+ def __init__(self, default_configuration='Release'):
+ self._default_configuration = default_configuration
+
+ def build_directory(self, configuration):
+ return "/build"
+
+ def build_dumprendertree(self, configuration):
+ return True
+
+ def default_configuration(self):
+ return self._default_configuration
+
+ def path_from_webkit_base(self, *comps):
+ return "/" + "/".join(list(comps))
+
+ def webkit_base_dir(self):
+ return "/"
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/config_standalone.py b/Tools/Scripts/webkitpy/layout_tests/port/config_standalone.py
new file mode 100644
index 0000000..3dec3b9
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/config_standalone.py
@@ -0,0 +1,70 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""FIXME: This script is used by
+config_unittest.test_default_configuration__standalone() to read the
+default configuration to work around any possible caching / reset bugs. See
+https://bugs.webkit.org/show_bug?id=49360 for the motivation. We can remove
+this test when we remove the global configuration cache in config.py."""
+
+import os
+import unittest
+import sys
+
+
+# Ensure that webkitpy is in PYTHONPATH.
+this_dir = os.path.abspath(sys.path[0])
+up = os.path.dirname
+script_dir = up(up(up(this_dir)))
+if script_dir not in sys.path:
+ sys.path.append(script_dir)
+
+from webkitpy.common.system import executive
+from webkitpy.common.system import executive_mock
+from webkitpy.common.system import filesystem
+from webkitpy.common.system import filesystem_mock
+
+import config
+
+
+def main(argv=None):
+ if not argv:
+ argv = sys.argv
+
+ if len(argv) == 3 and argv[1] == '--mock':
+ e = executive_mock.MockExecutive2(output='foo')
+ fs = filesystem_mock.MockFileSystem({'foo/Configuration': argv[2]})
+ else:
+ e = executive.Executive()
+ fs = filesystem.FileSystem()
+
+ c = config.Config(e, fs)
+ print c.default_configuration()
+
+if __name__ == '__main__':
+ main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/config_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/config_unittest.py
new file mode 100644
index 0000000..2cce3cc
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/config_unittest.py
@@ -0,0 +1,202 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import sys
+import unittest
+
+from webkitpy.common.system import executive
+from webkitpy.common.system import executive_mock
+from webkitpy.common.system import filesystem
+from webkitpy.common.system import filesystem_mock
+from webkitpy.common.system import outputcapture
+
+import config
+
+
+def mock_run_command(arg_list):
+ # Set this to True to test actual output (where possible).
+ integration_test = False
+ if integration_test:
+ return executive.Executive().run_command(arg_list)
+
+ if 'webkit-build-directory' in arg_list[1]:
+ return mock_webkit_build_directory(arg_list[2:])
+ return 'Error'
+
+
+def mock_webkit_build_directory(arg_list):
+ if arg_list == ['--top-level']:
+ return '/WebKitBuild'
+ elif arg_list == ['--configuration', '--debug']:
+ return '/WebKitBuild/Debug'
+ elif arg_list == ['--configuration', '--release']:
+ return '/WebKitBuild/Release'
+ return 'Error'
+
+
+class ConfigTest(unittest.TestCase):
+ def tearDown(self):
+ config.clear_cached_configuration()
+
+ def make_config(self, output='', files={}, exit_code=0, exception=None,
+ run_command_fn=None):
+ e = executive_mock.MockExecutive2(output=output, exit_code=exit_code,
+ exception=exception,
+ run_command_fn=run_command_fn)
+ fs = filesystem_mock.MockFileSystem(files)
+ return config.Config(e, fs)
+
+ def assert_configuration(self, contents, expected):
+ # This tests that a configuration file containing
+ # _contents_ ends up being interpreted as _expected_.
+ c = self.make_config('foo', {'foo/Configuration': contents})
+ self.assertEqual(c.default_configuration(), expected)
+
+ def test_build_directory(self):
+ # --top-level
+ c = self.make_config(run_command_fn=mock_run_command)
+ self.assertTrue(c.build_directory(None).endswith('WebKitBuild'))
+
+ # Test again to check caching
+ self.assertTrue(c.build_directory(None).endswith('WebKitBuild'))
+
+ # Test other values
+ self.assertTrue(c.build_directory('Release').endswith('/Release'))
+ self.assertTrue(c.build_directory('Debug').endswith('/Debug'))
+ self.assertRaises(KeyError, c.build_directory, 'Unknown')
+
+ def test_build_dumprendertree__success(self):
+ c = self.make_config(exit_code=0)
+ self.assertTrue(c.build_dumprendertree("Debug"))
+ self.assertTrue(c.build_dumprendertree("Release"))
+ self.assertRaises(KeyError, c.build_dumprendertree, "Unknown")
+
+ def test_build_dumprendertree__failure(self):
+ c = self.make_config(exit_code=-1)
+
+ # FIXME: Build failures should log errors. However, the message we
+ # get depends on how we're being called; as a standalone test,
+ # we'll get the "no handlers found" message. As part of
+ # test-webkitpy, we get the actual message. Really, we need
+ # outputcapture to install its own handler.
+ oc = outputcapture.OutputCapture()
+ oc.capture_output()
+ self.assertFalse(c.build_dumprendertree('Debug'))
+ oc.restore_output()
+
+ oc.capture_output()
+ self.assertFalse(c.build_dumprendertree('Release'))
+ oc.restore_output()
+
+ def test_default_configuration__release(self):
+ self.assert_configuration('Release', 'Release')
+
+ def test_default_configuration__debug(self):
+ self.assert_configuration('Debug', 'Debug')
+
+ def test_default_configuration__deployment(self):
+ self.assert_configuration('Deployment', 'Release')
+
+ def test_default_configuration__development(self):
+ self.assert_configuration('Development', 'Debug')
+
+ def test_default_configuration__notfound(self):
+ # This tests what happens if the default configuration file
+ # doesn't exist.
+ c = self.make_config(output='foo', files={'foo/Configuration': None})
+ self.assertEqual(c.default_configuration(), "Release")
+
+ def test_default_configuration__unknown(self):
+ # Ignore the warning about an unknown configuration value.
+ oc = outputcapture.OutputCapture()
+ oc.capture_output()
+ self.assert_configuration('Unknown', 'Unknown')
+ oc.restore_output()
+
+ def test_default_configuration__standalone(self):
+ # FIXME: This test runs a standalone python script to test
+ # reading the default configuration to work around any possible
+ # caching / reset bugs. See https://bugs.webkit.org/show_bug?id=49360
+ # for the motivation. We can remove this test when we remove the
+ # global configuration cache in config.py.
+ e = executive.Executive()
+ fs = filesystem.FileSystem()
+ c = config.Config(e, fs)
+ script = c.path_from_webkit_base('Tools', 'Scripts',
+ 'webkitpy', 'layout_tests', 'port', 'config_standalone.py')
+
+ # Note: don't use 'Release' here, since that's the normal default.
+ expected = 'Debug'
+
+ args = [sys.executable, script, '--mock', expected]
+ actual = e.run_command(args).rstrip()
+ self.assertEqual(actual, expected)
+
+ def test_default_configuration__no_perl(self):
+ # We need perl to run webkit-build-directory to find out where the
+ # default configuration file is. See what happens if perl isn't
+ # installed. (We should get the default value, 'Release').
+ c = self.make_config(exception=OSError)
+ actual = c.default_configuration()
+ self.assertEqual(actual, 'Release')
+
+ def test_default_configuration__scripterror(self):
+ # We run webkit-build-directory to find out where the default
+ # configuration file is. See what happens if that script fails.
+ # (We should get the default value, 'Release').
+ c = self.make_config(exception=executive.ScriptError())
+ actual = c.default_configuration()
+ self.assertEqual(actual, 'Release')
+
+ def test_path_from_webkit_base(self):
+ # FIXME: We use a real filesystem here. Should this move to a
+ # mocked one?
+ c = config.Config(executive.Executive(), filesystem.FileSystem())
+ self.assertTrue(c.path_from_webkit_base('foo'))
+
+ def test_webkit_base_dir(self):
+ # FIXME: We use a real filesystem here. Should this move to a
+ # mocked one?
+ c = config.Config(executive.Executive(), filesystem.FileSystem())
+ base_dir = c.webkit_base_dir()
+ self.assertTrue(base_dir)
+ self.assertNotEqual(base_dir[-1], '/')
+
+ orig_cwd = os.getcwd()
+ os.chdir(os.environ['HOME'])
+ c = config.Config(executive.Executive(), filesystem.FileSystem())
+ try:
+ base_dir_2 = c.webkit_base_dir()
+ self.assertEqual(base_dir, base_dir_2)
+ finally:
+ os.chdir(orig_cwd)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py b/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py
new file mode 100644
index 0000000..4ed34e6
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the Google name nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""This is a test implementation of the Port interface that generates the
+ correct output for every test. It can be used for perf testing, because
+ it is pretty much a lower limit on how fast a port can possibly run.
+
+ This implementation acts as a wrapper around a real port (the real port
+ is held as a delegate object). To specify which port, use the port name
+ 'dryrun-XXX' (e.g., 'dryrun-chromium-mac-leopard'). If you use just
+ 'dryrun', it uses the default port.
+
+ Note that because this is really acting as a wrapper around the underlying
+ port, you must be able to run the underlying port as well
+ (check_build() and check_sys_deps() must pass and auxiliary binaries
+ like layout_test_helper and httpd must work).
+
+ This implementation also modifies the test expectations so that all
+ tests are either SKIPPED or expected to PASS."""
+
+from __future__ import with_statement
+
+import os
+import sys
+import time
+
+from webkitpy.layout_tests.layout_package import test_output
+
+import base
+import factory
+
+
+class DryRunPort(object):
+ """DryRun implementation of the Port interface."""
+
+ def __init__(self, **kwargs):
+ pfx = 'dryrun-'
+ if 'port_name' in kwargs:
+ if kwargs['port_name'].startswith(pfx):
+ kwargs['port_name'] = kwargs['port_name'][len(pfx):]
+ else:
+ kwargs['port_name'] = None
+ self.__delegate = factory.get(**kwargs)
+
+ def __getattr__(self, name):
+ return getattr(self.__delegate, name)
+
+ def check_build(self, needs_http):
+ return True
+
+ def check_sys_deps(self, needs_http):
+ return True
+
+ def start_helper(self):
+ pass
+
+ def start_http_server(self):
+ pass
+
+ def start_websocket_server(self):
+ pass
+
+ def stop_helper(self):
+ pass
+
+ def stop_http_server(self):
+ pass
+
+ def stop_websocket_server(self):
+ pass
+
+ def create_driver(self, worker_number):
+ return DryrunDriver(self, worker_number)
+
+
+class DryrunDriver(base.Driver):
+ """Dryrun implementation of the DumpRenderTree / Driver interface."""
+
+ def __init__(self, port, worker_number):
+ self._port = port
+ self._worker_number = worker_number
+
+ def cmd_line(self):
+ return ['None']
+
+ def poll(self):
+ return None
+
+ def run_test(self, test_input):
+ start_time = time.time()
+ text_output = self._port.expected_text(test_input.filename)
+
+ if test_input.image_hash is not None:
+ image = self._port.expected_image(test_input.filename)
+ hash = self._port.expected_checksum(test_input.filename)
+ else:
+ image = None
+ hash = None
+ return test_output.TestOutput(text_output, image, hash, False,
+ time.time() - start_time, False, None)
+
+ def start(self):
+ pass
+
+ def stop(self):
+ pass
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/factory.py b/Tools/Scripts/webkitpy/layout_tests/port/factory.py
new file mode 100644
index 0000000..6935744
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/factory.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Factory method to retrieve the appropriate port implementation."""
+
+
+import sys
+
+ALL_PORT_NAMES = ['test', 'dryrun', 'mac', 'win', 'gtk', 'qt', 'chromium-mac',
+ 'chromium-linux', 'chromium-win', 'google-chrome-win',
+ 'google-chrome-mac', 'google-chrome-linux32', 'google-chrome-linux64']
+
+
+def get(port_name=None, options=None, **kwargs):
+ """Returns an object implementing the Port interface. If
+ port_name is None, this routine attempts to guess at the most
+ appropriate port on this platform."""
+ # Wrapped for backwards-compatibility
+ if port_name:
+ kwargs['port_name'] = port_name
+ if options:
+ kwargs['options'] = options
+ return _get_kwargs(**kwargs)
+
+
+def _get_kwargs(**kwargs):
+ port_to_use = kwargs.get('port_name', None)
+ options = kwargs.get('options', None)
+ if port_to_use is None:
+ if sys.platform == 'win32' or sys.platform == 'cygwin':
+ if options and hasattr(options, 'chromium') and options.chromium:
+ port_to_use = 'chromium-win'
+ else:
+ port_to_use = 'win'
+ elif sys.platform == 'linux2':
+ port_to_use = 'chromium-linux'
+ elif sys.platform == 'darwin':
+ if options and hasattr(options, 'chromium') and options.chromium:
+ port_to_use = 'chromium-mac'
+ else:
+ port_to_use = 'mac'
+
+ if port_to_use is None:
+ raise NotImplementedError('unknown port; sys.platform = "%s"' %
+ sys.platform)
+
+ if port_to_use == 'test':
+ import test
+ maker = test.TestPort
+ elif port_to_use.startswith('dryrun'):
+ import dryrun
+ maker = dryrun.DryRunPort
+ elif port_to_use.startswith('mac'):
+ import mac
+ maker = mac.MacPort
+ elif port_to_use.startswith('win'):
+ import win
+ maker = win.WinPort
+ elif port_to_use.startswith('gtk'):
+ import gtk
+ maker = gtk.GtkPort
+ elif port_to_use.startswith('qt'):
+ import qt
+ maker = qt.QtPort
+ elif port_to_use.startswith('chromium-gpu'):
+ import chromium_gpu
+ maker = chromium_gpu.get
+ elif port_to_use.startswith('chromium-mac'):
+ import chromium_mac
+ maker = chromium_mac.ChromiumMacPort
+ elif port_to_use.startswith('chromium-linux'):
+ import chromium_linux
+ maker = chromium_linux.ChromiumLinuxPort
+ elif port_to_use.startswith('chromium-win'):
+ import chromium_win
+ maker = chromium_win.ChromiumWinPort
+ elif port_to_use.startswith('google-chrome'):
+ import google_chrome
+ maker = google_chrome.GetGoogleChromePort
+ else:
+ raise NotImplementedError('unsupported port: %s' % port_to_use)
+ return maker(**kwargs)
+
+def get_all(options=None):
+ """Returns all the objects implementing the Port interface."""
+ return dict([(port_name, get(port_name, options=options))
+ for port_name in ALL_PORT_NAMES])
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py
new file mode 100644
index 0000000..978a557
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/factory_unittest.py
@@ -0,0 +1,188 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import sys
+import unittest
+
+from webkitpy.tool import mocktool
+
+import chromium_gpu
+import chromium_linux
+import chromium_mac
+import chromium_win
+import dryrun
+import factory
+import google_chrome
+import gtk
+import mac
+import qt
+import test
+import win
+
+
+class FactoryTest(unittest.TestCase):
+ """Test factory creates proper port object for the target.
+
+ Target is specified by port_name, sys.platform and options.
+
+ """
+ # FIXME: The ports themselves should expose what options they require,
+ # instead of passing generic "options".
+
+ def setUp(self):
+ self.real_sys_platform = sys.platform
+ self.webkit_options = mocktool.MockOptions(pixel_tests=False)
+ self.chromium_options = mocktool.MockOptions(pixel_tests=False,
+ chromium=True)
+
+ def tearDown(self):
+ sys.platform = self.real_sys_platform
+
+ def assert_port(self, port_name, expected_port, port_obj=None):
+ """Helper assert for port_name.
+
+ Args:
+ port_name: port name to get port object.
+ expected_port: class of expected port object.
+ port_obj: optional port object
+ """
+ port_obj = port_obj or factory.get(port_name=port_name)
+ self.assertTrue(isinstance(port_obj, expected_port))
+
+ def assert_platform_port(self, platform, options, expected_port):
+ """Helper assert for platform and options.
+
+ Args:
+ platform: sys.platform.
+ options: options to get port object.
+ expected_port: class of expected port object.
+
+ """
+ orig_platform = sys.platform
+ sys.platform = platform
+ self.assertTrue(isinstance(factory.get(options=options),
+ expected_port))
+ sys.platform = orig_platform
+
+ def test_test(self):
+ self.assert_port("test", test.TestPort)
+
+ def test_dryrun(self):
+ self.assert_port("dryrun-test", dryrun.DryRunPort)
+ self.assert_port("dryrun-mac", dryrun.DryRunPort)
+
+ def test_mac(self):
+ self.assert_port("mac", mac.MacPort)
+ self.assert_platform_port("darwin", None, mac.MacPort)
+ self.assert_platform_port("darwin", self.webkit_options, mac.MacPort)
+
+ def test_win(self):
+ self.assert_port("win", win.WinPort)
+ self.assert_platform_port("win32", None, win.WinPort)
+ self.assert_platform_port("win32", self.webkit_options, win.WinPort)
+ self.assert_platform_port("cygwin", None, win.WinPort)
+ self.assert_platform_port("cygwin", self.webkit_options, win.WinPort)
+
+ def test_google_chrome(self):
+ # The actual Chrome class names aren't available so we test that the
+ # objects we get are at least subclasses of the Chromium versions.
+ self.assert_port("google-chrome-linux32",
+ chromium_linux.ChromiumLinuxPort)
+ self.assert_port("google-chrome-linux64",
+ chromium_linux.ChromiumLinuxPort)
+ self.assert_port("google-chrome-win",
+ chromium_win.ChromiumWinPort)
+ self.assert_port("google-chrome-mac",
+ chromium_mac.ChromiumMacPort)
+
+ def test_gtk(self):
+ self.assert_port("gtk", gtk.GtkPort)
+
+ def test_qt(self):
+ self.assert_port("qt", qt.QtPort)
+
+ def test_chromium_gpu_linux(self):
+ self.assert_port("chromium-gpu-linux", chromium_gpu.ChromiumGpuLinuxPort)
+
+ def test_chromium_gpu_mac(self):
+ self.assert_port("chromium-gpu-mac", chromium_gpu.ChromiumGpuMacPort)
+
+ def test_chromium_gpu_win(self):
+ self.assert_port("chromium-gpu-win", chromium_gpu.ChromiumGpuWinPort)
+
+ def test_chromium_mac(self):
+ self.assert_port("chromium-mac", chromium_mac.ChromiumMacPort)
+ self.assert_platform_port("darwin", self.chromium_options,
+ chromium_mac.ChromiumMacPort)
+
+ def test_chromium_linux(self):
+ self.assert_port("chromium-linux", chromium_linux.ChromiumLinuxPort)
+ self.assert_platform_port("linux2", self.chromium_options,
+ chromium_linux.ChromiumLinuxPort)
+
+ def test_chromium_win(self):
+ self.assert_port("chromium-win", chromium_win.ChromiumWinPort)
+ self.assert_platform_port("win32", self.chromium_options,
+ chromium_win.ChromiumWinPort)
+ self.assert_platform_port("cygwin", self.chromium_options,
+ chromium_win.ChromiumWinPort)
+
+ def test_get_all_ports(self):
+ ports = factory.get_all()
+ for name in factory.ALL_PORT_NAMES:
+ self.assertTrue(name in ports.keys())
+ self.assert_port("test", test.TestPort, ports["test"])
+ self.assert_port("dryrun-test", dryrun.DryRunPort, ports["dryrun"])
+ self.assert_port("dryrun-mac", dryrun.DryRunPort, ports["dryrun"])
+ self.assert_port("mac", mac.MacPort, ports["mac"])
+ self.assert_port("win", win.WinPort, ports["win"])
+ self.assert_port("gtk", gtk.GtkPort, ports["gtk"])
+ self.assert_port("qt", qt.QtPort, ports["qt"])
+ self.assert_port("chromium-mac", chromium_mac.ChromiumMacPort,
+ ports["chromium-mac"])
+ self.assert_port("chromium-linux", chromium_linux.ChromiumLinuxPort,
+ ports["chromium-linux"])
+ self.assert_port("chromium-win", chromium_win.ChromiumWinPort,
+ ports["chromium-win"])
+
+ def test_unknown_specified(self):
+ # Test what happens when you specify an unknown port.
+ orig_platform = sys.platform
+ self.assertRaises(NotImplementedError, factory.get,
+ port_name='unknown')
+
+ def test_unknown_default(self):
+ # Test what happens when you're running on an unknown platform.
+ orig_platform = sys.platform
+ sys.platform = 'unknown'
+ self.assertRaises(NotImplementedError, factory.get)
+ sys.platform = orig_platform
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/google_chrome.py b/Tools/Scripts/webkitpy/layout_tests/port/google_chrome.py
new file mode 100644
index 0000000..8d94bb5
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/google_chrome.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import with_statement
+
+import codecs
+import os
+
+
+def _test_expectations_overrides(port, super):
+ # The chrome ports use the regular overrides plus anything in the
+ # official test_expectations as well. Hopefully we don't get collisions.
+ chromium_overrides = super.test_expectations_overrides(port)
+
+ # FIXME: It used to be that AssertionError would get raised by
+ # path_from_chromium_base() if we weren't in a Chromium checkout, but
+ # this changed in r60427. This should probably be changed back.
+ overrides_path = port.path_from_chromium_base('webkit', 'tools',
+ 'layout_tests', 'test_expectations_chrome.txt')
+ if not os.path.exists(overrides_path):
+ return chromium_overrides
+
+ with codecs.open(overrides_path, "r", "utf-8") as file:
+ if chromium_overrides:
+ return chromium_overrides + file.read()
+ else:
+ return file.read()
+
+def GetGoogleChromePort(**kwargs):
+ """Some tests have slightly different results when compiled as Google
+ Chrome vs Chromium. In those cases, we prepend an additional directory to
+ to the baseline paths."""
+ port_name = kwargs['port_name']
+ del kwargs['port_name']
+ if port_name == 'google-chrome-linux32':
+ import chromium_linux
+
+ class GoogleChromeLinux32Port(chromium_linux.ChromiumLinuxPort):
+ def baseline_search_path(self):
+ paths = chromium_linux.ChromiumLinuxPort.baseline_search_path(
+ self)
+ paths.insert(0, self._webkit_baseline_path(
+ 'google-chrome-linux32'))
+ return paths
+
+ def test_expectations_overrides(self):
+ return _test_expectations_overrides(self,
+ chromium_linux.ChromiumLinuxPort)
+
+ return GoogleChromeLinux32Port(**kwargs)
+ elif port_name == 'google-chrome-linux64':
+ import chromium_linux
+
+ class GoogleChromeLinux64Port(chromium_linux.ChromiumLinuxPort):
+ def baseline_search_path(self):
+ paths = chromium_linux.ChromiumLinuxPort.baseline_search_path(
+ self)
+ paths.insert(0, self._webkit_baseline_path(
+ 'google-chrome-linux64'))
+ return paths
+
+ def test_expectations_overrides(self):
+ return _test_expectations_overrides(self,
+ chromium_linux.ChromiumLinuxPort)
+
+ return GoogleChromeLinux64Port(**kwargs)
+ elif port_name.startswith('google-chrome-mac'):
+ import chromium_mac
+
+ class GoogleChromeMacPort(chromium_mac.ChromiumMacPort):
+ def baseline_search_path(self):
+ paths = chromium_mac.ChromiumMacPort.baseline_search_path(
+ self)
+ paths.insert(0, self._webkit_baseline_path(
+ 'google-chrome-mac'))
+ return paths
+
+ def test_expectations_overrides(self):
+ return _test_expectations_overrides(self,
+ chromium_mac.ChromiumMacPort)
+
+ return GoogleChromeMacPort(**kwargs)
+ elif port_name.startswith('google-chrome-win'):
+ import chromium_win
+
+ class GoogleChromeWinPort(chromium_win.ChromiumWinPort):
+ def baseline_search_path(self):
+ paths = chromium_win.ChromiumWinPort.baseline_search_path(
+ self)
+ paths.insert(0, self._webkit_baseline_path(
+ 'google-chrome-win'))
+ return paths
+
+ def test_expectations_overrides(self):
+ return _test_expectations_overrides(self,
+ chromium_win.ChromiumWinPort)
+
+ return GoogleChromeWinPort(**kwargs)
+ raise NotImplementedError('unsupported port: %s' % port_name)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/google_chrome_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/google_chrome_unittest.py
new file mode 100644
index 0000000..e60c274
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/google_chrome_unittest.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import codecs
+import os
+import unittest
+
+from webkitpy.common import newstringio
+
+import factory
+import google_chrome
+
+
+class GetGoogleChromePortTest(unittest.TestCase):
+ def test_get_google_chrome_port(self):
+ test_ports = ('google-chrome-linux32', 'google-chrome-linux64',
+ 'google-chrome-mac', 'google-chrome-win')
+ for port in test_ports:
+ self._verify_baseline_path(port, port)
+ self._verify_expectations_overrides(port)
+
+ self._verify_baseline_path('google-chrome-mac', 'google-chrome-mac-leopard')
+ self._verify_baseline_path('google-chrome-win', 'google-chrome-win-xp')
+ self._verify_baseline_path('google-chrome-win', 'google-chrome-win-vista')
+
+ def _verify_baseline_path(self, expected_path, port_name):
+ port = google_chrome.GetGoogleChromePort(port_name=port_name,
+ options=None)
+ path = port.baseline_search_path()[0]
+ self.assertEqual(expected_path, os.path.split(path)[1])
+
+ def _verify_expectations_overrides(self, port_name):
+ # FIXME: make this more robust when we have the Tree() abstraction.
+ # we should be able to test for the files existing or not, and
+ # be able to control the contents better.
+
+ chromium_port = factory.get("chromium-mac")
+ chromium_overrides = chromium_port.test_expectations_overrides()
+ port = google_chrome.GetGoogleChromePort(port_name=port_name,
+ options=None)
+
+ orig_exists = os.path.exists
+ orig_open = codecs.open
+ expected_string = "// hello, world\n"
+
+ def mock_exists_chrome_not_found(path):
+ if 'test_expectations_chrome.txt' in path:
+ return False
+ return orig_exists(path)
+
+ def mock_exists_chrome_found(path):
+ if 'test_expectations_chrome.txt' in path:
+ return True
+ return orig_exists(path)
+
+ def mock_open(path, mode, encoding):
+ if 'test_expectations_chrome.txt' in path:
+ return newstringio.StringIO(expected_string)
+ return orig_open(path, mode, encoding)
+
+ try:
+ os.path.exists = mock_exists_chrome_not_found
+ chrome_overrides = port.test_expectations_overrides()
+ self.assertEqual(chromium_overrides, chrome_overrides)
+
+ os.path.exists = mock_exists_chrome_found
+ codecs.open = mock_open
+ chrome_overrides = port.test_expectations_overrides()
+ if chromium_overrides:
+ self.assertEqual(chrome_overrides,
+ chromium_overrides + expected_string)
+ else:
+ self.assertEqual(chrome_overrides, expected_string)
+ finally:
+ os.path.exists = orig_exists
+ codecs.open = orig_open
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/gtk.py b/Tools/Scripts/webkitpy/layout_tests/port/gtk.py
new file mode 100644
index 0000000..a18fdff
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/gtk.py
@@ -0,0 +1,116 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the Google name nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""WebKit Gtk implementation of the Port interface."""
+
+import logging
+import os
+import signal
+
+from webkitpy.layout_tests.port.webkit import WebKitPort
+
+_log = logging.getLogger("webkitpy.layout_tests.port.gtk")
+
+
+class GtkPort(WebKitPort):
+ """WebKit Gtk implementation of the Port class."""
+
+ def __init__(self, **kwargs):
+ kwargs.setdefault('port_name', 'gtk')
+ WebKitPort.__init__(self, **kwargs)
+
+ def _tests_for_other_platforms(self):
+ # FIXME: This list could be dynamic based on platform name and
+ # pushed into base.Port.
+ # This really need to be automated.
+ return [
+ "platform/chromium",
+ "platform/win",
+ "platform/qt",
+ "platform/mac",
+ ]
+
+ def _path_to_apache_config_file(self):
+ # FIXME: This needs to detect the distribution and change config files.
+ return os.path.join(self.layout_tests_dir(), 'http', 'conf',
+ 'apache2-debian-httpd.conf')
+
+ def _shut_down_http_server(self, server_pid):
+ """Shut down the httpd web server. Blocks until it's fully
+ shut down.
+
+ Args:
+ server_pid: The process ID of the running server.
+ """
+ # server_pid is not set when "http_server.py stop" is run manually.
+ if server_pid is None:
+ # FIXME: This isn't ideal, since it could conflict with
+ # lighttpd processes not started by http_server.py,
+ # but good enough for now.
+ self._executive.kill_all('apache2')
+ else:
+ try:
+ os.kill(server_pid, signal.SIGTERM)
+ # TODO(mmoss) Maybe throw in a SIGKILL just to be sure?
+ except OSError:
+ # Sometimes we get a bad PID (e.g. from a stale httpd.pid
+ # file), so if kill fails on the given PID, just try to
+ # 'killall' web servers.
+ self._shut_down_http_server(None)
+
+ def _path_to_driver(self):
+ return self._build_path('Programs', 'DumpRenderTree')
+
+ def check_build(self, needs_http):
+ if not self._check_driver():
+ return False
+ return True
+
+ def _path_to_apache(self):
+ if self._is_redhat_based():
+ return '/usr/sbin/httpd'
+ else:
+ return '/usr/sbin/apache2'
+
+ def _path_to_apache_config_file(self):
+ if self._is_redhat_based():
+ config_name = 'fedora-httpd.conf'
+ else:
+ config_name = 'apache2-debian-httpd.conf'
+
+ return os.path.join(self.layout_tests_dir(), 'http', 'conf',
+ config_name)
+
+ def _path_to_wdiff(self):
+ if self._is_redhat_based():
+ return '/usr/bin/dwdiff'
+ else:
+ return '/usr/bin/wdiff'
+
+ def _is_redhat_based(self):
+ return os.path.exists(os.path.join('/etc', 'redhat-release'))
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/http_lock.py b/Tools/Scripts/webkitpy/layout_tests/port/http_lock.py
new file mode 100644
index 0000000..f5946b6
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/http_lock.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
+# Copyright (C) 2010 Andras Becsi (abecsi@inf.u-szeged.hu), University of Szeged
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UNIVERSITY OF SZEGED OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""This class helps to block NRWT threads when more NRWTs run
+http and websocket tests in a same time."""
+
+import glob
+import logging
+import os
+import sys
+import tempfile
+import time
+
+from webkitpy.common.system.executive import Executive
+from webkitpy.common.system.file_lock import FileLock
+from webkitpy.common.system.filesystem import FileSystem
+
+
+_log = logging.getLogger("webkitpy.layout_tests.port.http_lock")
+
+
+class HttpLock(object):
+
+ def __init__(self, lock_path, lock_file_prefix="WebKitHttpd.lock.",
+ guard_lock="WebKit.lock"):
+ self._lock_path = lock_path
+ if not self._lock_path:
+ self._lock_path = tempfile.gettempdir()
+ self._lock_file_prefix = lock_file_prefix
+ self._lock_file_path_prefix = os.path.join(self._lock_path,
+ self._lock_file_prefix)
+ self._guard_lock_file = os.path.join(self._lock_path, guard_lock)
+ self._guard_lock = FileLock(self._guard_lock_file)
+ self._process_lock_file_name = ""
+ self._executive = Executive()
+
+ def cleanup_http_lock(self):
+ """Delete the lock file if exists."""
+ if os.path.exists(self._process_lock_file_name):
+ _log.debug("Removing lock file: %s" % self._process_lock_file_name)
+ FileSystem().remove(self._process_lock_file_name)
+
+ def _extract_lock_number(self, lock_file_name):
+ """Return the lock number from lock file."""
+ prefix_length = len(self._lock_file_path_prefix)
+ return int(lock_file_name[prefix_length:])
+
+ def _lock_file_list(self):
+ """Return the list of lock files sequentially."""
+ lock_list = glob.glob(self._lock_file_path_prefix + '*')
+ lock_list.sort(key=self._extract_lock_number)
+ return lock_list
+
+ def _next_lock_number(self):
+ """Return the next available lock number."""
+ lock_list = self._lock_file_list()
+ if not lock_list:
+ return 0
+ return self._extract_lock_number(lock_list[-1]) + 1
+
+ def _curent_lock_pid(self):
+ """Return with the current lock pid. If the lock is not valid
+ it deletes the lock file."""
+ lock_list = self._lock_file_list()
+ if not lock_list:
+ return
+ try:
+ current_lock_file = open(lock_list[0], 'r')
+ current_pid = current_lock_file.readline()
+ current_lock_file.close()
+ if not (current_pid and self._executive.check_running_pid(int(current_pid))):
+ _log.debug("Removing stuck lock file: %s" % lock_list[0])
+ FileSystem().remove(lock_list[0])
+ return
+ except (IOError, OSError):
+ return
+ return int(current_pid)
+
+ def _create_lock_file(self):
+ """The lock files are used to schedule the running test sessions in first
+ come first served order. The guard lock ensures that the lock numbers are
+ sequential."""
+ if not os.path.exists(self._lock_path):
+ _log.debug("Lock directory does not exist: %s" % self._lock_path)
+ return False
+
+ if not self._guard_lock.acquire_lock():
+ _log.debug("Guard lock timed out!")
+ return False
+
+ self._process_lock_file_name = (self._lock_file_path_prefix +
+ str(self._next_lock_number()))
+ _log.debug("Creating lock file: %s" % self._process_lock_file_name)
+ lock_file = open(self._process_lock_file_name, 'w')
+ lock_file.write(str(os.getpid()))
+ lock_file.close()
+ self._guard_lock.release_lock()
+ return True
+
+
+ def wait_for_httpd_lock(self):
+ """Create a lock file and wait until it's turn comes. If something goes wrong
+ it wont do any locking."""
+ if not self._create_lock_file():
+ _log.debug("Warning, http locking failed!")
+ return
+
+ while self._curent_lock_pid() != os.getpid():
+ time.sleep(1)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/http_lock_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/http_lock_unittest.py
new file mode 100644
index 0000000..85c760a
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/http_lock_unittest.py
@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UNIVERSITY OF SZEGED OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import glob
+import http_lock
+import os
+import unittest
+
+
+class HttpLockTest(unittest.TestCase):
+
+ def __init__(self, testFunc):
+ self.http_lock_obj = http_lock.HttpLock(None, "WebKitTestHttpd.lock.", "WebKitTest.lock")
+ self.lock_file_path_prefix = os.path.join(self.http_lock_obj._lock_path,
+ self.http_lock_obj._lock_file_prefix)
+ self.lock_file_name = self.lock_file_path_prefix + "0"
+ self.guard_lock_file = self.http_lock_obj._guard_lock_file
+ self.clean_all_lockfile()
+ unittest.TestCase.__init__(self, testFunc)
+
+ def clean_all_lockfile(self):
+ if os.path.exists(self.guard_lock_file):
+ os.unlink(self.guard_lock_file)
+ lock_list = glob.glob(self.lock_file_path_prefix + '*')
+ for file_name in lock_list:
+ os.unlink(file_name)
+
+ def assertEqual(self, first, second):
+ if first != second:
+ self.clean_all_lockfile()
+ unittest.TestCase.assertEqual(self, first, second)
+
+ def _check_lock_file(self):
+ if os.path.exists(self.lock_file_name):
+ pid = os.getpid()
+ lock_file = open(self.lock_file_name, 'r')
+ lock_file_pid = lock_file.readline()
+ lock_file.close()
+ self.assertEqual(pid, int(lock_file_pid))
+ return True
+ return False
+
+ def test_lock_lifecycle(self):
+ self.http_lock_obj._create_lock_file()
+
+ self.assertEqual(True, self._check_lock_file())
+ self.assertEqual(1, self.http_lock_obj._next_lock_number())
+
+ self.http_lock_obj.cleanup_http_lock()
+
+ self.assertEqual(False, self._check_lock_file())
+ self.assertEqual(0, self.http_lock_obj._next_lock_number())
+
+ def test_extract_lock_number(self,):
+ lock_file_list = (
+ self.lock_file_path_prefix + "00",
+ self.lock_file_path_prefix + "9",
+ self.lock_file_path_prefix + "001",
+ self.lock_file_path_prefix + "021",
+ )
+
+ expected_number_list = (0, 9, 1, 21)
+
+ for lock_file, expected in zip(lock_file_list, expected_number_list):
+ self.assertEqual(self.http_lock_obj._extract_lock_number(lock_file), expected)
+
+ def test_lock_file_list(self):
+ lock_file_list = [
+ self.lock_file_path_prefix + "6",
+ self.lock_file_path_prefix + "1",
+ self.lock_file_path_prefix + "4",
+ self.lock_file_path_prefix + "3",
+ ]
+
+ expected_file_list = [
+ self.lock_file_path_prefix + "1",
+ self.lock_file_path_prefix + "3",
+ self.lock_file_path_prefix + "4",
+ self.lock_file_path_prefix + "6",
+ ]
+
+ for file_name in lock_file_list:
+ open(file_name, 'w')
+
+ self.assertEqual(self.http_lock_obj._lock_file_list(), expected_file_list)
+
+ for file_name in lock_file_list:
+ os.unlink(file_name)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/http_server.py b/Tools/Scripts/webkitpy/layout_tests/port/http_server.py
new file mode 100755
index 0000000..bd75e27
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/http_server.py
@@ -0,0 +1,233 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""A class to help start/stop the lighttpd server used by layout tests."""
+
+from __future__ import with_statement
+
+import codecs
+import logging
+import optparse
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+import urllib
+
+import factory
+import http_server_base
+
+_log = logging.getLogger("webkitpy.layout_tests.port.http_server")
+
+
+class HttpdNotStarted(Exception):
+ pass
+
+
+class Lighttpd(http_server_base.HttpServerBase):
+
+ def __init__(self, port_obj, output_dir, background=False, port=None,
+ root=None, run_background=None):
+ """Args:
+ output_dir: the absolute path to the layout test result directory
+ """
+ # Webkit tests
+ http_server_base.HttpServerBase.__init__(self, port_obj)
+ self._output_dir = output_dir
+ self._process = None
+ self._port = port
+ self._root = root
+ self._run_background = run_background
+ if self._port:
+ self._port = int(self._port)
+
+ try:
+ self._webkit_tests = os.path.join(
+ self._port_obj.layout_tests_dir(), 'http', 'tests')
+ self._js_test_resource = os.path.join(
+ self._port_obj.layout_tests_dir(), 'fast', 'js', 'resources')
+ except:
+ self._webkit_tests = None
+ self._js_test_resource = None
+
+ # Self generated certificate for SSL server (for client cert get
+ # <base-path>\chrome\test\data\ssl\certs\root_ca_cert.crt)
+ self._pem_file = os.path.join(
+ os.path.dirname(os.path.abspath(__file__)), 'httpd2.pem')
+
+ # One mapping where we can get to everything
+ self.VIRTUALCONFIG = []
+
+ if self._webkit_tests:
+ self.VIRTUALCONFIG.extend(
+ # Three mappings (one with SSL) for LayoutTests http tests
+ [{'port': 8000, 'docroot': self._webkit_tests},
+ {'port': 8080, 'docroot': self._webkit_tests},
+ {'port': 8443, 'docroot': self._webkit_tests,
+ 'sslcert': self._pem_file}])
+
+ def is_running(self):
+ return self._process != None
+
+ def start(self):
+ if self.is_running():
+ raise 'Lighttpd already running'
+
+ base_conf_file = self._port_obj.path_from_webkit_base('Tools',
+ 'Scripts', 'webkitpy', 'layout_tests', 'port', 'lighttpd.conf')
+ out_conf_file = os.path.join(self._output_dir, 'lighttpd.conf')
+ time_str = time.strftime("%d%b%Y-%H%M%S")
+ access_file_name = "access.log-" + time_str + ".txt"
+ access_log = os.path.join(self._output_dir, access_file_name)
+ log_file_name = "error.log-" + time_str + ".txt"
+ error_log = os.path.join(self._output_dir, log_file_name)
+
+ # Remove old log files. We only need to keep the last ones.
+ self.remove_log_files(self._output_dir, "access.log-")
+ self.remove_log_files(self._output_dir, "error.log-")
+
+ # Write out the config
+ with codecs.open(base_conf_file, "r", "utf-8") as file:
+ base_conf = file.read()
+
+ # FIXME: This should be re-worked so that this block can
+ # use with open() instead of a manual file.close() call.
+ # lighttpd.conf files seem to be UTF-8 without BOM:
+ # http://redmine.lighttpd.net/issues/992
+ f = codecs.open(out_conf_file, "w", "utf-8")
+ f.write(base_conf)
+
+ # Write out our cgi handlers. Run perl through env so that it
+ # processes the #! line and runs perl with the proper command
+ # line arguments. Emulate apache's mod_asis with a cat cgi handler.
+ f.write(('cgi.assign = ( ".cgi" => "/usr/bin/env",\n'
+ ' ".pl" => "/usr/bin/env",\n'
+ ' ".asis" => "/bin/cat",\n'
+ ' ".php" => "%s" )\n\n') %
+ self._port_obj._path_to_lighttpd_php())
+
+ # Setup log files
+ f.write(('server.errorlog = "%s"\n'
+ 'accesslog.filename = "%s"\n\n') % (error_log, access_log))
+
+ # Setup upload folders. Upload folder is to hold temporary upload files
+ # and also POST data. This is used to support XHR layout tests that
+ # does POST.
+ f.write(('server.upload-dirs = ( "%s" )\n\n') % (self._output_dir))
+
+ # Setup a link to where the js test templates are stored
+ f.write(('alias.url = ( "/js-test-resources" => "%s" )\n\n') %
+ (self._js_test_resource))
+
+ # dump out of virtual host config at the bottom.
+ if self._root:
+ if self._port:
+ # Have both port and root dir.
+ mappings = [{'port': self._port, 'docroot': self._root}]
+ else:
+ # Have only a root dir - set the ports as for LayoutTests.
+ # This is used in ui_tests to run http tests against a browser.
+
+ # default set of ports as for LayoutTests but with a
+ # specified root.
+ mappings = [{'port': 8000, 'docroot': self._root},
+ {'port': 8080, 'docroot': self._root},
+ {'port': 8443, 'docroot': self._root,
+ 'sslcert': self._pem_file}]
+ else:
+ mappings = self.VIRTUALCONFIG
+ for mapping in mappings:
+ ssl_setup = ''
+ if 'sslcert' in mapping:
+ ssl_setup = (' ssl.engine = "enable"\n'
+ ' ssl.pemfile = "%s"\n' % mapping['sslcert'])
+
+ f.write(('$SERVER["socket"] == "127.0.0.1:%d" {\n'
+ ' server.document-root = "%s"\n' +
+ ssl_setup +
+ '}\n\n') % (mapping['port'], mapping['docroot']))
+ f.close()
+
+ executable = self._port_obj._path_to_lighttpd()
+ module_path = self._port_obj._path_to_lighttpd_modules()
+ start_cmd = [executable,
+ # Newly written config file
+ '-f', os.path.join(self._output_dir, 'lighttpd.conf'),
+ # Where it can find its module dynamic libraries
+ '-m', module_path]
+
+ if not self._run_background:
+ start_cmd.append(# Don't background
+ '-D')
+
+ # Copy liblightcomp.dylib to /tmp/lighttpd/lib to work around the
+ # bug that mod_alias.so loads it from the hard coded path.
+ if sys.platform == 'darwin':
+ tmp_module_path = '/tmp/lighttpd/lib'
+ if not os.path.exists(tmp_module_path):
+ os.makedirs(tmp_module_path)
+ lib_file = 'liblightcomp.dylib'
+ shutil.copyfile(os.path.join(module_path, lib_file),
+ os.path.join(tmp_module_path, lib_file))
+
+ env = self._port_obj.setup_environ_for_server()
+ _log.debug('Starting http server')
+ # FIXME: Should use Executive.run_command
+ self._process = subprocess.Popen(start_cmd, env=env)
+
+ # Wait for server to start.
+ self.mappings = mappings
+ server_started = self.wait_for_action(
+ self.is_server_running_on_all_ports)
+
+ # Our process terminated already
+ if not server_started or self._process.returncode != None:
+ raise google.httpd_utils.HttpdNotStarted('Failed to start httpd.')
+
+ _log.debug("Server successfully started")
+
+ # TODO(deanm): Find a nicer way to shutdown cleanly. Our log files are
+ # probably not being flushed, etc... why doesn't our python have os.kill ?
+
+ def stop(self, force=False):
+ if not force and not self.is_running():
+ return
+
+ httpd_pid = None
+ if self._process:
+ httpd_pid = self._process.pid
+ self._port_obj._shut_down_http_server(httpd_pid)
+
+ if self._process:
+ # wait() is not threadsafe and can throw OSError due to:
+ # http://bugs.python.org/issue1731717
+ self._process.wait()
+ self._process = None
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/http_server_base.py b/Tools/Scripts/webkitpy/layout_tests/port/http_server_base.py
new file mode 100644
index 0000000..52a0403
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/http_server_base.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Base class with common routines between the Apache and Lighttpd servers."""
+
+import logging
+import os
+import time
+import urllib
+
+from webkitpy.common.system import filesystem
+
+_log = logging.getLogger("webkitpy.layout_tests.port.http_server_base")
+
+
+class HttpServerBase(object):
+
+ def __init__(self, port_obj):
+ self._port_obj = port_obj
+
+ def wait_for_action(self, action):
+ """Repeat the action for 20 seconds or until it succeeds. Returns
+ whether it succeeded."""
+ start_time = time.time()
+ while time.time() - start_time < 20:
+ if action():
+ return True
+ _log.debug("Waiting for action: %s" % action)
+ time.sleep(1)
+
+ return False
+
+ def is_server_running_on_all_ports(self):
+ """Returns whether the server is running on all the desired ports."""
+ for mapping in self.mappings:
+ if 'sslcert' in mapping:
+ http_suffix = 's'
+ else:
+ http_suffix = ''
+
+ url = 'http%s://127.0.0.1:%d/' % (http_suffix, mapping['port'])
+
+ try:
+ response = urllib.urlopen(url)
+ _log.debug("Server running at %s" % url)
+ except IOError, e:
+ _log.debug("Server NOT running at %s: %s" % (url, e))
+ return False
+
+ return True
+
+ def remove_log_files(self, folder, starts_with):
+ files = os.listdir(folder)
+ for file in files:
+ if file.startswith(starts_with):
+ full_path = os.path.join(folder, file)
+ filesystem.FileSystem().remove(full_path)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/httpd2.pem b/Tools/Scripts/webkitpy/layout_tests/port/httpd2.pem
new file mode 100644
index 0000000..6349b78
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/httpd2.pem
@@ -0,0 +1,41 @@
+-----BEGIN CERTIFICATE-----
+MIIEZDCCAkygAwIBAgIBATANBgkqhkiG9w0BAQUFADBgMRAwDgYDVQQDEwdUZXN0
+IENBMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMN
+TW91bnRhaW4gVmlldzESMBAGA1UEChMJQ2VydCBUZXN0MB4XDTA4MDcyODIyMzIy
+OFoXDTEzMDcyNzIyMzIyOFowSjELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlm
+b3JuaWExEjAQBgNVBAoTCUNlcnQgVGVzdDESMBAGA1UEAxMJMTI3LjAuMC4xMIGf
+MA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDQj2tPWPUgbuI4H3/3dnttqVbndwU3
+3BdRCd67DFM44GRrsjDSH4bY/EbFyX9D52d/iy6ZaAmDePcCz5k/fgP3DMujykYG
+qgNiV2ywxTlMj7NlN2C7SRt68fQMZr5iI7rypdxuaZt9lSMD3ENBffYtuLTyZd9a
+3JPJe1TaIab5GwIDAQABo4HCMIG/MAkGA1UdEwQCMAAwHQYDVR0OBBYEFCYLBv5K
+x5sLNVlpLh5FwTwhdDl7MIGSBgNVHSMEgYowgYeAFF3Of5nj1BlBMU/Gz7El9Vqv
+45cxoWSkYjBgMRAwDgYDVQQDEwdUZXN0IENBMQswCQYDVQQGEwJVUzETMBEGA1UE
+CBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzESMBAGA1UEChMJ
+Q2VydCBUZXN0ggkA1FGT1D/e2U4wDQYJKoZIhvcNAQEFBQADggIBAEtkVmLObUgk
+b2cIA2S+QDtifq1UgVfBbytvR2lFmnADOR55mo0gHQG3HHqq4g034LmoVXDHhUk8
+Gb6aFiv4QubmVhLXcUelTRXwiNvGzkW7pC6Jrq105hdPjzXMKTcmiLaopm5Fqfc7
+hj5Cn1Sjspc8pdeQjrbeMdvca7KlFrGP8YkwCU2xOOX9PiN9G0966BWfjnr/fZZp
++OQVuUFHdiAZwthEMuDpAAXHqYXIsermgdOpgJaA53cf8NqBV2QGhtFgtsJCRoiu
+7DKqhyRWBGyz19VIH2b7y+6qvQVxuHk19kKRM0nftw/yNcJnm7gtttespMUPsOMa
+a2SD1G0hm0TND6vxaBhgR3cVqpl/qIpAdFi00Tm7hTyYE7I43zPW03t+/DpCt3Um
+EMRZsQ90co5q+bcx/vQ7YAtwUh30uMb0wpibeyCwDp8cqNmSiRkEuc/FjTYes5t8
+5gR//WX1l0+qjrjusO9NmoLnq2Yk6UcioX+z+q6Z/dudGfqhLfeWD2Q0LWYA242C
+d7km5Y3KAt1PJdVsof/aiVhVdddY/OIEKTRQhWEdDbosy2eh16BCKXT2FFvhNDg1
+AYFvn6I8nj9IldMJiIc3DdhacEAEzRMeRgPdzAa1griKUGknxsyTyRii8ru0WS6w
+DCNrlDOVXdzYGEZooBI76BDVY0W0akjV
+-----END CERTIFICATE-----
+-----BEGIN RSA PRIVATE KEY-----
+MIICXQIBAAKBgQDQj2tPWPUgbuI4H3/3dnttqVbndwU33BdRCd67DFM44GRrsjDS
+H4bY/EbFyX9D52d/iy6ZaAmDePcCz5k/fgP3DMujykYGqgNiV2ywxTlMj7NlN2C7
+SRt68fQMZr5iI7rypdxuaZt9lSMD3ENBffYtuLTyZd9a3JPJe1TaIab5GwIDAQAB
+AoGANHXu8z2YIzlhE+bwhGm8MGBpKL3qhRuKjeriqMA36tWezOw8lY4ymEAU+Ulv
+BsCdaxqydQoTYou57m4TyUHEcxq9pq3H0zB0qL709DdHi/t4zbV9XIoAzC5v0/hG
+9+Ca29TwC02FCw+qLkNrtwCpwOcQmc+bPxqvFu1iMiahURECQQD2I/Hi2413CMZz
+TBjl8fMiVO9GhA2J0sc8Qi+YcgJakaLD9xcbaiLkTzPZDlA389C1b6Ia+poAr4YA
+Ve0FFbxpAkEA2OobayyHE/QtPEqoy6NLR57jirmVBNmSWWd4lAyL5UIHIYVttJZg
+8CLvbzaU/iDGwR+wKsM664rKPHEmtlyo4wJBAMeSqYO5ZOCJGu9NWjrHjM3fdAsG
+8zs2zhiLya+fcU0iHIksBW5TBmt71Jw/wMc9R5J1K0kYvFml98653O5si1ECQBCk
+RV4/mE1rmlzZzYFyEcB47DQkcM5ictvxGEsje0gnfKyRtAz6zI0f4QbDRUMJ+LWw
+XK+rMsYHa+SfOb0b9skCQQCLdeonsIpFDv/Uv+flHISy0WA+AFkLXrRkBKh6G/OD
+dMHaNevkJgUnpceVEnkrdenp5CcEoFTI17pd+nBgDm/B
+-----END RSA PRIVATE KEY-----
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/lighttpd.conf b/Tools/Scripts/webkitpy/layout_tests/port/lighttpd.conf
new file mode 100644
index 0000000..26ca22f
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/lighttpd.conf
@@ -0,0 +1,90 @@
+server.tag = "LightTPD/1.4.19 (Win32)"
+server.modules = ( "mod_accesslog",
+ "mod_alias",
+ "mod_cgi",
+ "mod_rewrite" )
+
+# default document root required
+server.document-root = "."
+
+# files to check for if .../ is requested
+index-file.names = ( "index.php", "index.pl", "index.cgi",
+ "index.html", "index.htm", "default.htm" )
+# mimetype mapping
+mimetype.assign = (
+ ".gif" => "image/gif",
+ ".jpg" => "image/jpeg",
+ ".jpeg" => "image/jpeg",
+ ".png" => "image/png",
+ ".svg" => "image/svg+xml",
+ ".css" => "text/css",
+ ".html" => "text/html",
+ ".htm" => "text/html",
+ ".xhtml" => "application/xhtml+xml",
+ ".xhtmlmp" => "application/vnd.wap.xhtml+xml",
+ ".js" => "application/x-javascript",
+ ".log" => "text/plain",
+ ".conf" => "text/plain",
+ ".text" => "text/plain",
+ ".txt" => "text/plain",
+ ".dtd" => "text/xml",
+ ".xml" => "text/xml",
+ ".manifest" => "text/cache-manifest",
+ )
+
+# Use the "Content-Type" extended attribute to obtain mime type if possible
+mimetype.use-xattr = "enable"
+
+##
+# which extensions should not be handle via static-file transfer
+#
+# .php, .pl, .fcgi are most often handled by mod_fastcgi or mod_cgi
+static-file.exclude-extensions = ( ".php", ".pl", ".cgi" )
+
+server.bind = "localhost"
+server.port = 8001
+
+## virtual directory listings
+dir-listing.activate = "enable"
+#dir-listing.encoding = "iso-8859-2"
+#dir-listing.external-css = "style/oldstyle.css"
+
+## enable debugging
+#debug.log-request-header = "enable"
+#debug.log-response-header = "enable"
+#debug.log-request-handling = "enable"
+#debug.log-file-not-found = "enable"
+
+#### SSL engine
+#ssl.engine = "enable"
+#ssl.pemfile = "server.pem"
+
+# Rewrite rule for utf-8 path test (LayoutTests/http/tests/uri/utf8-path.html)
+# See the apache rewrite rule at LayoutTests/http/tests/uri/intercept/.htaccess
+# Rewrite rule for LayoutTests/http/tests/appcache/cyrillic-uri.html.
+# See the apache rewrite rule at
+# LayoutTests/http/tests/appcache/resources/intercept/.htaccess
+url.rewrite-once = (
+ "^/uri/intercept/(.*)" => "/uri/resources/print-uri.php",
+ "^/appcache/resources/intercept/(.*)" => "/appcache/resources/print-uri.php"
+)
+
+# LayoutTests/http/tests/xmlhttprequest/response-encoding.html uses an htaccess
+# to override charset for reply2.txt, reply2.xml, and reply4.txt.
+$HTTP["url"] =~ "^/xmlhttprequest/resources/reply2.(txt|xml)" {
+ mimetype.assign = (
+ ".txt" => "text/plain; charset=windows-1251",
+ ".xml" => "text/xml; charset=windows-1251"
+ )
+}
+$HTTP["url"] =~ "^/xmlhttprequest/resources/reply4.txt" {
+ mimetype.assign = ( ".txt" => "text/plain; charset=koi8-r" )
+}
+
+# LayoutTests/http/tests/appcache/wrong-content-type.html uses an htaccess
+# to override mime type for wrong-content-type.manifest.
+$HTTP["url"] =~ "^/appcache/resources/wrong-content-type.manifest" {
+ mimetype.assign = ( ".manifest" => "text/plain" )
+}
+
+# Autogenerated test-specific config follows.
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mac.py b/Tools/Scripts/webkitpy/layout_tests/port/mac.py
new file mode 100644
index 0000000..696e339
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/mac.py
@@ -0,0 +1,152 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the Google name nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""WebKit Mac implementation of the Port interface."""
+
+import logging
+import os
+import platform
+import signal
+
+import webkitpy.common.system.ospath as ospath
+import webkitpy.layout_tests.port.server_process as server_process
+from webkitpy.layout_tests.port.webkit import WebKitPort, WebKitDriver
+
+_log = logging.getLogger("webkitpy.layout_tests.port.mac")
+
+
+class MacPort(WebKitPort):
+ """WebKit Mac implementation of the Port class."""
+
+ def __init__(self, **kwargs):
+ kwargs.setdefault('port_name', 'mac' + self.version())
+ WebKitPort.__init__(self, **kwargs)
+
+ def default_child_processes(self):
+ # FIXME: new-run-webkit-tests is unstable on Mac running more than
+ # four threads in parallel.
+ # See https://bugs.webkit.org/show_bug.cgi?id=36622
+ child_processes = WebKitPort.default_child_processes(self)
+ if child_processes > 4:
+ return 4
+ return child_processes
+
+ def baseline_search_path(self):
+ port_names = []
+ if self._name == 'mac-tiger':
+ port_names.append("mac-tiger")
+ if self._name in ('mac-tiger', 'mac-leopard'):
+ port_names.append("mac-leopard")
+ if self._name in ('mac-tiger', 'mac-leopard', 'mac-snowleopard'):
+ port_names.append("mac-snowleopard")
+ port_names.append("mac")
+ return map(self._webkit_baseline_path, port_names)
+
+ def path_to_test_expectations_file(self):
+ return self.path_from_webkit_base('LayoutTests', 'platform',
+ 'mac', 'test_expectations.txt')
+
+ def _skipped_file_paths(self):
+ # FIXME: This method will need to be made work for non-mac
+ # platforms and moved into base.Port.
+ skipped_files = []
+ if self._name in ('mac-tiger', 'mac-leopard', 'mac-snowleopard'):
+ skipped_files.append(os.path.join(
+ self._webkit_baseline_path(self._name), 'Skipped'))
+ skipped_files.append(os.path.join(self._webkit_baseline_path('mac'),
+ 'Skipped'))
+ return skipped_files
+
+ def test_platform_name(self):
+ return 'mac' + self.version()
+
+ def version(self):
+ os_version_string = platform.mac_ver()[0] # e.g. "10.5.6"
+ if not os_version_string:
+ return '-leopard'
+ release_version = int(os_version_string.split('.')[1])
+ if release_version == 4:
+ return '-tiger'
+ elif release_version == 5:
+ return '-leopard'
+ elif release_version == 6:
+ return '-snowleopard'
+ return ''
+
+ def _build_java_test_support(self):
+ java_tests_path = os.path.join(self.layout_tests_dir(), "java")
+ build_java = ["/usr/bin/make", "-C", java_tests_path]
+ if self._executive.run_command(build_java, return_exit_code=True):
+ _log.error("Failed to build Java support files: %s" % build_java)
+ return False
+ return True
+
+ def _check_port_build(self):
+ return self._build_java_test_support()
+
+ def _tests_for_other_platforms(self):
+ # The original run-webkit-tests builds up a "whitelist" of tests to
+ # run, and passes that to DumpRenderTree. new-run-webkit-tests assumes
+ # we run *all* tests and test_expectations.txt functions as a
+ # blacklist.
+ # FIXME: This list could be dynamic based on platform name and
+ # pushed into base.Port.
+ return [
+ "platform/chromium",
+ "platform/gtk",
+ "platform/qt",
+ "platform/win",
+ ]
+
+ def _path_to_apache_config_file(self):
+ return os.path.join(self.layout_tests_dir(), 'http', 'conf',
+ 'apache2-httpd.conf')
+
+ # FIXME: This doesn't have anything to do with WebKit.
+ def _shut_down_http_server(self, server_pid):
+ """Shut down the lighttpd web server. Blocks until it's fully
+ shut down.
+
+ Args:
+ server_pid: The process ID of the running server.
+ """
+ # server_pid is not set when "http_server.py stop" is run manually.
+ if server_pid is None:
+ # FIXME: This isn't ideal, since it could conflict with
+ # lighttpd processes not started by http_server.py,
+ # but good enough for now.
+ self._executive.kill_all('httpd')
+ else:
+ try:
+ os.kill(server_pid, signal.SIGTERM)
+ # FIXME: Maybe throw in a SIGKILL just to be sure?
+ except OSError:
+ # Sometimes we get a bad PID (e.g. from a stale httpd.pid
+ # file), so if kill fails on the given PID, just try to
+ # 'killall' web servers.
+ self._shut_down_http_server(None)
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py
new file mode 100644
index 0000000..d383a4c
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/mac_unittest.py
@@ -0,0 +1,81 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import StringIO
+import sys
+import unittest
+
+import mac
+import port_testcase
+
+
+class MacTest(port_testcase.PortTestCase):
+ def make_port(self, options=port_testcase.mock_options):
+ if sys.platform != 'darwin':
+ return None
+ port_obj = mac.MacPort(options=options)
+ port_obj._options.results_directory = port_obj.results_directory()
+ port_obj._options.configuration = 'Release'
+ return port_obj
+
+ def test_skipped_file_paths(self):
+ port = self.make_port()
+ if not port:
+ return
+ skipped_paths = port._skipped_file_paths()
+ # FIXME: _skipped_file_paths should return WebKit-relative paths.
+ # So to make it unit testable, we strip the WebKit directory from the path.
+ relative_paths = [path[len(port.path_from_webkit_base()):] for path in skipped_paths]
+ self.assertEqual(relative_paths, ['LayoutTests/platform/mac-leopard/Skipped', 'LayoutTests/platform/mac/Skipped'])
+
+ example_skipped_file = u"""
+# <rdar://problem/5647952> fast/events/mouseout-on-window.html needs mac DRT to issue mouse out events
+fast/events/mouseout-on-window.html
+
+# <rdar://problem/5643675> window.scrollTo scrolls a window with no scrollbars
+fast/events/attempt-scroll-with-no-scrollbars.html
+
+# see bug <rdar://problem/5646437> REGRESSION (r28015): svg/batik/text/smallFonts fails
+svg/batik/text/smallFonts.svg
+"""
+ example_skipped_tests = [
+ "fast/events/mouseout-on-window.html",
+ "fast/events/attempt-scroll-with-no-scrollbars.html",
+ "svg/batik/text/smallFonts.svg",
+ ]
+
+ def test_skipped_file_paths(self):
+ port = self.make_port()
+ if not port:
+ return
+ skipped_file = StringIO.StringIO(self.example_skipped_file)
+ self.assertEqual(port._tests_from_skipped_file(skipped_file), self.example_skipped_tests)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py b/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py
new file mode 100644
index 0000000..c4b36ac
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/port_testcase.py
@@ -0,0 +1,97 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit testing base class for Port implementations."""
+
+import os
+import tempfile
+import unittest
+
+from webkitpy.tool import mocktool
+mock_options = mocktool.MockOptions(results_directory='layout-test-results',
+ use_apache=True,
+ configuration='Release')
+
+# FIXME: This should be used for all ports, not just WebKit Mac. See
+# https://bugs.webkit.org/show_bug.cgi?id=50043 .
+
+class PortTestCase(unittest.TestCase):
+ """Tests the WebKit port implementation."""
+ def make_port(self, options=mock_options):
+ """Override in subclass."""
+ raise NotImplementedError()
+
+ def test_driver_cmd_line(self):
+ port = self.make_port()
+ if not port:
+ return
+ self.assertTrue(len(port.driver_cmd_line()))
+
+ def test_http_server(self):
+ port = self.make_port()
+ if not port:
+ return
+ port.start_http_server()
+ port.stop_http_server()
+
+ def test_image_diff(self):
+ port = self.make_port()
+ if not port:
+ return
+
+ # FIXME: not sure why this shouldn't always be True
+ #self.assertTrue(port.check_image_diff())
+ if not port.check_image_diff():
+ return
+
+ dir = port.layout_tests_dir()
+ file1 = os.path.join(dir, 'fast', 'css', 'button_center.png')
+ fh1 = file(file1)
+ contents1 = fh1.read()
+ file2 = os.path.join(dir, 'fast', 'css',
+ 'remove-shorthand-expected.png')
+ fh2 = file(file2)
+ contents2 = fh2.read()
+ tmpfile = tempfile.mktemp()
+
+ self.assertFalse(port.diff_image(contents1, contents1))
+ self.assertTrue(port.diff_image(contents1, contents2))
+
+ self.assertTrue(port.diff_image(contents1, contents2, tmpfile))
+ fh1.close()
+ fh2.close()
+ # FIXME: this may not be being written?
+ # self.assertTrue(os.path.exists(tmpfile))
+ # os.remove(tmpfile)
+
+ def test_websocket_server(self):
+ port = self.make_port()
+ if not port:
+ return
+ port.start_websocket_server()
+ port.stop_websocket_server()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/qt.py b/Tools/Scripts/webkitpy/layout_tests/port/qt.py
new file mode 100644
index 0000000..af94acc
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/qt.py
@@ -0,0 +1,119 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the Google name nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""QtWebKit implementation of the Port interface."""
+
+import logging
+import os
+import signal
+import sys
+
+import webkit
+
+from webkitpy.layout_tests.port.webkit import WebKitPort
+
+_log = logging.getLogger("webkitpy.layout_tests.port.qt")
+
+
+class QtPort(WebKitPort):
+ """QtWebKit implementation of the Port class."""
+
+ def __init__(self, **kwargs):
+ kwargs.setdefault('port_name', 'qt')
+ WebKitPort.__init__(self, **kwargs)
+
+ def baseline_search_path(self):
+ port_names = []
+ if sys.platform == 'linux2':
+ port_names.append("qt-linux")
+ elif sys.platform in ('win32', 'cygwin'):
+ port_names.append("qt-win")
+ elif sys.platform == 'darwin':
+ port_names.append("qt-mac")
+ port_names.append("qt")
+ return map(self._webkit_baseline_path, port_names)
+
+ def _tests_for_other_platforms(self):
+ # FIXME: This list could be dynamic based on platform name and
+ # pushed into base.Port.
+ # This really need to be automated.
+ return [
+ "platform/chromium",
+ "platform/win",
+ "platform/gtk",
+ "platform/mac",
+ ]
+
+ def _path_to_apache_config_file(self):
+ # FIXME: This needs to detect the distribution and change config files.
+ return os.path.join(self.layout_tests_dir(), 'http', 'conf',
+ 'apache2-debian-httpd.conf')
+
+ def _shut_down_http_server(self, server_pid):
+ """Shut down the httpd web server. Blocks until it's fully
+ shut down.
+
+ Args:
+ server_pid: The process ID of the running server.
+ """
+ # server_pid is not set when "http_server.py stop" is run manually.
+ if server_pid is None:
+ # FIXME: This isn't ideal, since it could conflict with
+ # lighttpd processes not started by http_server.py,
+ # but good enough for now.
+ self._executive.kill_all('apache2')
+ else:
+ try:
+ os.kill(server_pid, signal.SIGTERM)
+ # TODO(mmoss) Maybe throw in a SIGKILL just to be sure?
+ except OSError:
+ # Sometimes we get a bad PID (e.g. from a stale httpd.pid
+ # file), so if kill fails on the given PID, just try to
+ # 'killall' web servers.
+ self._shut_down_http_server(None)
+
+ def _build_driver(self):
+ # The Qt port builds DRT as part of the main build step
+ return True
+
+ def _path_to_driver(self):
+ return self._build_path('bin/DumpRenderTree')
+
+ def _path_to_image_diff(self):
+ return self._build_path('bin/ImageDiff')
+
+ def _path_to_webcore_library(self):
+ return self._build_path('lib/libQtWebKit.so')
+
+ def _runtime_feature_list(self):
+ return None
+
+ def setup_environ_for_server(self):
+ env = webkit.WebKitPort.setup_environ_for_server(self)
+ env['QTWEBKIT_PLUGIN_PATH'] = self._build_path('lib/plugins')
+ return env
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/server_process.py b/Tools/Scripts/webkitpy/layout_tests/port/server_process.py
new file mode 100644
index 0000000..5a0a40c
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/server_process.py
@@ -0,0 +1,225 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the Google name nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Package that implements the ServerProcess wrapper class"""
+
+import logging
+import os
+import select
+import signal
+import subprocess
+import sys
+import time
+if sys.platform != 'win32':
+ import fcntl
+
+from webkitpy.common.system.executive import Executive
+
+_log = logging.getLogger("webkitpy.layout_tests.port.server_process")
+
+
+class ServerProcess:
+ """This class provides a wrapper around a subprocess that
+ implements a simple request/response usage model. The primary benefit
+ is that reading responses takes a timeout, so that we don't ever block
+ indefinitely. The class also handles transparently restarting processes
+ as necessary to keep issuing commands."""
+
+ def __init__(self, port_obj, name, cmd, env=None, executive=Executive()):
+ self._port = port_obj
+ self._name = name
+ self._cmd = cmd
+ self._env = env
+ self._reset()
+ self._executive = executive
+
+ def _reset(self):
+ self._proc = None
+ self._output = ''
+ self.crashed = False
+ self.timed_out = False
+ self.error = ''
+
+ def _start(self):
+ if self._proc:
+ raise ValueError("%s already running" % self._name)
+ self._reset()
+ # close_fds is a workaround for http://bugs.python.org/issue2320
+ close_fds = sys.platform not in ('win32', 'cygwin')
+ self._proc = subprocess.Popen(self._cmd, stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ close_fds=close_fds,
+ env=self._env)
+ fd = self._proc.stdout.fileno()
+ fl = fcntl.fcntl(fd, fcntl.F_GETFL)
+ fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
+ fd = self._proc.stderr.fileno()
+ fl = fcntl.fcntl(fd, fcntl.F_GETFL)
+ fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
+
+ def handle_interrupt(self):
+ """This routine checks to see if the process crashed or exited
+ because of a keyboard interrupt and raises KeyboardInterrupt
+ accordingly."""
+ if self.crashed:
+ # This is hex code 0xc000001d, which is used for abrupt
+ # termination. This happens if we hit ctrl+c from the prompt
+ # and we happen to be waiting on the DumpRenderTree.
+ # sdoyon: Not sure for which OS and in what circumstances the
+ # above code is valid. What works for me under Linux to detect
+ # ctrl+c is for the subprocess returncode to be negative
+ # SIGINT. And that agrees with the subprocess documentation.
+ if (-1073741510 == self._proc.returncode or
+ - signal.SIGINT == self._proc.returncode):
+ raise KeyboardInterrupt
+ return
+
+ def poll(self):
+ """Check to see if the underlying process is running; returns None
+ if it still is (wrapper around subprocess.poll)."""
+ if self._proc:
+ # poll() is not threadsafe and can throw OSError due to:
+ # http://bugs.python.org/issue1731717
+ return self._proc.poll()
+ return None
+
+ def write(self, input):
+ """Write a request to the subprocess. The subprocess is (re-)start()'ed
+ if is not already running."""
+ if not self._proc:
+ self._start()
+ self._proc.stdin.write(input)
+
+ def read_line(self, timeout):
+ """Read a single line from the subprocess, waiting until the deadline.
+ If the deadline passes, the call times out. Note that even if the
+ subprocess has crashed or the deadline has passed, if there is output
+ pending, it will be returned.
+
+ Args:
+ timeout: floating-point number of seconds the call is allowed
+ to block for. A zero or negative number will attempt to read
+ any existing data, but will not block. There is no way to
+ block indefinitely.
+ Returns:
+ output: data returned, if any. If no data is available and the
+ call times out or crashes, an empty string is returned. Note
+ that the returned string includes the newline ('\n')."""
+ return self._read(timeout, size=0)
+
+ def read(self, timeout, size):
+ """Attempts to read size characters from the subprocess, waiting until
+ the deadline passes. If the deadline passes, any available data will be
+ returned. Note that even if the deadline has passed or if the
+ subprocess has crashed, any available data will still be returned.
+
+ Args:
+ timeout: floating-point number of seconds the call is allowed
+ to block for. A zero or negative number will attempt to read
+ any existing data, but will not block. There is no way to
+ block indefinitely.
+ size: amount of data to read. Must be a postive integer.
+ Returns:
+ output: data returned, if any. If no data is available, an empty
+ string is returned.
+ """
+ if size <= 0:
+ raise ValueError('ServerProcess.read() called with a '
+ 'non-positive size: %d ' % size)
+ return self._read(timeout, size)
+
+ def _read(self, timeout, size):
+ """Internal routine that actually does the read."""
+ index = -1
+ out_fd = self._proc.stdout.fileno()
+ err_fd = self._proc.stderr.fileno()
+ select_fds = (out_fd, err_fd)
+ deadline = time.time() + timeout
+ while not self.timed_out and not self.crashed:
+ # poll() is not threadsafe and can throw OSError due to:
+ # http://bugs.python.org/issue1731717
+ if self._proc.poll() != None:
+ self.crashed = True
+ self.handle_interrupt()
+
+ now = time.time()
+ if now > deadline:
+ self.timed_out = True
+
+ # Check to see if we have any output we can return.
+ if size and len(self._output) >= size:
+ index = size
+ elif size == 0:
+ index = self._output.find('\n') + 1
+
+ if index > 0 or self.crashed or self.timed_out:
+ output = self._output[0:index]
+ self._output = self._output[index:]
+ return output
+
+ # Nope - wait for more data.
+ (read_fds, write_fds, err_fds) = select.select(select_fds, [],
+ select_fds,
+ deadline - now)
+ try:
+ if out_fd in read_fds:
+ self._output += self._proc.stdout.read()
+ if err_fd in read_fds:
+ self.error += self._proc.stderr.read()
+ except IOError, e:
+ pass
+
+ def stop(self):
+ """Stop (shut down) the subprocess), if it is running."""
+ pid = self._proc.pid
+ self._proc.stdin.close()
+ self._proc.stdout.close()
+ if self._proc.stderr:
+ self._proc.stderr.close()
+ if sys.platform not in ('win32', 'cygwin'):
+ # Closing stdin/stdout/stderr hangs sometimes on OS X,
+ # (see restart(), above), and anyway we don't want to hang
+ # the harness if DumpRenderTree is buggy, so we wait a couple
+ # seconds to give DumpRenderTree a chance to clean up, but then
+ # force-kill the process if necessary.
+ KILL_TIMEOUT = 3.0
+ timeout = time.time() + KILL_TIMEOUT
+ # poll() is not threadsafe and can throw OSError due to:
+ # http://bugs.python.org/issue1731717
+ while self._proc.poll() is None and time.time() < timeout:
+ time.sleep(0.1)
+ # poll() is not threadsafe and can throw OSError due to:
+ # http://bugs.python.org/issue1731717
+ if self._proc.poll() is None:
+ _log.warning('stopping %s timed out, killing it' %
+ self._name)
+ self._executive.kill_process(self._proc.pid)
+ _log.warning('killed')
+ self._reset()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/test.py b/Tools/Scripts/webkitpy/layout_tests/port/test.py
new file mode 100644
index 0000000..935881c
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/test.py
@@ -0,0 +1,343 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the Google name nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Dummy Port implementation used for testing."""
+from __future__ import with_statement
+
+import codecs
+import fnmatch
+import os
+import sys
+import time
+
+from webkitpy.layout_tests.layout_package import test_output
+
+import base
+
+
+# This sets basic expectations for a test. Each individual expectation
+# can be overridden by a keyword argument in TestList.add().
+class TestInstance:
+ def __init__(self, name):
+ self.name = name
+ self.base = name[(name.rfind("/") + 1):name.rfind(".html")]
+ self.crash = False
+ self.exception = False
+ self.hang = False
+ self.keyboard = False
+ self.error = ''
+ self.timeout = False
+ self.actual_text = self.base + '-txt\n'
+ self.actual_checksum = self.base + '-checksum\n'
+ self.actual_image = self.base + '-png\n'
+ self.expected_text = self.actual_text
+ self.expected_checksum = self.actual_checksum
+ self.expected_image = self.actual_image
+
+
+# This is an in-memory list of tests, what we want them to produce, and
+# what we want to claim are the expected results.
+class TestList:
+ def __init__(self, port):
+ self.port = port
+ self.tests = {}
+
+ def add(self, name, **kwargs):
+ test = TestInstance(name)
+ for key, value in kwargs.items():
+ test.__dict__[key] = value
+ self.tests[name] = test
+
+ def keys(self):
+ return self.tests.keys()
+
+ def __contains__(self, item):
+ return item in self.tests
+
+ def __getitem__(self, item):
+ return self.tests[item]
+
+
+class TestPort(base.Port):
+ """Test implementation of the Port interface."""
+
+ def __init__(self, **kwargs):
+ base.Port.__init__(self, **kwargs)
+ tests = TestList(self)
+ tests.add('failures/expected/checksum.html',
+ actual_checksum='checksum_fail-checksum')
+ tests.add('failures/expected/crash.html', crash=True)
+ tests.add('failures/expected/exception.html', exception=True)
+ tests.add('failures/expected/timeout.html', timeout=True)
+ tests.add('failures/expected/hang.html', hang=True)
+ tests.add('failures/expected/missing_text.html',
+ expected_text=None)
+ tests.add('failures/expected/image.html',
+ actual_image='image_fail-png',
+ expected_image='image-png')
+ tests.add('failures/expected/image_checksum.html',
+ actual_checksum='image_checksum_fail-checksum',
+ actual_image='image_checksum_fail-png')
+ tests.add('failures/expected/keyboard.html',
+ keyboard=True)
+ tests.add('failures/expected/missing_check.html',
+ expected_checksum=None)
+ tests.add('failures/expected/missing_image.html',
+ expected_image=None)
+ tests.add('failures/expected/missing_text.html',
+ expected_text=None)
+ tests.add('failures/expected/newlines_leading.html',
+ expected_text="\nfoo\n",
+ actual_text="foo\n")
+ tests.add('failures/expected/newlines_trailing.html',
+ expected_text="foo\n\n",
+ actual_text="foo\n")
+ tests.add('failures/expected/newlines_with_excess_CR.html',
+ expected_text="foo\r\r\r\n",
+ actual_text="foo\n")
+ tests.add('failures/expected/text.html',
+ actual_text='text_fail-png')
+ tests.add('failures/unexpected/crash.html', crash=True)
+ tests.add('failures/unexpected/text-image-checksum.html',
+ actual_text='text-image-checksum_fail-txt',
+ actual_checksum='text-image-checksum_fail-checksum')
+ tests.add('failures/unexpected/timeout.html', timeout=True)
+ tests.add('http/tests/passes/text.html')
+ tests.add('http/tests/ssl/text.html')
+ tests.add('passes/error.html', error='stuff going to stderr')
+ tests.add('passes/image.html')
+ tests.add('passes/platform_image.html')
+ # Text output files contain "\r\n" on Windows. This may be
+ # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
+ tests.add('passes/text.html',
+ expected_text='\nfoo\n\n',
+ actual_text='\nfoo\r\n\r\r\n')
+ tests.add('websocket/tests/passes/text.html')
+ self._tests = tests
+
+ def baseline_path(self):
+ return os.path.join(self.layout_tests_dir(), 'platform',
+ self.name() + self.version())
+
+ def baseline_search_path(self):
+ return [self.baseline_path()]
+
+ def check_build(self, needs_http):
+ return True
+
+ def diff_image(self, expected_contents, actual_contents,
+ diff_filename=None):
+ diffed = actual_contents != expected_contents
+ if diffed and diff_filename:
+ with codecs.open(diff_filename, "w", "utf-8") as diff_fh:
+ diff_fh.write("< %s\n---\n> %s\n" %
+ (expected_contents, actual_contents))
+ return diffed
+
+ def expected_checksum(self, test):
+ test = self.relative_test_filename(test)
+ return self._tests[test].expected_checksum
+
+ def expected_image(self, test):
+ test = self.relative_test_filename(test)
+ return self._tests[test].expected_image
+
+ def expected_text(self, test):
+ test = self.relative_test_filename(test)
+ text = self._tests[test].expected_text
+ if not text:
+ text = ''
+ return text
+
+ def tests(self, paths):
+ # Test the idea of port-specific overrides for test lists. Also
+ # keep in memory to speed up the test harness.
+ if not paths:
+ paths = ['*']
+
+ matched_tests = []
+ for p in paths:
+ if self.path_isdir(p):
+ matched_tests.extend(fnmatch.filter(self._tests.keys(), p + '*'))
+ else:
+ matched_tests.extend(fnmatch.filter(self._tests.keys(), p))
+ layout_tests_dir = self.layout_tests_dir()
+ return set([os.path.join(layout_tests_dir, p) for p in matched_tests])
+
+ def path_exists(self, path):
+ # used by test_expectations.py and printing.py
+ rpath = self.relative_test_filename(path)
+ if rpath in self._tests:
+ return True
+ if self.path_isdir(rpath):
+ return True
+ if rpath.endswith('-expected.txt'):
+ test = rpath.replace('-expected.txt', '.html')
+ return (test in self._tests and
+ self._tests[test].expected_text)
+ if rpath.endswith('-expected.checksum'):
+ test = rpath.replace('-expected.checksum', '.html')
+ return (test in self._tests and
+ self._tests[test].expected_checksum)
+ if rpath.endswith('-expected.png'):
+ test = rpath.replace('-expected.png', '.html')
+ return (test in self._tests and
+ self._tests[test].expected_image)
+ return False
+
+ def layout_tests_dir(self):
+ return self.path_from_webkit_base('Tools', 'Scripts',
+ 'webkitpy', 'layout_tests', 'data')
+
+ def path_isdir(self, path):
+ # Used by test_expectations.py
+ #
+ # We assume that a path is a directory if we have any tests
+ # that whose prefix matches the path plus a directory modifier
+ # and not a file extension.
+ if path[-1] != '/':
+ path += '/'
+
+ # FIXME: Directories can have a dot in the name. We should
+ # probably maintain a white list of known cases like CSS2.1
+ # and check it here in the future.
+ if path.find('.') != -1:
+ # extension separator found, assume this is a file
+ return False
+
+ # strip out layout tests directory path if found. The tests
+ # keys are relative to it.
+ tests_dir = self.layout_tests_dir()
+ if path.startswith(tests_dir):
+ path = path[len(tests_dir) + 1:]
+
+ return any([t.startswith(path) for t in self._tests.keys()])
+
+ def test_dirs(self):
+ return ['passes', 'failures']
+
+ def name(self):
+ return self._name
+
+ def _path_to_wdiff(self):
+ return None
+
+ def results_directory(self):
+ return '/tmp/' + self.get_option('results_directory')
+
+ def setup_test_run(self):
+ pass
+
+ def create_driver(self, worker_number):
+ return TestDriver(self, worker_number)
+
+ def start_http_server(self):
+ pass
+
+ def start_websocket_server(self):
+ pass
+
+ def stop_http_server(self):
+ pass
+
+ def stop_websocket_server(self):
+ pass
+
+ def test_expectations(self):
+ """Returns the test expectations for this port.
+
+ Basically this string should contain the equivalent of a
+ test_expectations file. See test_expectations.py for more details."""
+ return """
+WONTFIX : failures/expected/checksum.html = IMAGE
+WONTFIX : failures/expected/crash.html = CRASH
+// This one actually passes because the checksums will match.
+WONTFIX : failures/expected/image.html = PASS
+WONTFIX : failures/expected/image_checksum.html = IMAGE
+WONTFIX : failures/expected/missing_check.html = MISSING PASS
+WONTFIX : failures/expected/missing_image.html = MISSING PASS
+WONTFIX : failures/expected/missing_text.html = MISSING PASS
+WONTFIX : failures/expected/newlines_leading.html = TEXT
+WONTFIX : failures/expected/newlines_trailing.html = TEXT
+WONTFIX : failures/expected/newlines_with_excess_CR.html = TEXT
+WONTFIX : failures/expected/text.html = TEXT
+WONTFIX : failures/expected/timeout.html = TIMEOUT
+WONTFIX SKIP : failures/expected/hang.html = TIMEOUT
+WONTFIX SKIP : failures/expected/keyboard.html = CRASH
+WONTFIX SKIP : failures/expected/exception.html = CRASH
+"""
+
+ def test_base_platform_names(self):
+ return ('mac', 'win')
+
+ def test_platform_name(self):
+ return 'mac'
+
+ def test_platform_names(self):
+ return self.test_base_platform_names()
+
+ def test_platform_name_to_name(self, test_platform_name):
+ return test_platform_name
+
+ def version(self):
+ return ''
+
+
+class TestDriver(base.Driver):
+ """Test/Dummy implementation of the DumpRenderTree interface."""
+
+ def __init__(self, port, worker_number):
+ self._port = port
+
+ def cmd_line(self):
+ return ['None']
+
+ def poll(self):
+ return True
+
+ def run_test(self, test_input):
+ start_time = time.time()
+ test_name = self._port.relative_test_filename(test_input.filename)
+ test = self._port._tests[test_name]
+ if test.keyboard:
+ raise KeyboardInterrupt
+ if test.exception:
+ raise ValueError('exception from ' + test_name)
+ if test.hang:
+ time.sleep((float(test_input.timeout) * 4) / 1000.0)
+ return test_output.TestOutput(test.actual_text, test.actual_image,
+ test.actual_checksum, test.crash,
+ time.time() - start_time, test.timeout,
+ test.error)
+
+ def start(self):
+ pass
+
+ def stop(self):
+ pass
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/test_files.py b/Tools/Scripts/webkitpy/layout_tests/port/test_files.py
new file mode 100644
index 0000000..2c0a7b6
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/test_files.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""This module is used to find all of the layout test files used by
+run-webkit-tests. It exposes one public function - find() -
+which takes an optional list of paths. If a list is passed in, the returned
+list of test files is constrained to those found under the paths passed in,
+i.e. calling find(["LayoutTests/fast"]) will only return files
+under that directory."""
+
+import glob
+import os
+import time
+
+from webkitpy.common.system import logutils
+
+
+_log = logutils.get_logger(__file__)
+
+
+# When collecting test cases, we include any file with these extensions.
+_supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.xhtmlmp', '.pl',
+ '.php', '.svg'])
+# When collecting test cases, skip these directories
+_skipped_directories = set(['.svn', '_svn', 'resources', 'script-tests'])
+
+
+def find(port, paths):
+ """Finds the set of tests under port.layout_tests_dir().
+
+ Args:
+ paths: a list of command line paths relative to the layout_tests_dir()
+ to limit the search to. glob patterns are ok.
+ """
+ gather_start_time = time.time()
+ paths_to_walk = set()
+ # if paths is empty, provide a pre-defined list.
+ if paths:
+ _log.debug("Gathering tests from: %s relative to %s" % (paths, port.layout_tests_dir()))
+ for path in paths:
+ # If there's an * in the name, assume it's a glob pattern.
+ path = os.path.join(port.layout_tests_dir(), path)
+ if path.find('*') > -1:
+ filenames = glob.glob(path)
+ paths_to_walk.update(filenames)
+ else:
+ paths_to_walk.add(path)
+ else:
+ _log.debug("Gathering tests from: %s" % port.layout_tests_dir())
+ paths_to_walk.add(port.layout_tests_dir())
+
+ # Now walk all the paths passed in on the command line and get filenames
+ test_files = set()
+ for path in paths_to_walk:
+ if os.path.isfile(path) and _is_test_file(path):
+ test_files.add(os.path.normpath(path))
+ continue
+
+ for root, dirs, files in os.walk(path):
+ # Don't walk skipped directories or their sub-directories.
+ if os.path.basename(root) in _skipped_directories:
+ del dirs[:]
+ continue
+ # This copy and for-in is slightly inefficient, but
+ # the extra walk avoidance consistently shaves .5 seconds
+ # off of total walk() time on my MacBook Pro.
+ for directory in dirs[:]:
+ if directory in _skipped_directories:
+ dirs.remove(directory)
+
+ for filename in files:
+ if _is_test_file(filename):
+ filename = os.path.join(root, filename)
+ filename = os.path.normpath(filename)
+ test_files.add(filename)
+
+ gather_time = time.time() - gather_start_time
+ _log.debug("Test gathering took %f seconds" % gather_time)
+
+ return test_files
+
+
+def _has_supported_extension(filename):
+ """Return true if filename is one of the file extensions we want to run a
+ test on."""
+ extension = os.path.splitext(filename)[1]
+ return extension in _supported_file_extensions
+
+
+def _is_reference_html_file(filename):
+ """Return true if the filename points to a reference HTML file."""
+ if (filename.endswith('-expected.html') or
+ filename.endswith('-expected-mismatch.html')):
+ _log.warn("Reftests are not supported - ignoring %s" % filename)
+ return True
+ return False
+
+
+def _is_test_file(filename):
+ """Return true if the filename points to a test file."""
+ return (_has_supported_extension(filename) and
+ not _is_reference_html_file(filename))
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py
new file mode 100644
index 0000000..83525c8
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py
@@ -0,0 +1,75 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import unittest
+
+import base
+import test_files
+
+
+class TestFilesTest(unittest.TestCase):
+ def test_find_no_paths_specified(self):
+ port = base.Port()
+ layout_tests_dir = port.layout_tests_dir()
+ port.layout_tests_dir = lambda: os.path.join(layout_tests_dir,
+ 'fast', 'html')
+ tests = test_files.find(port, [])
+ self.assertNotEqual(tests, 0)
+
+ def test_find_one_test(self):
+ port = base.Port()
+ # This is just a test picked at random but known to exist.
+ tests = test_files.find(port, ['fast/html/keygen.html'])
+ self.assertEqual(len(tests), 1)
+
+ def test_find_glob(self):
+ port = base.Port()
+ tests = test_files.find(port, ['fast/html/key*'])
+ self.assertEqual(len(tests), 1)
+
+ def test_find_with_skipped_directories(self):
+ port = base.Port()
+ tests = port.tests('userscripts')
+ self.assertTrue('userscripts/resources/frame1.html' not in tests)
+
+ def test_find_with_skipped_directories_2(self):
+ port = base.Port()
+ tests = test_files.find(port, ['userscripts/resources'])
+ self.assertEqual(tests, set([]))
+
+ def test_is_test_file(self):
+ self.assertTrue(test_files._is_test_file('foo.html'))
+ self.assertTrue(test_files._is_test_file('foo.shtml'))
+ self.assertFalse(test_files._is_test_file('foo.png'))
+ self.assertFalse(test_files._is_test_file('foo-expected.html'))
+ self.assertFalse(test_files._is_test_file('foo-expected-mismatch.html'))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/webkit.py b/Tools/Scripts/webkitpy/layout_tests/port/webkit.py
new file mode 100644
index 0000000..afdebeb
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/webkit.py
@@ -0,0 +1,497 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+# Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the Google name nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""WebKit implementations of the Port interface."""
+
+
+from __future__ import with_statement
+
+import codecs
+import logging
+import os
+import re
+import shutil
+import signal
+import sys
+import time
+import webbrowser
+import operator
+import tempfile
+import shutil
+
+import webkitpy.common.system.ospath as ospath
+import webkitpy.layout_tests.layout_package.test_output as test_output
+import webkitpy.layout_tests.port.base as base
+import webkitpy.layout_tests.port.server_process as server_process
+
+_log = logging.getLogger("webkitpy.layout_tests.port.webkit")
+
+
+class WebKitPort(base.Port):
+ """WebKit implementation of the Port class."""
+
+ def __init__(self, **kwargs):
+ base.Port.__init__(self, **kwargs)
+ self._cached_apache_path = None
+
+ # FIXME: disable pixel tests until they are run by default on the
+ # build machines.
+ self.set_option_default('pixel_tests', False)
+
+ def baseline_path(self):
+ return self._webkit_baseline_path(self._name)
+
+ def baseline_search_path(self):
+ return [self._webkit_baseline_path(self._name)]
+
+ def path_to_test_expectations_file(self):
+ return os.path.join(self._webkit_baseline_path(self._name),
+ 'test_expectations.txt')
+
+ # Only needed by ports which maintain versioned test expectations (like mac-tiger vs. mac-leopard)
+ def version(self):
+ return ''
+
+ def _build_driver(self):
+ configuration = self.get_option('configuration')
+ return self._config.build_dumprendertree(configuration)
+
+ def _check_driver(self):
+ driver_path = self._path_to_driver()
+ if not os.path.exists(driver_path):
+ _log.error("DumpRenderTree was not found at %s" % driver_path)
+ return False
+ return True
+
+ def check_build(self, needs_http):
+ if self.get_option('build') and not self._build_driver():
+ return False
+ if not self._check_driver():
+ return False
+ if self.get_option('pixel_tests'):
+ if not self.check_image_diff():
+ return False
+ if not self._check_port_build():
+ return False
+ return True
+
+ def _check_port_build(self):
+ # Ports can override this method to do additional checks.
+ return True
+
+ def check_image_diff(self, override_step=None, logging=True):
+ image_diff_path = self._path_to_image_diff()
+ if not os.path.exists(image_diff_path):
+ _log.error("ImageDiff was not found at %s" % image_diff_path)
+ return False
+ return True
+
+ def diff_image(self, expected_contents, actual_contents,
+ diff_filename=None):
+ """Return True if the two files are different. Also write a delta
+ image of the two images into |diff_filename| if it is not None."""
+
+ # Handle the case where the test didn't actually generate an image.
+ if not actual_contents:
+ return True
+
+ sp = self._diff_image_request(expected_contents, actual_contents)
+ return self._diff_image_reply(sp, diff_filename)
+
+ def _diff_image_request(self, expected_contents, actual_contents):
+ # FIXME: use self.get_option('tolerance') and
+ # self.set_option_default('tolerance', 0.1) once that behaves correctly
+ # with default values.
+ if self.get_option('tolerance') is not None:
+ tolerance = self.get_option('tolerance')
+ else:
+ tolerance = 0.1
+ command = [self._path_to_image_diff(), '--tolerance', str(tolerance)]
+ sp = server_process.ServerProcess(self, 'ImageDiff', command)
+
+ sp.write('Content-Length: %d\n%sContent-Length: %d\n%s' %
+ (len(actual_contents), actual_contents,
+ len(expected_contents), expected_contents))
+
+ return sp
+
+ def _diff_image_reply(self, sp, diff_filename):
+ timeout = 2.0
+ deadline = time.time() + timeout
+ output = sp.read_line(timeout)
+ while not sp.timed_out and not sp.crashed and output:
+ if output.startswith('Content-Length'):
+ m = re.match('Content-Length: (\d+)', output)
+ content_length = int(m.group(1))
+ timeout = deadline - time.time()
+ output = sp.read(timeout, content_length)
+ break
+ elif output.startswith('diff'):
+ break
+ else:
+ timeout = deadline - time.time()
+ output = sp.read_line(deadline)
+
+ result = True
+ if output.startswith('diff'):
+ m = re.match('diff: (.+)% (passed|failed)', output)
+ if m.group(2) == 'passed':
+ result = False
+ elif output and diff_filename:
+ with open(diff_filename, 'w') as file:
+ file.write(output)
+ elif sp.timed_out:
+ _log.error("ImageDiff timed out")
+ elif sp.crashed:
+ _log.error("ImageDiff crashed")
+ sp.stop()
+ return result
+
+ def results_directory(self):
+ # Results are store relative to the built products to make it easy
+ # to have multiple copies of webkit checked out and built.
+ return self._build_path(self.get_option('results_directory'))
+
+ def setup_test_run(self):
+ # This port doesn't require any specific configuration.
+ pass
+
+ def create_driver(self, worker_number):
+ return WebKitDriver(self, worker_number)
+
+ def test_base_platform_names(self):
+ # At the moment we don't use test platform names, but we have
+ # to return something.
+ return ('mac', 'win')
+
+ def _tests_for_other_platforms(self):
+ raise NotImplementedError('WebKitPort._tests_for_other_platforms')
+ # The original run-webkit-tests builds up a "whitelist" of tests to
+ # run, and passes that to DumpRenderTree. new-run-webkit-tests assumes
+ # we run *all* tests and test_expectations.txt functions as a
+ # blacklist.
+ # FIXME: This list could be dynamic based on platform name and
+ # pushed into base.Port.
+ return [
+ "platform/chromium",
+ "platform/gtk",
+ "platform/qt",
+ "platform/win",
+ ]
+
+ def _runtime_feature_list(self):
+ """Return the supported features of DRT. If a port doesn't support
+ this DRT switch, it has to override this method to return None"""
+ driver_path = self._path_to_driver()
+ feature_list = ' '.join(os.popen(driver_path + " --print-supported-features 2>&1").readlines())
+ if "SupportedFeatures:" in feature_list:
+ return feature_list
+ return None
+
+ def _supported_symbol_list(self):
+ """Return the supported symbols of WebCore."""
+ webcore_library_path = self._path_to_webcore_library()
+ if not webcore_library_path:
+ return None
+ symbol_list = ' '.join(os.popen("nm " + webcore_library_path).readlines())
+ return symbol_list
+
+ def _directories_for_features(self):
+ """Return the supported feature dictionary. The keys are the
+ features and the values are the directories in lists."""
+ directories_for_features = {
+ "Accelerated Compositing": ["compositing"],
+ "3D Rendering": ["animations/3d", "transforms/3d"],
+ }
+ return directories_for_features
+
+ def _directories_for_symbols(self):
+ """Return the supported feature dictionary. The keys are the
+ symbols and the values are the directories in lists."""
+ directories_for_symbol = {
+ "MathMLElement": ["mathml"],
+ "GraphicsLayer": ["compositing"],
+ "WebCoreHas3DRendering": ["animations/3d", "transforms/3d"],
+ "WebGLShader": ["fast/canvas/webgl", "compositing/webgl", "http/tests/canvas/webgl"],
+ "WMLElement": ["http/tests/wml", "fast/wml", "wml"],
+ "parseWCSSInputProperty": ["fast/wcss"],
+ "isXHTMLMPDocument": ["fast/xhtmlmp"],
+ }
+ return directories_for_symbol
+
+ def _skipped_tests_for_unsupported_features(self):
+ """Return the directories of unsupported tests. Search for the
+ symbols in the symbol_list, if found add the corresponding
+ directories to the skipped directory list."""
+ feature_list = self._runtime_feature_list()
+ directories = self._directories_for_features()
+
+ # if DRT feature detection not supported
+ if not feature_list:
+ feature_list = self._supported_symbol_list()
+ directories = self._directories_for_symbols()
+
+ if not feature_list:
+ return []
+
+ skipped_directories = [directories[feature]
+ for feature in directories.keys()
+ if feature not in feature_list]
+ return reduce(operator.add, skipped_directories)
+
+ def _tests_for_disabled_features(self):
+ # FIXME: This should use the feature detection from
+ # webkitperl/features.pm to match run-webkit-tests.
+ # For now we hard-code a list of features known to be disabled on
+ # the Mac platform.
+ disabled_feature_tests = [
+ "fast/xhtmlmp",
+ "http/tests/wml",
+ "mathml",
+ "wml",
+ ]
+ # FIXME: webarchive tests expect to read-write from
+ # -expected.webarchive files instead of .txt files.
+ # This script doesn't know how to do that yet, so pretend they're
+ # just "disabled".
+ webarchive_tests = [
+ "webarchive",
+ "svg/webarchive",
+ "http/tests/webarchive",
+ "svg/custom/image-with-prefix-in-webarchive.svg",
+ ]
+ unsupported_feature_tests = self._skipped_tests_for_unsupported_features()
+ return disabled_feature_tests + webarchive_tests + unsupported_feature_tests
+
+ def _tests_from_skipped_file(self, skipped_file):
+ tests_to_skip = []
+ for line in skipped_file.readlines():
+ line = line.strip()
+ if line.startswith('#') or not len(line):
+ continue
+ tests_to_skip.append(line)
+ return tests_to_skip
+
+ def _skipped_file_paths(self):
+ return [os.path.join(self._webkit_baseline_path(self._name),
+ 'Skipped')]
+
+ def _expectations_from_skipped_files(self):
+ tests_to_skip = []
+ for filename in self._skipped_file_paths():
+ if not os.path.exists(filename):
+ _log.warn("Failed to open Skipped file: %s" % filename)
+ continue
+ with codecs.open(filename, "r", "utf-8") as skipped_file:
+ tests_to_skip.extend(self._tests_from_skipped_file(skipped_file))
+ return tests_to_skip
+
+ def test_expectations(self):
+ # The WebKit mac port uses a combination of a test_expectations file
+ # and 'Skipped' files.
+ expectations_path = self.path_to_test_expectations_file()
+ with codecs.open(expectations_path, "r", "utf-8") as file:
+ return file.read() + self._skips()
+
+ def _skips(self):
+ # Each Skipped file contains a list of files
+ # or directories to be skipped during the test run. The total list
+ # of tests to skipped is given by the contents of the generic
+ # Skipped file found in platform/X plus a version-specific file
+ # found in platform/X-version. Duplicate entries are allowed.
+ # This routine reads those files and turns contents into the
+ # format expected by test_expectations.
+
+ tests_to_skip = self.skipped_layout_tests()
+ skip_lines = map(lambda test_path: "BUG_SKIPPED SKIP : %s = FAIL" %
+ test_path, tests_to_skip)
+ return "\n".join(skip_lines)
+
+ def skipped_layout_tests(self):
+ # Use a set to allow duplicates
+ tests_to_skip = set(self._expectations_from_skipped_files())
+ tests_to_skip.update(self._tests_for_other_platforms())
+ tests_to_skip.update(self._tests_for_disabled_features())
+ return tests_to_skip
+
+ def test_platform_name(self):
+ return self._name + self.version()
+
+ def test_platform_names(self):
+ return self.test_base_platform_names() + (
+ 'mac-tiger', 'mac-leopard', 'mac-snowleopard')
+
+ def _build_path(self, *comps):
+ return self._filesystem.join(self._config.build_directory(
+ self.get_option('configuration')), *comps)
+
+ def _path_to_driver(self):
+ return self._build_path('DumpRenderTree')
+
+ def _path_to_webcore_library(self):
+ return None
+
+ def _path_to_helper(self):
+ return None
+
+ def _path_to_image_diff(self):
+ return self._build_path('ImageDiff')
+
+ def _path_to_wdiff(self):
+ # FIXME: This does not exist on a default Mac OS X Leopard install.
+ return 'wdiff'
+
+ def _path_to_apache(self):
+ if not self._cached_apache_path:
+ # The Apache binary path can vary depending on OS and distribution
+ # See http://wiki.apache.org/httpd/DistrosDefaultLayout
+ for path in ["/usr/sbin/httpd", "/usr/sbin/apache2"]:
+ if os.path.exists(path):
+ self._cached_apache_path = path
+ break
+
+ if not self._cached_apache_path:
+ _log.error("Could not find apache. Not installed or unknown path.")
+
+ return self._cached_apache_path
+
+
+class WebKitDriver(base.Driver):
+ """WebKit implementation of the DumpRenderTree interface."""
+
+ def __init__(self, port, worker_number):
+ self._worker_number = worker_number
+ self._port = port
+ self._driver_tempdir = tempfile.mkdtemp(prefix='DumpRenderTree-')
+
+ def __del__(self):
+ shutil.rmtree(self._driver_tempdir)
+
+ def cmd_line(self):
+ cmd = self._command_wrapper(self._port.get_option('wrapper'))
+ cmd += [self._port._path_to_driver(), '-']
+
+ if self._port.get_option('pixel_tests'):
+ cmd.append('--pixel-tests')
+
+ return cmd
+
+ def start(self):
+ environment = self._port.setup_environ_for_server()
+ environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path()
+ environment['DUMPRENDERTREE_TEMP'] = self._driver_tempdir
+ self._server_process = server_process.ServerProcess(self._port,
+ "DumpRenderTree", self.cmd_line(), environment)
+
+ def poll(self):
+ return self._server_process.poll()
+
+ def restart(self):
+ self._server_process.stop()
+ self._server_process.start()
+ return
+
+ # FIXME: This function is huge.
+ def run_test(self, test_input):
+ uri = self._port.filename_to_uri(test_input.filename)
+ if uri.startswith("file:///"):
+ command = uri[7:]
+ else:
+ command = uri
+
+ if test_input.image_hash:
+ command += "'" + test_input.image_hash
+ command += "\n"
+
+ start_time = time.time()
+ self._server_process.write(command)
+
+ have_seen_content_type = False
+ actual_image_hash = None
+ output = str() # Use a byte array for output, even though it should be UTF-8.
+ image = str()
+
+ timeout = int(test_input.timeout) / 1000.0
+ deadline = time.time() + timeout
+ line = self._server_process.read_line(timeout)
+ while (not self._server_process.timed_out
+ and not self._server_process.crashed
+ and line.rstrip() != "#EOF"):
+ if (line.startswith('Content-Type:') and not
+ have_seen_content_type):
+ have_seen_content_type = True
+ else:
+ # Note: Text output from DumpRenderTree is always UTF-8.
+ # However, some tests (e.g. webarchives) spit out binary
+ # data instead of text. So to make things simple, we
+ # always treat the output as binary.
+ output += line
+ line = self._server_process.read_line(timeout)
+ timeout = deadline - time.time()
+
+ # Now read a second block of text for the optional image data
+ remaining_length = -1
+ HASH_HEADER = 'ActualHash: '
+ LENGTH_HEADER = 'Content-Length: '
+ line = self._server_process.read_line(timeout)
+ while (not self._server_process.timed_out
+ and not self._server_process.crashed
+ and line.rstrip() != "#EOF"):
+ if line.startswith(HASH_HEADER):
+ actual_image_hash = line[len(HASH_HEADER):].strip()
+ elif line.startswith('Content-Type:'):
+ pass
+ elif line.startswith(LENGTH_HEADER):
+ timeout = deadline - time.time()
+ content_length = int(line[len(LENGTH_HEADER):])
+ image = self._server_process.read(timeout, content_length)
+ timeout = deadline - time.time()
+ line = self._server_process.read_line(timeout)
+
+ error_lines = self._server_process.error.splitlines()
+ # FIXME: This is a hack. It is unclear why sometimes
+ # we do not get any error lines from the server_process
+ # probably we are not flushing stderr.
+ if error_lines and error_lines[-1] == "#EOF":
+ error_lines.pop() # Remove the expected "#EOF"
+ error = "\n".join(error_lines)
+ # FIXME: This seems like the wrong section of code to be doing
+ # this reset in.
+ self._server_process.error = ""
+ return test_output.TestOutput(output, image, actual_image_hash,
+ self._server_process.crashed,
+ time.time() - start_time,
+ self._server_process.timed_out,
+ error)
+
+ def stop(self):
+ if self._server_process:
+ self._server_process.stop()
+ self._server_process = None
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py b/Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py
new file mode 100644
index 0000000..7b68310
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UNIVERSITY OF SZEGED OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import unittest
+
+from webkitpy.layout_tests.port.webkit import WebKitPort
+
+
+class TestWebKitPort(WebKitPort):
+ def __init__(self, symbol_list=None, feature_list=None):
+ self.symbol_list = symbol_list
+ self.feature_list = feature_list
+
+ def _runtime_feature_list(self):
+ return self.feature_list
+
+ def _supported_symbol_list(self):
+ return self.symbol_list
+
+ def _tests_for_other_platforms(self):
+ return ["media", ]
+
+ def _tests_for_disabled_features(self):
+ return ["accessibility", ]
+
+ def _skipped_file_paths(self):
+ return []
+
+class WebKitPortTest(unittest.TestCase):
+
+ def test_skipped_directories_for_symbols(self):
+ supported_symbols = ["GraphicsLayer", "WebCoreHas3DRendering", "isXHTMLMPDocument", "fooSymbol"]
+ expected_directories = set(["mathml", "fast/canvas/webgl", "compositing/webgl", "http/tests/canvas/webgl", "http/tests/wml", "fast/wml", "wml", "fast/wcss"])
+ result_directories = set(TestWebKitPort(supported_symbols, None)._skipped_tests_for_unsupported_features())
+ self.assertEqual(result_directories, expected_directories)
+
+ def test_skipped_directories_for_features(self):
+ supported_features = ["Accelerated Compositing", "Foo Feature"]
+ expected_directories = set(["animations/3d", "transforms/3d"])
+ result_directories = set(TestWebKitPort(None, supported_features)._skipped_tests_for_unsupported_features())
+ self.assertEqual(result_directories, expected_directories)
+
+ def test_skipped_layout_tests(self):
+ self.assertEqual(TestWebKitPort(None, None).skipped_layout_tests(),
+ set(["media", "accessibility"]))
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/websocket_server.py b/Tools/Scripts/webkitpy/layout_tests/port/websocket_server.py
new file mode 100644
index 0000000..926bc04
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/websocket_server.py
@@ -0,0 +1,257 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""A class to help start/stop the PyWebSocket server used by layout tests."""
+
+
+from __future__ import with_statement
+
+import codecs
+import logging
+import optparse
+import os
+import subprocess
+import sys
+import tempfile
+import time
+import urllib
+
+import factory
+import http_server
+
+from webkitpy.common.system.executive import Executive
+from webkitpy.thirdparty.autoinstalled.pywebsocket import mod_pywebsocket
+
+
+_log = logging.getLogger("webkitpy.layout_tests.port.websocket_server")
+
+_WS_LOG_PREFIX = 'pywebsocket.ws.log-'
+_WSS_LOG_PREFIX = 'pywebsocket.wss.log-'
+
+_DEFAULT_WS_PORT = 8880
+_DEFAULT_WSS_PORT = 9323
+
+
+def url_is_alive(url):
+ """Checks to see if we get an http response from |url|.
+ We poll the url 20 times with a 0.5 second delay. If we don't
+ get a reply in that time, we give up and assume the httpd
+ didn't start properly.
+
+ Args:
+ url: The URL to check.
+ Return:
+ True if the url is alive.
+ """
+ sleep_time = 0.5
+ wait_time = 10
+ while wait_time > 0:
+ try:
+ response = urllib.urlopen(url)
+ # Server is up and responding.
+ return True
+ except IOError:
+ pass
+ # Wait for sleep_time before trying again.
+ wait_time -= sleep_time
+ time.sleep(sleep_time)
+
+ return False
+
+
+class PyWebSocketNotStarted(Exception):
+ pass
+
+
+class PyWebSocketNotFound(Exception):
+ pass
+
+
+class PyWebSocket(http_server.Lighttpd):
+
+ def __init__(self, port_obj, output_dir, port=_DEFAULT_WS_PORT,
+ root=None, use_tls=False,
+ pidfile=None):
+ """Args:
+ output_dir: the absolute path to the layout test result directory
+ """
+ http_server.Lighttpd.__init__(self, port_obj, output_dir,
+ port=_DEFAULT_WS_PORT,
+ root=root)
+ self._output_dir = output_dir
+ self._process = None
+ self._port = port
+ self._root = root
+ self._use_tls = use_tls
+ self._private_key = self._pem_file
+ self._certificate = self._pem_file
+ if self._port:
+ self._port = int(self._port)
+ if self._use_tls:
+ self._server_name = 'PyWebSocket(Secure)'
+ else:
+ self._server_name = 'PyWebSocket'
+ self._pidfile = pidfile
+ self._wsout = None
+
+ # Webkit tests
+ if self._root:
+ self._layout_tests = os.path.abspath(self._root)
+ self._web_socket_tests = os.path.abspath(
+ os.path.join(self._root, 'http', 'tests',
+ 'websocket', 'tests'))
+ else:
+ try:
+ self._layout_tests = self._port_obj.layout_tests_dir()
+ self._web_socket_tests = os.path.join(self._layout_tests,
+ 'http', 'tests', 'websocket', 'tests')
+ except:
+ self._web_socket_tests = None
+
+ def start(self):
+ if not self._web_socket_tests:
+ _log.info('No need to start %s server.' % self._server_name)
+ return
+ if self.is_running():
+ raise PyWebSocketNotStarted('%s is already running.' %
+ self._server_name)
+
+ time_str = time.strftime('%d%b%Y-%H%M%S')
+ if self._use_tls:
+ log_prefix = _WSS_LOG_PREFIX
+ else:
+ log_prefix = _WS_LOG_PREFIX
+ log_file_name = log_prefix + time_str
+
+ # Remove old log files. We only need to keep the last ones.
+ self.remove_log_files(self._output_dir, log_prefix)
+
+ error_log = os.path.join(self._output_dir, log_file_name + "-err.txt")
+
+ output_log = os.path.join(self._output_dir, log_file_name + "-out.txt")
+ self._wsout = codecs.open(output_log, "w", "utf-8")
+
+ python_interp = sys.executable
+ pywebsocket_base = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.dirname(
+ os.path.abspath(__file__)))), 'thirdparty',
+ 'autoinstalled', 'pywebsocket')
+ pywebsocket_script = os.path.join(pywebsocket_base, 'mod_pywebsocket',
+ 'standalone.py')
+ start_cmd = [
+ python_interp, '-u', pywebsocket_script,
+ '--server-host', '127.0.0.1',
+ '--port', str(self._port),
+ '--document-root', os.path.join(self._layout_tests, 'http', 'tests'),
+ '--scan-dir', self._web_socket_tests,
+ '--cgi-paths', '/websocket/tests',
+ '--log-file', error_log,
+ ]
+
+ handler_map_file = os.path.join(self._web_socket_tests,
+ 'handler_map.txt')
+ if os.path.exists(handler_map_file):
+ _log.debug('Using handler_map_file: %s' % handler_map_file)
+ start_cmd.append('--websock-handlers-map-file')
+ start_cmd.append(handler_map_file)
+ else:
+ _log.warning('No handler_map_file found')
+
+ if self._use_tls:
+ start_cmd.extend(['-t', '-k', self._private_key,
+ '-c', self._certificate])
+
+ env = self._port_obj.setup_environ_for_server()
+ env['PYTHONPATH'] = (pywebsocket_base + os.path.pathsep +
+ env.get('PYTHONPATH', ''))
+
+ _log.debug('Starting %s server on %d.' % (
+ self._server_name, self._port))
+ _log.debug('cmdline: %s' % ' '.join(start_cmd))
+ # FIXME: We should direct this call through Executive for testing.
+ # Note: Not thread safe: http://bugs.python.org/issue2320
+ self._process = subprocess.Popen(start_cmd,
+ stdin=open(os.devnull, 'r'),
+ stdout=self._wsout,
+ stderr=subprocess.STDOUT,
+ env=env)
+
+ if self._use_tls:
+ url = 'https'
+ else:
+ url = 'http'
+ url = url + '://127.0.0.1:%d/' % self._port
+ if not url_is_alive(url):
+ if self._process.returncode == None:
+ # FIXME: We should use a non-static Executive for easier
+ # testing.
+ Executive().kill_process(self._process.pid)
+ with codecs.open(output_log, "r", "utf-8") as fp:
+ for line in fp:
+ _log.error(line)
+ raise PyWebSocketNotStarted(
+ 'Failed to start %s server on port %s.' %
+ (self._server_name, self._port))
+
+ # Our process terminated already
+ if self._process.returncode != None:
+ raise PyWebSocketNotStarted(
+ 'Failed to start %s server.' % self._server_name)
+ if self._pidfile:
+ with codecs.open(self._pidfile, "w", "ascii") as file:
+ file.write("%d" % self._process.pid)
+
+ def stop(self, force=False):
+ if not force and not self.is_running():
+ return
+
+ pid = None
+ if self._process:
+ pid = self._process.pid
+ elif self._pidfile:
+ with codecs.open(self._pidfile, "r", "ascii") as file:
+ pid = int(file.read().strip())
+
+ if not pid:
+ raise PyWebSocketNotFound(
+ 'Failed to find %s server pid.' % self._server_name)
+
+ _log.debug('Shutting down %s server %d.' % (self._server_name, pid))
+ # FIXME: We should use a non-static Executive for easier testing.
+ Executive().kill_process(pid)
+
+ if self._process:
+ # wait() is not threadsafe and can throw OSError due to:
+ # http://bugs.python.org/issue1731717
+ self._process.wait()
+ self._process = None
+
+ if self._wsout:
+ self._wsout.close()
+ self._wsout = None
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/win.py b/Tools/Scripts/webkitpy/layout_tests/port/win.py
new file mode 100644
index 0000000..9e30155
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/port/win.py
@@ -0,0 +1,75 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the Google name nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""WebKit Win implementation of the Port interface."""
+
+import logging
+import os
+
+from webkitpy.layout_tests.port.webkit import WebKitPort
+
+_log = logging.getLogger("webkitpy.layout_tests.port.win")
+
+
+class WinPort(WebKitPort):
+ """WebKit Win implementation of the Port class."""
+
+ def __init__(self, **kwargs):
+ kwargs.setdefault('port_name', 'win')
+ WebKitPort.__init__(self, **kwargs)
+
+ def baseline_search_path(self):
+ # Based on code from old-run-webkit-tests expectedDirectoryForTest()
+ port_names = ["win", "mac-snowleopard", "mac"]
+ return map(self._webkit_baseline_path, port_names)
+
+ def _tests_for_other_platforms(self):
+ # FIXME: This list could be dynamic based on platform name and
+ # pushed into base.Port.
+ # This really need to be automated.
+ return [
+ "platform/chromium",
+ "platform/gtk",
+ "platform/qt",
+ "platform/mac",
+ ]
+
+ def _path_to_apache_config_file(self):
+ return os.path.join(self.layout_tests_dir(), 'http', 'conf',
+ 'cygwin-httpd.conf')
+
+ def _shut_down_http_server(self, server_pid):
+ """Shut down the httpd web server. Blocks until it's fully
+ shut down.
+
+ Args:
+ server_pid: The process ID of the running server.
+ """
+ # Looks like we ignore server_pid.
+ # Copy/pasted from chromium-win.
+ self._executive.kill_all("httpd.exe")
diff --git a/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py b/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py
new file mode 100644
index 0000000..4d8b7c9
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py
@@ -0,0 +1,966 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Rebaselining tool that automatically produces baselines for all platforms.
+
+The script does the following for each platform specified:
+ 1. Compile a list of tests that need rebaselining.
+ 2. Download test result archive from buildbot for the platform.
+ 3. Extract baselines from the archive file for all identified files.
+ 4. Add new baselines to SVN repository.
+ 5. For each test that has been rebaselined, remove this platform option from
+ the test in test_expectation.txt. If no other platforms remain after
+ removal, delete the rebaselined test from the file.
+
+At the end, the script generates a html that compares old and new baselines.
+"""
+
+from __future__ import with_statement
+
+import codecs
+import copy
+import logging
+import optparse
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+import urllib
+import zipfile
+
+from webkitpy.common.system import path
+from webkitpy.common.system import user
+from webkitpy.common.system.executive import Executive, ScriptError
+import webkitpy.common.checkout.scm as scm
+
+import port
+from layout_package import test_expectations
+
+_log = logging.getLogger("webkitpy.layout_tests."
+ "rebaseline_chromium_webkit_tests")
+
+BASELINE_SUFFIXES = ['.txt', '.png', '.checksum']
+REBASELINE_PLATFORM_ORDER = ['mac', 'win', 'win-xp', 'win-vista', 'linux']
+ARCHIVE_DIR_NAME_DICT = {'win': 'Webkit_Win',
+ 'win-vista': 'webkit-dbg-vista',
+ 'win-xp': 'Webkit_Win',
+ 'mac': 'Webkit_Mac10_5',
+ 'linux': 'webkit-rel-linux64',
+ 'win-canary': 'webkit-rel-webkit-org',
+ 'win-vista-canary': 'webkit-dbg-vista',
+ 'win-xp-canary': 'webkit-rel-webkit-org',
+ 'mac-canary': 'webkit-rel-mac-webkit-org',
+ 'linux-canary': 'webkit-rel-linux-webkit-org'}
+
+
+def log_dashed_string(text, platform, logging_level=logging.INFO):
+ """Log text message with dashes on both sides."""
+
+ msg = text
+ if platform:
+ msg += ': ' + platform
+ if len(msg) < 78:
+ dashes = '-' * ((78 - len(msg)) / 2)
+ msg = '%s %s %s' % (dashes, msg, dashes)
+
+ if logging_level == logging.ERROR:
+ _log.error(msg)
+ elif logging_level == logging.WARNING:
+ _log.warn(msg)
+ else:
+ _log.info(msg)
+
+
+def setup_html_directory(html_directory):
+ """Setup the directory to store html results.
+
+ All html related files are stored in the "rebaseline_html" subdirectory.
+
+ Args:
+ html_directory: parent directory that stores the rebaselining results.
+ If None, a temp directory is created.
+
+ Returns:
+ the directory that stores the html related rebaselining results.
+ """
+
+ if not html_directory:
+ html_directory = tempfile.mkdtemp()
+ elif not os.path.exists(html_directory):
+ os.mkdir(html_directory)
+
+ html_directory = os.path.join(html_directory, 'rebaseline_html')
+ _log.info('Html directory: "%s"', html_directory)
+
+ if os.path.exists(html_directory):
+ shutil.rmtree(html_directory, True)
+ _log.info('Deleted file at html directory: "%s"', html_directory)
+
+ if not os.path.exists(html_directory):
+ os.mkdir(html_directory)
+ return html_directory
+
+
+def get_result_file_fullpath(html_directory, baseline_filename, platform,
+ result_type):
+ """Get full path of the baseline result file.
+
+ Args:
+ html_directory: directory that stores the html related files.
+ baseline_filename: name of the baseline file.
+ platform: win, linux or mac
+ result_type: type of the baseline result: '.txt', '.png'.
+
+ Returns:
+ Full path of the baseline file for rebaselining result comparison.
+ """
+
+ base, ext = os.path.splitext(baseline_filename)
+ result_filename = '%s-%s-%s%s' % (base, platform, result_type, ext)
+ fullpath = os.path.join(html_directory, result_filename)
+ _log.debug(' Result file full path: "%s".', fullpath)
+ return fullpath
+
+
+class Rebaseliner(object):
+ """Class to produce new baselines for a given platform."""
+
+ REVISION_REGEX = r'<a href=\"(\d+)/\">'
+
+ def __init__(self, running_port, target_port, platform, options):
+ """
+ Args:
+ running_port: the Port the script is running on.
+ target_port: the Port the script uses to find port-specific
+ configuration information like the test_expectations.txt
+ file location and the list of test platforms.
+ platform: the test platform to rebaseline
+ options: the command-line options object."""
+ self._platform = platform
+ self._options = options
+ self._port = running_port
+ self._target_port = target_port
+ self._rebaseline_port = port.get(
+ self._target_port.test_platform_name_to_name(platform), options)
+ self._rebaselining_tests = []
+ self._rebaselined_tests = []
+
+ # Create tests and expectations helper which is used to:
+ # -. compile list of tests that need rebaselining.
+ # -. update the tests in test_expectations file after rebaseline
+ # is done.
+ expectations_str = self._rebaseline_port.test_expectations()
+ self._test_expectations = \
+ test_expectations.TestExpectations(self._rebaseline_port,
+ None,
+ expectations_str,
+ self._platform,
+ False,
+ False)
+ self._scm = scm.default_scm()
+
+ def run(self, backup):
+ """Run rebaseline process."""
+
+ log_dashed_string('Compiling rebaselining tests', self._platform)
+ if not self._compile_rebaselining_tests():
+ return True
+
+ log_dashed_string('Downloading archive', self._platform)
+ archive_file = self._download_buildbot_archive()
+ _log.info('')
+ if not archive_file:
+ _log.error('No archive found.')
+ return False
+
+ log_dashed_string('Extracting and adding new baselines',
+ self._platform)
+ if not self._extract_and_add_new_baselines(archive_file):
+ return False
+
+ log_dashed_string('Updating rebaselined tests in file',
+ self._platform)
+ self._update_rebaselined_tests_in_file(backup)
+ _log.info('')
+
+ if len(self._rebaselining_tests) != len(self._rebaselined_tests):
+ _log.warning('NOT ALL TESTS THAT NEED REBASELINING HAVE BEEN '
+ 'REBASELINED.')
+ _log.warning(' Total tests needing rebaselining: %d',
+ len(self._rebaselining_tests))
+ _log.warning(' Total tests rebaselined: %d',
+ len(self._rebaselined_tests))
+ return False
+
+ _log.warning('All tests needing rebaselining were successfully '
+ 'rebaselined.')
+
+ return True
+
+ def get_rebaselining_tests(self):
+ return self._rebaselining_tests
+
+ def _compile_rebaselining_tests(self):
+ """Compile list of tests that need rebaselining for the platform.
+
+ Returns:
+ List of tests that need rebaselining or
+ None if there is no such test.
+ """
+
+ self._rebaselining_tests = \
+ self._test_expectations.get_rebaselining_failures()
+ if not self._rebaselining_tests:
+ _log.warn('No tests found that need rebaselining.')
+ return None
+
+ _log.info('Total number of tests needing rebaselining '
+ 'for "%s": "%d"', self._platform,
+ len(self._rebaselining_tests))
+
+ test_no = 1
+ for test in self._rebaselining_tests:
+ _log.info(' %d: %s', test_no, test)
+ test_no += 1
+
+ return self._rebaselining_tests
+
+ def _get_latest_revision(self, url):
+ """Get the latest layout test revision number from buildbot.
+
+ Args:
+ url: Url to retrieve layout test revision numbers.
+
+ Returns:
+ latest revision or
+ None on failure.
+ """
+
+ _log.debug('Url to retrieve revision: "%s"', url)
+
+ f = urllib.urlopen(url)
+ content = f.read()
+ f.close()
+
+ revisions = re.findall(self.REVISION_REGEX, content)
+ if not revisions:
+ _log.error('Failed to find revision, content: "%s"', content)
+ return None
+
+ revisions.sort(key=int)
+ _log.info('Latest revision: "%s"', revisions[len(revisions) - 1])
+ return revisions[len(revisions) - 1]
+
+ def _get_archive_dir_name(self, platform, webkit_canary):
+ """Get name of the layout test archive directory.
+
+ Returns:
+ Directory name or
+ None on failure
+ """
+
+ if webkit_canary:
+ platform += '-canary'
+
+ if platform in ARCHIVE_DIR_NAME_DICT:
+ return ARCHIVE_DIR_NAME_DICT[platform]
+ else:
+ _log.error('Cannot find platform key %s in archive '
+ 'directory name dictionary', platform)
+ return None
+
+ def _get_archive_url(self):
+ """Generate the url to download latest layout test archive.
+
+ Returns:
+ Url to download archive or
+ None on failure
+ """
+
+ if self._options.force_archive_url:
+ return self._options.force_archive_url
+
+ dir_name = self._get_archive_dir_name(self._platform,
+ self._options.webkit_canary)
+ if not dir_name:
+ return None
+
+ _log.debug('Buildbot platform dir name: "%s"', dir_name)
+
+ url_base = '%s/%s/' % (self._options.archive_url, dir_name)
+ latest_revision = self._get_latest_revision(url_base)
+ if latest_revision is None or latest_revision <= 0:
+ return None
+ archive_url = ('%s%s/layout-test-results.zip' % (url_base,
+ latest_revision))
+ _log.info('Archive url: "%s"', archive_url)
+ return archive_url
+
+ def _download_buildbot_archive(self):
+ """Download layout test archive file from buildbot.
+
+ Returns:
+ True if download succeeded or
+ False otherwise.
+ """
+
+ url = self._get_archive_url()
+ if url is None:
+ return None
+
+ fn = urllib.urlretrieve(url)[0]
+ _log.info('Archive downloaded and saved to file: "%s"', fn)
+ return fn
+
+ def _extract_and_add_new_baselines(self, archive_file):
+ """Extract new baselines from archive and add them to SVN repository.
+
+ Args:
+ archive_file: full path to the archive file.
+
+ Returns:
+ List of tests that have been rebaselined or
+ None on failure.
+ """
+
+ zip_file = zipfile.ZipFile(archive_file, 'r')
+ zip_namelist = zip_file.namelist()
+
+ _log.debug('zip file namelist:')
+ for name in zip_namelist:
+ _log.debug(' ' + name)
+
+ platform = self._rebaseline_port.test_platform_name_to_name(
+ self._platform)
+ _log.debug('Platform dir: "%s"', platform)
+
+ test_no = 1
+ self._rebaselined_tests = []
+ for test in self._rebaselining_tests:
+ _log.info('Test %d: %s', test_no, test)
+
+ found = False
+ scm_error = False
+ test_basename = os.path.splitext(test)[0]
+ for suffix in BASELINE_SUFFIXES:
+ archive_test_name = ('layout-test-results/%s-actual%s' %
+ (test_basename, suffix))
+ _log.debug(' Archive test file name: "%s"',
+ archive_test_name)
+ if not archive_test_name in zip_namelist:
+ _log.info(' %s file not in archive.', suffix)
+ continue
+
+ found = True
+ _log.info(' %s file found in archive.', suffix)
+
+ # Extract new baseline from archive and save it to a temp file.
+ data = zip_file.read(archive_test_name)
+ temp_fd, temp_name = tempfile.mkstemp(suffix)
+ f = os.fdopen(temp_fd, 'wb')
+ f.write(data)
+ f.close()
+
+ expected_filename = '%s-expected%s' % (test_basename, suffix)
+ expected_fullpath = os.path.join(
+ self._rebaseline_port.baseline_path(), expected_filename)
+ expected_fullpath = os.path.normpath(expected_fullpath)
+ _log.debug(' Expected file full path: "%s"',
+ expected_fullpath)
+
+ # TODO(victorw): for now, the rebaselining tool checks whether
+ # or not THIS baseline is duplicate and should be skipped.
+ # We could improve the tool to check all baselines in upper
+ # and lower
+ # levels and remove all duplicated baselines.
+ if self._is_dup_baseline(temp_name,
+ expected_fullpath,
+ test,
+ suffix,
+ self._platform):
+ os.remove(temp_name)
+ self._delete_baseline(expected_fullpath)
+ continue
+
+ # Create the new baseline directory if it doesn't already
+ # exist.
+ self._port.maybe_make_directory(
+ os.path.dirname(expected_fullpath))
+
+ shutil.move(temp_name, expected_fullpath)
+
+ if 0 != self._scm.add(expected_fullpath, return_exit_code=True):
+ # FIXME: print detailed diagnose messages
+ scm_error = True
+ elif suffix != '.checksum':
+ self._create_html_baseline_files(expected_fullpath)
+
+ if not found:
+ _log.warn(' No new baselines found in archive.')
+ else:
+ if scm_error:
+ _log.warn(' Failed to add baselines to your repository.')
+ else:
+ _log.info(' Rebaseline succeeded.')
+ self._rebaselined_tests.append(test)
+
+ test_no += 1
+
+ zip_file.close()
+ os.remove(archive_file)
+
+ return self._rebaselined_tests
+
+ def _is_dup_baseline(self, new_baseline, baseline_path, test, suffix,
+ platform):
+ """Check whether a baseline is duplicate and can fallback to same
+ baseline for another platform. For example, if a test has same
+ baseline on linux and windows, then we only store windows
+ baseline and linux baseline will fallback to the windows version.
+
+ Args:
+ expected_filename: baseline expectation file name.
+ test: test name.
+ suffix: file suffix of the expected results, including dot;
+ e.g. '.txt' or '.png'.
+ platform: baseline platform 'mac', 'win' or 'linux'.
+
+ Returns:
+ True if the baseline is unnecessary.
+ False otherwise.
+ """
+ test_filepath = os.path.join(self._target_port.layout_tests_dir(),
+ test)
+ all_baselines = self._rebaseline_port.expected_baselines(
+ test_filepath, suffix, True)
+ for (fallback_dir, fallback_file) in all_baselines:
+ if fallback_dir and fallback_file:
+ fallback_fullpath = os.path.normpath(
+ os.path.join(fallback_dir, fallback_file))
+ if fallback_fullpath.lower() != baseline_path.lower():
+ with codecs.open(new_baseline, "r",
+ None) as file_handle1:
+ new_output = file_handle1.read()
+ with codecs.open(fallback_fullpath, "r",
+ None) as file_handle2:
+ fallback_output = file_handle2.read()
+ is_image = baseline_path.lower().endswith('.png')
+ if not self._diff_baselines(new_output, fallback_output,
+ is_image):
+ _log.info(' Found same baseline at %s',
+ fallback_fullpath)
+ return True
+ else:
+ return False
+
+ return False
+
+ def _diff_baselines(self, output1, output2, is_image):
+ """Check whether two baselines are different.
+
+ Args:
+ output1, output2: contents of the baselines to compare.
+
+ Returns:
+ True if two files are different or have different extensions.
+ False otherwise.
+ """
+
+ if is_image:
+ return self._port.diff_image(output1, output2, None)
+ else:
+ return self._port.compare_text(output1, output2)
+
+ def _delete_baseline(self, filename):
+ """Remove the file from repository and delete it from disk.
+
+ Args:
+ filename: full path of the file to delete.
+ """
+
+ if not filename or not os.path.isfile(filename):
+ return
+ self._scm.delete(filename)
+
+ def _update_rebaselined_tests_in_file(self, backup):
+ """Update the rebaselined tests in test expectations file.
+
+ Args:
+ backup: if True, backup the original test expectations file.
+
+ Returns:
+ no
+ """
+
+ if self._rebaselined_tests:
+ new_expectations = (
+ self._test_expectations.remove_platform_from_expectations(
+ self._rebaselined_tests, self._platform))
+ path = self._target_port.path_to_test_expectations_file()
+ if backup:
+ date_suffix = time.strftime('%Y%m%d%H%M%S',
+ time.localtime(time.time()))
+ backup_file = ('%s.orig.%s' % (path, date_suffix))
+ if os.path.exists(backup_file):
+ os.remove(backup_file)
+ _log.info('Saving original file to "%s"', backup_file)
+ os.rename(path, backup_file)
+ # FIXME: What encoding are these files?
+ # Or is new_expectations always a byte array?
+ with open(path, "w") as file:
+ file.write(new_expectations)
+ # self._scm.add(path)
+ else:
+ _log.info('No test was rebaselined so nothing to remove.')
+
+ def _create_html_baseline_files(self, baseline_fullpath):
+ """Create baseline files (old, new and diff) in html directory.
+
+ The files are used to compare the rebaselining results.
+
+ Args:
+ baseline_fullpath: full path of the expected baseline file.
+ """
+
+ if not baseline_fullpath or not os.path.exists(baseline_fullpath):
+ return
+
+ # Copy the new baseline to html directory for result comparison.
+ baseline_filename = os.path.basename(baseline_fullpath)
+ new_file = get_result_file_fullpath(self._options.html_directory,
+ baseline_filename, self._platform,
+ 'new')
+ shutil.copyfile(baseline_fullpath, new_file)
+ _log.info(' Html: copied new baseline file from "%s" to "%s".',
+ baseline_fullpath, new_file)
+
+ # Get the old baseline from the repository and save to the html directory.
+ try:
+ output = self._scm.show_head(baseline_fullpath)
+ except ScriptError, e:
+ _log.info(e)
+ output = ""
+
+ if (not output) or (output.upper().rstrip().endswith(
+ 'NO SUCH FILE OR DIRECTORY')):
+ _log.info(' No base file: "%s"', baseline_fullpath)
+ return
+ base_file = get_result_file_fullpath(self._options.html_directory,
+ baseline_filename, self._platform,
+ 'old')
+ # We should be using an explicit encoding here.
+ with open(base_file, "wb") as file:
+ file.write(output)
+ _log.info(' Html: created old baseline file: "%s".',
+ base_file)
+
+ # Get the diff between old and new baselines and save to the html dir.
+ if baseline_filename.upper().endswith('.TXT'):
+ output = self._scm.diff_for_file(baseline_fullpath, log=_log)
+ if output:
+ diff_file = get_result_file_fullpath(
+ self._options.html_directory, baseline_filename,
+ self._platform, 'diff')
+ with open(diff_file, 'wb') as file:
+ file.write(output)
+ _log.info(' Html: created baseline diff file: "%s".',
+ diff_file)
+
+
+class HtmlGenerator(object):
+ """Class to generate rebaselining result comparison html."""
+
+ HTML_REBASELINE = ('<html>'
+ '<head>'
+ '<style>'
+ 'body {font-family: sans-serif;}'
+ '.mainTable {background: #666666;}'
+ '.mainTable td , .mainTable th {background: white;}'
+ '.detail {margin-left: 10px; margin-top: 3px;}'
+ '</style>'
+ '<title>Rebaselining Result Comparison (%(time)s)'
+ '</title>'
+ '</head>'
+ '<body>'
+ '<h2>Rebaselining Result Comparison (%(time)s)</h2>'
+ '%(body)s'
+ '</body>'
+ '</html>')
+ HTML_NO_REBASELINING_TESTS = (
+ '<p>No tests found that need rebaselining.</p>')
+ HTML_TABLE_TEST = ('<table class="mainTable" cellspacing=1 cellpadding=5>'
+ '%s</table><br>')
+ HTML_TR_TEST = ('<tr>'
+ '<th style="background-color: #CDECDE; border-bottom: '
+ '1px solid black; font-size: 18pt; font-weight: bold" '
+ 'colspan="5">'
+ '<a href="%s">%s</a>'
+ '</th>'
+ '</tr>')
+ HTML_TEST_DETAIL = ('<div class="detail">'
+ '<tr>'
+ '<th width="100">Baseline</th>'
+ '<th width="100">Platform</th>'
+ '<th width="200">Old</th>'
+ '<th width="200">New</th>'
+ '<th width="150">Difference</th>'
+ '</tr>'
+ '%s'
+ '</div>')
+ HTML_TD_NOLINK = '<td align=center><a>%s</a></td>'
+ HTML_TD_LINK = '<td align=center><a href="%(uri)s">%(name)s</a></td>'
+ HTML_TD_LINK_IMG = ('<td><a href="%(uri)s">'
+ '<img style="width: 200" src="%(uri)s" /></a></td>')
+ HTML_TR = '<tr>%s</tr>'
+
+ def __init__(self, target_port, options, platforms, rebaselining_tests,
+ executive):
+ self._html_directory = options.html_directory
+ self._target_port = target_port
+ self._platforms = platforms
+ self._rebaselining_tests = rebaselining_tests
+ self._executive = executive
+ self._html_file = os.path.join(options.html_directory,
+ 'rebaseline.html')
+
+ def abspath_to_uri(self, filename):
+ """Converts an absolute path to a file: URI."""
+ return path.abspath_to_uri(filename, self._executive)
+
+ def generate_html(self):
+ """Generate html file for rebaselining result comparison."""
+
+ _log.info('Generating html file')
+
+ html_body = ''
+ if not self._rebaselining_tests:
+ html_body += self.HTML_NO_REBASELINING_TESTS
+ else:
+ tests = list(self._rebaselining_tests)
+ tests.sort()
+
+ test_no = 1
+ for test in tests:
+ _log.info('Test %d: %s', test_no, test)
+ html_body += self._generate_html_for_one_test(test)
+
+ html = self.HTML_REBASELINE % ({'time': time.asctime(),
+ 'body': html_body})
+ _log.debug(html)
+
+ with codecs.open(self._html_file, "w", "utf-8") as file:
+ file.write(html)
+
+ _log.info('Baseline comparison html generated at "%s"',
+ self._html_file)
+
+ def show_html(self):
+ """Launch the rebaselining html in brwoser."""
+
+ _log.info('Launching html: "%s"', self._html_file)
+ user.User().open_url(self._html_file)
+ _log.info('Html launched.')
+
+ def _generate_baseline_links(self, test_basename, suffix, platform):
+ """Generate links for baseline results (old, new and diff).
+
+ Args:
+ test_basename: base filename of the test
+ suffix: baseline file suffixes: '.txt', '.png'
+ platform: win, linux or mac
+
+ Returns:
+ html links for showing baseline results (old, new and diff)
+ """
+
+ baseline_filename = '%s-expected%s' % (test_basename, suffix)
+ _log.debug(' baseline filename: "%s"', baseline_filename)
+
+ new_file = get_result_file_fullpath(self._html_directory,
+ baseline_filename, platform, 'new')
+ _log.info(' New baseline file: "%s"', new_file)
+ if not os.path.exists(new_file):
+ _log.info(' No new baseline file: "%s"', new_file)
+ return ''
+
+ old_file = get_result_file_fullpath(self._html_directory,
+ baseline_filename, platform, 'old')
+ _log.info(' Old baseline file: "%s"', old_file)
+ if suffix == '.png':
+ html_td_link = self.HTML_TD_LINK_IMG
+ else:
+ html_td_link = self.HTML_TD_LINK
+
+ links = ''
+ if os.path.exists(old_file):
+ links += html_td_link % {
+ 'uri': self.abspath_to_uri(old_file),
+ 'name': baseline_filename}
+ else:
+ _log.info(' No old baseline file: "%s"', old_file)
+ links += self.HTML_TD_NOLINK % ''
+
+ links += html_td_link % {'uri': self.abspath_to_uri(new_file),
+ 'name': baseline_filename}
+
+ diff_file = get_result_file_fullpath(self._html_directory,
+ baseline_filename, platform,
+ 'diff')
+ _log.info(' Baseline diff file: "%s"', diff_file)
+ if os.path.exists(diff_file):
+ links += html_td_link % {'uri': self.abspath_to_uri(diff_file),
+ 'name': 'Diff'}
+ else:
+ _log.info(' No baseline diff file: "%s"', diff_file)
+ links += self.HTML_TD_NOLINK % ''
+
+ return links
+
+ def _generate_html_for_one_test(self, test):
+ """Generate html for one rebaselining test.
+
+ Args:
+ test: layout test name
+
+ Returns:
+ html that compares baseline results for the test.
+ """
+
+ test_basename = os.path.basename(os.path.splitext(test)[0])
+ _log.info(' basename: "%s"', test_basename)
+ rows = []
+ for suffix in BASELINE_SUFFIXES:
+ if suffix == '.checksum':
+ continue
+
+ _log.info(' Checking %s files', suffix)
+ for platform in self._platforms:
+ links = self._generate_baseline_links(test_basename, suffix,
+ platform)
+ if links:
+ row = self.HTML_TD_NOLINK % self._get_baseline_result_type(
+ suffix)
+ row += self.HTML_TD_NOLINK % platform
+ row += links
+ _log.debug(' html row: %s', row)
+
+ rows.append(self.HTML_TR % row)
+
+ if rows:
+ test_path = os.path.join(self._target_port.layout_tests_dir(),
+ test)
+ html = self.HTML_TR_TEST % (self.abspath_to_uri(test_path), test)
+ html += self.HTML_TEST_DETAIL % ' '.join(rows)
+
+ _log.debug(' html for test: %s', html)
+ return self.HTML_TABLE_TEST % html
+
+ return ''
+
+ def _get_baseline_result_type(self, suffix):
+ """Name of the baseline result type."""
+
+ if suffix == '.png':
+ return 'Pixel'
+ elif suffix == '.txt':
+ return 'Render Tree'
+ else:
+ return 'Other'
+
+
+def get_host_port_object(options):
+ """Return a port object for the platform we're running on."""
+ # The only thing we really need on the host is a way to diff
+ # text files and image files, which means we need to check that some
+ # version of ImageDiff has been built. We will look for either Debug
+ # or Release versions of the default port on the platform.
+ options.configuration = "Release"
+ port_obj = port.get(None, options)
+ if not port_obj.check_image_diff(override_step=None, logging=False):
+ _log.debug('No release version of the image diff binary was found.')
+ options.configuration = "Debug"
+ port_obj = port.get(None, options)
+ if not port_obj.check_image_diff(override_step=None, logging=False):
+ _log.error('No version of image diff was found. Check your build.')
+ return None
+ else:
+ _log.debug('Found the debug version of the image diff binary.')
+ else:
+ _log.debug('Found the release version of the image diff binary.')
+ return port_obj
+
+
+def parse_options(args):
+ """Parse options and return a pair of host options and target options."""
+ option_parser = optparse.OptionParser()
+ option_parser.add_option('-v', '--verbose',
+ action='store_true',
+ default=False,
+ help='include debug-level logging.')
+
+ option_parser.add_option('-q', '--quiet',
+ action='store_true',
+ help='Suppress result HTML viewing')
+
+ option_parser.add_option('-p', '--platforms',
+ default='mac,win,win-xp,win-vista,linux',
+ help=('Comma delimited list of platforms '
+ 'that need rebaselining.'))
+
+ option_parser.add_option('-u', '--archive_url',
+ default=('http://build.chromium.org/f/chromium/'
+ 'layout_test_results'),
+ help=('Url to find the layout test result archive'
+ ' file.'))
+ option_parser.add_option('-U', '--force_archive_url',
+ help=('Url of result zip file. This option is for debugging '
+ 'purposes'))
+
+ option_parser.add_option('-w', '--webkit_canary',
+ action='store_true',
+ default=False,
+ help=('If True, pull baselines from webkit.org '
+ 'canary bot.'))
+
+ option_parser.add_option('-b', '--backup',
+ action='store_true',
+ default=False,
+ help=('Whether or not to backup the original test'
+ ' expectations file after rebaseline.'))
+
+ option_parser.add_option('-d', '--html_directory',
+ default='',
+ help=('The directory that stores the results for '
+ 'rebaselining comparison.'))
+
+ option_parser.add_option('', '--use_drt',
+ action='store_true',
+ default=False,
+ help=('Use ImageDiff from DumpRenderTree instead '
+ 'of image_diff for pixel tests.'))
+
+ option_parser.add_option('', '--target-platform',
+ default='chromium',
+ help=('The target platform to rebaseline '
+ '("mac", "chromium", "qt", etc.). Defaults '
+ 'to "chromium".'))
+ options = option_parser.parse_args(args)[0]
+
+ target_options = copy.copy(options)
+ if options.target_platform == 'chromium':
+ target_options.chromium = True
+ options.tolerance = 0
+
+ return (options, target_options)
+
+
+def main(executive=Executive()):
+ """Main function to produce new baselines."""
+
+ (options, target_options) = parse_options(sys.argv[1:])
+
+ # We need to create three different Port objects over the life of this
+ # script. |target_port_obj| is used to determine configuration information:
+ # location of the expectations file, names of ports to rebaseline, etc.
+ # |port_obj| is used for runtime functionality like actually diffing
+ # Then we create a rebaselining port to actual find and manage the
+ # baselines.
+ target_port_obj = port.get(None, target_options)
+
+ # Set up our logging format.
+ log_level = logging.INFO
+ if options.verbose:
+ log_level = logging.DEBUG
+ logging.basicConfig(level=log_level,
+ format=('%(asctime)s %(filename)s:%(lineno)-3d '
+ '%(levelname)s %(message)s'),
+ datefmt='%y%m%d %H:%M:%S')
+
+ host_port_obj = get_host_port_object(options)
+ if not host_port_obj:
+ sys.exit(1)
+
+ # Verify 'platforms' option is valid.
+ if not options.platforms:
+ _log.error('Invalid "platforms" option. --platforms must be '
+ 'specified in order to rebaseline.')
+ sys.exit(1)
+ platforms = [p.strip().lower() for p in options.platforms.split(',')]
+ for platform in platforms:
+ if not platform in REBASELINE_PLATFORM_ORDER:
+ _log.error('Invalid platform: "%s"' % (platform))
+ sys.exit(1)
+
+ # Adjust the platform order so rebaseline tool is running at the order of
+ # 'mac', 'win' and 'linux'. This is in same order with layout test baseline
+ # search paths. It simplifies how the rebaseline tool detects duplicate
+ # baselines. Check _IsDupBaseline method for details.
+ rebaseline_platforms = []
+ for platform in REBASELINE_PLATFORM_ORDER:
+ if platform in platforms:
+ rebaseline_platforms.append(platform)
+
+ options.html_directory = setup_html_directory(options.html_directory)
+
+ rebaselining_tests = set()
+ backup = options.backup
+ for platform in rebaseline_platforms:
+ rebaseliner = Rebaseliner(host_port_obj, target_port_obj,
+ platform, options)
+
+ _log.info('')
+ log_dashed_string('Rebaseline started', platform)
+ if rebaseliner.run(backup):
+ # Only need to backup one original copy of test expectation file.
+ backup = False
+ log_dashed_string('Rebaseline done', platform)
+ else:
+ log_dashed_string('Rebaseline failed', platform, logging.ERROR)
+
+ rebaselining_tests |= set(rebaseliner.get_rebaselining_tests())
+
+ _log.info('')
+ log_dashed_string('Rebaselining result comparison started', None)
+ html_generator = HtmlGenerator(target_port_obj,
+ options,
+ rebaseline_platforms,
+ rebaselining_tests,
+ executive=executive)
+ html_generator.generate_html()
+ if not options.quiet:
+ html_generator.show_html()
+ log_dashed_string('Rebaselining result comparison done', None)
+
+ sys.exit(0)
+
+if '__main__' == __name__:
+ main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py b/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py
new file mode 100644
index 0000000..7c55b94
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py
@@ -0,0 +1,157 @@
+#!/usr/bin/python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit tests for rebaseline_chromium_webkit_tests.py."""
+
+import os
+import sys
+import unittest
+
+from webkitpy.tool import mocktool
+from webkitpy.layout_tests import port
+from webkitpy.layout_tests import rebaseline_chromium_webkit_tests
+from webkitpy.common.system.executive import Executive, ScriptError
+
+
+class MockPort(object):
+ def __init__(self, image_diff_exists):
+ self.image_diff_exists = image_diff_exists
+
+ def check_image_diff(self, override_step, logging):
+ return self.image_diff_exists
+
+
+def get_mock_get(config_expectations):
+ def mock_get(port_name, options):
+ return MockPort(config_expectations[options.configuration])
+ return mock_get
+
+
+class TestGetHostPortObject(unittest.TestCase):
+ def assert_result(self, release_present, debug_present, valid_port_obj):
+ # Tests whether we get a valid port object returned when we claim
+ # that Image diff is (or isn't) present in the two configs.
+ port.get = get_mock_get({'Release': release_present,
+ 'Debug': debug_present})
+ options = mocktool.MockOptions(configuration=None,
+ html_directory=None)
+ port_obj = rebaseline_chromium_webkit_tests.get_host_port_object(
+ options)
+ if valid_port_obj:
+ self.assertNotEqual(port_obj, None)
+ else:
+ self.assertEqual(port_obj, None)
+
+ def test_get_host_port_object(self):
+ # Save the normal port.get() function for future testing.
+ old_get = port.get
+
+ # Test whether we get a valid port object back for the four
+ # possible cases of having ImageDiffs built. It should work when
+ # there is at least one binary present.
+ self.assert_result(False, False, False)
+ self.assert_result(True, False, True)
+ self.assert_result(False, True, True)
+ self.assert_result(True, True, True)
+
+ # Restore the normal port.get() function.
+ port.get = old_get
+
+
+class TestRebaseliner(unittest.TestCase):
+ def make_rebaseliner(self):
+ options = mocktool.MockOptions(configuration=None,
+ html_directory=None)
+ host_port_obj = port.get('test', options)
+ target_options = options
+ target_port_obj = port.get('test', target_options)
+ platform = 'test'
+ return rebaseline_chromium_webkit_tests.Rebaseliner(
+ host_port_obj, target_port_obj, platform, options)
+
+ def test_parse_options(self):
+ (options, target_options) = rebaseline_chromium_webkit_tests.parse_options([])
+ self.assertTrue(target_options.chromium)
+ self.assertEqual(options.tolerance, 0)
+
+ (options, target_options) = rebaseline_chromium_webkit_tests.parse_options(['--target-platform', 'qt'])
+ self.assertFalse(hasattr(target_options, 'chromium'))
+ self.assertEqual(options.tolerance, 0)
+
+ def test_noop(self):
+ # this method tests that was can at least instantiate an object, even
+ # if there is nothing to do.
+ rebaseliner = self.make_rebaseliner()
+ self.assertNotEqual(rebaseliner, None)
+
+ def test_diff_baselines_txt(self):
+ rebaseliner = self.make_rebaseliner()
+ output = rebaseliner._port.expected_text(
+ os.path.join(rebaseliner._port.layout_tests_dir(),
+ 'passes/text.html'))
+ self.assertFalse(rebaseliner._diff_baselines(output, output,
+ is_image=False))
+
+ def test_diff_baselines_png(self):
+ rebaseliner = self.make_rebaseliner()
+ image = rebaseliner._port.expected_image(
+ os.path.join(rebaseliner._port.layout_tests_dir(),
+ 'passes/image.html'))
+ self.assertFalse(rebaseliner._diff_baselines(image, image,
+ is_image=True))
+
+
+class TestHtmlGenerator(unittest.TestCase):
+ def make_generator(self, tests):
+ return rebaseline_chromium_webkit_tests.HtmlGenerator(
+ target_port=None,
+ options=mocktool.MockOptions(configuration=None,
+ html_directory='/tmp'),
+ platforms=['mac'],
+ rebaselining_tests=tests,
+ executive=Executive())
+
+ def test_generate_baseline_links(self):
+ orig_platform = sys.platform
+ orig_exists = os.path.exists
+
+ try:
+ sys.platform = 'darwin'
+ os.path.exists = lambda x: True
+ generator = self.make_generator(["foo.txt"])
+ links = generator._generate_baseline_links("foo", ".txt", "mac")
+ expected_links = '<td align=center><a href="file:///tmp/foo-expected-mac-old.txt">foo-expected.txt</a></td><td align=center><a href="file:///tmp/foo-expected-mac-new.txt">foo-expected.txt</a></td><td align=center><a href="file:///tmp/foo-expected-mac-diff.txt">Diff</a></td>'
+ self.assertEqual(links, expected_links)
+ finally:
+ sys.platform = orig_platform
+ os.path.exists = orig_exists
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
new file mode 100755
index 0000000..f7e5330
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
@@ -0,0 +1,434 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Run layout tests."""
+
+from __future__ import with_statement
+
+import codecs
+import errno
+import logging
+import optparse
+import os
+import signal
+import sys
+
+from layout_package import printing
+from layout_package import test_runner
+
+from webkitpy.common.system import user
+from webkitpy.thirdparty import simplejson
+
+import port
+
+_log = logging.getLogger(__name__)
+
+
+def run(port, options, args, regular_output=sys.stderr,
+ buildbot_output=sys.stdout):
+ """Run the tests.
+
+ Args:
+ port: Port object for port-specific behavior
+ options: a dictionary of command line options
+ args: a list of sub directories or files to test
+ regular_output: a stream-like object that we can send logging/debug
+ output to
+ buildbot_output: a stream-like object that we can write all output that
+ is intended to be parsed by the buildbot to
+ Returns:
+ the number of unexpected results that occurred, or -1 if there is an
+ error.
+
+ """
+ warnings = _set_up_derived_options(port, options)
+
+ printer = printing.Printer(port, options, regular_output, buildbot_output,
+ int(options.child_processes), options.experimental_fully_parallel)
+ for w in warnings:
+ _log.warning(w)
+
+ if options.help_printing:
+ printer.help_printing()
+ printer.cleanup()
+ return 0
+
+ last_unexpected_results = _gather_unexpected_results(options)
+ if options.print_last_failures:
+ printer.write("\n".join(last_unexpected_results) + "\n")
+ printer.cleanup()
+ return 0
+
+ # We wrap any parts of the run that are slow or likely to raise exceptions
+ # in a try/finally to ensure that we clean up the logging configuration.
+ num_unexpected_results = -1
+ try:
+ runner = test_runner.TestRunner(port, options, printer)
+ runner._print_config()
+
+ printer.print_update("Collecting tests ...")
+ try:
+ runner.collect_tests(args, last_unexpected_results)
+ except IOError, e:
+ if e.errno == errno.ENOENT:
+ return -1
+ raise
+
+ printer.print_update("Parsing expectations ...")
+ if options.lint_test_files:
+ return runner.lint()
+ runner.parse_expectations(port.test_platform_name(),
+ options.configuration == 'Debug')
+
+ printer.print_update("Checking build ...")
+ if not port.check_build(runner.needs_http()):
+ _log.error("Build check failed")
+ return -1
+
+ result_summary = runner.set_up_run()
+ if result_summary:
+ num_unexpected_results = runner.run(result_summary)
+ runner.clean_up_run()
+ _log.debug("Testing completed, Exit status: %d" %
+ num_unexpected_results)
+ finally:
+ printer.cleanup()
+
+ return num_unexpected_results
+
+
+def _set_up_derived_options(port_obj, options):
+ """Sets the options values that depend on other options values."""
+ # We return a list of warnings to print after the printer is initialized.
+ warnings = []
+
+ if options.worker_model == 'old-inline':
+ if options.child_processes and int(options.child_processes) > 1:
+ warnings.append("--worker-model=old-inline overrides --child-processes")
+ options.child_processes = "1"
+ if not options.child_processes:
+ options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
+ str(port_obj.default_child_processes()))
+
+ if not options.configuration:
+ options.configuration = port_obj.default_configuration()
+
+ if options.pixel_tests is None:
+ options.pixel_tests = True
+
+ if not options.use_apache:
+ options.use_apache = sys.platform in ('darwin', 'linux2')
+
+ if not os.path.isabs(options.results_directory):
+ # This normalizes the path to the build dir.
+ # FIXME: how this happens is not at all obvious; this is a dumb
+ # interface and should be cleaned up.
+ options.results_directory = port_obj.results_directory()
+
+ if not options.time_out_ms:
+ if options.configuration == "Debug":
+ options.time_out_ms = str(2 * test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
+ else:
+ options.time_out_ms = str(test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
+
+ options.slow_time_out_ms = str(5 * int(options.time_out_ms))
+ return warnings
+
+
+def _gather_unexpected_results(options):
+ """Returns the unexpected results from the previous run, if any."""
+ last_unexpected_results = []
+ if options.print_last_failures or options.retest_last_failures:
+ unexpected_results_filename = os.path.join(
+ options.results_directory, "unexpected_results.json")
+ with codecs.open(unexpected_results_filename, "r", "utf-8") as file:
+ results = simplejson.load(file)
+ last_unexpected_results = results['tests'].keys()
+ return last_unexpected_results
+
+
+def _compat_shim_callback(option, opt_str, value, parser):
+ print "Ignoring unsupported option: %s" % opt_str
+
+
+def _compat_shim_option(option_name, **kwargs):
+ return optparse.make_option(option_name, action="callback",
+ callback=_compat_shim_callback,
+ help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
+
+
+def parse_args(args=None):
+ """Provides a default set of command line args.
+
+ Returns a tuple of options, args from optparse"""
+
+ # FIXME: All of these options should be stored closer to the code which
+ # FIXME: actually uses them. configuration_options should move
+ # FIXME: to WebKitPort and be shared across all scripts.
+ configuration_options = [
+ optparse.make_option("-t", "--target", dest="configuration",
+ help="(DEPRECATED)"),
+ # FIXME: --help should display which configuration is default.
+ optparse.make_option('--debug', action='store_const', const='Debug',
+ dest="configuration",
+ help='Set the configuration to Debug'),
+ optparse.make_option('--release', action='store_const',
+ const='Release', dest="configuration",
+ help='Set the configuration to Release'),
+ # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
+ ]
+
+ print_options = printing.print_options()
+
+ # FIXME: These options should move onto the ChromiumPort.
+ chromium_options = [
+ optparse.make_option("--chromium", action="store_true", default=False,
+ help="use the Chromium port"),
+ optparse.make_option("--startup-dialog", action="store_true",
+ default=False, help="create a dialog on DumpRenderTree startup"),
+ optparse.make_option("--gp-fault-error-box", action="store_true",
+ default=False, help="enable Windows GP fault error box"),
+ optparse.make_option("--multiple-loads",
+ type="int", help="turn on multiple loads of each test"),
+ optparse.make_option("--js-flags",
+ type="string", help="JavaScript flags to pass to tests"),
+ optparse.make_option("--nocheck-sys-deps", action="store_true",
+ default=False,
+ help="Don't check the system dependencies (themes)"),
+ optparse.make_option("--use-test-shell", action="store_true",
+ default=False,
+ help="Use test_shell instead of DRT"),
+ optparse.make_option("--accelerated-compositing",
+ action="store_true",
+ help="Use hardware-accelated compositing for rendering"),
+ optparse.make_option("--no-accelerated-compositing",
+ action="store_false",
+ dest="accelerated_compositing",
+ help="Don't use hardware-accelerated compositing for rendering"),
+ optparse.make_option("--accelerated-2d-canvas",
+ action="store_true",
+ help="Use hardware-accelerated 2D Canvas calls"),
+ optparse.make_option("--no-accelerated-2d-canvas",
+ action="store_false",
+ dest="accelerated_2d_canvas",
+ help="Don't use hardware-accelerated 2D Canvas calls"),
+ ]
+
+ # Missing Mac-specific old-run-webkit-tests options:
+ # FIXME: Need: -g, --guard for guard malloc support on Mac.
+ # FIXME: Need: -l --leaks Enable leaks checking.
+ # FIXME: Need: --sample-on-timeout Run sample on timeout
+
+ old_run_webkit_tests_compat = [
+ # NRWT doesn't generate results by default anyway.
+ _compat_shim_option("--no-new-test-results"),
+ # NRWT doesn't sample on timeout yet anyway.
+ _compat_shim_option("--no-sample-on-timeout"),
+ # FIXME: NRWT needs to support remote links eventually.
+ _compat_shim_option("--use-remote-links-to-tests"),
+ ]
+
+ results_options = [
+ # NEED for bots: --use-remote-links-to-tests Link to test files
+ # within the SVN repository in the results.
+ optparse.make_option("-p", "--pixel-tests", action="store_true",
+ dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
+ optparse.make_option("--no-pixel-tests", action="store_false",
+ dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
+ optparse.make_option("--tolerance",
+ help="Ignore image differences less than this percentage (some "
+ "ports may ignore this option)", type="float"),
+ optparse.make_option("--results-directory",
+ default="layout-test-results",
+ help="Output results directory source dir, relative to Debug or "
+ "Release"),
+ optparse.make_option("--new-baseline", action="store_true",
+ default=False, help="Save all generated results as new baselines "
+ "into the platform directory, overwriting whatever's "
+ "already there."),
+ optparse.make_option("--reset-results", action="store_true",
+ default=False, help="Reset any existing baselines to the "
+ "generated results"),
+ optparse.make_option("--no-show-results", action="store_false",
+ default=True, dest="show_results",
+ help="Don't launch a browser with results after the tests "
+ "are done"),
+ # FIXME: We should have a helper function to do this sort of
+ # deprectated mapping and automatically log, etc.
+ optparse.make_option("--noshow-results", action="store_false",
+ dest="show_results",
+ help="Deprecated, same as --no-show-results."),
+ optparse.make_option("--no-launch-safari", action="store_false",
+ dest="show_results",
+ help="old-run-webkit-tests compat, same as --noshow-results."),
+ # old-run-webkit-tests:
+ # --[no-]launch-safari Launch (or do not launch) Safari to display
+ # test results (default: launch)
+ optparse.make_option("--full-results-html", action="store_true",
+ default=False,
+ help="Show all failures in results.html, rather than only "
+ "regressions"),
+ optparse.make_option("--clobber-old-results", action="store_true",
+ default=False, help="Clobbers test results from previous runs."),
+ optparse.make_option("--platform",
+ help="Override the platform for expected results"),
+ optparse.make_option("--no-record-results", action="store_false",
+ default=True, dest="record_results",
+ help="Don't record the results."),
+ # old-run-webkit-tests also has HTTP toggle options:
+ # --[no-]http Run (or do not run) http tests
+ # (default: run)
+ ]
+
+ test_options = [
+ optparse.make_option("--build", dest="build",
+ action="store_true", default=True,
+ help="Check to ensure the DumpRenderTree build is up-to-date "
+ "(default)."),
+ optparse.make_option("--no-build", dest="build",
+ action="store_false", help="Don't check to see if the "
+ "DumpRenderTree build is up-to-date."),
+ optparse.make_option("-n", "--dry-run", action="store_true",
+ default=False,
+ help="Do everything but actually run the tests or upload results."),
+ # old-run-webkit-tests has --valgrind instead of wrapper.
+ optparse.make_option("--wrapper",
+ help="wrapper command to insert before invocations of "
+ "DumpRenderTree; option is split on whitespace before "
+ "running. (Example: --wrapper='valgrind --smc-check=all')"),
+ # old-run-webkit-tests:
+ # -i|--ignore-tests Comma-separated list of directories
+ # or tests to ignore
+ optparse.make_option("--test-list", action="append",
+ help="read list of tests to run from file", metavar="FILE"),
+ # old-run-webkit-tests uses --skipped==[default|ignore|only]
+ # instead of --force:
+ optparse.make_option("--force", action="store_true", default=False,
+ help="Run all tests, even those marked SKIP in the test list"),
+ optparse.make_option("--use-apache", action="store_true",
+ default=False, help="Whether to use apache instead of lighttpd."),
+ optparse.make_option("--time-out-ms",
+ help="Set the timeout for each test"),
+ # old-run-webkit-tests calls --randomize-order --random:
+ optparse.make_option("--randomize-order", action="store_true",
+ default=False, help=("Run tests in random order (useful "
+ "for tracking down corruption)")),
+ optparse.make_option("--run-chunk",
+ help=("Run a specified chunk (n:l), the nth of len l, "
+ "of the layout tests")),
+ optparse.make_option("--run-part", help=("Run a specified part (n:m), "
+ "the nth of m parts, of the layout tests")),
+ # old-run-webkit-tests calls --batch-size: --nthly n
+ # Restart DumpRenderTree every n tests (default: 1000)
+ optparse.make_option("--batch-size",
+ help=("Run a the tests in batches (n), after every n tests, "
+ "DumpRenderTree is relaunched."), type="int", default=0),
+ # old-run-webkit-tests calls --run-singly: -1|--singly
+ # Isolate each test case run (implies --nthly 1 --verbose)
+ optparse.make_option("--run-singly", action="store_true",
+ default=False, help="run a separate DumpRenderTree for each test"),
+ optparse.make_option("--child-processes",
+ help="Number of DumpRenderTrees to run in parallel."),
+ # FIXME: Display default number of child processes that will run.
+ optparse.make_option("--worker-model", action="store",
+ default="old-threads", help=("controls worker model. Valid values "
+ "are 'old-inline', 'old-threads'.")),
+ optparse.make_option("--experimental-fully-parallel",
+ action="store_true", default=False,
+ help="run all tests in parallel"),
+ optparse.make_option("--exit-after-n-failures", type="int", nargs=1,
+ help="Exit after the first N failures instead of running all "
+ "tests"),
+ optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
+ nargs=1, help="Exit after the first N crashes instead of running "
+ "all tests"),
+ # FIXME: consider: --iterations n
+ # Number of times to run the set of tests (e.g. ABCABCABC)
+ optparse.make_option("--print-last-failures", action="store_true",
+ default=False, help="Print the tests in the last run that "
+ "had unexpected failures (or passes) and then exit."),
+ optparse.make_option("--retest-last-failures", action="store_true",
+ default=False, help="re-test the tests in the last run that "
+ "had unexpected failures (or passes)."),
+ optparse.make_option("--retry-failures", action="store_true",
+ default=True,
+ help="Re-try any tests that produce unexpected results (default)"),
+ optparse.make_option("--no-retry-failures", action="store_false",
+ dest="retry_failures",
+ help="Don't re-try any tests that produce unexpected results."),
+ ]
+
+ misc_options = [
+ optparse.make_option("--lint-test-files", action="store_true",
+ default=False, help=("Makes sure the test files parse for all "
+ "configurations. Does not run any tests.")),
+ ]
+
+ # FIXME: Move these into json_results_generator.py
+ results_json_options = [
+ optparse.make_option("--master-name", help="The name of the buildbot master."),
+ optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
+ help=("The name of the builder shown on the waterfall running "
+ "this script e.g. WebKit.")),
+ optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
+ help=("The name of the builder used in its path, e.g. "
+ "webkit-rel.")),
+ optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
+ help=("The build number of the builder running this script.")),
+ optparse.make_option("--test-results-server", default="",
+ help=("If specified, upload results json files to this appengine "
+ "server.")),
+ optparse.make_option("--upload-full-results",
+ action="store_true",
+ default=False,
+ help="If true, upload full json results to server."),
+ ]
+
+ option_list = (configuration_options + print_options +
+ chromium_options + results_options + test_options +
+ misc_options + results_json_options +
+ old_run_webkit_tests_compat)
+ option_parser = optparse.OptionParser(option_list=option_list)
+
+ return option_parser.parse_args(args)
+
+
+def main():
+ options, args = parse_args()
+ port_obj = port.get(options.platform, options)
+ return run(port_obj, options, args)
+
+
+if '__main__' == __name__:
+ try:
+ sys.exit(main())
+ except KeyboardInterrupt:
+ # this mirrors what the shell normally does
+ sys.exit(signal.SIGINT + 128)
diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py
new file mode 100644
index 0000000..2bfac2f
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py
@@ -0,0 +1,545 @@
+#!/usr/bin/python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit tests for run_webkit_tests."""
+
+import codecs
+import itertools
+import logging
+import os
+import Queue
+import shutil
+import sys
+import tempfile
+import thread
+import time
+import threading
+import unittest
+
+from webkitpy.common import array_stream
+from webkitpy.common.system import outputcapture
+from webkitpy.common.system import user
+from webkitpy.layout_tests import port
+from webkitpy.layout_tests import run_webkit_tests
+from webkitpy.layout_tests.layout_package import dump_render_tree_thread
+from webkitpy.layout_tests.port.test import TestPort, TestDriver
+from webkitpy.python24.versioning import compare_version
+from webkitpy.test.skip import skip_if
+
+from webkitpy.thirdparty.mock import Mock
+
+
+class MockUser():
+ def __init__(self):
+ self.url = None
+
+ def open_url(self, url):
+ self.url = url
+
+
+def parse_args(extra_args=None, record_results=False, tests_included=False,
+ print_nothing=True):
+ extra_args = extra_args or []
+ if print_nothing:
+ args = ['--print', 'nothing']
+ else:
+ args = []
+ if not '--platform' in extra_args:
+ args.extend(['--platform', 'test'])
+ if not record_results:
+ args.append('--no-record-results')
+ if not '--child-processes' in extra_args:
+ args.extend(['--worker-model', 'old-inline'])
+ args.extend(extra_args)
+ if not tests_included:
+ # We use the glob to test that globbing works.
+ args.extend(['passes',
+ 'http/tests',
+ 'websocket/tests',
+ 'failures/expected/*'])
+ return run_webkit_tests.parse_args(args)
+
+
+def passing_run(extra_args=None, port_obj=None, record_results=False,
+ tests_included=False):
+ options, parsed_args = parse_args(extra_args, record_results,
+ tests_included)
+ if not port_obj:
+ port_obj = port.get(port_name=options.platform, options=options,
+ user=MockUser())
+ res = run_webkit_tests.run(port_obj, options, parsed_args)
+ return res == 0
+
+
+def logging_run(extra_args=None, port_obj=None, tests_included=False):
+ options, parsed_args = parse_args(extra_args=extra_args,
+ record_results=False,
+ tests_included=tests_included,
+ print_nothing=False)
+ user = MockUser()
+ if not port_obj:
+ port_obj = port.get(port_name=options.platform, options=options,
+ user=user)
+
+ res, buildbot_output, regular_output = run_and_capture(port_obj, options,
+ parsed_args)
+ return (res, buildbot_output, regular_output, user)
+
+
+def run_and_capture(port_obj, options, parsed_args):
+ oc = outputcapture.OutputCapture()
+ try:
+ oc.capture_output()
+ buildbot_output = array_stream.ArrayStream()
+ regular_output = array_stream.ArrayStream()
+ res = run_webkit_tests.run(port_obj, options, parsed_args,
+ buildbot_output=buildbot_output,
+ regular_output=regular_output)
+ finally:
+ oc.restore_output()
+ return (res, buildbot_output, regular_output)
+
+
+def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False):
+ extra_args = extra_args or []
+ if not tests_included:
+ # Not including http tests since they get run out of order (that
+ # behavior has its own test, see test_get_test_file_queue)
+ extra_args = ['passes', 'failures'] + extra_args
+ options, parsed_args = parse_args(extra_args, tests_included=True)
+
+ user = MockUser()
+
+ test_batches = []
+
+ class RecordingTestDriver(TestDriver):
+ def __init__(self, port, worker_number):
+ TestDriver.__init__(self, port, worker_number)
+ self._current_test_batch = None
+
+ def poll(self):
+ # So that we don't create a new driver for every test
+ return None
+
+ def stop(self):
+ self._current_test_batch = None
+
+ def run_test(self, test_input):
+ if self._current_test_batch is None:
+ self._current_test_batch = []
+ test_batches.append(self._current_test_batch)
+ test_name = self._port.relative_test_filename(test_input.filename)
+ self._current_test_batch.append(test_name)
+ return TestDriver.run_test(self, test_input)
+
+ class RecordingTestPort(TestPort):
+ def create_driver(self, worker_number):
+ return RecordingTestDriver(self, worker_number)
+
+ recording_port = RecordingTestPort(options=options, user=user)
+ run_and_capture(recording_port, options, parsed_args)
+
+ if flatten_batches:
+ return list(itertools.chain(*test_batches))
+
+ return test_batches
+
+
+class MainTest(unittest.TestCase):
+ def test_accelerated_compositing(self):
+ # This just tests that we recognize the command line args
+ self.assertTrue(passing_run(['--accelerated-compositing']))
+ self.assertTrue(passing_run(['--no-accelerated-compositing']))
+
+ def test_accelerated_2d_canvas(self):
+ # This just tests that we recognize the command line args
+ self.assertTrue(passing_run(['--accelerated-2d-canvas']))
+ self.assertTrue(passing_run(['--no-accelerated-2d-canvas']))
+
+ def test_basic(self):
+ self.assertTrue(passing_run())
+
+ def test_batch_size(self):
+ batch_tests_run = get_tests_run(['--batch-size', '2'])
+ for batch in batch_tests_run:
+ self.assertTrue(len(batch) <= 2, '%s had too many tests' % ', '.join(batch))
+
+ def test_child_process_1(self):
+ (res, buildbot_output, regular_output, user) = logging_run(
+ ['--print', 'config', '--child-processes', '1'])
+ self.assertTrue('Running one DumpRenderTree\n'
+ in regular_output.get())
+
+ def test_child_processes_2(self):
+ (res, buildbot_output, regular_output, user) = logging_run(
+ ['--print', 'config', '--child-processes', '2'])
+ self.assertTrue('Running 2 DumpRenderTrees in parallel\n'
+ in regular_output.get())
+
+ def test_dryrun(self):
+ batch_tests_run = get_tests_run(['--dry-run'])
+ self.assertEqual(batch_tests_run, [])
+
+ batch_tests_run = get_tests_run(['-n'])
+ self.assertEqual(batch_tests_run, [])
+
+ def test_exception_raised(self):
+ self.assertRaises(ValueError, logging_run,
+ ['failures/expected/exception.html'], tests_included=True)
+
+ def test_full_results_html(self):
+ # FIXME: verify html?
+ self.assertTrue(passing_run(['--full-results-html']))
+
+ def test_help_printing(self):
+ res, out, err, user = logging_run(['--help-printing'])
+ self.assertEqual(res, 0)
+ self.assertTrue(out.empty())
+ self.assertFalse(err.empty())
+
+ def test_hung_thread(self):
+ res, out, err, user = logging_run(['--run-singly', '--time-out-ms=50',
+ 'failures/expected/hang.html'],
+ tests_included=True)
+ self.assertEqual(res, 0)
+ self.assertFalse(out.empty())
+ self.assertFalse(err.empty())
+
+ def test_keyboard_interrupt(self):
+ # Note that this also tests running a test marked as SKIP if
+ # you specify it explicitly.
+ self.assertRaises(KeyboardInterrupt, logging_run,
+ ['failures/expected/keyboard.html'], tests_included=True)
+
+ def test_last_results(self):
+ passing_run(['--clobber-old-results'], record_results=True)
+ (res, buildbot_output, regular_output, user) = logging_run(
+ ['--print-last-failures'])
+ self.assertEqual(regular_output.get(), ['\n\n'])
+ self.assertEqual(buildbot_output.get(), [])
+
+ def test_lint_test_files(self):
+ res, out, err, user = logging_run(['--lint-test-files'])
+ self.assertEqual(res, 0)
+ self.assertTrue(out.empty())
+ self.assertTrue(any(['Lint succeeded' in msg for msg in err.get()]))
+
+ def test_lint_test_files__errors(self):
+ options, parsed_args = parse_args(['--lint-test-files'])
+ user = MockUser()
+ port_obj = port.get(options.platform, options=options, user=user)
+ port_obj.test_expectations = lambda: "# syntax error"
+ res, out, err = run_and_capture(port_obj, options, parsed_args)
+
+ self.assertEqual(res, -1)
+ self.assertTrue(out.empty())
+ self.assertTrue(any(['Lint failed' in msg for msg in err.get()]))
+
+ def test_no_tests_found(self):
+ res, out, err, user = logging_run(['resources'], tests_included=True)
+ self.assertEqual(res, -1)
+ self.assertTrue(out.empty())
+ self.assertTrue('No tests to run.\n' in err.get())
+
+ def test_no_tests_found_2(self):
+ res, out, err, user = logging_run(['foo'], tests_included=True)
+ self.assertEqual(res, -1)
+ self.assertTrue(out.empty())
+ self.assertTrue('No tests to run.\n' in err.get())
+
+ def test_randomize_order(self):
+ # FIXME: verify order was shuffled
+ self.assertTrue(passing_run(['--randomize-order']))
+
+ def test_run_chunk(self):
+ # Test that we actually select the right chunk
+ all_tests_run = get_tests_run(flatten_batches=True)
+ chunk_tests_run = get_tests_run(['--run-chunk', '1:4'], flatten_batches=True)
+ self.assertEquals(all_tests_run[4:8], chunk_tests_run)
+
+ # Test that we wrap around if the number of tests is not evenly divisible by the chunk size
+ tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
+ chunk_tests_run = get_tests_run(['--run-chunk', '1:3'] + tests_to_run, tests_included=True, flatten_batches=True)
+ self.assertEquals(['passes/text.html', 'passes/error.html', 'passes/image.html'], chunk_tests_run)
+
+ def test_run_force(self):
+ # This raises an exception because we run
+ # failures/expected/exception.html, which is normally SKIPped.
+ self.assertRaises(ValueError, logging_run, ['--force'])
+
+ def test_run_part(self):
+ # Test that we actually select the right part
+ tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
+ tests_run = get_tests_run(['--run-part', '1:2'] + tests_to_run, tests_included=True, flatten_batches=True)
+ self.assertEquals(['passes/error.html', 'passes/image.html'], tests_run)
+
+ # Test that we wrap around if the number of tests is not evenly divisible by the chunk size
+ # (here we end up with 3 parts, each with 2 tests, and we only have 4 tests total, so the
+ # last part repeats the first two tests).
+ chunk_tests_run = get_tests_run(['--run-part', '3:3'] + tests_to_run, tests_included=True, flatten_batches=True)
+ self.assertEquals(['passes/error.html', 'passes/image.html'], chunk_tests_run)
+
+ def test_run_singly(self):
+ batch_tests_run = get_tests_run(['--run-singly'])
+ for batch in batch_tests_run:
+ self.assertEquals(len(batch), 1, '%s had too many tests' % ', '.join(batch))
+
+ def test_single_file(self):
+ tests_run = get_tests_run(['passes/text.html'], tests_included=True, flatten_batches=True)
+ self.assertEquals(['passes/text.html'], tests_run)
+
+ def test_test_list(self):
+ filename = tempfile.mktemp()
+ tmpfile = file(filename, mode='w+')
+ tmpfile.write('passes/text.html')
+ tmpfile.close()
+ tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True)
+ self.assertEquals(['passes/text.html'], tests_run)
+ os.remove(filename)
+ res, out, err, user = logging_run(['--test-list=%s' % filename],
+ tests_included=True)
+ self.assertEqual(res, -1)
+ self.assertFalse(err.empty())
+
+ def test_unexpected_failures(self):
+ # Run tests including the unexpected failures.
+ self._url_opened = None
+ res, out, err, user = logging_run(tests_included=True)
+ self.assertEqual(res, 3)
+ self.assertFalse(out.empty())
+ self.assertFalse(err.empty())
+ self.assertEqual(user.url, '/tmp/layout-test-results/results.html')
+
+ def test_exit_after_n_failures(self):
+ # Unexpected failures should result in tests stopping.
+ tests_run = get_tests_run([
+ 'failures/unexpected/text-image-checksum.html',
+ 'passes/text.html',
+ '--exit-after-n-failures', '1',
+ ],
+ tests_included=True,
+ flatten_batches=True)
+ self.assertEquals(['failures/unexpected/text-image-checksum.html'], tests_run)
+
+ # But we'll keep going for expected ones.
+ tests_run = get_tests_run([
+ 'failures/expected/text.html',
+ 'passes/text.html',
+ '--exit-after-n-failures', '1',
+ ],
+ tests_included=True,
+ flatten_batches=True)
+ self.assertEquals(['failures/expected/text.html', 'passes/text.html'], tests_run)
+
+ def test_exit_after_n_crashes(self):
+ # Unexpected crashes should result in tests stopping.
+ tests_run = get_tests_run([
+ 'failures/unexpected/crash.html',
+ 'passes/text.html',
+ '--exit-after-n-crashes-or-timeouts', '1',
+ ],
+ tests_included=True,
+ flatten_batches=True)
+ self.assertEquals(['failures/unexpected/crash.html'], tests_run)
+
+ # Same with timeouts.
+ tests_run = get_tests_run([
+ 'failures/unexpected/timeout.html',
+ 'passes/text.html',
+ '--exit-after-n-crashes-or-timeouts', '1',
+ ],
+ tests_included=True,
+ flatten_batches=True)
+ self.assertEquals(['failures/unexpected/timeout.html'], tests_run)
+
+ # But we'll keep going for expected ones.
+ tests_run = get_tests_run([
+ 'failures/expected/crash.html',
+ 'passes/text.html',
+ '--exit-after-n-crashes-or-timeouts', '1',
+ ],
+ tests_included=True,
+ flatten_batches=True)
+ self.assertEquals(['failures/expected/crash.html', 'passes/text.html'], tests_run)
+
+ def test_results_directory_absolute(self):
+ # We run a configuration that should fail, to generate output, then
+ # look for what the output results url was.
+
+ tmpdir = tempfile.mkdtemp()
+ res, out, err, user = logging_run(['--results-directory=' + tmpdir],
+ tests_included=True)
+ self.assertEqual(user.url, os.path.join(tmpdir, 'results.html'))
+ shutil.rmtree(tmpdir, ignore_errors=True)
+
+ def test_results_directory_default(self):
+ # We run a configuration that should fail, to generate output, then
+ # look for what the output results url was.
+
+ # This is the default location.
+ res, out, err, user = logging_run(tests_included=True)
+ self.assertEqual(user.url, '/tmp/layout-test-results/results.html')
+
+ def test_results_directory_relative(self):
+ # We run a configuration that should fail, to generate output, then
+ # look for what the output results url was.
+
+ res, out, err, user = logging_run(['--results-directory=foo'],
+ tests_included=True)
+ self.assertEqual(user.url, '/tmp/foo/results.html')
+
+ def test_tolerance(self):
+ class ImageDiffTestPort(TestPort):
+ def diff_image(self, expected_contents, actual_contents,
+ diff_filename=None):
+ self.tolerance_used_for_diff_image = self._options.tolerance
+ return True
+
+ def get_port_for_run(args):
+ options, parsed_args = run_webkit_tests.parse_args(args)
+ test_port = ImageDiffTestPort(options=options, user=MockUser())
+ passing_run(args, port_obj=test_port, tests_included=True)
+ return test_port
+
+ base_args = ['--pixel-tests', 'failures/expected/*']
+
+ # If we pass in an explicit tolerance argument, then that will be used.
+ test_port = get_port_for_run(base_args + ['--tolerance', '.1'])
+ self.assertEqual(0.1, test_port.tolerance_used_for_diff_image)
+ test_port = get_port_for_run(base_args + ['--tolerance', '0'])
+ self.assertEqual(0, test_port.tolerance_used_for_diff_image)
+
+ # Otherwise the port's default tolerance behavior (including ignoring it)
+ # should be used.
+ test_port = get_port_for_run(base_args)
+ self.assertEqual(None, test_port.tolerance_used_for_diff_image)
+
+ def test_worker_model__inline(self):
+ self.assertTrue(passing_run(['--worker-model', 'old-inline']))
+
+ def test_worker_model__threads(self):
+ self.assertTrue(passing_run(['--worker-model', 'old-threads']))
+
+ def test_worker_model__unknown(self):
+ self.assertRaises(ValueError, logging_run,
+ ['--worker-model', 'unknown'])
+
+MainTest = skip_if(MainTest, sys.platform == 'cygwin' and compare_version(sys, '2.6')[0] < 0, 'new-run-webkit-tests tests hang on Cygwin Python 2.5.2')
+
+
+
+def _mocked_open(original_open, file_list):
+ def _wrapper(name, mode, encoding):
+ if name.find("-expected.") != -1 and mode.find("w") != -1:
+ # we don't want to actually write new baselines, so stub these out
+ name.replace('\\', '/')
+ file_list.append(name)
+ return original_open(os.devnull, mode, encoding)
+ return original_open(name, mode, encoding)
+ return _wrapper
+
+
+class RebaselineTest(unittest.TestCase):
+ def assertBaselines(self, file_list, file):
+ "assert that the file_list contains the baselines."""
+ for ext in [".txt", ".png", ".checksum"]:
+ baseline = file + "-expected" + ext
+ self.assertTrue(any(f.find(baseline) != -1 for f in file_list))
+
+ # FIXME: Add tests to ensure that we're *not* writing baselines when we're not
+ # supposed to be.
+
+ def disabled_test_reset_results(self):
+ # FIXME: This test is disabled until we can rewrite it to use a
+ # mock filesystem.
+ #
+ # Test that we update expectations in place. If the expectation
+ # is missing, update the expected generic location.
+ file_list = []
+ passing_run(['--pixel-tests',
+ '--reset-results',
+ 'passes/image.html',
+ 'failures/expected/missing_image.html'],
+ tests_included=True)
+ self.assertEqual(len(file_list), 6)
+ self.assertBaselines(file_list,
+ "data/passes/image")
+ self.assertBaselines(file_list,
+ "data/failures/expected/missing_image")
+
+ def disabled_test_new_baseline(self):
+ # FIXME: This test is disabled until we can rewrite it to use a
+ # mock filesystem.
+ #
+ # Test that we update the platform expectations. If the expectation
+ # is mssing, then create a new expectation in the platform dir.
+ file_list = []
+ original_open = codecs.open
+ try:
+ # Test that we update the platform expectations. If the expectation
+ # is mssing, then create a new expectation in the platform dir.
+ file_list = []
+ codecs.open = _mocked_open(original_open, file_list)
+ passing_run(['--pixel-tests',
+ '--new-baseline',
+ 'passes/image.html',
+ 'failures/expected/missing_image.html'],
+ tests_included=True)
+ self.assertEqual(len(file_list), 6)
+ self.assertBaselines(file_list,
+ "data/platform/test/passes/image")
+ self.assertBaselines(file_list,
+ "data/platform/test/failures/expected/missing_image")
+ finally:
+ codecs.open = original_open
+
+
+class DryrunTest(unittest.TestCase):
+ # FIXME: it's hard to know which platforms are safe to test; the
+ # chromium platforms require a chromium checkout, and the mac platform
+ # requires fcntl, so it can't be tested on win32, etc. There is
+ # probably a better way of handling this.
+ def test_darwin(self):
+ if sys.platform != "darwin":
+ return
+
+ self.assertTrue(passing_run(['--platform', 'test']))
+ self.assertTrue(passing_run(['--platform', 'dryrun',
+ 'fast/html']))
+ self.assertTrue(passing_run(['--platform', 'dryrun-mac',
+ 'fast/html']))
+
+ def test_test(self):
+ self.assertTrue(passing_run(['--platform', 'dryrun-test',
+ '--pixel-tests']))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/__init__.py b/Tools/Scripts/webkitpy/layout_tests/test_types/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/test_types/__init__.py
diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/image_diff.py b/Tools/Scripts/webkitpy/layout_tests/test_types/image_diff.py
new file mode 100644
index 0000000..da466c8
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/test_types/image_diff.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Compares the image output of a test to the expected image output.
+
+Compares hashes for the generated and expected images. If the output doesn't
+match, returns FailureImageHashMismatch and outputs both hashes into the layout
+test results directory.
+"""
+
+from __future__ import with_statement
+
+import codecs
+import errno
+import logging
+import os
+import shutil
+
+from webkitpy.layout_tests.layout_package import test_failures
+from webkitpy.layout_tests.test_types import test_type_base
+
+# Cache whether we have the image_diff executable available.
+_compare_available = True
+_compare_msg_printed = False
+
+_log = logging.getLogger("webkitpy.layout_tests.test_types.image_diff")
+
+
+class ImageDiff(test_type_base.TestTypeBase):
+
+ def _save_baseline_files(self, filename, image, image_hash,
+ generate_new_baseline):
+ """Saves new baselines for the PNG and checksum.
+
+ Args:
+ filename: test filename
+ image: a image output
+ image_hash: a checksum of the image
+ generate_new_baseline: whether to generate a new, platform-specific
+ baseline, or update the existing one
+ """
+ self._save_baseline_data(filename, image, ".png", encoding=None,
+ generate_new_baseline=generate_new_baseline)
+ self._save_baseline_data(filename, image_hash, ".checksum",
+ encoding="ascii",
+ generate_new_baseline=generate_new_baseline)
+
+ def _copy_image(self, filename, actual_image, expected_image):
+ self.write_output_files(filename, '.png',
+ output=actual_image, expected=expected_image,
+ encoding=None, print_text_diffs=False)
+
+ def _copy_image_hash(self, filename, actual_image_hash, expected_image_hash):
+ self.write_output_files(filename, '.checksum',
+ actual_image_hash, expected_image_hash,
+ encoding="ascii", print_text_diffs=False)
+
+ def _create_diff_image(self, port, filename, actual_image, expected_image):
+ """Creates the visual diff of the expected/actual PNGs.
+
+ Returns True if the images are different.
+ """
+ diff_filename = self.output_filename(filename,
+ self.FILENAME_SUFFIX_COMPARE)
+ return port.diff_image(actual_image, expected_image, diff_filename)
+
+ def compare_output(self, port, filename, test_args, actual_test_output,
+ expected_test_output):
+ """Implementation of CompareOutput that checks the output image and
+ checksum against the expected files from the LayoutTest directory.
+ """
+ failures = []
+
+ # If we didn't produce a hash file, this test must be text-only.
+ if actual_test_output.image_hash is None:
+ return failures
+
+ # If we're generating a new baseline, we pass.
+ if test_args.new_baseline or test_args.reset_results:
+ self._save_baseline_files(filename, actual_test_output.image,
+ actual_test_output.image_hash,
+ test_args.new_baseline)
+ return failures
+
+ if not expected_test_output.image:
+ # Report a missing expected PNG file.
+ self._copy_image(filename, actual_test_output.image, expected_image=None)
+ self._copy_image_hash(filename, actual_test_output.image_hash,
+ expected_test_output.image_hash)
+ failures.append(test_failures.FailureMissingImage())
+ return failures
+ if not expected_test_output.image_hash:
+ # Report a missing expected checksum file.
+ self._copy_image(filename, actual_test_output.image,
+ expected_test_output.image)
+ self._copy_image_hash(filename, actual_test_output.image_hash,
+ expected_image_hash=None)
+ failures.append(test_failures.FailureMissingImageHash())
+ return failures
+
+ if actual_test_output.image_hash == expected_test_output.image_hash:
+ # Hash matched (no diff needed, okay to return).
+ return failures
+
+ self._copy_image(filename, actual_test_output.image,
+ expected_test_output.image)
+ self._copy_image_hash(filename, actual_test_output.image_hash,
+ expected_test_output.image_hash)
+
+ # Even though we only use the result in one codepath below but we
+ # still need to call CreateImageDiff for other codepaths.
+ images_are_different = self._create_diff_image(port, filename,
+ actual_test_output.image,
+ expected_test_output.image)
+ if not images_are_different:
+ failures.append(test_failures.FailureImageHashIncorrect())
+ else:
+ failures.append(test_failures.FailureImageHashMismatch())
+
+ return failures
diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py b/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py
new file mode 100644
index 0000000..4b96b3a
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py
@@ -0,0 +1,223 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Defines the interface TestTypeBase which other test types inherit from.
+
+Also defines the TestArguments "struct" to pass them additional arguments.
+"""
+
+from __future__ import with_statement
+
+import codecs
+import cgi
+import errno
+import logging
+import os.path
+
+_log = logging.getLogger("webkitpy.layout_tests.test_types.test_type_base")
+
+
+class TestArguments(object):
+ """Struct-like wrapper for additional arguments needed by
+ specific tests."""
+ # Whether to save new baseline results.
+ new_baseline = False
+
+ # Path to the actual PNG file generated by pixel tests
+ png_path = None
+
+ # Value of checksum generated by pixel tests.
+ hash = None
+
+ # Whether to use wdiff to generate by-word diffs.
+ wdiff = False
+
+# Python bug workaround. See the wdiff code in WriteOutputFiles for an
+# explanation.
+_wdiff_available = True
+
+
+class TestTypeBase(object):
+
+ # Filename pieces when writing failures to the test results directory.
+ FILENAME_SUFFIX_ACTUAL = "-actual"
+ FILENAME_SUFFIX_EXPECTED = "-expected"
+ FILENAME_SUFFIX_DIFF = "-diff"
+ FILENAME_SUFFIX_WDIFF = "-wdiff.html"
+ FILENAME_SUFFIX_PRETTY_PATCH = "-pretty-diff.html"
+ FILENAME_SUFFIX_COMPARE = "-diff.png"
+
+ def __init__(self, port, root_output_dir):
+ """Initialize a TestTypeBase object.
+
+ Args:
+ port: object implementing port-specific information and methods
+ root_output_dir: The unix style path to the output dir.
+ """
+ self._root_output_dir = root_output_dir
+ self._port = port
+
+ def _make_output_directory(self, filename):
+ """Creates the output directory (if needed) for a given test
+ filename."""
+ output_filename = os.path.join(self._root_output_dir,
+ self._port.relative_test_filename(filename))
+ self._port.maybe_make_directory(os.path.split(output_filename)[0])
+
+ def _save_baseline_data(self, filename, data, modifier, encoding,
+ generate_new_baseline=True):
+ """Saves a new baseline file into the port's baseline directory.
+
+ The file will be named simply "<test>-expected<modifier>", suitable for
+ use as the expected results in a later run.
+
+ Args:
+ filename: path to the test file
+ data: result to be saved as the new baseline
+ modifier: type of the result file, e.g. ".txt" or ".png"
+ encoding: file encoding (none, "utf-8", etc.)
+ generate_new_baseline: whether to enerate a new, platform-specific
+ baseline, or update the existing one
+ """
+
+ if generate_new_baseline:
+ relative_dir = os.path.dirname(
+ self._port.relative_test_filename(filename))
+ baseline_path = self._port.baseline_path()
+ output_dir = os.path.join(baseline_path, relative_dir)
+ output_file = os.path.basename(os.path.splitext(filename)[0] +
+ self.FILENAME_SUFFIX_EXPECTED + modifier)
+ self._port.maybe_make_directory(output_dir)
+ output_path = os.path.join(output_dir, output_file)
+ _log.debug('writing new baseline result "%s"' % (output_path))
+ else:
+ output_path = self._port.expected_filename(filename, modifier)
+ _log.debug('resetting baseline result "%s"' % output_path)
+
+ self._port.update_baseline(output_path, data, encoding)
+
+ def output_filename(self, filename, modifier):
+ """Returns a filename inside the output dir that contains modifier.
+
+ For example, if filename is c:/.../fast/dom/foo.html and modifier is
+ "-expected.txt", the return value is
+ c:/cygwin/tmp/layout-test-results/fast/dom/foo-expected.txt
+
+ Args:
+ filename: absolute filename to test file
+ modifier: a string to replace the extension of filename with
+
+ Return:
+ The absolute windows path to the output filename
+ """
+ output_filename = os.path.join(self._root_output_dir,
+ self._port.relative_test_filename(filename))
+ return os.path.splitext(output_filename)[0] + modifier
+
+ def compare_output(self, port, filename, test_args, actual_test_output,
+ expected_test_output):
+ """Method that compares the output from the test with the
+ expected value.
+
+ This is an abstract method to be implemented by all sub classes.
+
+ Args:
+ port: object implementing port-specific information and methods
+ filename: absolute filename to test file
+ test_args: a TestArguments object holding optional additional
+ arguments
+ actual_test_output: a TestOutput object which represents actual test
+ output
+ expected_test_output: a TestOutput object which represents a expected
+ test output
+
+ Return:
+ a list of TestFailure objects, empty if the test passes
+ """
+ raise NotImplementedError
+
+ def _write_into_file_at_path(self, file_path, contents, encoding):
+ """This method assumes that byte_array is already encoded
+ into the right format."""
+ open_mode = 'w'
+ if encoding is None:
+ open_mode = 'w+b'
+ with codecs.open(file_path, open_mode, encoding=encoding) as file:
+ file.write(contents)
+
+ def write_output_files(self, filename, file_type,
+ output, expected, encoding,
+ print_text_diffs=False):
+ """Writes the test output, the expected output and optionally the diff
+ between the two to files in the results directory.
+
+ The full output filename of the actual, for example, will be
+ <filename>-actual<file_type>
+ For instance,
+ my_test-actual.txt
+
+ Args:
+ filename: The test filename
+ file_type: A string describing the test output file type, e.g. ".txt"
+ output: A string containing the test output
+ expected: A string containing the expected test output
+ print_text_diffs: True for text diffs. (FIXME: We should be able to get this from the file type?)
+ """
+ self._make_output_directory(filename)
+ actual_filename = self.output_filename(filename, self.FILENAME_SUFFIX_ACTUAL + file_type)
+ expected_filename = self.output_filename(filename, self.FILENAME_SUFFIX_EXPECTED + file_type)
+ # FIXME: This function is poorly designed. We should be passing in some sort of
+ # encoding information from the callers.
+ if output:
+ self._write_into_file_at_path(actual_filename, output, encoding)
+ if expected:
+ self._write_into_file_at_path(expected_filename, expected, encoding)
+
+ if not output or not expected:
+ return
+
+ if not print_text_diffs:
+ return
+
+ # Note: We pass encoding=None for all diff writes, as we treat diff
+ # output as binary. Diff output may contain multiple files in
+ # conflicting encodings.
+ diff = self._port.diff_text(expected, output, expected_filename, actual_filename)
+ diff_filename = self.output_filename(filename, self.FILENAME_SUFFIX_DIFF + file_type)
+ self._write_into_file_at_path(diff_filename, diff, encoding=None)
+
+ # Shell out to wdiff to get colored inline diffs.
+ wdiff = self._port.wdiff_text(expected_filename, actual_filename)
+ wdiff_filename = self.output_filename(filename, self.FILENAME_SUFFIX_WDIFF)
+ self._write_into_file_at_path(wdiff_filename, wdiff, encoding=None)
+
+ # Use WebKit's PrettyPatch.rb to get an HTML diff.
+ pretty_patch = self._port.pretty_patch_text(diff_filename)
+ pretty_patch_filename = self.output_filename(filename, self.FILENAME_SUFFIX_PRETTY_PATCH)
+ self._write_into_file_at_path(pretty_patch_filename, pretty_patch, encoding=None)
diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py b/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py
new file mode 100644
index 0000000..5dbfcb6
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py
@@ -0,0 +1,47 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+""""Tests stray tests not covered by regular code paths."""
+
+import test_type_base
+import unittest
+
+from webkitpy.thirdparty.mock import Mock
+
+
+class Test(unittest.TestCase):
+
+ def test_compare_output_notimplemented(self):
+ test_type = test_type_base.TestTypeBase(None, None)
+ self.assertRaises(NotImplementedError, test_type.compare_output,
+ None, "foo.txt", '',
+ test_type_base.TestArguments(), 'Debug')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/text_diff.py b/Tools/Scripts/webkitpy/layout_tests/test_types/text_diff.py
new file mode 100644
index 0000000..ad25262
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/test_types/text_diff.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Compares the text output of a test to the expected text output.
+
+If the output doesn't match, returns FailureTextMismatch and outputs the diff
+files into the layout test results directory.
+"""
+
+from __future__ import with_statement
+
+import codecs
+import errno
+import logging
+import os.path
+
+from webkitpy.layout_tests.layout_package import test_failures
+from webkitpy.layout_tests.test_types import test_type_base
+
+_log = logging.getLogger("webkitpy.layout_tests.test_types.text_diff")
+
+
+class TestTextDiff(test_type_base.TestTypeBase):
+
+ def _get_normalized_output_text(self, output):
+ """Returns the normalized text output, i.e. the output in which
+ the end-of-line characters are normalized to "\n"."""
+ # Running tests on Windows produces "\r\n". The "\n" part is helpfully
+ # changed to "\r\n" by our system (Python/Cygwin), resulting in
+ # "\r\r\n", when, in fact, we wanted to compare the text output with
+ # the normalized text expectation files.
+ return output.replace("\r\r\n", "\r\n").replace("\r\n", "\n")
+
+ def compare_output(self, port, filename, test_args, actual_test_output,
+ expected_test_output):
+ """Implementation of CompareOutput that checks the output text against
+ the expected text from the LayoutTest directory."""
+ failures = []
+
+ # If we're generating a new baseline, we pass.
+ if test_args.new_baseline or test_args.reset_results:
+ # Although all test_shell/DumpRenderTree output should be utf-8,
+ # we do not ever decode it inside run-webkit-tests. For some tests
+ # DumpRenderTree may not output utf-8 text (e.g. webarchives).
+ self._save_baseline_data(filename, actual_test_output.text,
+ ".txt", encoding=None,
+ generate_new_baseline=test_args.new_baseline)
+ return failures
+
+ # Normalize text to diff
+ actual_text = self._get_normalized_output_text(actual_test_output.text)
+ # Assuming expected_text is already normalized.
+ expected_text = expected_test_output.text
+
+ # Write output files for new tests, too.
+ if port.compare_text(actual_text, expected_text):
+ # Text doesn't match, write output files.
+ self.write_output_files(filename, ".txt", actual_text,
+ expected_text, encoding=None,
+ print_text_diffs=True)
+
+ if expected_text == '':
+ failures.append(test_failures.FailureMissingResult())
+ else:
+ failures.append(test_failures.FailureTextMismatch())
+
+ return failures
diff --git a/Tools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests.py b/Tools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests.py
new file mode 100755
index 0000000..f4c8098
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests.py
@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+from __future__ import with_statement
+
+import glob
+import logging
+import optparse
+import os
+import re
+import sys
+import webkitpy.common.checkout.scm as scm
+
+_log = logging.getLogger("webkitpy.layout_tests."
+ "update-webgl-conformance-tests")
+
+
+def remove_first_line_comment(text):
+ return re.compile(r'^<!--.*?-->\s*', re.DOTALL).sub('', text)
+
+
+def translate_includes(text):
+ # Mapping of single filename to relative path under WebKit root.
+ # Assumption: these filenames are globally unique.
+ include_mapping = {
+ "js-test-style.css": "../../js/resources",
+ "js-test-pre.js": "../../js/resources",
+ "js-test-post.js": "../../js/resources",
+ "desktop-gl-constants.js": "resources",
+ }
+
+ for filename, path in include_mapping.items():
+ search = r'(?:[^"\'= ]*/)?' + re.escape(filename)
+ replace = os.path.join(path, filename)
+ text = re.sub(search, replace, text)
+
+ return text
+
+
+def translate_khronos_test(text):
+ """
+ This method translates the contents of a Khronos test to a WebKit test.
+ """
+
+ translateFuncs = [
+ remove_first_line_comment,
+ translate_includes,
+ ]
+
+ for f in translateFuncs:
+ text = f(text)
+
+ return text
+
+
+def update_file(in_filename, out_dir):
+ # check in_filename exists
+ # check out_dir exists
+ out_filename = os.path.join(out_dir, os.path.basename(in_filename))
+
+ _log.debug("Processing " + in_filename)
+ with open(in_filename, 'r') as in_file:
+ with open(out_filename, 'w') as out_file:
+ out_file.write(translate_khronos_test(in_file.read()))
+
+
+def update_directory(in_dir, out_dir):
+ for filename in glob.glob(os.path.join(in_dir, '*.html')):
+ update_file(os.path.join(in_dir, filename), out_dir)
+
+
+def default_out_dir():
+ current_scm = scm.detect_scm_system(os.path.dirname(sys.argv[0]))
+ if not current_scm:
+ return os.getcwd()
+ root_dir = current_scm.checkout_root
+ if not root_dir:
+ return os.getcwd()
+ out_dir = os.path.join(root_dir, "LayoutTests/fast/canvas/webgl")
+ if os.path.isdir(out_dir):
+ return out_dir
+ return os.getcwd()
+
+
+def configure_logging(options):
+ """Configures the logging system."""
+ log_fmt = '%(levelname)s: %(message)s'
+ log_datefmt = '%y%m%d %H:%M:%S'
+ log_level = logging.INFO
+ if options.verbose:
+ log_fmt = ('%(asctime)s %(filename)s:%(lineno)-4d %(levelname)s '
+ '%(message)s')
+ log_level = logging.DEBUG
+ logging.basicConfig(level=log_level, format=log_fmt,
+ datefmt=log_datefmt)
+
+
+def option_parser():
+ usage = "usage: %prog [options] (input file or directory)"
+ parser = optparse.OptionParser(usage=usage)
+ parser.add_option('-v', '--verbose',
+ action='store_true',
+ default=False,
+ help='include debug-level logging')
+ parser.add_option('-o', '--output',
+ action='store',
+ type='string',
+ default=default_out_dir(),
+ metavar='DIR',
+ help='specify an output directory to place files '
+ 'in [default: %default]')
+ return parser
+
+
+def main():
+ parser = option_parser()
+ (options, args) = parser.parse_args()
+ configure_logging(options)
+
+ if len(args) == 0:
+ _log.error("Must specify an input directory or filename.")
+ parser.print_help()
+ return 1
+
+ in_name = args[0]
+ if os.path.isfile(in_name):
+ update_file(in_name, options.output)
+ elif os.path.isdir(in_name):
+ update_directory(in_name, options.output)
+ else:
+ _log.error("'%s' is not a directory or a file.", in_name)
+ return 2
+
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/Tools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests_unittest.py b/Tools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests_unittest.py
new file mode 100644
index 0000000..7393b70
--- /dev/null
+++ b/Tools/Scripts/webkitpy/layout_tests/update_webgl_conformance_tests_unittest.py
@@ -0,0 +1,102 @@
+#!/usr/bin/python
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit tests for update_webgl_conformance_tests."""
+
+import unittest
+from webkitpy.layout_tests import update_webgl_conformance_tests as webgl
+
+
+def construct_script(name):
+ return "<script src=\"" + name + "\"></script>\n"
+
+
+def construct_style(name):
+ return "<link rel=\"stylesheet\" href=\"" + name + "\">"
+
+
+class TestTranslation(unittest.TestCase):
+ def assert_unchanged(self, text):
+ self.assertEqual(text, webgl.translate_khronos_test(text))
+
+ def assert_translate(self, input, output):
+ self.assertEqual(output, webgl.translate_khronos_test(input))
+
+ def test_simple_unchanged(self):
+ self.assert_unchanged("")
+ self.assert_unchanged("<html></html>")
+
+ def test_header_strip(self):
+ single_line_header = "<!-- single line header. -->"
+ multi_line_header = """<!-- this is a multi-line
+ header. it should all be removed too.
+ -->"""
+ text = "<html></html>"
+ self.assert_translate(single_line_header, "")
+ self.assert_translate(single_line_header + text, text)
+ self.assert_translate(multi_line_header + text, text)
+
+ def dont_strip_other_headers(self):
+ self.assert_unchanged("<html>\n<!-- don't remove comments on other lines. -->\n</html>")
+
+ def test_include_rewriting(self):
+ # Mappings to None are unchanged
+ styles = {
+ "../resources/js-test-style.css": "../../js/resources/js-test-style.css",
+ "fail.css": None,
+ "resources/stylesheet.css": None,
+ "../resources/style.css": None,
+ }
+ scripts = {
+ "../resources/js-test-pre.js": "../../js/resources/js-test-pre.js",
+ "../resources/js-test-post.js": "../../js/resources/js-test-post.js",
+ "../resources/desktop-gl-constants.js": "resources/desktop-gl-constants.js",
+
+ "resources/shadow-offset.js": None,
+ "../resources/js-test-post-async.js": None,
+ }
+
+ input_text = ""
+ output_text = ""
+ for input, output in styles.items():
+ input_text += construct_style(input)
+ output_text += construct_style(output if output else input)
+ for input, output in scripts.items():
+ input_text += construct_script(input)
+ output_text += construct_script(output if output else input)
+
+ head = '<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML//EN">\n<html>\n<head>\n'
+ foot = '</head>\n<body>\n</body>\n</html>'
+ input_text = head + input_text + foot
+ output_text = head + output_text + foot
+ self.assert_translate(input_text, output_text)
+
+
+if __name__ == '__main__':
+ unittest.main()