diff options
Diffstat (limited to 'WebKitTools/Scripts/webkitpy/layout_tests/port')
15 files changed, 552 insertions, 140 deletions
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py index 0dda774..9125f9e 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/base.py @@ -120,7 +120,7 @@ class Port(object): def check_image_diff(self, override_step=None, logging=True): """This routine is used to check whether image_diff binary exists.""" - raise NotImplemented('Port.check_image_diff') + raise NotImplementedError('Port.check_image_diff') def compare_text(self, expected_text, actual_text): """Return whether or not the two strings are *not* equal. This @@ -141,26 +141,9 @@ class Port(object): |tolerance| should be a percentage value (0.0 - 100.0). If it is omitted, the port default tolerance value is used. - While this is a generic routine, we include it in the Port - interface so that it can be overriden for testing purposes.""" - executable = self._path_to_image_diff() - - if diff_filename: - cmd = [executable, '--diff', expected_filename, actual_filename, - diff_filename] - else: - cmd = [executable, expected_filename, actual_filename] + """ + raise NotImplementedError('Port.diff_image') - result = True - try: - if self._executive.run_command(cmd, return_exit_code=True) == 0: - return False - except OSError, e: - if e.errno == errno.ENOENT or e.errno == errno.EACCES: - _compare_available = False - else: - raise e - return result def diff_text(self, expected_text, actual_text, expected_filename, actual_filename): @@ -305,6 +288,18 @@ class Port(object): """Return the absolute path to the top of the LayoutTests directory.""" return self.path_from_webkit_base('LayoutTests') + def skips_layout_test(self, test_name): + """Figures out if the givent test is being skipped or not. + + Test categories are handled as well.""" + for test_or_category in self.skipped_layout_tests(): + if test_or_category == test_name: + return True + category = os.path.join(self.layout_tests_dir(), test_or_category) + if os.path.isdir(category) and test_name.startswith(test_or_category): + return True + return False + def maybe_make_directory(self, *path): """Creates the specified directory if it doesn't already exist.""" try: @@ -341,84 +336,16 @@ class Port(object): if the port does not use expectations files.""" raise NotImplementedError('Port.path_to_test_expectations_file') - def remove_directory(self, *path): - """Recursively removes a directory, even if it's marked read-only. - - Remove the directory located at *path, if it exists. - - shutil.rmtree() doesn't work on Windows if any of the files - or directories are read-only, which svn repositories and - some .svn files are. We need to be able to force the files - to be writable (i.e., deletable) as we traverse the tree. - - Even with all this, Windows still sometimes fails to delete a file, - citing a permission error (maybe something to do with antivirus - scans or disk indexing). The best suggestion any of the user - forums had was to wait a bit and try again, so we do that too. - It's hand-waving, but sometimes it works. :/ - """ - file_path = os.path.join(*path) - if not os.path.exists(file_path): - return - - win32 = False - if sys.platform == 'win32': - win32 = True - # Some people don't have the APIs installed. In that case we'll do - # without. - try: - win32api = __import__('win32api') - win32con = __import__('win32con') - except ImportError: - win32 = False - - def remove_with_retry(rmfunc, path): - os.chmod(path, os.stat.S_IWRITE) - if win32: - win32api.SetFileAttributes(path, - win32con.FILE_ATTRIBUTE_NORMAL) - try: - return rmfunc(path) - except EnvironmentError, e: - if e.errno != errno.EACCES: - raise - print 'Failed to delete %s: trying again' % repr(path) - time.sleep(0.1) - return rmfunc(path) - else: - - def remove_with_retry(rmfunc, path): - if os.path.islink(path): - return os.remove(path) - else: - return rmfunc(path) - - for root, dirs, files in os.walk(file_path, topdown=False): - # For POSIX: making the directory writable guarantees - # removability. Windows will ignore the non-read-only - # bits in the chmod value. - os.chmod(root, 0770) - for name in files: - remove_with_retry(os.remove, os.path.join(root, name)) - for name in dirs: - remove_with_retry(os.rmdir, os.path.join(root, name)) - - remove_with_retry(os.rmdir, file_path) - - def test_platform_name(self): - return self._name - def relative_test_filename(self, filename): """Relative unix-style path for a filename under the LayoutTests directory. Filenames outside the LayoutTests directory should raise an error.""" - # FIXME This should assert() here but cannot due to printing_unittest.Testprinter - # assert(filename.startswith(self.layout_tests_dir())) + assert(filename.startswith(self.layout_tests_dir())) return filename[len(self.layout_tests_dir()) + 1:] def results_directory(self): """Absolute path to the place to store the test results.""" - raise NotImplemented('Port.results_directory') + raise NotImplementedError('Port.results_directory') def setup_test_run(self): """Perform port-specific work at the beginning of a test run.""" @@ -608,7 +535,6 @@ class Port(object): _wdiff_available = False return "" raise - assert(False) # Should never be reached. _pretty_patch_error_html = "Failed to run PrettyPatch, see error console." @@ -636,7 +562,7 @@ class Port(object): return self._pretty_patch_error_html def _webkit_build_directory(self, args): - args = [self.script_path("webkit-build-directory")] + args + args = ["perl", self.script_path("webkit-build-directory")] + args return self._executive.run_command(args).rstrip() def _configuration_file_path(self): @@ -691,6 +617,10 @@ class Port(object): """Returns the full path to the test driver (DumpRenderTree).""" raise NotImplementedError('Port.path_to_driver') + def _path_to_webcore_library(self): + """Returns the full path to a built copy of WebCore.""" + raise NotImplementedError('Port.path_to_webcore_library') + def _path_to_helper(self): """Returns the full path to the layout_test_helper binary, which is used to help configure the system for the test run, or None @@ -808,10 +738,5 @@ class Driver: if it has exited.""" raise NotImplementedError('Driver.poll') - def returncode(self): - """Returns the system-specific returncode if the Driver has stopped or - exited.""" - raise NotImplementedError('Driver.returncode') - def stop(self): raise NotImplementedError('Driver.stop') diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py index f821353..1cc426f 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py @@ -27,15 +27,49 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import base -import unittest +import os +import StringIO +import sys import tempfile +import unittest from webkitpy.common.system.executive import Executive, ScriptError from webkitpy.thirdparty.mock import Mock -class PortTest(unittest.TestCase): +# FIXME: This makes StringIO objects work with "with". Remove +# when we upgrade to 2.6. +class NewStringIO(StringIO.StringIO): + def __enter__(self): + return self + + def __exit__(self, type, value, traceback): + pass + + +class MockExecutive(): + def __init__(self, exception): + self._exception = exception + + def run_command(self, *args, **kwargs): + raise self._exception + +class UnitTestPort(base.Port): + """Subclass of base.Port used for unit testing.""" + def __init__(self, configuration_contents=None, executive_exception=None): + base.Port.__init__(self) + self._configuration_contents = configuration_contents + if executive_exception: + self._executive = MockExecutive(executive_exception) + + def _open_configuration_file(self): + if self._configuration_contents: + return NewStringIO(self._configuration_contents) + return base.Port._open_configuration_file(self) + + +class PortTest(unittest.TestCase): def test_format_wdiff_output_as_html(self): output = "OUTPUT %s %s %s" % (base.Port._WDIFF_DEL, base.Port._WDIFF_ADD, base.Port._WDIFF_END) html = base.Port()._format_wdiff_output_as_html(output) @@ -63,6 +97,26 @@ class PortTest(unittest.TestCase): new_file.flush() return new_file + def test_pretty_patch_os_error(self): + port = UnitTestPort(executive_exception=OSError) + self.assertEqual(port.pretty_patch_text("patch.txt"), + port._pretty_patch_error_html) + + # This tests repeated calls to make sure we cache the result. + self.assertEqual(port.pretty_patch_text("patch.txt"), + port._pretty_patch_error_html) + + def test_pretty_patch_script_error(self): + # FIXME: This is some ugly white-box test hacking ... + base._pretty_patch_available = True + port = UnitTestPort(executive_exception=ScriptError) + self.assertEqual(port.pretty_patch_text("patch.txt"), + port._pretty_patch_error_html) + + # This tests repeated calls to make sure we cache the result. + self.assertEqual(port.pretty_patch_text("patch.txt"), + port._pretty_patch_error_html) + def test_run_wdiff(self): executive = Executive() # This may fail on some systems. We could ask the port @@ -109,6 +163,79 @@ class PortTest(unittest.TestCase): self.assertFalse(base._wdiff_available) base._wdiff_available = True + def test_default_configuration_notfound(self): + port = UnitTestPort() + self.assertEqual(port.default_configuration(), "Release") + + def test_layout_tests_skipping(self): + port = base.Port() + port.skipped_layout_tests = lambda: ['foo/bar.html', 'media'] + self.assertTrue(port.skips_layout_test('foo/bar.html')) + self.assertTrue(port.skips_layout_test('media/video-zoom.html')) + self.assertFalse(port.skips_layout_test('foo/foo.html')) + + def test_default_configuration_found(self): + port = UnitTestPort(configuration_contents="Debug") + self.assertEqual(port.default_configuration(), "Debug") + + def test_default_configuration_unknown(self): + port = UnitTestPort(configuration_contents="weird_value") + self.assertEqual(port.default_configuration(), "weird_value") + + def test_setup_test_run(self): + port = base.Port() + # This routine is a no-op. We just test it for coverage. + port.setup_test_run() + + +class VirtualTest(unittest.TestCase): + """Tests that various methods expected to be virtual are.""" + def assertVirtual(self, method, *args, **kwargs): + self.assertRaises(NotImplementedError, method, *args, **kwargs) + + def test_virtual_methods(self): + port = base.Port() + self.assertVirtual(port.baseline_path) + self.assertVirtual(port.baseline_search_path) + self.assertVirtual(port.check_build, None) + self.assertVirtual(port.check_image_diff) + self.assertVirtual(port.create_driver, None, None) + self.assertVirtual(port.diff_image, None, None) + self.assertVirtual(port.path_to_test_expectations_file) + self.assertVirtual(port.test_platform_name) + self.assertVirtual(port.results_directory) + self.assertVirtual(port.show_html_results_file, None) + self.assertVirtual(port.test_expectations) + self.assertVirtual(port.test_base_platform_names) + self.assertVirtual(port.test_platform_name) + self.assertVirtual(port.test_platforms) + self.assertVirtual(port.test_platform_name_to_name, None) + self.assertVirtual(port.version) + self.assertVirtual(port._path_to_apache) + self.assertVirtual(port._path_to_apache_config_file) + self.assertVirtual(port._path_to_driver) + self.assertVirtual(port._path_to_helper) + self.assertVirtual(port._path_to_image_diff) + self.assertVirtual(port._path_to_lighttpd) + self.assertVirtual(port._path_to_lighttpd_modules) + self.assertVirtual(port._path_to_lighttpd_php) + self.assertVirtual(port._path_to_wdiff) + self.assertVirtual(port._shut_down_http_server, None) + + def test_virtual_driver_method(self): + self.assertRaises(NotImplementedError, base.Driver, base.Port, "", None) + self.assertVirtual(base.Driver, base.Port, "", None) + + def test_virtual_driver_methods(self): + class VirtualDriver(base.Driver): + def __init__(self): + pass + + driver = VirtualDriver() + self.assertVirtual(driver.run_test, None, None, None) + self.assertVirtual(driver.poll) + self.assertVirtual(driver.stop) + class DriverTest(unittest.TestCase): @@ -124,3 +251,7 @@ class DriverTest(unittest.TestCase): command_with_spaces = "valgrind --smc-check=\"check with spaces!\" --foo" expected_parse = ["valgrind", "--smc-check=check with spaces!", "--foo"] self._assert_wrapper(command_with_spaces, expected_parse) + + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py index 6cfc0b8..896eab1 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py @@ -46,6 +46,8 @@ import base import http_server from webkitpy.common.system.executive import Executive +from webkitpy.layout_tests.layout_package import test_files +from webkitpy.layout_tests.layout_package import test_expectations # Chromium DRT on OSX uses WebKitDriver. if sys.platform == 'darwin': @@ -124,6 +126,26 @@ class ChromiumPort(base.Port): return check_file_exists(image_diff_path, 'image diff exe', override_step, logging) + def diff_image(self, expected_filename, actual_filename, + diff_filename=None, tolerance=0): + executable = self._path_to_image_diff() + if diff_filename: + cmd = [executable, '--diff', expected_filename, actual_filename, + diff_filename] + else: + cmd = [executable, expected_filename, actual_filename] + + result = True + try: + if self._executive.run_command(cmd, return_exit_code=True) == 0: + return False + except OSError, e: + if e.errno == errno.ENOENT or e.errno == errno.EACCES: + _compare_available = False + else: + raise e + return result + def driver_name(self): return "test_shell" @@ -162,7 +184,7 @@ class ChromiumPort(base.Port): shutil.rmtree(cachedir) def show_results_html_file(self, results_filename): - uri = self.filename_to_uri(results_filename) + uri = self.get_absolute_path(results_filename) if self._options.use_drt: # FIXME: This should use User.open_url webbrowser.open(uri, new=1) @@ -233,6 +255,24 @@ class ChromiumPort(base.Port): with codecs.open(overrides_path, "r", "utf-8") as file: return file.read() + drt_overrides + def skipped_layout_tests(self, extra_test_files=None): + expectations_str = self.test_expectations() + overrides_str = self.test_expectations_overrides() + test_platform_name = self.test_platform_name() + is_debug_mode = False + + all_test_files = test_files.gather_test_files(self, '*') + if extra_test_files: + all_test_files.update(extra_test_files) + + expectations = test_expectations.TestExpectations( + self, all_test_files, expectations_str, test_platform_name, + is_debug_mode, is_lint_mode=True, + tests_are_present=False, overrides=overrides_str) + tests_dir = self.layout_tests_dir() + return [self.relative_test_filename(test) + for test in expectations.get_tests_with_result_type(test_expectations.SKIP)] + def test_platform_names(self): return self.test_base_platform_names() + ('win-xp', 'win-vista', 'win-7') @@ -330,9 +370,6 @@ class ChromiumDriver(base.Driver): # http://bugs.python.org/issue1731717 return self._proc.poll() - def returncode(self): - return self._proc.returncode - def _write_command_and_read_line(self, input=None): """Returns a tuple: (line, did_crash)""" try: diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py index a32eafd..7a005b1 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/chromium_unittest.py @@ -32,6 +32,7 @@ import chromium_mac import chromium_win import unittest import StringIO +import os from webkitpy.thirdparty.mock import Mock @@ -95,3 +96,22 @@ class ChromiumDriverTest(unittest.TestCase): '/xcodebuild/Release/ImageDiff')) # FIXME: Figure out how this is going to work on Windows. #port = chromium_win.ChromiumWinPort('test-port', options=MockOptions()) + + def test_skipped_layout_tests(self): + class MockOptions: + def __init__(self): + self.use_drt = True + + port = chromium_linux.ChromiumLinuxPort('test-port', options=MockOptions()) + + fake_test = os.path.join(port.layout_tests_dir(), "fast/js/not-good.js") + + port.test_expectations = lambda: """BUG_TEST SKIP : fast/js/not-good.js = TEXT +DEFER LINUX WIN : fast/js/very-good.js = TIMEOUT PASS""" + port.test_expectations_overrides = lambda: '' + + skipped_tests = port.skipped_layout_tests(extra_test_files=[fake_test, ]) + self.assertTrue("fast/js/not-good.js" in skipped_tests) + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py index e01bd2f..1af01ad 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py @@ -70,14 +70,11 @@ def _read_file(path, mode='r'): def _write_file(path, contents, mode='w'): """Write the string to the specified path. - Returns nothing if the write fails, instead of raising an IOError. + Writes should never fail, so we may raise IOError. """ - try: - with open(path, mode) as f: + with open(path, mode) as f: f.write(contents) - except IOError: - pass class DryRunPort(object): @@ -134,9 +131,6 @@ class DryrunDriver(base.Driver): def poll(self): return None - def returncode(self): - return 0 - def run_test(self, uri, timeoutms, image_hash): test_name = self._uri_to_test(uri) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/factory.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/factory.py index 258bf33..5704f65 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/factory.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/factory.py @@ -32,6 +32,10 @@ import sys +ALL_PORT_NAMES = ['test', 'dryrun', 'mac', 'win', 'gtk', 'qt', 'chromium-mac', + 'chromium-linux', 'chromium-win', 'google-chrome-win', + 'google-chrome-mac', 'google-chrome-linux32', 'google-chrome-linux64'] + def get(port_name=None, options=None): """Returns an object implementing the Port interface. If @@ -88,3 +92,9 @@ def get(port_name=None, options=None): return google_chrome.GetGoogleChromePort(port_name, options) raise NotImplementedError('unsupported port: %s' % port_to_use) + + +def get_all(options=None): + """Returns all the objects implementing the Port interface.""" + return dict([(port_name, get(port_name, options=options)) + for port_name in ALL_PORT_NAMES]) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/factory_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/factory_unittest.py index d8dffdf..c0a4c5e 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/factory_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/factory_unittest.py @@ -34,6 +34,7 @@ import chromium_mac import chromium_win import dryrun import factory +import google_chrome import gtk import mac import qt @@ -69,16 +70,16 @@ class FactoryTest(unittest.TestCase): def tearDown(self): sys.platform = self.real_sys_platform - def assert_port(self, port_name, expected_port): + def assert_port(self, port_name, expected_port, port_obj=None): """Helper assert for port_name. Args: port_name: port name to get port object. expected_port: class of expected port object. - + port_obj: optional port object """ - self.assertTrue(isinstance(factory.get(port_name=port_name), - expected_port)) + port_obj = port_obj or factory.get(port_name=port_name) + self.assertTrue(isinstance(port_obj, expected_port)) def assert_platform_port(self, platform, options, expected_port): """Helper assert for platform and options. @@ -114,6 +115,18 @@ class FactoryTest(unittest.TestCase): self.assert_platform_port("cygwin", None, win.WinPort) self.assert_platform_port("cygwin", self.webkit_options, win.WinPort) + def test_google_chrome(self): + # The actual Chrome class names aren't available so we test that the + # objects we get are at least subclasses of the Chromium versions. + self.assert_port("google-chrome-linux32", + chromium_linux.ChromiumLinuxPort) + self.assert_port("google-chrome-linux64", + chromium_linux.ChromiumLinuxPort) + self.assert_port("google-chrome-win", + chromium_win.ChromiumWinPort) + self.assert_port("google-chrome-mac", + chromium_mac.ChromiumMacPort) + def test_gtk(self): self.assert_port("gtk", gtk.GtkPort) @@ -136,3 +149,38 @@ class FactoryTest(unittest.TestCase): chromium_win.ChromiumWinPort) self.assert_platform_port("cygwin", self.chromium_options, chromium_win.ChromiumWinPort) + + def test_get_all_ports(self): + ports = factory.get_all() + for name in factory.ALL_PORT_NAMES: + self.assertTrue(name in ports.keys()) + self.assert_port("test", test.TestPort, ports["test"]) + self.assert_port("dryrun-test", dryrun.DryRunPort, ports["dryrun"]) + self.assert_port("dryrun-mac", dryrun.DryRunPort, ports["dryrun"]) + self.assert_port("mac", mac.MacPort, ports["mac"]) + self.assert_port("win", win.WinPort, ports["win"]) + self.assert_port("gtk", gtk.GtkPort, ports["gtk"]) + self.assert_port("qt", qt.QtPort, ports["qt"]) + self.assert_port("chromium-mac", chromium_mac.ChromiumMacPort, + ports["chromium-mac"]) + self.assert_port("chromium-linux", chromium_linux.ChromiumLinuxPort, + ports["chromium-linux"]) + self.assert_port("chromium-win", chromium_win.ChromiumWinPort, + ports["chromium-win"]) + + def test_unknown_specified(self): + # Test what happens when you specify an unknown port. + orig_platform = sys.platform + self.assertRaises(NotImplementedError, factory.get, + port_name='unknown') + + def test_unknown_default(self): + # Test what happens when you're running on an unknown platform. + orig_platform = sys.platform + sys.platform = 'unknown' + self.assertRaises(NotImplementedError, factory.get) + sys.platform = orig_platform + + +if __name__ == '__main__': + unittest.main() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/lighttpd.conf b/WebKitTools/Scripts/webkitpy/layout_tests/port/lighttpd.conf index 2e9c82e..26ca22f 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/lighttpd.conf +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/lighttpd.conf @@ -22,7 +22,7 @@ mimetype.assign = ( ".htm" => "text/html", ".xhtml" => "application/xhtml+xml", ".xhtmlmp" => "application/vnd.wap.xhtml+xml", - ".js" => "text/javascript", + ".js" => "application/x-javascript", ".log" => "text/plain", ".conf" => "text/plain", ".text" => "text/plain", diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/mac_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/mac_unittest.py index ae7d40c..327b19e 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/mac_unittest.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/mac_unittest.py @@ -26,14 +26,27 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import StringIO +import sys import unittest + import mac -import StringIO +import port_testcase + -class MacTest(unittest.TestCase): +class MacTest(port_testcase.PortTestCase): + def make_port(self, options=port_testcase.MockOptions()): + if sys.platform != 'darwin': + return None + port_obj = mac.MacPort(options=options) + port_obj._options.results_directory = port_obj.results_directory() + port_obj._options.configuration = 'Release' + return port_obj def test_skipped_file_paths(self): - port = mac.MacPort() + port = self.make_port() + if not port: + return skipped_paths = port._skipped_file_paths() # FIXME: _skipped_file_paths should return WebKit-relative paths. # So to make it unit testable, we strip the WebKit directory from the path. @@ -57,7 +70,9 @@ svg/batik/text/smallFonts.svg ] def test_skipped_file_paths(self): - port = mac.MacPort() + port = self.make_port() + if not port: + return skipped_file = StringIO.StringIO(self.example_skipped_file) self.assertEqual(port._tests_from_skipped_file(skipped_file), self.example_skipped_tests) diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py new file mode 100644 index 0000000..2d650f5 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py @@ -0,0 +1,88 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Unit testing base class for Port implementations.""" + +import os +import tempfile +import unittest + + +class MockOptions(object): + def __init__(self, + results_directory='layout-test-results', + use_apache=True, + configuration='Release'): + self.results_directory = results_directory + self.use_apache = use_apache + self.configuration = configuration + + +class PortTestCase(unittest.TestCase): + """Tests the WebKit port implementation.""" + def make_port(self, options=MockOptions()): + """Override in subclass.""" + raise NotImplementedError() + + def test_http_server(self): + port = self.make_port() + if not port: + return + port.start_http_server() + port.stop_http_server() + + def test_image_diff(self): + port = self.make_port() + if not port: + return + + # FIXME: not sure why this shouldn't always be True + #self.assertTrue(port.check_image_diff()) + if not port.check_image_diff(): + return + + dir = port.layout_tests_dir() + file1 = os.path.join(dir, 'fast', 'css', 'button_center.png') + file2 = os.path.join(dir, 'fast', 'css', + 'remove-shorthand-expected.png') + tmpfile = tempfile.mktemp() + + self.assertFalse(port.diff_image(file1, file1)) + self.assertTrue(port.diff_image(file1, file2)) + + self.assertTrue(port.diff_image(file1, file2, tmpfile)) + # FIXME: this may not be being written? + # self.assertTrue(os.path.exists(tmpfile)) + # os.remove(tmpfile) + + def test_websocket_server(self): + port = self.make_port() + if not port: + return + port.start_websocket_server() + port.stop_websocket_server() diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/qt.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/qt.py index 41b2ba0..158c633 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/qt.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/qt.py @@ -93,6 +93,12 @@ class QtPort(WebKitPort): def _path_to_driver(self): return self._build_path('bin/DumpRenderTree') + def _path_to_webcore_library(self): + return self._build_path('lib/libQtWebKit.so') + + def _runtime_feature_list(self): + return None + def setup_environ_for_server(self): env = webkit.WebKitPort.setup_environ_for_server(self) env['QTWEBKIT_PLUGIN_PATH'] = self._build_path('lib/plugins') diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py index 62ca693..8e0bc11 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py @@ -29,7 +29,6 @@ """Package that implements the ServerProcess wrapper class""" -import fcntl import logging import os import select @@ -37,6 +36,8 @@ import signal import subprocess import sys import time +if sys.platform != 'win32': + import fcntl from webkitpy.common.system.executive import Executive @@ -109,14 +110,6 @@ class ServerProcess: return self._proc.poll() return None - def returncode(self): - """Returns the exit code from the subprcoess; returns None if the - process hasn't exited (this is a wrapper around subprocess.returncode). - """ - if self._proc: - return self._proc.returncode - return None - def write(self, input): """Write a request to the subprocess. The subprocess is (re-)start()'ed if is not already running.""" diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py index e309334..a3a16c3 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/test.py @@ -76,6 +76,9 @@ class TestPort(base.Port): def options(self): return self._options + def skipped_layout_tests(self): + return [] + def path_to_test_expectations_file(self): return self.path_from_webkit_base('WebKitTools', 'Scripts', 'webkitpy', 'layout_tests', 'data', 'platform', 'test', @@ -145,9 +148,6 @@ class TestDriver(base.Driver): def poll(self): return True - def returncode(self): - return 0 - def run_test(self, uri, timeoutms, image_hash): basename = uri[(uri.rfind("/") + 1):uri.rfind(".html")] @@ -179,6 +179,7 @@ class TestDriver(base.Driver): raise ValueError('exception from ' + basename) crash = 'crash' in basename + timeout = 'timeout' in basename or 'hang' in basename timeout = 'timeout' in basename if 'text' in basename: output = basename + '_failed-txt\n' @@ -196,6 +197,9 @@ class TestDriver(base.Driver): f.write(basename + "-png\n") if 'checksum' in basename: checksum = basename + "_failed-checksum\n" + + if 'hang' in basename: + time.sleep((float(timeoutms) * 4) / 1000.0) else: crash = False timeout = False diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py index cedc028..b085ceb 100644 --- a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # Copyright (C) 2010 Google Inc. All rights reserved. +# Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are @@ -41,6 +42,9 @@ import signal import sys import time import webbrowser +import operator +import tempfile +import shutil from webkitpy.common.system.executive import Executive @@ -217,6 +221,66 @@ class WebKitPort(base.Port): "platform/win", ] + def _runtime_feature_list(self): + """Return the supported features of DRT. If a port doesn't support + this DRT switch, it has to override this method to return None""" + driver_path = self._path_to_driver() + feature_list = ' '.join(os.popen(driver_path + " --print-supported-features 2>&1").readlines()) + if "SupportedFeatures:" in feature_list: + return feature_list + return None + + def _supported_symbol_list(self): + """Return the supported symbols of WebCore.""" + webcore_library_path = self._path_to_webcore_library() + if not webcore_library_path: + return None + symbol_list = ' '.join(os.popen("nm " + webcore_library_path).readlines()) + return symbol_list + + def _directories_for_features(self): + """Return the supported feature dictionary. The keys are the + features and the values are the directories in lists.""" + directories_for_features = { + "Accelerated Compositing": ["compositing"], + "3D Rendering": ["animations/3d", "transforms/3d"], + } + return directories_for_features + + def _directories_for_symbols(self): + """Return the supported feature dictionary. The keys are the + symbols and the values are the directories in lists.""" + directories_for_symbol = { + "MathMLElement": ["mathml"], + "GraphicsLayer": ["compositing"], + "WebCoreHas3DRendering": ["animations/3d", "transforms/3d"], + "WebGLShader": ["fast/canvas/webgl"], + "WMLElement": ["http/tests/wml", "fast/wml", "wml"], + "parseWCSSInputProperty": ["fast/wcss"], + "isXHTMLMPDocument": ["fast/xhtmlmp"], + } + return directories_for_symbol + + def _skipped_tests_for_unsupported_features(self): + """Return the directories of unsupported tests. Search for the + symbols in the symbol_list, if found add the corresponding + directories to the skipped directory list.""" + feature_list = self._runtime_feature_list() + directories = self._directories_for_features() + + # if DRT feature detection not supported + if not feature_list: + feature_list = self._supported_symbol_list() + directories = self._directories_for_symbols() + + if not feature_list: + return [] + + skipped_directories = [directories[feature] + for feature in directories.keys() + if feature not in feature_list] + return reduce(operator.add, skipped_directories) + def _tests_for_disabled_features(self): # FIXME: This should use the feature detection from # webkitperl/features.pm to match run-webkit-tests. @@ -238,7 +302,8 @@ class WebKitPort(base.Port): "http/tests/webarchive", "svg/custom/image-with-prefix-in-webarchive.svg", ] - return disabled_feature_tests + webarchive_tests + unsupported_feature_tests = self._skipped_tests_for_unsupported_features() + return disabled_feature_tests + webarchive_tests + unsupported_feature_tests def _tests_from_skipped_file(self, skipped_file): tests_to_skip = [] @@ -279,14 +344,17 @@ class WebKitPort(base.Port): # This routine reads those files and turns contents into the # format expected by test_expectations. + tests_to_skip = self.skipped_layout_tests() + skip_lines = map(lambda test_path: "BUG_SKIPPED SKIP : %s = FAIL" % + test_path, tests_to_skip) + return "\n".join(skip_lines) + + def skipped_layout_tests(self): # Use a set to allow duplicates tests_to_skip = set(self._expectations_from_skipped_files()) - tests_to_skip.update(self._tests_for_other_platforms()) tests_to_skip.update(self._tests_for_disabled_features()) - skip_lines = map(lambda test_path: "BUG_SKIPPED SKIP : %s = FAIL" % - test_path, tests_to_skip) - return "\n".join(skip_lines) + return tests_to_skip def test_platform_name(self): return self._name + self.version() @@ -306,6 +374,9 @@ class WebKitPort(base.Port): def _path_to_driver(self): return self._build_path('DumpRenderTree') + def _path_to_webcore_library(self): + return None + def _path_to_helper(self): return None @@ -338,6 +409,10 @@ class WebKitDriver(base.Driver): self._port = port # FIXME: driver_options is never used. self._image_path = image_path + self._driver_tempdir = tempfile.mkdtemp(prefix='DumpRenderTree-') + + def __del__(self): + shutil.rmtree(self._driver_tempdir) def start(self): command = [] @@ -348,6 +423,7 @@ class WebKitDriver(base.Driver): command.append('--pixel-tests') environment = self._port.setup_environ_for_server() environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path() + environment['DUMPRENDERTREE_TEMP'] = self._driver_tempdir self._server_process = server_process.ServerProcess(self._port, "DumpRenderTree", command, environment) @@ -359,9 +435,6 @@ class WebKitDriver(base.Driver): self._server_process.start() return - def returncode(self): - return self._server_process.returncode() - # FIXME: This function is huge. def run_test(self, uri, timeoutms, image_hash): if uri.startswith("file:///"): diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py new file mode 100644 index 0000000..fbfadc3 --- /dev/null +++ b/WebKitTools/Scripts/webkitpy/layout_tests/port/webkit_unittest.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF SZEGED ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL UNIVERSITY OF SZEGED OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.layout_tests.port.webkit import WebKitPort + + +class TestWebKitPort(WebKitPort): + def __init__(self, symbol_list=None, feature_list=None): + self.symbol_list = symbol_list + self.feature_list = feature_list + + def _runtime_feature_list(self): + return self.feature_list + + def _supported_symbol_list(self): + return self.symbol_list + + def _tests_for_other_platforms(self): + return ["media", ] + + def _tests_for_disabled_features(self): + return ["accessibility", ] + + def _skipped_file_paths(self): + return [] + +class WebKitPortTest(unittest.TestCase): + + def test_skipped_directories_for_symbols(self): + supported_symbols = ["GraphicsLayer", "WebCoreHas3DRendering", "isXHTMLMPDocument", "fooSymbol"] + expected_directories = set(["mathml", "fast/canvas/webgl", "http/tests/wml", "fast/wml", "wml", "fast/wcss"]) + result_directories = set(TestWebKitPort(supported_symbols, None)._skipped_tests_for_unsupported_features()) + self.assertEqual(result_directories, expected_directories) + + def test_skipped_directories_for_features(self): + supported_features = ["Accelerated Compositing", "Foo Feature"] + expected_directories = set(["animations/3d", "transforms/3d"]) + result_directories = set(TestWebKitPort(None, supported_features)._skipped_tests_for_unsupported_features()) + self.assertEqual(result_directories, expected_directories) + + def test_skipped_layout_tests(self): + self.assertEqual(TestWebKitPort(None, None).skipped_layout_tests(), + set(["media", "accessibility"])) |