diff options
Diffstat (limited to 'Tools/Scripts/webkitpy/layout_tests/port/test.py')
-rw-r--r-- | Tools/Scripts/webkitpy/layout_tests/port/test.py | 343 |
1 files changed, 343 insertions, 0 deletions
diff --git a/Tools/Scripts/webkitpy/layout_tests/port/test.py b/Tools/Scripts/webkitpy/layout_tests/port/test.py new file mode 100644 index 0000000..935881c --- /dev/null +++ b/Tools/Scripts/webkitpy/layout_tests/port/test.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the Google name nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Dummy Port implementation used for testing.""" +from __future__ import with_statement + +import codecs +import fnmatch +import os +import sys +import time + +from webkitpy.layout_tests.layout_package import test_output + +import base + + +# This sets basic expectations for a test. Each individual expectation +# can be overridden by a keyword argument in TestList.add(). +class TestInstance: + def __init__(self, name): + self.name = name + self.base = name[(name.rfind("/") + 1):name.rfind(".html")] + self.crash = False + self.exception = False + self.hang = False + self.keyboard = False + self.error = '' + self.timeout = False + self.actual_text = self.base + '-txt\n' + self.actual_checksum = self.base + '-checksum\n' + self.actual_image = self.base + '-png\n' + self.expected_text = self.actual_text + self.expected_checksum = self.actual_checksum + self.expected_image = self.actual_image + + +# This is an in-memory list of tests, what we want them to produce, and +# what we want to claim are the expected results. +class TestList: + def __init__(self, port): + self.port = port + self.tests = {} + + def add(self, name, **kwargs): + test = TestInstance(name) + for key, value in kwargs.items(): + test.__dict__[key] = value + self.tests[name] = test + + def keys(self): + return self.tests.keys() + + def __contains__(self, item): + return item in self.tests + + def __getitem__(self, item): + return self.tests[item] + + +class TestPort(base.Port): + """Test implementation of the Port interface.""" + + def __init__(self, **kwargs): + base.Port.__init__(self, **kwargs) + tests = TestList(self) + tests.add('failures/expected/checksum.html', + actual_checksum='checksum_fail-checksum') + tests.add('failures/expected/crash.html', crash=True) + tests.add('failures/expected/exception.html', exception=True) + tests.add('failures/expected/timeout.html', timeout=True) + tests.add('failures/expected/hang.html', hang=True) + tests.add('failures/expected/missing_text.html', + expected_text=None) + tests.add('failures/expected/image.html', + actual_image='image_fail-png', + expected_image='image-png') + tests.add('failures/expected/image_checksum.html', + actual_checksum='image_checksum_fail-checksum', + actual_image='image_checksum_fail-png') + tests.add('failures/expected/keyboard.html', + keyboard=True) + tests.add('failures/expected/missing_check.html', + expected_checksum=None) + tests.add('failures/expected/missing_image.html', + expected_image=None) + tests.add('failures/expected/missing_text.html', + expected_text=None) + tests.add('failures/expected/newlines_leading.html', + expected_text="\nfoo\n", + actual_text="foo\n") + tests.add('failures/expected/newlines_trailing.html', + expected_text="foo\n\n", + actual_text="foo\n") + tests.add('failures/expected/newlines_with_excess_CR.html', + expected_text="foo\r\r\r\n", + actual_text="foo\n") + tests.add('failures/expected/text.html', + actual_text='text_fail-png') + tests.add('failures/unexpected/crash.html', crash=True) + tests.add('failures/unexpected/text-image-checksum.html', + actual_text='text-image-checksum_fail-txt', + actual_checksum='text-image-checksum_fail-checksum') + tests.add('failures/unexpected/timeout.html', timeout=True) + tests.add('http/tests/passes/text.html') + tests.add('http/tests/ssl/text.html') + tests.add('passes/error.html', error='stuff going to stderr') + tests.add('passes/image.html') + tests.add('passes/platform_image.html') + # Text output files contain "\r\n" on Windows. This may be + # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling. + tests.add('passes/text.html', + expected_text='\nfoo\n\n', + actual_text='\nfoo\r\n\r\r\n') + tests.add('websocket/tests/passes/text.html') + self._tests = tests + + def baseline_path(self): + return os.path.join(self.layout_tests_dir(), 'platform', + self.name() + self.version()) + + def baseline_search_path(self): + return [self.baseline_path()] + + def check_build(self, needs_http): + return True + + def diff_image(self, expected_contents, actual_contents, + diff_filename=None): + diffed = actual_contents != expected_contents + if diffed and diff_filename: + with codecs.open(diff_filename, "w", "utf-8") as diff_fh: + diff_fh.write("< %s\n---\n> %s\n" % + (expected_contents, actual_contents)) + return diffed + + def expected_checksum(self, test): + test = self.relative_test_filename(test) + return self._tests[test].expected_checksum + + def expected_image(self, test): + test = self.relative_test_filename(test) + return self._tests[test].expected_image + + def expected_text(self, test): + test = self.relative_test_filename(test) + text = self._tests[test].expected_text + if not text: + text = '' + return text + + def tests(self, paths): + # Test the idea of port-specific overrides for test lists. Also + # keep in memory to speed up the test harness. + if not paths: + paths = ['*'] + + matched_tests = [] + for p in paths: + if self.path_isdir(p): + matched_tests.extend(fnmatch.filter(self._tests.keys(), p + '*')) + else: + matched_tests.extend(fnmatch.filter(self._tests.keys(), p)) + layout_tests_dir = self.layout_tests_dir() + return set([os.path.join(layout_tests_dir, p) for p in matched_tests]) + + def path_exists(self, path): + # used by test_expectations.py and printing.py + rpath = self.relative_test_filename(path) + if rpath in self._tests: + return True + if self.path_isdir(rpath): + return True + if rpath.endswith('-expected.txt'): + test = rpath.replace('-expected.txt', '.html') + return (test in self._tests and + self._tests[test].expected_text) + if rpath.endswith('-expected.checksum'): + test = rpath.replace('-expected.checksum', '.html') + return (test in self._tests and + self._tests[test].expected_checksum) + if rpath.endswith('-expected.png'): + test = rpath.replace('-expected.png', '.html') + return (test in self._tests and + self._tests[test].expected_image) + return False + + def layout_tests_dir(self): + return self.path_from_webkit_base('Tools', 'Scripts', + 'webkitpy', 'layout_tests', 'data') + + def path_isdir(self, path): + # Used by test_expectations.py + # + # We assume that a path is a directory if we have any tests + # that whose prefix matches the path plus a directory modifier + # and not a file extension. + if path[-1] != '/': + path += '/' + + # FIXME: Directories can have a dot in the name. We should + # probably maintain a white list of known cases like CSS2.1 + # and check it here in the future. + if path.find('.') != -1: + # extension separator found, assume this is a file + return False + + # strip out layout tests directory path if found. The tests + # keys are relative to it. + tests_dir = self.layout_tests_dir() + if path.startswith(tests_dir): + path = path[len(tests_dir) + 1:] + + return any([t.startswith(path) for t in self._tests.keys()]) + + def test_dirs(self): + return ['passes', 'failures'] + + def name(self): + return self._name + + def _path_to_wdiff(self): + return None + + def results_directory(self): + return '/tmp/' + self.get_option('results_directory') + + def setup_test_run(self): + pass + + def create_driver(self, worker_number): + return TestDriver(self, worker_number) + + def start_http_server(self): + pass + + def start_websocket_server(self): + pass + + def stop_http_server(self): + pass + + def stop_websocket_server(self): + pass + + def test_expectations(self): + """Returns the test expectations for this port. + + Basically this string should contain the equivalent of a + test_expectations file. See test_expectations.py for more details.""" + return """ +WONTFIX : failures/expected/checksum.html = IMAGE +WONTFIX : failures/expected/crash.html = CRASH +// This one actually passes because the checksums will match. +WONTFIX : failures/expected/image.html = PASS +WONTFIX : failures/expected/image_checksum.html = IMAGE +WONTFIX : failures/expected/missing_check.html = MISSING PASS +WONTFIX : failures/expected/missing_image.html = MISSING PASS +WONTFIX : failures/expected/missing_text.html = MISSING PASS +WONTFIX : failures/expected/newlines_leading.html = TEXT +WONTFIX : failures/expected/newlines_trailing.html = TEXT +WONTFIX : failures/expected/newlines_with_excess_CR.html = TEXT +WONTFIX : failures/expected/text.html = TEXT +WONTFIX : failures/expected/timeout.html = TIMEOUT +WONTFIX SKIP : failures/expected/hang.html = TIMEOUT +WONTFIX SKIP : failures/expected/keyboard.html = CRASH +WONTFIX SKIP : failures/expected/exception.html = CRASH +""" + + def test_base_platform_names(self): + return ('mac', 'win') + + def test_platform_name(self): + return 'mac' + + def test_platform_names(self): + return self.test_base_platform_names() + + def test_platform_name_to_name(self, test_platform_name): + return test_platform_name + + def version(self): + return '' + + +class TestDriver(base.Driver): + """Test/Dummy implementation of the DumpRenderTree interface.""" + + def __init__(self, port, worker_number): + self._port = port + + def cmd_line(self): + return ['None'] + + def poll(self): + return True + + def run_test(self, test_input): + start_time = time.time() + test_name = self._port.relative_test_filename(test_input.filename) + test = self._port._tests[test_name] + if test.keyboard: + raise KeyboardInterrupt + if test.exception: + raise ValueError('exception from ' + test_name) + if test.hang: + time.sleep((float(test_input.timeout) * 4) / 1000.0) + return test_output.TestOutput(test.actual_text, test.actual_image, + test.actual_checksum, test.crash, + time.time() - start_time, test.timeout, + test.error) + + def start(self): + pass + + def stop(self): + pass |