diff options
Diffstat (limited to 'Tools/Scripts/webkitpy/layout_tests/test_types')
5 files changed, 509 insertions, 0 deletions
diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/__init__.py b/Tools/Scripts/webkitpy/layout_tests/test_types/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/Tools/Scripts/webkitpy/layout_tests/test_types/__init__.py diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/image_diff.py b/Tools/Scripts/webkitpy/layout_tests/test_types/image_diff.py new file mode 100644 index 0000000..da466c8 --- /dev/null +++ b/Tools/Scripts/webkitpy/layout_tests/test_types/image_diff.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Compares the image output of a test to the expected image output. + +Compares hashes for the generated and expected images. If the output doesn't +match, returns FailureImageHashMismatch and outputs both hashes into the layout +test results directory. +""" + +from __future__ import with_statement + +import codecs +import errno +import logging +import os +import shutil + +from webkitpy.layout_tests.layout_package import test_failures +from webkitpy.layout_tests.test_types import test_type_base + +# Cache whether we have the image_diff executable available. +_compare_available = True +_compare_msg_printed = False + +_log = logging.getLogger("webkitpy.layout_tests.test_types.image_diff") + + +class ImageDiff(test_type_base.TestTypeBase): + + def _save_baseline_files(self, filename, image, image_hash, + generate_new_baseline): + """Saves new baselines for the PNG and checksum. + + Args: + filename: test filename + image: a image output + image_hash: a checksum of the image + generate_new_baseline: whether to generate a new, platform-specific + baseline, or update the existing one + """ + self._save_baseline_data(filename, image, ".png", encoding=None, + generate_new_baseline=generate_new_baseline) + self._save_baseline_data(filename, image_hash, ".checksum", + encoding="ascii", + generate_new_baseline=generate_new_baseline) + + def _copy_image(self, filename, actual_image, expected_image): + self.write_output_files(filename, '.png', + output=actual_image, expected=expected_image, + encoding=None, print_text_diffs=False) + + def _copy_image_hash(self, filename, actual_image_hash, expected_image_hash): + self.write_output_files(filename, '.checksum', + actual_image_hash, expected_image_hash, + encoding="ascii", print_text_diffs=False) + + def _create_diff_image(self, port, filename, actual_image, expected_image): + """Creates the visual diff of the expected/actual PNGs. + + Returns True if the images are different. + """ + diff_filename = self.output_filename(filename, + self.FILENAME_SUFFIX_COMPARE) + return port.diff_image(actual_image, expected_image, diff_filename) + + def compare_output(self, port, filename, test_args, actual_test_output, + expected_test_output): + """Implementation of CompareOutput that checks the output image and + checksum against the expected files from the LayoutTest directory. + """ + failures = [] + + # If we didn't produce a hash file, this test must be text-only. + if actual_test_output.image_hash is None: + return failures + + # If we're generating a new baseline, we pass. + if test_args.new_baseline or test_args.reset_results: + self._save_baseline_files(filename, actual_test_output.image, + actual_test_output.image_hash, + test_args.new_baseline) + return failures + + if not expected_test_output.image: + # Report a missing expected PNG file. + self._copy_image(filename, actual_test_output.image, expected_image=None) + self._copy_image_hash(filename, actual_test_output.image_hash, + expected_test_output.image_hash) + failures.append(test_failures.FailureMissingImage()) + return failures + if not expected_test_output.image_hash: + # Report a missing expected checksum file. + self._copy_image(filename, actual_test_output.image, + expected_test_output.image) + self._copy_image_hash(filename, actual_test_output.image_hash, + expected_image_hash=None) + failures.append(test_failures.FailureMissingImageHash()) + return failures + + if actual_test_output.image_hash == expected_test_output.image_hash: + # Hash matched (no diff needed, okay to return). + return failures + + self._copy_image(filename, actual_test_output.image, + expected_test_output.image) + self._copy_image_hash(filename, actual_test_output.image_hash, + expected_test_output.image_hash) + + # Even though we only use the result in one codepath below but we + # still need to call CreateImageDiff for other codepaths. + images_are_different = self._create_diff_image(port, filename, + actual_test_output.image, + expected_test_output.image) + if not images_are_different: + failures.append(test_failures.FailureImageHashIncorrect()) + else: + failures.append(test_failures.FailureImageHashMismatch()) + + return failures diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py b/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py new file mode 100644 index 0000000..4b96b3a --- /dev/null +++ b/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Defines the interface TestTypeBase which other test types inherit from. + +Also defines the TestArguments "struct" to pass them additional arguments. +""" + +from __future__ import with_statement + +import codecs +import cgi +import errno +import logging +import os.path + +_log = logging.getLogger("webkitpy.layout_tests.test_types.test_type_base") + + +class TestArguments(object): + """Struct-like wrapper for additional arguments needed by + specific tests.""" + # Whether to save new baseline results. + new_baseline = False + + # Path to the actual PNG file generated by pixel tests + png_path = None + + # Value of checksum generated by pixel tests. + hash = None + + # Whether to use wdiff to generate by-word diffs. + wdiff = False + +# Python bug workaround. See the wdiff code in WriteOutputFiles for an +# explanation. +_wdiff_available = True + + +class TestTypeBase(object): + + # Filename pieces when writing failures to the test results directory. + FILENAME_SUFFIX_ACTUAL = "-actual" + FILENAME_SUFFIX_EXPECTED = "-expected" + FILENAME_SUFFIX_DIFF = "-diff" + FILENAME_SUFFIX_WDIFF = "-wdiff.html" + FILENAME_SUFFIX_PRETTY_PATCH = "-pretty-diff.html" + FILENAME_SUFFIX_COMPARE = "-diff.png" + + def __init__(self, port, root_output_dir): + """Initialize a TestTypeBase object. + + Args: + port: object implementing port-specific information and methods + root_output_dir: The unix style path to the output dir. + """ + self._root_output_dir = root_output_dir + self._port = port + + def _make_output_directory(self, filename): + """Creates the output directory (if needed) for a given test + filename.""" + output_filename = os.path.join(self._root_output_dir, + self._port.relative_test_filename(filename)) + self._port.maybe_make_directory(os.path.split(output_filename)[0]) + + def _save_baseline_data(self, filename, data, modifier, encoding, + generate_new_baseline=True): + """Saves a new baseline file into the port's baseline directory. + + The file will be named simply "<test>-expected<modifier>", suitable for + use as the expected results in a later run. + + Args: + filename: path to the test file + data: result to be saved as the new baseline + modifier: type of the result file, e.g. ".txt" or ".png" + encoding: file encoding (none, "utf-8", etc.) + generate_new_baseline: whether to enerate a new, platform-specific + baseline, or update the existing one + """ + + if generate_new_baseline: + relative_dir = os.path.dirname( + self._port.relative_test_filename(filename)) + baseline_path = self._port.baseline_path() + output_dir = os.path.join(baseline_path, relative_dir) + output_file = os.path.basename(os.path.splitext(filename)[0] + + self.FILENAME_SUFFIX_EXPECTED + modifier) + self._port.maybe_make_directory(output_dir) + output_path = os.path.join(output_dir, output_file) + _log.debug('writing new baseline result "%s"' % (output_path)) + else: + output_path = self._port.expected_filename(filename, modifier) + _log.debug('resetting baseline result "%s"' % output_path) + + self._port.update_baseline(output_path, data, encoding) + + def output_filename(self, filename, modifier): + """Returns a filename inside the output dir that contains modifier. + + For example, if filename is c:/.../fast/dom/foo.html and modifier is + "-expected.txt", the return value is + c:/cygwin/tmp/layout-test-results/fast/dom/foo-expected.txt + + Args: + filename: absolute filename to test file + modifier: a string to replace the extension of filename with + + Return: + The absolute windows path to the output filename + """ + output_filename = os.path.join(self._root_output_dir, + self._port.relative_test_filename(filename)) + return os.path.splitext(output_filename)[0] + modifier + + def compare_output(self, port, filename, test_args, actual_test_output, + expected_test_output): + """Method that compares the output from the test with the + expected value. + + This is an abstract method to be implemented by all sub classes. + + Args: + port: object implementing port-specific information and methods + filename: absolute filename to test file + test_args: a TestArguments object holding optional additional + arguments + actual_test_output: a TestOutput object which represents actual test + output + expected_test_output: a TestOutput object which represents a expected + test output + + Return: + a list of TestFailure objects, empty if the test passes + """ + raise NotImplementedError + + def _write_into_file_at_path(self, file_path, contents, encoding): + """This method assumes that byte_array is already encoded + into the right format.""" + open_mode = 'w' + if encoding is None: + open_mode = 'w+b' + with codecs.open(file_path, open_mode, encoding=encoding) as file: + file.write(contents) + + def write_output_files(self, filename, file_type, + output, expected, encoding, + print_text_diffs=False): + """Writes the test output, the expected output and optionally the diff + between the two to files in the results directory. + + The full output filename of the actual, for example, will be + <filename>-actual<file_type> + For instance, + my_test-actual.txt + + Args: + filename: The test filename + file_type: A string describing the test output file type, e.g. ".txt" + output: A string containing the test output + expected: A string containing the expected test output + print_text_diffs: True for text diffs. (FIXME: We should be able to get this from the file type?) + """ + self._make_output_directory(filename) + actual_filename = self.output_filename(filename, self.FILENAME_SUFFIX_ACTUAL + file_type) + expected_filename = self.output_filename(filename, self.FILENAME_SUFFIX_EXPECTED + file_type) + # FIXME: This function is poorly designed. We should be passing in some sort of + # encoding information from the callers. + if output: + self._write_into_file_at_path(actual_filename, output, encoding) + if expected: + self._write_into_file_at_path(expected_filename, expected, encoding) + + if not output or not expected: + return + + if not print_text_diffs: + return + + # Note: We pass encoding=None for all diff writes, as we treat diff + # output as binary. Diff output may contain multiple files in + # conflicting encodings. + diff = self._port.diff_text(expected, output, expected_filename, actual_filename) + diff_filename = self.output_filename(filename, self.FILENAME_SUFFIX_DIFF + file_type) + self._write_into_file_at_path(diff_filename, diff, encoding=None) + + # Shell out to wdiff to get colored inline diffs. + wdiff = self._port.wdiff_text(expected_filename, actual_filename) + wdiff_filename = self.output_filename(filename, self.FILENAME_SUFFIX_WDIFF) + self._write_into_file_at_path(wdiff_filename, wdiff, encoding=None) + + # Use WebKit's PrettyPatch.rb to get an HTML diff. + pretty_patch = self._port.pretty_patch_text(diff_filename) + pretty_patch_filename = self.output_filename(filename, self.FILENAME_SUFFIX_PRETTY_PATCH) + self._write_into_file_at_path(pretty_patch_filename, pretty_patch, encoding=None) diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py b/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py new file mode 100644 index 0000000..5dbfcb6 --- /dev/null +++ b/Tools/Scripts/webkitpy/layout_tests/test_types/test_type_base_unittest.py @@ -0,0 +1,47 @@ +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +""""Tests stray tests not covered by regular code paths.""" + +import test_type_base +import unittest + +from webkitpy.thirdparty.mock import Mock + + +class Test(unittest.TestCase): + + def test_compare_output_notimplemented(self): + test_type = test_type_base.TestTypeBase(None, None) + self.assertRaises(NotImplementedError, test_type.compare_output, + None, "foo.txt", '', + test_type_base.TestArguments(), 'Debug') + + +if __name__ == '__main__': + unittest.main() diff --git a/Tools/Scripts/webkitpy/layout_tests/test_types/text_diff.py b/Tools/Scripts/webkitpy/layout_tests/test_types/text_diff.py new file mode 100644 index 0000000..ad25262 --- /dev/null +++ b/Tools/Scripts/webkitpy/layout_tests/test_types/text_diff.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# Copyright (C) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Compares the text output of a test to the expected text output. + +If the output doesn't match, returns FailureTextMismatch and outputs the diff +files into the layout test results directory. +""" + +from __future__ import with_statement + +import codecs +import errno +import logging +import os.path + +from webkitpy.layout_tests.layout_package import test_failures +from webkitpy.layout_tests.test_types import test_type_base + +_log = logging.getLogger("webkitpy.layout_tests.test_types.text_diff") + + +class TestTextDiff(test_type_base.TestTypeBase): + + def _get_normalized_output_text(self, output): + """Returns the normalized text output, i.e. the output in which + the end-of-line characters are normalized to "\n".""" + # Running tests on Windows produces "\r\n". The "\n" part is helpfully + # changed to "\r\n" by our system (Python/Cygwin), resulting in + # "\r\r\n", when, in fact, we wanted to compare the text output with + # the normalized text expectation files. + return output.replace("\r\r\n", "\r\n").replace("\r\n", "\n") + + def compare_output(self, port, filename, test_args, actual_test_output, + expected_test_output): + """Implementation of CompareOutput that checks the output text against + the expected text from the LayoutTest directory.""" + failures = [] + + # If we're generating a new baseline, we pass. + if test_args.new_baseline or test_args.reset_results: + # Although all test_shell/DumpRenderTree output should be utf-8, + # we do not ever decode it inside run-webkit-tests. For some tests + # DumpRenderTree may not output utf-8 text (e.g. webarchives). + self._save_baseline_data(filename, actual_test_output.text, + ".txt", encoding=None, + generate_new_baseline=test_args.new_baseline) + return failures + + # Normalize text to diff + actual_text = self._get_normalized_output_text(actual_test_output.text) + # Assuming expected_text is already normalized. + expected_text = expected_test_output.text + + # Write output files for new tests, too. + if port.compare_text(actual_text, expected_text): + # Text doesn't match, write output files. + self.write_output_files(filename, ".txt", actual_text, + expected_text, encoding=None, + print_text_diffs=True) + + if expected_text == '': + failures.append(test_failures.FailureMissingResult()) + else: + failures.append(test_failures.FailureTextMismatch()) + + return failures |