path: root/Tools/Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py

#!/usr/bin/python
# Copyright (C) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Unit tests for test_expectations.py."""

import unittest

from webkitpy.layout_tests import port
from webkitpy.layout_tests.port import base
from webkitpy.layout_tests.layout_package.test_expectations import *

class FunctionsTest(unittest.TestCase):
    def test_result_was_expected(self):
        # test basics
        self.assertEquals(result_was_expected(PASS, set([PASS]),
                                              False, False), True)
        self.assertEquals(result_was_expected(TEXT, set([PASS]),
                                              False, False), False)

        # test handling of FAIL expectations
        self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]),
                                              False, False), True)
        self.assertEquals(result_was_expected(IMAGE, set([FAIL]),
                                              False, False), True)
        self.assertEquals(result_was_expected(TEXT, set([FAIL]),
                                              False, False), True)
        self.assertEquals(result_was_expected(CRASH, set([FAIL]),
                                              False, False), False)

        # test handling of SKIPped tests and results
        self.assertEquals(result_was_expected(SKIP, set([CRASH]),
                                              False, True), True)
        self.assertEquals(result_was_expected(SKIP, set([CRASH]),
                                              False, False), False)

        # test handling of MISSING results and the REBASELINE modifier
        self.assertEquals(result_was_expected(MISSING, set([PASS]),
                                              True, False), True)
        self.assertEquals(result_was_expected(MISSING, set([PASS]),
                                              False, False), False)

    def test_remove_pixel_failures(self):
        self.assertEquals(remove_pixel_failures(set([TEXT])),
                          set([TEXT]))
        self.assertEquals(remove_pixel_failures(set([PASS])),
                          set([PASS]))
        self.assertEquals(remove_pixel_failures(set([IMAGE])),
                          set([PASS]))
        self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])),
                          set([TEXT]))
        self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
                          set([PASS, CRASH]))


class Base(unittest.TestCase):
    # Note that all of these tests are written assuming the configuration
    # being tested is Windows XP, Release build.

    def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
        self._port = port.get('test-win-xp', None)
        self._fs = self._port._filesystem
        self._exp = None
        unittest.TestCase.__init__(self, testFunc)

    def get_test(self, test_name):
        return self._fs.join(self._port.layout_tests_dir(), test_name)

    def get_basic_tests(self):
        return [self.get_test('failures/expected/text.html'),
                self.get_test('failures/expected/image_checksum.html'),
                self.get_test('failures/expected/crash.html'),
                self.get_test('failures/expected/missing_text.html'),
                self.get_test('failures/expected/image.html'),
                self.get_test('passes/text.html')]

    def get_basic_expectations(self):
        return """
BUG_TEST : failures/expected/text.html = TEXT
BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE
"""

    def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
        test_config = self._port.test_configuration()
        self._exp = TestExpectations(self._port,
             tests=self.get_basic_tests(),
             expectations=expectations,
             test_config=test_config,
             is_lint_mode=is_lint_mode,
             overrides=overrides)

    def assert_exp(self, test, result):
        self.assertEquals(self._exp.get_expectations(self.get_test(test)),
                          set([result]))


class BasicTests(Base):
    def test_basic(self):
        self.parse_exp(self.get_basic_expectations())
        self.assert_exp('failures/expected/text.html', TEXT)
        self.assert_exp('failures/expected/image_checksum.html', IMAGE)
        self.assert_exp('passes/text.html', PASS)
        self.assert_exp('failures/expected/image.html', PASS)


class MiscTests(Base):
    def test_multiple_results(self):
        self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
        self.assertEqual(self._exp.get_expectations(
            self.get_test('failures/expected/text.html')),
            set([TEXT, CRASH]))

    def test_category_expectations(self):
        # This test checks unknown tests are not present in the
        # expectations and that known test part of a test category is
        # present in the expectations.
        exp_str = """
BUGX WONTFIX : failures/expected = IMAGE
"""
        self.parse_exp(exp_str)
        test_name = 'failures/expected/unknown-test.html'
        unknown_test = self.get_test(test_name)
        self.assertRaises(KeyError, self._exp.get_expectations,
                          unknown_test)
        self.assert_exp('failures/expected/crash.html', IMAGE)

    def test_get_options(self):
        self.parse_exp(self.get_basic_expectations())
        self.assertEqual(self._exp.get_options(
                         self.get_test('passes/text.html')), [])

    def test_expectations_json_for_all_platforms(self):
        self.parse_exp(self.get_basic_expectations())
        json_str = self._exp.get_expectations_json_for_all_platforms()
        # FIXME: test actual content?
        self.assertTrue(json_str)

    def test_get_expectations_string(self):
        self.parse_exp(self.get_basic_expectations())
        self.assertEquals(self._exp.get_expectations_string(
                          self.get_test('failures/expected/text.html')),
                          'TEXT')

    def test_expectation_to_string(self):
        # Normal cases are handled by other tests.
        self.parse_exp(self.get_basic_expectations())
        self.assertRaises(ValueError, self._exp.expectation_to_string,
                          -1)

    def test_get_test_set(self):
        # Handle some corner cases for this routine not covered by other tests.
        self.parse_exp(self.get_basic_expectations())
        s = self._exp._expected_failures.get_test_set(WONTFIX)
        self.assertEqual(s,
            set([self.get_test('failures/expected/crash.html'),
                 self.get_test('failures/expected/image_checksum.html')]))
        s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH)
        self.assertEqual(s,
            set([self.get_test('failures/expected/crash.html')]))
        s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH,
                                                      include_skips=False)
        self.assertEqual(s, set([]))

    def test_parse_error_fatal(self):
        try:
            self.parse_exp("""FOO : failures/expected/text.html = TEXT
SKIP : failures/expected/image.html""")
            self.assertFalse(True, "ParseError wasn't raised")
        except ParseError, e:
            self.assertTrue(e.fatal)
            exp_errors = [u"Line:1 Unrecognized option 'foo' failures/expected/text.html",
                          u"Line:2 Missing expectations. [' failures/expected/image.html']"]
            self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
            self.assertEqual(e.errors, exp_errors)

    def test_parse_error_nonfatal(self):
        try:
            self.parse_exp('SKIP : failures/expected/text.html = TEXT',
                           is_lint_mode=True)
            self.assertFalse(True, "ParseError wasn't raised")
        except ParseError, e:
            self.assertFalse(e.fatal)
            exp_errors = [u'Line:1 Test lacks BUG modifier. failures/expected/text.html']
            self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
            self.assertEqual(e.errors, exp_errors)

    def test_overrides(self):
        self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
                       "BUG_OVERRIDE : failures/expected/text.html = IMAGE")
        self.assert_exp('failures/expected/text.html', IMAGE)

    def test_overrides__duplicate(self):
        self.assertRaises(ParseError, self.parse_exp,
             "BUG_EXP: failures/expected/text.html = TEXT",
             """
BUG_OVERRIDE : failures/expected/text.html = IMAGE
BUG_OVERRIDE : failures/expected/text.html = CRASH
""")

    def test_pixel_tests_flag(self):
        def match(test, result, pixel_tests_enabled):
            return self._exp.matches_an_expected_result(
                self.get_test(test), result, pixel_tests_enabled)

        self.parse_exp(self.get_basic_expectations())
        self.assertTrue(match('failures/expected/text.html', TEXT, True))
        self.assertTrue(match('failures/expected/text.html', TEXT, False))
        self.assertFalse(match('failures/expected/text.html', CRASH, True))
        self.assertFalse(match('failures/expected/text.html', CRASH, False))
        self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
                              True))
        self.assertTrue(match('failures/expected/image_checksum.html', PASS,
                              False))
        self.assertTrue(match('failures/expected/crash.html', SKIP, False))
        self.assertTrue(match('passes/text.html', PASS, False))

    def test_more_specific_override_resets_skip(self):
        self.parse_exp("BUGX SKIP : failures/expected = TEXT\n"
                       "BUGX : failures/expected/text.html = IMAGE\n")
        self.assert_exp('failures/expected/text.html', IMAGE)
        self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
                                                     'failures/expected/text.html') in
                         self._exp.get_tests_with_result_type(SKIP))

class ExpectationSyntaxTests(Base):
    def test_missing_expectation(self):
        # This is missing the expectation.
        self.assertRaises(ParseError, self.parse_exp,
                          'BUG_TEST: failures/expected/text.html')

    def test_missing_colon(self):
        # This is missing the modifiers and the ':'
        self.assertRaises(ParseError, self.parse_exp,
                          'failures/expected/text.html = TEXT')

    def disabled_test_too_many_colons(self):
        # FIXME: Enable this test and fix the underlying bug.
        self.assertRaises(ParseError, self.parse_exp,
                          'BUG_TEST: failures/expected/text.html = PASS :')

    def test_too_many_equals_signs(self):
        self.assertRaises(ParseError, self.parse_exp,
                          'BUG_TEST: failures/expected/text.html = TEXT = IMAGE')

    def test_unrecognized_expectation(self):
        self.assertRaises(ParseError, self.parse_exp,
                          'BUG_TEST: failures/expected/text.html = UNKNOWN')

    def test_macro(self):
        exp_str = """
BUG_TEST WIN-XP : failures/expected/text.html = TEXT
"""
        self.parse_exp(exp_str)
        self.assert_exp('failures/expected/text.html', TEXT)


class SemanticTests(Base):
    def test_bug_format(self):
        self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT')

    def test_missing_bugid(self):
        # This should log a non-fatal error.
        self.parse_exp('SLOW : failures/expected/text.html = TEXT')
        self.assertEqual(
            len(self._exp._expected_failures.get_non_fatal_errors()), 1)

    def test_slow_and_timeout(self):
        # A test cannot be SLOW and expected to TIMEOUT.
        self.assertRaises(ParseError, self.parse_exp,
            'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT')

    def test_rebaseline(self):
        # Can't lint a file w/ 'REBASELINE' in it.
        self.assertRaises(ParseError, self.parse_exp,
            'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
            is_lint_mode=True)

    def test_duplicates(self):
        self.assertRaises(ParseError, self.parse_exp, """
BUG_EXP : failures/expected/text.html = TEXT
BUG_EXP : failures/expected/text.html = IMAGE""")

        self.assertRaises(ParseError, self.parse_exp,
            self.get_basic_expectations(), overrides="""
BUG_OVERRIDE : failures/expected/text.html = TEXT
BUG_OVERRIDE : failures/expected/text.html = IMAGE""", )

    def test_missing_file(self):
        # This should log a non-fatal error.
        self.parse_exp('BUG_TEST : missing_file.html = TEXT')
        self.assertEqual(
            len(self._exp._expected_failures.get_non_fatal_errors()), 1)


class PrecedenceTests(Base):
    def test_file_over_directory(self):
        # This tests handling precedence of specific lines over directories
        # and tests expectations covering entire directories.
        exp_str = """
BUGX : failures/expected/text.html = TEXT
BUGX WONTFIX : failures/expected = IMAGE
"""
        self.parse_exp(exp_str)
        self.assert_exp('failures/expected/text.html', TEXT)
        self.assert_exp('failures/expected/crash.html', IMAGE)

        exp_str = """
BUGX WONTFIX : failures/expected = IMAGE
BUGX : failures/expected/text.html = TEXT
"""
        self.parse_exp(exp_str)
        self.assert_exp('failures/expected/text.html', TEXT)
        self.assert_exp('failures/expected/crash.html', IMAGE)

    def test_ambiguous(self):
        self.assertRaises(ParseError, self.parse_exp, """
BUG_TEST RELEASE : passes/text.html = PASS
BUG_TEST WIN : passes/text.html = FAIL
""")

    def test_more_modifiers(self):
        exp_str = """
BUG_TEST RELEASE : passes/text.html = PASS
BUG_TEST WIN RELEASE : passes/text.html = TEXT
"""
        self.assertRaises(ParseError, self.parse_exp, exp_str)

    def test_order_in_file(self):
        exp_str = """
BUG_TEST WIN RELEASE : passes/text.html = TEXT
BUG_TEST RELEASE : passes/text.html = PASS
"""
        self.assertRaises(ParseError, self.parse_exp, exp_str)

    def test_version_overrides(self):
        exp_str = """
BUG_TEST WIN : passes/text.html = PASS
BUG_TEST WIN XP : passes/text.html = TEXT
"""
        self.assertRaises(ParseError, self.parse_exp, exp_str)

    def test_macro_overrides(self):
        exp_str = """
BUG_TEST WIN : passes/text.html = PASS
BUG_TEST WIN-XP : passes/text.html = TEXT
"""
        self.assertRaises(ParseError, self.parse_exp, exp_str)


class RebaseliningTest(Base):
    """Test rebaselining-specific functionality."""
    def assertRemove(self, input_expectations, tests, expected_expectations):
        self.parse_exp(input_expectations)
        actual_expectations = self._exp.remove_rebaselined_tests(tests)
        self.assertEqual(expected_expectations, actual_expectations)

    def test_remove(self):
        self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n'
                          'BUGY : failures/expected/image.html = IMAGE\n'
                          'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
                          ['failures/expected/text.html'],
                          'BUGY : failures/expected/image.html = IMAGE\n'
                          'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n')

    def test_no_get_rebaselining_failures(self):
        self.parse_exp(self.get_basic_expectations())
        self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)


class ModifierTests(unittest.TestCase):
    def setUp(self):
        port_obj = port.get('test-win-xp', None)
        self.config = port_obj.test_configuration()
        self.matcher = ModifierMatcher(self.config)

    def match(self, modifiers, expected_num_matches=-1, values=None, num_errors=0):
        matcher = self.matcher
        if values:
            matcher = ModifierMatcher(self.FakeTestConfiguration(values))
        match_result = matcher.match(modifiers)
        self.assertEqual(len(match_result.warnings), 0)
        self.assertEqual(len(match_result.errors), num_errors)
        self.assertEqual(match_result.num_matches, expected_num_matches,
             'match(%s, %s) returned -> %d, expected %d' %
             (modifiers, str(self.config.values()),
              match_result.num_matches, expected_num_matches))

    def test_bad_match_modifier(self):
        self.match(['foo'], num_errors=1)

    def test_none(self):
        self.match([], 0)

    def test_one(self):
        self.match(['xp'], 1)
        self.match(['win'], 1)
        self.match(['release'], 1)
        self.match(['cpu'], 1)
        self.match(['x86'], 1)
        self.match(['leopard'], -1)
        self.match(['gpu'], -1)
        self.match(['debug'], -1)

    def test_two(self):
        self.match(['xp', 'release'], 2)
        self.match(['win7', 'release'], -1)
        self.match(['win7', 'xp'], 1)

    def test_three(self):
        self.match(['win7', 'xp', 'release'], 2)
        self.match(['xp', 'debug', 'x86'], -1)
        self.match(['xp', 'release', 'x86'], 3)
        self.match(['xp', 'cpu', 'release'], 3)

    def test_four(self):
        self.match(['xp', 'release', 'cpu', 'x86'], 4)
        self.match(['win7', 'xp', 'release', 'cpu'], 3)
        self.match(['win7', 'xp', 'debug', 'cpu'], -1)

    def test_case_insensitivity(self):
        self.match(['Win'], num_errors=1)
        self.match(['WIN'], num_errors=1)
        self.match(['win'], 1)

    def test_duplicates(self):
        self.match(['release', 'release'], num_errors=1)
        self.match(['win-xp', 'xp'], num_errors=1)
        self.match(['win-xp', 'win-xp'], num_errors=1)
        self.match(['xp', 'release', 'xp', 'release'], num_errors=2)
        self.match(['rebaseline', 'rebaseline'], num_errors=1)

    def test_unknown_option(self):
        self.match(['vms'], num_errors=1)

    def test_duplicate_bugs(self):
        # BUG* regexes can appear multiple times.
        self.match(['bugfoo', 'bugbar'], 0)

    def test_invalid_combinations(self):
        # FIXME: This should probably raise an error instead of NO_MATCH.
        self.match(['mac', 'xp'], num_errors=0)

    def test_regexes_are_ignored(self):
        self.match(['bug123xy', 'rebaseline', 'wontfix', 'slow', 'skip'], 0)

    def test_none_is_invalid(self):
        self.match(['none'], num_errors=1)


if __name__ == '__main__':
    unittest.main()