#!/usr/bin/python # Copyright (C) 2010 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """Unit tests for test_expectations.py.""" import unittest from webkitpy.layout_tests import port from webkitpy.layout_tests.port import base from webkitpy.layout_tests.layout_package.test_expectations import * class FunctionsTest(unittest.TestCase): def test_result_was_expected(self): # test basics self.assertEquals(result_was_expected(PASS, set([PASS]), False, False), True) self.assertEquals(result_was_expected(TEXT, set([PASS]), False, False), False) # test handling of FAIL expectations self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]), False, False), True) self.assertEquals(result_was_expected(IMAGE, set([FAIL]), False, False), True) self.assertEquals(result_was_expected(TEXT, set([FAIL]), False, False), True) self.assertEquals(result_was_expected(CRASH, set([FAIL]), False, False), False) # test handling of SKIPped tests and results self.assertEquals(result_was_expected(SKIP, set([CRASH]), False, True), True) self.assertEquals(result_was_expected(SKIP, set([CRASH]), False, False), False) # test handling of MISSING results and the REBASELINE modifier self.assertEquals(result_was_expected(MISSING, set([PASS]), True, False), True) self.assertEquals(result_was_expected(MISSING, set([PASS]), False, False), False) def test_remove_pixel_failures(self): self.assertEquals(remove_pixel_failures(set([TEXT])), set([TEXT])) self.assertEquals(remove_pixel_failures(set([PASS])), set([PASS])) self.assertEquals(remove_pixel_failures(set([IMAGE])), set([PASS])) self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])), set([TEXT])) self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])), set([PASS, CRASH])) class Base(unittest.TestCase): # Note that all of these tests are written assuming the configuration # being tested is Windows XP, Release build. def __init__(self, testFunc, setUp=None, tearDown=None, description=None): self._port = port.get('test-win-xp', None) self._fs = self._port._filesystem self._exp = None unittest.TestCase.__init__(self, testFunc) def get_test(self, test_name): return self._fs.join(self._port.layout_tests_dir(), test_name) def get_basic_tests(self): return [self.get_test('failures/expected/text.html'), self.get_test('failures/expected/image_checksum.html'), self.get_test('failures/expected/crash.html'), self.get_test('failures/expected/missing_text.html'), self.get_test('failures/expected/image.html'), self.get_test('passes/text.html')] def get_basic_expectations(self): return """ BUG_TEST : failures/expected/text.html = TEXT BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE """ def parse_exp(self, expectations, overrides=None, is_lint_mode=False): test_config = self._port.test_configuration() self._exp = TestExpectations(self._port, tests=self.get_basic_tests(), expectations=expectations, test_config=test_config, is_lint_mode=is_lint_mode, overrides=overrides) def assert_exp(self, test, result): self.assertEquals(self._exp.get_expectations(self.get_test(test)), set([result])) class BasicTests(Base): def test_basic(self): self.parse_exp(self.get_basic_expectations()) self.assert_exp('failures/expected/text.html', TEXT) self.assert_exp('failures/expected/image_checksum.html', IMAGE) self.assert_exp('passes/text.html', PASS) self.assert_exp('failures/expected/image.html', PASS) class MiscTests(Base): def test_multiple_results(self): self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH') self.assertEqual(self._exp.get_expectations( self.get_test('failures/expected/text.html')), set([TEXT, CRASH])) def test_category_expectations(self): # This test checks unknown tests are not present in the # expectations and that known test part of a test category is # present in the expectations. exp_str = """ BUGX WONTFIX : failures/expected = IMAGE """ self.parse_exp(exp_str) test_name = 'failures/expected/unknown-test.html' unknown_test = self.get_test(test_name) self.assertRaises(KeyError, self._exp.get_expectations, unknown_test) self.assert_exp('failures/expected/crash.html', IMAGE) def test_get_options(self): self.parse_exp(self.get_basic_expectations()) self.assertEqual(self._exp.get_options( self.get_test('passes/text.html')), []) def test_expectations_json_for_all_platforms(self): self.parse_exp(self.get_basic_expectations()) json_str = self._exp.get_expectations_json_for_all_platforms() # FIXME: test actual content? self.assertTrue(json_str) def test_get_expectations_string(self): self.parse_exp(self.get_basic_expectations()) self.assertEquals(self._exp.get_expectations_string( self.get_test('failures/expected/text.html')), 'TEXT') def test_expectation_to_string(self): # Normal cases are handled by other tests. self.parse_exp(self.get_basic_expectations()) self.assertRaises(ValueError, self._exp.expectation_to_string, -1) def test_get_test_set(self): # Handle some corner cases for this routine not covered by other tests. self.parse_exp(self.get_basic_expectations()) s = self._exp._expected_failures.get_test_set(WONTFIX) self.assertEqual(s, set([self.get_test('failures/expected/crash.html'), self.get_test('failures/expected/image_checksum.html')])) s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH) self.assertEqual(s, set([self.get_test('failures/expected/crash.html')])) s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH, include_skips=False) self.assertEqual(s, set([])) def test_parse_error_fatal(self): try: self.parse_exp("""FOO : failures/expected/text.html = TEXT SKIP : failures/expected/image.html""") self.assertFalse(True, "ParseError wasn't raised") except ParseError, e: self.assertTrue(e.fatal) exp_errors = [u"Line:1 Unrecognized option 'foo' failures/expected/text.html", u"Line:2 Missing expectations. [' failures/expected/image.html']"] self.assertEqual(str(e), '\n'.join(map(str, exp_errors))) self.assertEqual(e.errors, exp_errors) def test_parse_error_nonfatal(self): try: self.parse_exp('SKIP : failures/expected/text.html = TEXT', is_lint_mode=True) self.assertFalse(True, "ParseError wasn't raised") except ParseError, e: self.assertFalse(e.fatal) exp_errors = [u'Line:1 Test lacks BUG modifier. failures/expected/text.html'] self.assertEqual(str(e), '\n'.join(map(str, exp_errors))) self.assertEqual(e.errors, exp_errors) def test_overrides(self): self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT", "BUG_OVERRIDE : failures/expected/text.html = IMAGE") self.assert_exp('failures/expected/text.html', IMAGE) def test_overrides__duplicate(self): self.assertRaises(ParseError, self.parse_exp, "BUG_EXP: failures/expected/text.html = TEXT", """ BUG_OVERRIDE : failures/expected/text.html = IMAGE BUG_OVERRIDE : failures/expected/text.html = CRASH """) def test_pixel_tests_flag(self): def match(test, result, pixel_tests_enabled): return self._exp.matches_an_expected_result( self.get_test(test), result, pixel_tests_enabled) self.parse_exp(self.get_basic_expectations()) self.assertTrue(match('failures/expected/text.html', TEXT, True)) self.assertTrue(match('failures/expected/text.html', TEXT, False)) self.assertFalse(match('failures/expected/text.html', CRASH, True)) self.assertFalse(match('failures/expected/text.html', CRASH, False)) self.assertTrue(match('failures/expected/image_checksum.html', IMAGE, True)) self.assertTrue(match('failures/expected/image_checksum.html', PASS, False)) self.assertTrue(match('failures/expected/crash.html', SKIP, False)) self.assertTrue(match('passes/text.html', PASS, False)) def test_more_specific_override_resets_skip(self): self.parse_exp("BUGX SKIP : failures/expected = TEXT\n" "BUGX : failures/expected/text.html = IMAGE\n") self.assert_exp('failures/expected/text.html', IMAGE) self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(), 'failures/expected/text.html') in self._exp.get_tests_with_result_type(SKIP)) class ExpectationSyntaxTests(Base): def test_missing_expectation(self): # This is missing the expectation. self.assertRaises(ParseError, self.parse_exp, 'BUG_TEST: failures/expected/text.html') def test_missing_colon(self): # This is missing the modifiers and the ':' self.assertRaises(ParseError, self.parse_exp, 'failures/expected/text.html = TEXT') def disabled_test_too_many_colons(self): # FIXME: Enable this test and fix the underlying bug. self.assertRaises(ParseError, self.parse_exp, 'BUG_TEST: failures/expected/text.html = PASS :') def test_too_many_equals_signs(self): self.assertRaises(ParseError, self.parse_exp, 'BUG_TEST: failures/expected/text.html = TEXT = IMAGE') def test_unrecognized_expectation(self): self.assertRaises(ParseError, self.parse_exp, 'BUG_TEST: failures/expected/text.html = UNKNOWN') def test_macro(self): exp_str = """ BUG_TEST WIN-XP : failures/expected/text.html = TEXT """ self.parse_exp(exp_str) self.assert_exp('failures/expected/text.html', TEXT) class SemanticTests(Base): def test_bug_format(self): self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT') def test_missing_bugid(self): # This should log a non-fatal error. self.parse_exp('SLOW : failures/expected/text.html = TEXT') self.assertEqual( len(self._exp._expected_failures.get_non_fatal_errors()), 1) def test_slow_and_timeout(self): # A test cannot be SLOW and expected to TIMEOUT. self.assertRaises(ParseError, self.parse_exp, 'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT') def test_rebaseline(self): # Can't lint a file w/ 'REBASELINE' in it. self.assertRaises(ParseError, self.parse_exp, 'BUG_TEST REBASELINE : failures/expected/text.html = TEXT', is_lint_mode=True) def test_duplicates(self): self.assertRaises(ParseError, self.parse_exp, """ BUG_EXP : failures/expected/text.html = TEXT BUG_EXP : failures/expected/text.html = IMAGE""") self.assertRaises(ParseError, self.parse_exp, self.get_basic_expectations(), overrides=""" BUG_OVERRIDE : failures/expected/text.html = TEXT BUG_OVERRIDE : failures/expected/text.html = IMAGE""", ) def test_missing_file(self): # This should log a non-fatal error. self.parse_exp('BUG_TEST : missing_file.html = TEXT') self.assertEqual( len(self._exp._expected_failures.get_non_fatal_errors()), 1) class PrecedenceTests(Base): def test_file_over_directory(self): # This tests handling precedence of specific lines over directories # and tests expectations covering entire directories. exp_str = """ BUGX : failures/expected/text.html = TEXT BUGX WONTFIX : failures/expected = IMAGE """ self.parse_exp(exp_str) self.assert_exp('failures/expected/text.html', TEXT) self.assert_exp('failures/expected/crash.html', IMAGE) exp_str = """ BUGX WONTFIX : failures/expected = IMAGE BUGX : failures/expected/text.html = TEXT """ self.parse_exp(exp_str) self.assert_exp('failures/expected/text.html', TEXT) self.assert_exp('failures/expected/crash.html', IMAGE) def test_ambiguous(self): self.assertRaises(ParseError, self.parse_exp, """ BUG_TEST RELEASE : passes/text.html = PASS BUG_TEST WIN : passes/text.html = FAIL """) def test_more_modifiers(self): exp_str = """ BUG_TEST RELEASE : passes/text.html = PASS BUG_TEST WIN RELEASE : passes/text.html = TEXT """ self.assertRaises(ParseError, self.parse_exp, exp_str) def test_order_in_file(self): exp_str = """ BUG_TEST WIN RELEASE : passes/text.html = TEXT BUG_TEST RELEASE : passes/text.html = PASS """ self.assertRaises(ParseError, self.parse_exp, exp_str) def test_version_overrides(self): exp_str = """ BUG_TEST WIN : passes/text.html = PASS BUG_TEST WIN XP : passes/text.html = TEXT """ self.assertRaises(ParseError, self.parse_exp, exp_str) def test_macro_overrides(self): exp_str = """ BUG_TEST WIN : passes/text.html = PASS BUG_TEST WIN-XP : passes/text.html = TEXT """ self.assertRaises(ParseError, self.parse_exp, exp_str) class RebaseliningTest(Base): """Test rebaselining-specific functionality.""" def assertRemove(self, platform, input_expectations, expected_expectations): self.parse_exp(input_expectations) test = self.get_test('failures/expected/text.html') actual_expectations = self._exp.remove_platform_from_expectations( test, platform) self.assertEqual(expected_expectations, actual_expectations) def test_no_get_rebaselining_failures(self): self.parse_exp(self.get_basic_expectations()) self.assertEqual(len(self._exp.get_rebaselining_failures()), 0) def test_get_rebaselining_failures_expand(self): self.parse_exp(""" BUG_TEST REBASELINE : failures/expected/text.html = TEXT """) self.assertEqual(len(self._exp.get_rebaselining_failures()), 1) def test_remove_expand(self): self.assertRemove('mac', 'BUGX REBASELINE : failures/expected/text.html = TEXT\n', 'BUGX REBASELINE WIN : failures/expected/text.html = TEXT\n' 'BUGX REBASELINE WIN-XP : failures/expected/text.html = TEXT\n') def test_remove_mac_win(self): self.assertRemove('mac', 'BUGX REBASELINE MAC WIN : failures/expected/text.html = TEXT\n', 'BUGX REBASELINE WIN : failures/expected/text.html = TEXT\n') def test_remove_mac_mac(self): self.assertRemove('mac', 'BUGX REBASELINE MAC : failures/expected/text.html = TEXT\n', '') def test_remove_nothing(self): self.assertRemove('mac', '\n\n', '\n\n') class ModifierTests(unittest.TestCase): def setUp(self): port_obj = port.get('test-win-xp', None) self.config = port_obj.test_configuration() self.matcher = ModifierMatcher(self.config) def match(self, modifiers, expected_num_matches=-1, values=None, num_errors=0): matcher = self.matcher if values: matcher = ModifierMatcher(self.FakeTestConfiguration(values)) match_result = matcher.match(modifiers) self.assertEqual(len(match_result.warnings), 0) self.assertEqual(len(match_result.errors), num_errors) self.assertEqual(match_result.num_matches, expected_num_matches, 'match(%s, %s) returned -> %d, expected %d' % (modifiers, str(self.config.values()), match_result.num_matches, expected_num_matches)) def test_bad_match_modifier(self): self.match(['foo'], num_errors=1) def test_none(self): self.match([], 0) def test_one(self): self.match(['xp'], 1) self.match(['win'], 1) self.match(['release'], 1) self.match(['cpu'], 1) self.match(['x86'], 1) self.match(['leopard'], -1) self.match(['gpu'], -1) self.match(['debug'], -1) def test_two(self): self.match(['xp', 'release'], 2) self.match(['win7', 'release'], -1) self.match(['win7', 'xp'], 1) def test_three(self): self.match(['win7', 'xp', 'release'], 2) self.match(['xp', 'debug', 'x86'], -1) self.match(['xp', 'release', 'x86'], 3) self.match(['xp', 'cpu', 'release'], 3) def test_four(self): self.match(['xp', 'release', 'cpu', 'x86'], 4) self.match(['win7', 'xp', 'release', 'cpu'], 3) self.match(['win7', 'xp', 'debug', 'cpu'], -1) def test_case_insensitivity(self): self.match(['Win'], num_errors=1) self.match(['WIN'], num_errors=1) self.match(['win'], 1) def test_duplicates(self): self.match(['release', 'release'], num_errors=1) self.match(['win-xp', 'xp'], num_errors=1) self.match(['win-xp', 'win-xp'], num_errors=1) self.match(['xp', 'release', 'xp', 'release'], num_errors=2) self.match(['rebaseline', 'rebaseline'], num_errors=1) def test_unknown_option(self): self.match(['vms'], num_errors=1) def test_duplicate_bugs(self): # BUG* regexes can appear multiple times. self.match(['bugfoo', 'bugbar'], 0) def test_invalid_combinations(self): # FIXME: This should probably raise an error instead of NO_MATCH. self.match(['mac', 'xp'], num_errors=0) def test_regexes_are_ignored(self): self.match(['bug123xy', 'rebaseline', 'wontfix', 'slow', 'skip'], 0) def test_none_is_invalid(self): self.match(['none'], num_errors=1) if __name__ == '__main__': unittest.main()