diff options
Diffstat (limited to 'WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py')
| -rw-r--r-- | WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py | 606 |
1 files changed, 0 insertions, 606 deletions
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py deleted file mode 100644 index 27a6a29..0000000 --- a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py +++ /dev/null @@ -1,606 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2010 Google Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Unit tests for printing.py.""" - -import os -import optparse -import pdb -import sys -import unittest -import logging - -from webkitpy.common import array_stream -from webkitpy.common.system import logtesting -from webkitpy.layout_tests import port -from webkitpy.layout_tests.layout_package import printing -from webkitpy.layout_tests.layout_package import test_results -from webkitpy.layout_tests.layout_package import test_expectations -from webkitpy.layout_tests.layout_package import test_failures -from webkitpy.layout_tests import run_webkit_tests - - -def get_options(args): - print_options = printing.print_options() - option_parser = optparse.OptionParser(option_list=print_options) - return option_parser.parse_args(args) - - -class TestUtilityFunctions(unittest.TestCase): - def test_configure_logging(self): - options, args = get_options([]) - stream = array_stream.ArrayStream() - handler = printing._configure_logging(stream, options.verbose) - logging.info("this should be logged") - self.assertFalse(stream.empty()) - - stream.reset() - logging.debug("this should not be logged") - self.assertTrue(stream.empty()) - - printing._restore_logging(handler) - - stream.reset() - options, args = get_options(['--verbose']) - handler = printing._configure_logging(stream, options.verbose) - logging.debug("this should be logged") - self.assertFalse(stream.empty()) - printing._restore_logging(handler) - - def test_print_options(self): - options, args = get_options([]) - self.assertTrue(options is not None) - - def test_parse_print_options(self): - def test_switches(args, expected_switches_str, - verbose=False, child_processes=1, - is_fully_parallel=False): - options, args = get_options(args) - if expected_switches_str: - expected_switches = set(expected_switches_str.split(',')) - else: - expected_switches = set() - switches = printing.parse_print_options(options.print_options, - verbose, - child_processes, - is_fully_parallel) - self.assertEqual(expected_switches, switches) - - # test that we default to the default set of switches - test_switches([], printing.PRINT_DEFAULT) - - # test that verbose defaults to everything - test_switches([], printing.PRINT_EVERYTHING, verbose=True) - - # test that --print default does what it's supposed to - test_switches(['--print', 'default'], printing.PRINT_DEFAULT) - - # test that --print nothing does what it's supposed to - test_switches(['--print', 'nothing'], None) - - # test that --print everything does what it's supposed to - test_switches(['--print', 'everything'], printing.PRINT_EVERYTHING) - - # this tests that '--print X' overrides '--verbose' - test_switches(['--print', 'actual'], 'actual', verbose=True) - - - -class Testprinter(unittest.TestCase): - def get_printer(self, args=None, single_threaded=False, - is_fully_parallel=False): - printing_options = printing.print_options() - option_parser = optparse.OptionParser(option_list=printing_options) - options, args = option_parser.parse_args(args) - self._port = port.get('test', options) - nproc = 2 - if single_threaded: - nproc = 1 - - regular_output = array_stream.ArrayStream() - buildbot_output = array_stream.ArrayStream() - printer = printing.Printer(self._port, options, regular_output, - buildbot_output, single_threaded, - is_fully_parallel) - return printer, regular_output, buildbot_output - - def get_result(self, test, result_type=test_expectations.PASS, run_time=0): - failures = [] - if result_type == test_expectations.TIMEOUT: - failures = [test_failures.FailureTimeout()] - elif result_type == test_expectations.CRASH: - failures = [test_failures.FailureCrash()] - path = os.path.join(self._port.layout_tests_dir(), test) - return test_results.TestResult(path, failures, run_time, - total_time_for_all_diffs=0, - time_for_diffs=0) - - def get_result_summary(self, tests, expectations_str): - test_paths = [os.path.join(self._port.layout_tests_dir(), test) for - test in tests] - expectations = test_expectations.TestExpectations( - self._port, test_paths, expectations_str, - self._port.test_platform_name(), is_debug_mode=False, - is_lint_mode=False) - - rs = run_webkit_tests.ResultSummary(expectations, test_paths) - return test_paths, rs, expectations - - def test_help_printer(self): - # Here and below we'll call the "regular" printer err and the - # buildbot printer out; this corresponds to how things run on the - # bots with stderr and stdout. - printer, err, out = self.get_printer() - - # This routine should print something to stdout. testing what it is - # is kind of pointless. - printer.help_printing() - self.assertFalse(err.empty()) - self.assertTrue(out.empty()) - - def do_switch_tests(self, method_name, switch, to_buildbot, - message='hello', exp_err=None, exp_bot=None): - def do_helper(method_name, switch, message, exp_err, exp_bot): - printer, err, bot = self.get_printer(['--print', switch]) - getattr(printer, method_name)(message) - self.assertEqual(err.get(), exp_err) - self.assertEqual(bot.get(), exp_bot) - - if to_buildbot: - if exp_err is None: - exp_err = [] - if exp_bot is None: - exp_bot = [message + "\n"] - else: - if exp_err is None: - exp_err = [message + "\n"] - if exp_bot is None: - exp_bot = [] - do_helper(method_name, 'nothing', 'hello', [], []) - do_helper(method_name, switch, 'hello', exp_err, exp_bot) - do_helper(method_name, 'everything', 'hello', exp_err, exp_bot) - - def test_configure_and_cleanup(self): - # This test verifies that calling cleanup repeatedly and deleting - # the object is safe. - printer, err, out = self.get_printer(['--print', 'everything']) - printer.cleanup() - printer.cleanup() - printer = None - - def test_print_actual(self): - # Actual results need to be logged to the buildbot's stream. - self.do_switch_tests('print_actual', 'actual', to_buildbot=True) - - def test_print_actual_buildbot(self): - # FIXME: Test that the format of the actual results matches what the - # buildbot is expecting. - pass - - def test_print_config(self): - self.do_switch_tests('print_config', 'config', to_buildbot=False) - - def test_print_expected(self): - self.do_switch_tests('print_expected', 'expected', to_buildbot=False) - - def test_print_timing(self): - self.do_switch_tests('print_timing', 'timing', to_buildbot=False) - - def test_print_update(self): - # Note that there shouldn't be a carriage return here; updates() - # are meant to be overwritten. - self.do_switch_tests('print_update', 'updates', to_buildbot=False, - message='hello', exp_err=['hello']) - - def test_print_one_line_summary(self): - printer, err, out = self.get_printer(['--print', 'nothing']) - printer.print_one_line_summary(1, 1, 0) - self.assertTrue(err.empty()) - - printer, err, out = self.get_printer(['--print', 'one-line-summary']) - printer.print_one_line_summary(1, 1, 0) - self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"]) - - printer, err, out = self.get_printer(['--print', 'everything']) - printer.print_one_line_summary(1, 1, 0) - self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"]) - - err.reset() - printer.print_one_line_summary(2, 1, 1) - self.assertEquals(err.get(), - ["1 test ran as expected, 1 didn't:\n", "\n"]) - - err.reset() - printer.print_one_line_summary(3, 2, 1) - self.assertEquals(err.get(), - ["2 tests ran as expected, 1 didn't:\n", "\n"]) - - err.reset() - printer.print_one_line_summary(3, 2, 0) - self.assertEquals(err.get(), - ['\n', "2 tests ran as expected (1 didn't run).\n", - '\n']) - - - def test_print_test_result(self): - # Note here that we don't use meaningful exp_str and got_str values; - # the actual contents of the string are treated opaquely by - # print_test_result() when tracing, and usually we don't want - # to test what exactly is printed, just that something - # was printed (or that nothing was printed). - # - # FIXME: this is actually some goofy layering; it would be nice - # we could refactor it so that the args weren't redundant. Maybe - # the TestResult should contain what was expected, and the - # strings could be derived from the TestResult? - printer, err, out = self.get_printer(['--print', 'nothing']) - result = self.get_result('passes/image.html') - printer.print_test_result(result, expected=False, exp_str='', - got_str='') - self.assertTrue(err.empty()) - - printer, err, out = self.get_printer(['--print', 'unexpected']) - printer.print_test_result(result, expected=True, exp_str='', - got_str='') - self.assertTrue(err.empty()) - printer.print_test_result(result, expected=False, exp_str='', - got_str='') - self.assertEquals(err.get(), - [' passes/image.html -> unexpected pass\n']) - - printer, err, out = self.get_printer(['--print', 'everything']) - printer.print_test_result(result, expected=True, exp_str='', - got_str='') - self.assertTrue(err.empty()) - - printer.print_test_result(result, expected=False, exp_str='', - got_str='') - self.assertEquals(err.get(), - [' passes/image.html -> unexpected pass\n']) - - printer, err, out = self.get_printer(['--print', 'nothing']) - printer.print_test_result(result, expected=False, exp_str='', - got_str='') - self.assertTrue(err.empty()) - - printer, err, out = self.get_printer(['--print', - 'trace-unexpected']) - printer.print_test_result(result, expected=True, exp_str='', - got_str='') - self.assertTrue(err.empty()) - - printer, err, out = self.get_printer(['--print', - 'trace-unexpected']) - printer.print_test_result(result, expected=False, exp_str='', - got_str='') - self.assertFalse(err.empty()) - - printer, err, out = self.get_printer(['--print', - 'trace-unexpected']) - result = self.get_result("passes/text.html") - printer.print_test_result(result, expected=False, exp_str='', - got_str='') - self.assertFalse(err.empty()) - - err.reset() - printer.print_test_result(result, expected=False, exp_str='', - got_str='') - self.assertFalse(err.empty()) - - printer, err, out = self.get_printer(['--print', 'trace-everything']) - result = self.get_result('passes/image.html') - printer.print_test_result(result, expected=True, exp_str='', - got_str='') - result = self.get_result('failures/expected/missing_text.html') - printer.print_test_result(result, expected=True, exp_str='', - got_str='') - result = self.get_result('failures/expected/missing_check.html') - printer.print_test_result(result, expected=True, exp_str='', - got_str='') - result = self.get_result('failures/expected/missing_image.html') - printer.print_test_result(result, expected=True, exp_str='', - got_str='') - self.assertFalse(err.empty()) - - err.reset() - printer.print_test_result(result, expected=False, exp_str='', - got_str='') - - def test_print_progress(self): - expectations = '' - - # test that we print nothing - printer, err, out = self.get_printer(['--print', 'nothing']) - tests = ['passes/text.html', 'failures/expected/timeout.html', - 'failures/expected/crash.html'] - paths, rs, exp = self.get_result_summary(tests, expectations) - - printer.print_progress(rs, False, paths) - self.assertTrue(out.empty()) - self.assertTrue(err.empty()) - - printer.print_progress(rs, True, paths) - self.assertTrue(out.empty()) - self.assertTrue(err.empty()) - - # test regular functionality - printer, err, out = self.get_printer(['--print', - 'one-line-progress']) - printer.print_progress(rs, False, paths) - self.assertTrue(out.empty()) - self.assertFalse(err.empty()) - - err.reset() - out.reset() - printer.print_progress(rs, True, paths) - self.assertFalse(err.empty()) - self.assertTrue(out.empty()) - - def test_print_progress__detailed(self): - tests = ['passes/text.html', 'failures/expected/timeout.html', - 'failures/expected/crash.html'] - expectations = 'failures/expected/timeout.html = TIMEOUT' - - # first, test that it is disabled properly - # should still print one-line-progress - printer, err, out = self.get_printer( - ['--print', 'detailed-progress'], single_threaded=False) - paths, rs, exp = self.get_result_summary(tests, expectations) - printer.print_progress(rs, False, paths) - self.assertFalse(err.empty()) - self.assertTrue(out.empty()) - - # now test the enabled paths - printer, err, out = self.get_printer( - ['--print', 'detailed-progress'], single_threaded=True) - paths, rs, exp = self.get_result_summary(tests, expectations) - printer.print_progress(rs, False, paths) - self.assertFalse(err.empty()) - self.assertTrue(out.empty()) - - err.reset() - out.reset() - printer.print_progress(rs, True, paths) - self.assertFalse(err.empty()) - self.assertTrue(out.empty()) - - rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False) - rs.add(self.get_result('failures/expected/timeout.html'), True) - rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True) - err.reset() - out.reset() - printer.print_progress(rs, False, paths) - self.assertFalse(err.empty()) - self.assertTrue(out.empty()) - - # We only clear the meter when retrying w/ detailed-progress. - err.reset() - out.reset() - printer.print_progress(rs, True, paths) - self.assertFalse(err.empty()) - self.assertTrue(out.empty()) - - printer, err, out = self.get_printer( - ['--print', 'detailed-progress,unexpected'], single_threaded=True) - paths, rs, exp = self.get_result_summary(tests, expectations) - printer.print_progress(rs, False, paths) - self.assertFalse(err.empty()) - self.assertTrue(out.empty()) - - err.reset() - out.reset() - printer.print_progress(rs, True, paths) - self.assertFalse(err.empty()) - self.assertTrue(out.empty()) - - rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False) - rs.add(self.get_result('failures/expected/timeout.html'), True) - rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True) - err.reset() - out.reset() - printer.print_progress(rs, False, paths) - self.assertFalse(err.empty()) - self.assertTrue(out.empty()) - - # We only clear the meter when retrying w/ detailed-progress. - err.reset() - out.reset() - printer.print_progress(rs, True, paths) - self.assertFalse(err.empty()) - self.assertTrue(out.empty()) - - def test_write_nothing(self): - printer, err, out = self.get_printer(['--print', 'nothing']) - printer.write("foo") - self.assertTrue(err.empty()) - - def test_write_misc(self): - printer, err, out = self.get_printer(['--print', 'misc']) - printer.write("foo") - self.assertFalse(err.empty()) - err.reset() - printer.write("foo", "config") - self.assertTrue(err.empty()) - - def test_write_everything(self): - printer, err, out = self.get_printer(['--print', 'everything']) - printer.write("foo") - self.assertFalse(err.empty()) - err.reset() - printer.write("foo", "config") - self.assertFalse(err.empty()) - - def test_write_verbose(self): - printer, err, out = self.get_printer(['--verbose']) - printer.write("foo") - self.assertTrue(not err.empty() and "foo" in err.get()[0]) - self.assertTrue(out.empty()) - - def test_print_unexpected_results(self): - # This routine is the only one that prints stuff that the bots - # care about. - # - # FIXME: there's some weird layering going on here. It seems - # like we shouldn't be both using an expectations string and - # having to specify whether or not the result was expected. - # This whole set of tests should probably be rewritten. - # - # FIXME: Plus, the fact that we're having to call into - # run_webkit_tests is clearly a layering inversion. - def get_unexpected_results(expected, passing, flaky): - """Return an unexpected results summary matching the input description. - - There are a lot of different combinations of test results that - can be tested; this routine produces various combinations based - on the values of the input flags. - - Args - expected: whether the tests ran as expected - passing: whether the tests should all pass - flaky: whether the tests should be flaky (if False, they - produce the same results on both runs; if True, they - all pass on the second run). - - """ - paths, rs, exp = self.get_result_summary(tests, expectations) - if expected: - rs.add(self.get_result('passes/text.html', test_expectations.PASS), - expected) - rs.add(self.get_result('failures/expected/timeout.html', - test_expectations.TIMEOUT), expected) - rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), - expected) - elif passing: - rs.add(self.get_result('passes/text.html'), expected) - rs.add(self.get_result('failures/expected/timeout.html'), expected) - rs.add(self.get_result('failures/expected/crash.html'), expected) - else: - rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), - expected) - rs.add(self.get_result('failures/expected/timeout.html', - test_expectations.CRASH), expected) - rs.add(self.get_result('failures/expected/crash.html', - test_expectations.TIMEOUT), - expected) - retry = rs - if flaky: - paths, retry, exp = self.get_result_summary(tests, - expectations) - retry.add(self.get_result('passes/text.html'), True) - retry.add(self.get_result('failures/expected/timeout.html'), True) - retry.add(self.get_result('failures/expected/crash.html'), True) - unexpected_results = run_webkit_tests.summarize_unexpected_results( - self._port, exp, rs, retry) - return unexpected_results - - tests = ['passes/text.html', 'failures/expected/timeout.html', - 'failures/expected/crash.html'] - expectations = '' - - printer, err, out = self.get_printer(['--print', 'nothing']) - ur = get_unexpected_results(expected=False, passing=False, flaky=False) - printer.print_unexpected_results(ur) - self.assertTrue(err.empty()) - self.assertTrue(out.empty()) - - printer, err, out = self.get_printer(['--print', - 'unexpected-results']) - - # test everything running as expected - ur = get_unexpected_results(expected=True, passing=False, flaky=False) - printer.print_unexpected_results(ur) - self.assertTrue(err.empty()) - self.assertTrue(out.empty()) - - # test failures - err.reset() - out.reset() - ur = get_unexpected_results(expected=False, passing=False, flaky=False) - printer.print_unexpected_results(ur) - self.assertTrue(err.empty()) - self.assertFalse(out.empty()) - - # test unexpected flaky results - err.reset() - out.reset() - ur = get_unexpected_results(expected=False, passing=True, flaky=False) - printer.print_unexpected_results(ur) - self.assertTrue(err.empty()) - self.assertFalse(out.empty()) - - # test unexpected passes - err.reset() - out.reset() - ur = get_unexpected_results(expected=False, passing=False, flaky=True) - printer.print_unexpected_results(ur) - self.assertTrue(err.empty()) - self.assertFalse(out.empty()) - - err.reset() - out.reset() - printer, err, out = self.get_printer(['--print', 'everything']) - ur = get_unexpected_results(expected=False, passing=False, flaky=False) - printer.print_unexpected_results(ur) - self.assertTrue(err.empty()) - self.assertFalse(out.empty()) - - expectations = """ -failures/expected/crash.html = CRASH -failures/expected/timeout.html = TIMEOUT -""" - err.reset() - out.reset() - ur = get_unexpected_results(expected=False, passing=False, flaky=False) - printer.print_unexpected_results(ur) - self.assertTrue(err.empty()) - self.assertFalse(out.empty()) - - err.reset() - out.reset() - ur = get_unexpected_results(expected=False, passing=True, flaky=False) - printer.print_unexpected_results(ur) - self.assertTrue(err.empty()) - self.assertFalse(out.empty()) - - # Test handling of --verbose as well. - err.reset() - out.reset() - printer, err, out = self.get_printer(['--verbose']) - ur = get_unexpected_results(expected=False, passing=False, flaky=False) - printer.print_unexpected_results(ur) - self.assertTrue(err.empty()) - self.assertFalse(out.empty()) - - def test_print_unexpected_results_buildbot(self): - # FIXME: Test that print_unexpected_results() produces the printer the - # buildbot is expecting. - pass - -if __name__ == '__main__': - unittest.main() |