path: root/Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py

#!/usr/bin/python
# Copyright (C) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Unit tests for printing.py."""

import os
import optparse
import pdb
import sys
import unittest
import logging

from webkitpy.common import array_stream
from webkitpy.common.system import logtesting
from webkitpy.layout_tests import port

from webkitpy.layout_tests.layout_package import printing
from webkitpy.layout_tests.layout_package import result_summary
from webkitpy.layout_tests.layout_package import test_expectations
from webkitpy.layout_tests.layout_package import test_failures
from webkitpy.layout_tests.layout_package import test_results
from webkitpy.layout_tests.layout_package import test_runner


def get_options(args):
    print_options = printing.print_options()
    option_parser = optparse.OptionParser(option_list=print_options)
    return option_parser.parse_args(args)


class TestUtilityFunctions(unittest.TestCase):
    def test_configure_logging(self):
        options, args = get_options([])
        stream = array_stream.ArrayStream()
        handler = printing._configure_logging(stream, options.verbose)
        logging.info("this should be logged")
        self.assertFalse(stream.empty())

        stream.reset()
        logging.debug("this should not be logged")
        self.assertTrue(stream.empty())

        printing._restore_logging(handler)

        stream.reset()
        options, args = get_options(['--verbose'])
        handler = printing._configure_logging(stream, options.verbose)
        logging.debug("this should be logged")
        self.assertFalse(stream.empty())
        printing._restore_logging(handler)

    def test_print_options(self):
        options, args = get_options([])
        self.assertTrue(options is not None)

    def test_parse_print_options(self):
        def test_switches(args, expected_switches_str,
                          verbose=False, child_processes=1,
                          is_fully_parallel=False):
            options, args = get_options(args)
            if expected_switches_str:
                expected_switches = set(expected_switches_str.split(','))
            else:
                expected_switches = set()
            switches = printing.parse_print_options(options.print_options,
                                                    verbose,
                                                    child_processes,
                                                    is_fully_parallel)
            self.assertEqual(expected_switches, switches)

        # test that we default to the default set of switches
        test_switches([], printing.PRINT_DEFAULT)

        # test that verbose defaults to everything
        test_switches([], printing.PRINT_EVERYTHING, verbose=True)

        # test that --print default does what it's supposed to
        test_switches(['--print', 'default'], printing.PRINT_DEFAULT)

        # test that --print nothing does what it's supposed to
        test_switches(['--print', 'nothing'], None)

        # test that --print everything does what it's supposed to
        test_switches(['--print', 'everything'], printing.PRINT_EVERYTHING)

        # this tests that '--print X' overrides '--verbose'
        test_switches(['--print', 'actual'], 'actual', verbose=True)



class  Testprinter(unittest.TestCase):
    def get_printer(self, args=None, single_threaded=False,
                   is_fully_parallel=False):
        printing_options = printing.print_options()
        option_parser = optparse.OptionParser(option_list=printing_options)
        options, args = option_parser.parse_args(args)
        self._port = port.get('test', options)
        nproc = 2
        if single_threaded:
            nproc = 1

        regular_output = array_stream.ArrayStream()
        buildbot_output = array_stream.ArrayStream()
        printer = printing.Printer(self._port, options, regular_output,
                                   buildbot_output, single_threaded,
                                   is_fully_parallel)
        return printer, regular_output, buildbot_output

    def get_result(self, test, result_type=test_expectations.PASS, run_time=0):
        failures = []
        if result_type == test_expectations.TIMEOUT:
            failures = [test_failures.FailureTimeout()]
        elif result_type == test_expectations.CRASH:
            failures = [test_failures.FailureCrash()]
        path = os.path.join(self._port.layout_tests_dir(), test)
        return test_results.TestResult(path, failures=failures, test_run_time=run_time)

    def get_result_summary(self, tests, expectations_str):
        test_paths = [os.path.join(self._port.layout_tests_dir(), test) for
                      test in tests]
        expectations = test_expectations.TestExpectations(
            self._port, test_paths, expectations_str,
            self._port.test_platform_name(), is_debug_mode=False,
            is_lint_mode=False)

        rs = result_summary.ResultSummary(expectations, test_paths)
        return test_paths, rs, expectations

    def test_help_printer(self):
        # Here and below we'll call the "regular" printer err and the
        # buildbot printer out; this corresponds to how things run on the
        # bots with stderr and stdout.
        printer, err, out = self.get_printer()

        # This routine should print something to stdout. testing what it is
        # is kind of pointless.
        printer.help_printing()
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

    def do_switch_tests(self, method_name, switch, to_buildbot,
                        message='hello', exp_err=None, exp_bot=None):
        def do_helper(method_name, switch, message, exp_err, exp_bot):
            printer, err, bot = self.get_printer(['--print', switch])
            getattr(printer, method_name)(message)
            self.assertEqual(err.get(), exp_err)
            self.assertEqual(bot.get(), exp_bot)

        if to_buildbot:
            if exp_err is None:
                exp_err = []
            if exp_bot is None:
                exp_bot = [message + "\n"]
        else:
            if exp_err is None:
                exp_err = [message + "\n"]
            if exp_bot is None:
                exp_bot = []
        do_helper(method_name, 'nothing', 'hello', [], [])
        do_helper(method_name, switch, 'hello', exp_err, exp_bot)
        do_helper(method_name, 'everything', 'hello', exp_err, exp_bot)

    def test_configure_and_cleanup(self):
        # This test verifies that calling cleanup repeatedly and deleting
        # the object is safe.
        printer, err, out = self.get_printer(['--print', 'everything'])
        printer.cleanup()
        printer.cleanup()
        printer = None

    def test_print_actual(self):
        # Actual results need to be logged to the buildbot's stream.
        self.do_switch_tests('print_actual', 'actual', to_buildbot=True)

    def test_print_actual_buildbot(self):
        # FIXME: Test that the format of the actual results matches what the
        # buildbot is expecting.
        pass

    def test_print_config(self):
        self.do_switch_tests('print_config', 'config', to_buildbot=False)

    def test_print_expected(self):
        self.do_switch_tests('print_expected', 'expected', to_buildbot=False)

    def test_print_timing(self):
        self.do_switch_tests('print_timing', 'timing', to_buildbot=False)

    def test_print_update(self):
        # Note that there shouldn't be a carriage return here; updates()
        # are meant to be overwritten.
        self.do_switch_tests('print_update', 'updates', to_buildbot=False,
                             message='hello', exp_err=['hello'])

    def test_print_one_line_summary(self):
        printer, err, out = self.get_printer(['--print', 'nothing'])
        printer.print_one_line_summary(1, 1, 0)
        self.assertTrue(err.empty())

        printer, err, out = self.get_printer(['--print', 'one-line-summary'])
        printer.print_one_line_summary(1, 1, 0)
        self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])

        printer, err, out = self.get_printer(['--print', 'everything'])
        printer.print_one_line_summary(1, 1, 0)
        self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])

        err.reset()
        printer.print_one_line_summary(2, 1, 1)
        self.assertEquals(err.get(),
                          ["1 test ran as expected, 1 didn't:\n", "\n"])

        err.reset()
        printer.print_one_line_summary(3, 2, 1)
        self.assertEquals(err.get(),
                          ["2 tests ran as expected, 1 didn't:\n", "\n"])

        err.reset()
        printer.print_one_line_summary(3, 2, 0)
        self.assertEquals(err.get(),
                          ['\n', "2 tests ran as expected (1 didn't run).\n",
                           '\n'])


    def test_print_test_result(self):
        # Note here that we don't use meaningful exp_str and got_str values;
        # the actual contents of the string are treated opaquely by
        # print_test_result() when tracing, and usually we don't want
        # to test what exactly is printed, just that something
        # was printed (or that nothing was printed).
        #
        # FIXME: this is actually some goofy layering; it would be nice
        # we could refactor it so that the args weren't redundant. Maybe
        # the TestResult should contain what was expected, and the
        # strings could be derived from the TestResult?
        printer, err, out = self.get_printer(['--print', 'nothing'])
        result = self.get_result('passes/image.html')
        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')
        self.assertTrue(err.empty())

        printer, err, out = self.get_printer(['--print', 'unexpected'])
        printer.print_test_result(result, expected=True, exp_str='',
                                  got_str='')
        self.assertTrue(err.empty())
        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')
        self.assertEquals(err.get(),
                          ['  passes/image.html -> unexpected pass\n'])

        printer, err, out = self.get_printer(['--print', 'everything'])
        printer.print_test_result(result, expected=True, exp_str='',
                                  got_str='')
        self.assertTrue(err.empty())

        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')
        self.assertEquals(err.get(),
                          ['  passes/image.html -> unexpected pass\n'])

        printer, err, out = self.get_printer(['--print', 'nothing'])
        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')
        self.assertTrue(err.empty())

        printer, err, out = self.get_printer(['--print',
                                              'trace-unexpected'])
        printer.print_test_result(result, expected=True, exp_str='',
                                  got_str='')
        self.assertTrue(err.empty())

        printer, err, out = self.get_printer(['--print',
                                              'trace-unexpected'])
        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')
        self.assertFalse(err.empty())

        printer, err, out = self.get_printer(['--print',
                                              'trace-unexpected'])
        result = self.get_result("passes/text.html")
        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')
        self.assertFalse(err.empty())

        err.reset()
        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')
        self.assertFalse(err.empty())

        printer, err, out = self.get_printer(['--print', 'trace-everything'])
        result = self.get_result('passes/image.html')
        printer.print_test_result(result, expected=True, exp_str='',
                                  got_str='')
        result = self.get_result('failures/expected/missing_text.html')
        printer.print_test_result(result, expected=True, exp_str='',
                                  got_str='')
        result = self.get_result('failures/expected/missing_check.html')
        printer.print_test_result(result, expected=True, exp_str='',
                                  got_str='')
        result = self.get_result('failures/expected/missing_image.html')
        printer.print_test_result(result, expected=True, exp_str='',
                                  got_str='')
        self.assertFalse(err.empty())

        err.reset()
        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')

    def test_print_progress(self):
        expectations = ''

        # test that we print nothing
        printer, err, out = self.get_printer(['--print', 'nothing'])
        tests = ['passes/text.html', 'failures/expected/timeout.html',
                 'failures/expected/crash.html']
        paths, rs, exp = self.get_result_summary(tests, expectations)

        printer.print_progress(rs, False, paths)
        self.assertTrue(out.empty())
        self.assertTrue(err.empty())

        printer.print_progress(rs, True, paths)
        self.assertTrue(out.empty())
        self.assertTrue(err.empty())

        # test regular functionality
        printer, err, out = self.get_printer(['--print',
                                              'one-line-progress'])
        printer.print_progress(rs, False, paths)
        self.assertTrue(out.empty())
        self.assertFalse(err.empty())

        err.reset()
        out.reset()
        printer.print_progress(rs, True, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

    def test_print_progress__detailed(self):
        tests = ['passes/text.html', 'failures/expected/timeout.html',
                 'failures/expected/crash.html']
        expectations = 'failures/expected/timeout.html = TIMEOUT'

        # first, test that it is disabled properly
        # should still print one-line-progress
        printer, err, out = self.get_printer(
            ['--print', 'detailed-progress'], single_threaded=False)
        paths, rs, exp = self.get_result_summary(tests, expectations)
        printer.print_progress(rs, False, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        # now test the enabled paths
        printer, err, out = self.get_printer(
            ['--print', 'detailed-progress'], single_threaded=True)
        paths, rs, exp = self.get_result_summary(tests, expectations)
        printer.print_progress(rs, False, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        err.reset()
        out.reset()
        printer.print_progress(rs, True, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False)
        rs.add(self.get_result('failures/expected/timeout.html'), True)
        rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True)
        err.reset()
        out.reset()
        printer.print_progress(rs, False, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        # We only clear the meter when retrying w/ detailed-progress.
        err.reset()
        out.reset()
        printer.print_progress(rs, True, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        printer, err, out = self.get_printer(
            ['--print', 'detailed-progress,unexpected'], single_threaded=True)
        paths, rs, exp = self.get_result_summary(tests, expectations)
        printer.print_progress(rs, False, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        err.reset()
        out.reset()
        printer.print_progress(rs, True, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False)
        rs.add(self.get_result('failures/expected/timeout.html'), True)
        rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True)
        err.reset()
        out.reset()
        printer.print_progress(rs, False, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        # We only clear the meter when retrying w/ detailed-progress.
        err.reset()
        out.reset()
        printer.print_progress(rs, True, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

    def test_write_nothing(self):
        printer, err, out = self.get_printer(['--print', 'nothing'])
        printer.write("foo")
        self.assertTrue(err.empty())

    def test_write_misc(self):
        printer, err, out = self.get_printer(['--print', 'misc'])
        printer.write("foo")
        self.assertFalse(err.empty())
        err.reset()
        printer.write("foo", "config")
        self.assertTrue(err.empty())

    def test_write_everything(self):
        printer, err, out = self.get_printer(['--print', 'everything'])
        printer.write("foo")
        self.assertFalse(err.empty())
        err.reset()
        printer.write("foo", "config")
        self.assertFalse(err.empty())

    def test_write_verbose(self):
        printer, err, out = self.get_printer(['--verbose'])
        printer.write("foo")
        self.assertTrue(not err.empty() and "foo" in err.get()[0])
        self.assertTrue(out.empty())

    def test_print_unexpected_results(self):
        # This routine is the only one that prints stuff that the bots
        # care about.
        #
        # FIXME: there's some weird layering going on here. It seems
        # like we shouldn't be both using an expectations string and
        # having to specify whether or not the result was expected.
        # This whole set of tests should probably be rewritten.
        #
        # FIXME: Plus, the fact that we're having to call into
        # run_webkit_tests is clearly a layering inversion.
        def get_unexpected_results(expected, passing, flaky):
            """Return an unexpected results summary matching the input description.

            There are a lot of different combinations of test results that
            can be tested; this routine produces various combinations based
            on the values of the input flags.

            Args
                expected: whether the tests ran as expected
                passing: whether the tests should all pass
                flaky: whether the tests should be flaky (if False, they
                    produce the same results on both runs; if True, they
                    all pass on the second run).

            """
            paths, rs, exp = self.get_result_summary(tests, expectations)
            if expected:
                rs.add(self.get_result('passes/text.html', test_expectations.PASS),
                       expected)
                rs.add(self.get_result('failures/expected/timeout.html',
                       test_expectations.TIMEOUT), expected)
                rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH),
                   expected)
            elif passing:
                rs.add(self.get_result('passes/text.html'), expected)
                rs.add(self.get_result('failures/expected/timeout.html'), expected)
                rs.add(self.get_result('failures/expected/crash.html'), expected)
            else:
                rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT),
                       expected)
                rs.add(self.get_result('failures/expected/timeout.html',
                       test_expectations.CRASH), expected)
                rs.add(self.get_result('failures/expected/crash.html',
                                  test_expectations.TIMEOUT),
                   expected)
            retry = rs
            if flaky:
                paths, retry, exp = self.get_result_summary(tests,
                                                expectations)
                retry.add(self.get_result('passes/text.html'), True)
                retry.add(self.get_result('failures/expected/timeout.html'), True)
                retry.add(self.get_result('failures/expected/crash.html'), True)
            unexpected_results = test_runner.summarize_unexpected_results(
                self._port, exp, rs, retry)
            return unexpected_results

        tests = ['passes/text.html', 'failures/expected/timeout.html',
                 'failures/expected/crash.html']
        expectations = ''

        printer, err, out = self.get_printer(['--print', 'nothing'])
        ur = get_unexpected_results(expected=False, passing=False, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertTrue(out.empty())

        printer, err, out = self.get_printer(['--print',
                                              'unexpected-results'])

        # test everything running as expected
        ur = get_unexpected_results(expected=True, passing=False, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertTrue(out.empty())

        # test failures
        err.reset()
        out.reset()
        ur = get_unexpected_results(expected=False, passing=False, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertFalse(out.empty())

        # test unexpected flaky results
        err.reset()
        out.reset()
        ur = get_unexpected_results(expected=False, passing=True, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertFalse(out.empty())

        # test unexpected passes
        err.reset()
        out.reset()
        ur = get_unexpected_results(expected=False, passing=False, flaky=True)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertFalse(out.empty())

        err.reset()
        out.reset()
        printer, err, out = self.get_printer(['--print', 'everything'])
        ur = get_unexpected_results(expected=False, passing=False, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertFalse(out.empty())

        expectations = """
failures/expected/crash.html = CRASH
failures/expected/timeout.html = TIMEOUT
"""
        err.reset()
        out.reset()
        ur = get_unexpected_results(expected=False, passing=False, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertFalse(out.empty())

        err.reset()
        out.reset()
        ur = get_unexpected_results(expected=False, passing=True, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertFalse(out.empty())

        # Test handling of --verbose as well.
        err.reset()
        out.reset()
        printer, err, out = self.get_printer(['--verbose'])
        ur = get_unexpected_results(expected=False, passing=False, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertFalse(out.empty())

    def test_print_unexpected_results_buildbot(self):
        # FIXME: Test that print_unexpected_results() produces the printer the
        # buildbot is expecting.
        pass

if __name__ == '__main__':
    unittest.main()