path: root/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

#!/usr/bin/env python
# Copyright (C) 2010 Google Inc. All rights reserved.
# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Run layout tests."""

import errno
import logging
import optparse
import os
import signal
import sys

from layout_package import json_results_generator
from layout_package import printing
from layout_package import test_runner
from layout_package import test_runner2

from webkitpy.common.system import user
from webkitpy.thirdparty import simplejson

import port

_log = logging.getLogger(__name__)


def run(port, options, args, regular_output=sys.stderr,
        buildbot_output=sys.stdout):
    """Run the tests.

    Args:
      port: Port object for port-specific behavior
      options: a dictionary of command line options
      args: a list of sub directories or files to test
      regular_output: a stream-like object that we can send logging/debug
          output to
      buildbot_output: a stream-like object that we can write all output that
          is intended to be parsed by the buildbot to
    Returns:
      the number of unexpected results that occurred, or -1 if there is an
          error.

    """
    warnings = _set_up_derived_options(port, options)

    printer = printing.Printer(port, options, regular_output, buildbot_output,
        int(options.child_processes), options.experimental_fully_parallel)
    for w in warnings:
        _log.warning(w)

    if options.help_printing:
        printer.help_printing()
        printer.cleanup()
        return 0

    last_unexpected_results = _gather_unexpected_results(port._filesystem, options)
    if options.print_last_failures:
        printer.write("\n".join(last_unexpected_results) + "\n")
        printer.cleanup()
        return 0

    # We wrap any parts of the run that are slow or likely to raise exceptions
    # in a try/finally to ensure that we clean up the logging configuration.
    num_unexpected_results = -1
    try:
        if options.worker_model in ('inline', 'threads', 'processes'):
            runner = test_runner2.TestRunner2(port, options, printer)
        else:
            runner = test_runner.TestRunner(port, options, printer)

        runner._print_config()

        printer.print_update("Collecting tests ...")
        try:
            runner.collect_tests(args, last_unexpected_results)
        except IOError, e:
            if e.errno == errno.ENOENT:
                return -1
            raise

        if options.lint_test_files:
            return runner.lint()

        printer.print_update("Parsing expectations ...")
        runner.parse_expectations()

        printer.print_update("Checking build ...")
        if not port.check_build(runner.needs_http()):
            _log.error("Build check failed")
            return -1

        result_summary = runner.set_up_run()
        if result_summary:
            num_unexpected_results = runner.run(result_summary)
            runner.clean_up_run()
            _log.debug("Testing completed, Exit status: %d" %
                       num_unexpected_results)
    finally:
        printer.cleanup()

    return num_unexpected_results


def _set_up_derived_options(port_obj, options):
    """Sets the options values that depend on other options values."""
    # We return a list of warnings to print after the printer is initialized.
    warnings = []

    if options.worker_model is None:
        options.worker_model = port_obj.default_worker_model()

    if options.worker_model in ('inline', 'old-inline'):
        if options.child_processes and int(options.child_processes) > 1:
            warnings.append("--worker-model=%s overrides --child-processes" % options.worker_model)
        options.child_processes = "1"
    if not options.child_processes:
        options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
                                                 str(port_obj.default_child_processes()))

    if not options.configuration:
        options.configuration = port_obj.default_configuration()

    if options.pixel_tests is None:
        options.pixel_tests = True

    if not options.use_apache:
        options.use_apache = sys.platform in ('darwin', 'linux2')

    if not port_obj._filesystem.isabs(options.results_directory):
        # This normalizes the path to the build dir.
        # FIXME: how this happens is not at all obvious; this is a dumb
        # interface and should be cleaned up.
        options.results_directory = port_obj.results_directory()

    if not options.time_out_ms:
        if options.configuration == "Debug":
            options.time_out_ms = str(2 * test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
        else:
            options.time_out_ms = str(test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)

    options.slow_time_out_ms = str(5 * int(options.time_out_ms))
    return warnings


def _gather_unexpected_results(filesystem, options):
    """Returns the unexpected results from the previous run, if any."""
    last_unexpected_results = []
    if options.print_last_failures or options.retest_last_failures:
        unexpected_results_filename = filesystem.join(options.results_directory, "unexpected_results.json")
        if filesystem.exists(unexpected_results_filename):
            results = json_results_generator.load_json(filesystem, unexpected_results_filename)
            last_unexpected_results = results['tests'].keys()
    return last_unexpected_results


def _compat_shim_callback(option, opt_str, value, parser):
    print "Ignoring unsupported option: %s" % opt_str


def _compat_shim_option(option_name, **kwargs):
    return optparse.make_option(option_name, action="callback",
        callback=_compat_shim_callback,
        help="Ignored, for old-run-webkit-tests compat only.", **kwargs)


def parse_args(args=None):
    """Provides a default set of command line args.

    Returns a tuple of options, args from optparse"""

    # FIXME: All of these options should be stored closer to the code which
    # FIXME: actually uses them. configuration_options should move
    # FIXME: to WebKitPort and be shared across all scripts.
    configuration_options = [
        optparse.make_option("-t", "--target", dest="configuration",
                             help="(DEPRECATED)"),
        # FIXME: --help should display which configuration is default.
        optparse.make_option('--debug', action='store_const', const='Debug',
                             dest="configuration",
                             help='Set the configuration to Debug'),
        optparse.make_option('--release', action='store_const',
                             const='Release', dest="configuration",
                             help='Set the configuration to Release'),
        # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
    ]

    print_options = printing.print_options()

    # FIXME: These options should move onto the ChromiumPort.
    chromium_options = [
        optparse.make_option("--chromium", action="store_true", default=False,
            help="use the Chromium port"),
        optparse.make_option("--startup-dialog", action="store_true",
            default=False, help="create a dialog on DumpRenderTree startup"),
        optparse.make_option("--gp-fault-error-box", action="store_true",
            default=False, help="enable Windows GP fault error box"),
        optparse.make_option("--js-flags",
            type="string", help="JavaScript flags to pass to tests"),
        optparse.make_option("--stress-opt", action="store_true",
            default=False,
            help="Enable additional stress test to JavaScript optimization"),
        optparse.make_option("--stress-deopt", action="store_true",
            default=False,
            help="Enable additional stress test to JavaScript optimization"),
        optparse.make_option("--nocheck-sys-deps", action="store_true",
            default=False,
            help="Don't check the system dependencies (themes)"),
        optparse.make_option("--accelerated-compositing",
            action="store_true",
            help="Use hardware-accelerated compositing for rendering"),
        optparse.make_option("--no-accelerated-compositing",
            action="store_false",
            dest="accelerated_compositing",
            help="Don't use hardware-accelerated compositing for rendering"),
        optparse.make_option("--accelerated-2d-canvas",
            action="store_true",
            help="Use hardware-accelerated 2D Canvas calls"),
        optparse.make_option("--no-accelerated-2d-canvas",
            action="store_false",
            dest="accelerated_2d_canvas",
            help="Don't use hardware-accelerated 2D Canvas calls"),
        optparse.make_option("--enable-hardware-gpu",
            action="store_true",
            default=False,
            help="Run graphics tests on real GPU hardware vs software"),
    ]

    # Missing Mac-specific old-run-webkit-tests options:
    # FIXME: Need: -g, --guard for guard malloc support on Mac.
    # FIXME: Need: -l --leaks    Enable leaks checking.
    # FIXME: Need: --sample-on-timeout Run sample on timeout

    old_run_webkit_tests_compat = [
        # NRWT doesn't generate results by default anyway.
        _compat_shim_option("--no-new-test-results"),
        # NRWT doesn't sample on timeout yet anyway.
        _compat_shim_option("--no-sample-on-timeout"),
        # FIXME: NRWT needs to support remote links eventually.
        _compat_shim_option("--use-remote-links-to-tests"),
    ]

    results_options = [
        # NEED for bots: --use-remote-links-to-tests Link to test files
        # within the SVN repository in the results.
        optparse.make_option("-p", "--pixel-tests", action="store_true",
            dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
        optparse.make_option("--no-pixel-tests", action="store_false",
            dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
        optparse.make_option("--tolerance",
            help="Ignore image differences less than this percentage (some "
                "ports may ignore this option)", type="float"),
        optparse.make_option("--results-directory",
            default="layout-test-results",
            help="Output results directory source dir, relative to Debug or "
                 "Release"),
        optparse.make_option("--build-directory",
            help="Path to the directory under which build files are kept (should not include configuration)"),
        optparse.make_option("--new-baseline", action="store_true",
            default=False, help="Save all generated results as new baselines "
                 "into the platform directory, overwriting whatever's "
                 "already there."),
        optparse.make_option("--reset-results", action="store_true",
            default=False, help="Reset any existing baselines to the "
                 "generated results"),
        optparse.make_option("--no-show-results", action="store_false",
            default=True, dest="show_results",
            help="Don't launch a browser with results after the tests "
                 "are done"),
        # FIXME: We should have a helper function to do this sort of
        # deprectated mapping and automatically log, etc.
        optparse.make_option("--noshow-results", action="store_false",
            dest="show_results",
            help="Deprecated, same as --no-show-results."),
        optparse.make_option("--no-launch-safari", action="store_false",
            dest="show_results",
            help="old-run-webkit-tests compat, same as --noshow-results."),
        # old-run-webkit-tests:
        # --[no-]launch-safari    Launch (or do not launch) Safari to display
        #                         test results (default: launch)
        optparse.make_option("--full-results-html", action="store_true",
            default=False,
            help="Show all failures in results.html, rather than only "
                 "regressions"),
        optparse.make_option("--clobber-old-results", action="store_true",
            default=False, help="Clobbers test results from previous runs."),
        optparse.make_option("--platform",
            help="Override the platform for expected results"),
        optparse.make_option("--no-record-results", action="store_false",
            default=True, dest="record_results",
            help="Don't record the results."),
        # old-run-webkit-tests also has HTTP toggle options:
        # --[no-]http                     Run (or do not run) http tests
        #                                 (default: run)
    ]

    test_options = [
        optparse.make_option("--build", dest="build",
            action="store_true", default=True,
            help="Check to ensure the DumpRenderTree build is up-to-date "
                 "(default)."),
        optparse.make_option("--no-build", dest="build",
            action="store_false", help="Don't check to see if the "
                                       "DumpRenderTree build is up-to-date."),
        optparse.make_option("-n", "--dry-run", action="store_true",
            default=False,
            help="Do everything but actually run the tests or upload results."),
        # old-run-webkit-tests has --valgrind instead of wrapper.
        optparse.make_option("--wrapper",
            help="wrapper command to insert before invocations of "
                 "DumpRenderTree; option is split on whitespace before "
                 "running. (Example: --wrapper='valgrind --smc-check=all')"),
        # old-run-webkit-tests:
        # -i|--ignore-tests               Comma-separated list of directories
        #                                 or tests to ignore
        optparse.make_option("--test-list", action="append",
            help="read list of tests to run from file", metavar="FILE"),
        # old-run-webkit-tests uses --skipped==[default|ignore|only]
        # instead of --force:
        optparse.make_option("--force", action="store_true", default=False,
            help="Run all tests, even those marked SKIP in the test list"),
        optparse.make_option("--use-apache", action="store_true",
            default=False, help="Whether to use apache instead of lighttpd."),
        optparse.make_option("--time-out-ms",
            help="Set the timeout for each test"),
        # old-run-webkit-tests calls --randomize-order --random:
        optparse.make_option("--randomize-order", action="store_true",
            default=False, help=("Run tests in random order (useful "
                                "for tracking down corruption)")),
        optparse.make_option("--run-chunk",
            help=("Run a specified chunk (n:l), the nth of len l, "
                 "of the layout tests")),
        optparse.make_option("--run-part", help=("Run a specified part (n:m), "
                  "the nth of m parts, of the layout tests")),
        # old-run-webkit-tests calls --batch-size: --nthly n
        #   Restart DumpRenderTree every n tests (default: 1000)
        optparse.make_option("--batch-size",
            help=("Run a the tests in batches (n), after every n tests, "
                  "DumpRenderTree is relaunched."), type="int", default=0),
        # old-run-webkit-tests calls --run-singly: -1|--singly
        # Isolate each test case run (implies --nthly 1 --verbose)
        optparse.make_option("--run-singly", action="store_true",
            default=False, help="run a separate DumpRenderTree for each test"),
        optparse.make_option("--child-processes",
            help="Number of DumpRenderTrees to run in parallel."),
        # FIXME: Display default number of child processes that will run.
        optparse.make_option("--worker-model", action="store",
            default=None, help=("controls worker model. Valid values are 'old-inline', "
                                "'old-threads', 'inline', 'threads', and 'processes'.")),
        optparse.make_option("--experimental-fully-parallel",
            action="store_true", default=False,
            help="run all tests in parallel"),
        optparse.make_option("--exit-after-n-failures", type="int", default=500,
            help="Exit after the first N failures instead of running all "
            "tests"),
        optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
            default=20, help="Exit after the first N crashes instead of "
            "running all tests"),
        # FIXME: consider: --iterations n
        #      Number of times to run the set of tests (e.g. ABCABCABC)
        optparse.make_option("--print-last-failures", action="store_true",
            default=False, help="Print the tests in the last run that "
            "had unexpected failures (or passes) and then exit."),
        optparse.make_option("--retest-last-failures", action="store_true",
            default=False, help="re-test the tests in the last run that "
            "had unexpected failures (or passes)."),
        optparse.make_option("--retry-failures", action="store_true",
            default=True,
            help="Re-try any tests that produce unexpected results (default)"),
        optparse.make_option("--no-retry-failures", action="store_false",
            dest="retry_failures",
            help="Don't re-try any tests that produce unexpected results."),
    ]

    misc_options = [
        optparse.make_option("--lint-test-files", action="store_true",
        default=False, help=("Makes sure the test files parse for all "
                            "configurations. Does not run any tests.")),
    ]

    # FIXME: Move these into json_results_generator.py
    results_json_options = [
        optparse.make_option("--master-name", help="The name of the buildbot master."),
        optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
            help=("The name of the builder shown on the waterfall running "
                  "this script e.g. WebKit.")),
        optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
            help=("The name of the builder used in its path, e.g. "
                  "webkit-rel.")),
        optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
            help=("The build number of the builder running this script.")),
        optparse.make_option("--test-results-server", default="",
            help=("If specified, upload results json files to this appengine "
                  "server.")),
    ]

    option_list = (configuration_options + print_options +
                   chromium_options + results_options + test_options +
                   misc_options + results_json_options +
                   old_run_webkit_tests_compat)
    option_parser = optparse.OptionParser(option_list=option_list)

    return option_parser.parse_args(args)


def main():
    options, args = parse_args()
    port_obj = port.get(options.platform, options)
    return run(port_obj, options, args)


if '__main__' == __name__:
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        # this mirrors what the shell normally does
        sys.exit(signal.SIGINT + 128)