summaryrefslogtreecommitdiffstats
path: root/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
blob: 2d55b93d27b8006c37bb286395ce85e0b4341def (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
#!/usr/bin/env python
# Copyright (C) 2010 Google Inc. All rights reserved.
# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Run layout tests."""

import errno
import logging
import optparse
import os
import signal
import sys

from layout_package import printing
from layout_package import test_runner
from layout_package import test_runner2

from webkitpy.common.system import user
from webkitpy.thirdparty import simplejson

import port

_log = logging.getLogger(__name__)


def run(port, options, args, regular_output=sys.stderr,
        buildbot_output=sys.stdout):
    """Run the tests.

    Args:
      port: Port object for port-specific behavior
      options: a dictionary of command line options
      args: a list of sub directories or files to test
      regular_output: a stream-like object that we can send logging/debug
          output to
      buildbot_output: a stream-like object that we can write all output that
          is intended to be parsed by the buildbot to
    Returns:
      the number of unexpected results that occurred, or -1 if there is an
          error.

    """
    warnings = _set_up_derived_options(port, options)

    printer = printing.Printer(port, options, regular_output, buildbot_output,
        int(options.child_processes), options.experimental_fully_parallel)
    for w in warnings:
        _log.warning(w)

    if options.help_printing:
        printer.help_printing()
        printer.cleanup()
        return 0

    last_unexpected_results = _gather_unexpected_results(port._filesystem, options)
    if options.print_last_failures:
        printer.write("\n".join(last_unexpected_results) + "\n")
        printer.cleanup()
        return 0

    # We wrap any parts of the run that are slow or likely to raise exceptions
    # in a try/finally to ensure that we clean up the logging configuration.
    num_unexpected_results = -1
    try:
        if options.worker_model in ('inline', 'threads', 'processes'):
            runner = test_runner2.TestRunner2(port, options, printer)
        else:
            runner = test_runner.TestRunner(port, options, printer)

        runner._print_config()

        printer.print_update("Collecting tests ...")
        try:
            runner.collect_tests(args, last_unexpected_results)
        except IOError, e:
            if e.errno == errno.ENOENT:
                return -1
            raise

        if options.lint_test_files:
            return runner.lint()

        printer.print_update("Parsing expectations ...")
        runner.parse_expectations()

        printer.print_update("Checking build ...")
        if not port.check_build(runner.needs_http()):
            _log.error("Build check failed")
            return -1

        result_summary = runner.set_up_run()
        if result_summary:
            num_unexpected_results = runner.run(result_summary)
            runner.clean_up_run()
            _log.debug("Testing completed, Exit status: %d" %
                       num_unexpected_results)
    finally:
        printer.cleanup()

    return num_unexpected_results


def _set_up_derived_options(port_obj, options):
    """Sets the options values that depend on other options values."""
    # We return a list of warnings to print after the printer is initialized.
    warnings = []

    if options.worker_model is None:
        options.worker_model = port_obj.default_worker_model()

    if options.worker_model in ('inline', 'old-inline'):
        if options.child_processes and int(options.child_processes) > 1:
            warnings.append("--worker-model=%s overrides --child-processes" % options.worker_model)
        options.child_processes = "1"
    if not options.child_processes:
        options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
                                                 str(port_obj.default_child_processes()))

    if not options.configuration:
        options.configuration = port_obj.default_configuration()

    if options.pixel_tests is None:
        options.pixel_tests = True

    if not options.use_apache:
        options.use_apache = sys.platform in ('darwin', 'linux2')

    if not port_obj._filesystem.isabs(options.results_directory):
        # This normalizes the path to the build dir.
        # FIXME: how this happens is not at all obvious; this is a dumb
        # interface and should be cleaned up.
        options.results_directory = port_obj.results_directory()

    if not options.time_out_ms:
        if options.configuration == "Debug":
            options.time_out_ms = str(2 * test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
        else:
            options.time_out_ms = str(test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)

    options.slow_time_out_ms = str(5 * int(options.time_out_ms))
    return warnings


def _gather_unexpected_results(filesystem, options):
    """Returns the unexpected results from the previous run, if any."""
    last_unexpected_results = []
    if options.print_last_failures or options.retest_last_failures:
        unexpected_results_filename = filesystem.join(
            options.results_directory, "unexpected_results.json")
        if filesystem.exists(unexpected_results_filename):
            content = filesystem.read_text_file(unexpected_results_filename)
            results = simplejson.loads(content)
            last_unexpected_results = results['tests'].keys()
    return last_unexpected_results


def _compat_shim_callback(option, opt_str, value, parser):
    print "Ignoring unsupported option: %s" % opt_str


def _compat_shim_option(option_name, **kwargs):
    return optparse.make_option(option_name, action="callback",
        callback=_compat_shim_callback,
        help="Ignored, for old-run-webkit-tests compat only.", **kwargs)


def parse_args(args=None):
    """Provides a default set of command line args.

    Returns a tuple of options, args from optparse"""

    # FIXME: All of these options should be stored closer to the code which
    # FIXME: actually uses them. configuration_options should move
    # FIXME: to WebKitPort and be shared across all scripts.
    configuration_options = [
        optparse.make_option("-t", "--target", dest="configuration",
                             help="(DEPRECATED)"),
        # FIXME: --help should display which configuration is default.
        optparse.make_option('--debug', action='store_const', const='Debug',
                             dest="configuration",
                             help='Set the configuration to Debug'),
        optparse.make_option('--release', action='store_const',
                             const='Release', dest="configuration",
                             help='Set the configuration to Release'),
        # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
    ]

    print_options = printing.print_options()

    # FIXME: These options should move onto the ChromiumPort.
    chromium_options = [
        optparse.make_option("--chromium", action="store_true", default=False,
            help="use the Chromium port"),
        optparse.make_option("--startup-dialog", action="store_true",
            default=False, help="create a dialog on DumpRenderTree startup"),
        optparse.make_option("--gp-fault-error-box", action="store_true",
            default=False, help="enable Windows GP fault error box"),
        optparse.make_option("--js-flags",
            type="string", help="JavaScript flags to pass to tests"),
        optparse.make_option("--stress-opt", action="store_true",
            default=False,
            help="Enable additional stress test to JavaScript optimization"),
        optparse.make_option("--stress-deopt", action="store_true",
            default=False,
            help="Enable additional stress test to JavaScript optimization"),
        optparse.make_option("--nocheck-sys-deps", action="store_true",
            default=False,
            help="Don't check the system dependencies (themes)"),
        optparse.make_option("--accelerated-compositing",
            action="store_true",
            help="Use hardware-accelated compositing for rendering"),
        optparse.make_option("--no-accelerated-compositing",
            action="store_false",
            dest="accelerated_compositing",
            help="Don't use hardware-accelerated compositing for rendering"),
        optparse.make_option("--accelerated-2d-canvas",
            action="store_true",
            help="Use hardware-accelerated 2D Canvas calls"),
        optparse.make_option("--no-accelerated-2d-canvas",
            action="store_false",
            dest="accelerated_2d_canvas",
            help="Don't use hardware-accelerated 2D Canvas calls"),
        optparse.make_option("--enable-hardware-gpu",
            action="store_true",
            default=False,
            help="Run graphics tests on real GPU hardware vs software"),
    ]

    # Missing Mac-specific old-run-webkit-tests options:
    # FIXME: Need: -g, --guard for guard malloc support on Mac.
    # FIXME: Need: -l --leaks    Enable leaks checking.
    # FIXME: Need: --sample-on-timeout Run sample on timeout

    old_run_webkit_tests_compat = [
        # NRWT doesn't generate results by default anyway.
        _compat_shim_option("--no-new-test-results"),
        # NRWT doesn't sample on timeout yet anyway.
        _compat_shim_option("--no-sample-on-timeout"),
        # FIXME: NRWT needs to support remote links eventually.
        _compat_shim_option("--use-remote-links-to-tests"),
    ]

    results_options = [
        # NEED for bots: --use-remote-links-to-tests Link to test files
        # within the SVN repository in the results.
        optparse.make_option("-p", "--pixel-tests", action="store_true",
            dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
        optparse.make_option("--no-pixel-tests", action="store_false",
            dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
        optparse.make_option("--tolerance",
            help="Ignore image differences less than this percentage (some "
                "ports may ignore this option)", type="float"),
        optparse.make_option("--results-directory",
            default="layout-test-results",
            help="Output results directory source dir, relative to Debug or "
                 "Release"),
        optparse.make_option("--build-directory",
            help="Path to the directory under which build files are kept (should not include configuration)"),
        optparse.make_option("--new-baseline", action="store_true",
            default=False, help="Save all generated results as new baselines "
                 "into the platform directory, overwriting whatever's "
                 "already there."),
        optparse.make_option("--reset-results", action="store_true",
            default=False, help="Reset any existing baselines to the "
                 "generated results"),
        optparse.make_option("--no-show-results", action="store_false",
            default=True, dest="show_results",
            help="Don't launch a browser with results after the tests "
                 "are done"),
        # FIXME: We should have a helper function to do this sort of
        # deprectated mapping and automatically log, etc.
        optparse.make_option("--noshow-results", action="store_false",
            dest="show_results",
            help="Deprecated, same as --no-show-results."),
        optparse.make_option("--no-launch-safari", action="store_false",
            dest="show_results",
            help="old-run-webkit-tests compat, same as --noshow-results."),
        # old-run-webkit-tests:
        # --[no-]launch-safari    Launch (or do not launch) Safari to display
        #                         test results (default: launch)
        optparse.make_option("--full-results-html", action="store_true",
            default=False,
            help="Show all failures in results.html, rather than only "
                 "regressions"),
        optparse.make_option("--clobber-old-results", action="store_true",
            default=False, help="Clobbers test results from previous runs."),
        optparse.make_option("--platform",
            help="Override the platform for expected results"),
        optparse.make_option("--no-record-results", action="store_false",
            default=True, dest="record_results",
            help="Don't record the results."),
        # old-run-webkit-tests also has HTTP toggle options:
        # --[no-]http                     Run (or do not run) http tests
        #                                 (default: run)
    ]

    test_options = [
        optparse.make_option("--build", dest="build",
            action="store_true", default=True,
            help="Check to ensure the DumpRenderTree build is up-to-date "
                 "(default)."),
        optparse.make_option("--no-build", dest="build",
            action="store_false", help="Don't check to see if the "
                                       "DumpRenderTree build is up-to-date."),
        optparse.make_option("-n", "--dry-run", action="store_true",
            default=False,
            help="Do everything but actually run the tests or upload results."),
        # old-run-webkit-tests has --valgrind instead of wrapper.
        optparse.make_option("--wrapper",
            help="wrapper command to insert before invocations of "
                 "DumpRenderTree; option is split on whitespace before "
                 "running. (Example: --wrapper='valgrind --smc-check=all')"),
        # old-run-webkit-tests:
        # -i|--ignore-tests               Comma-separated list of directories
        #                                 or tests to ignore
        optparse.make_option("--test-list", action="append",
            help="read list of tests to run from file", metavar="FILE"),
        # old-run-webkit-tests uses --skipped==[default|ignore|only]
        # instead of --force:
        optparse.make_option("--force", action="store_true", default=False,
            help="Run all tests, even those marked SKIP in the test list"),
        optparse.make_option("--use-apache", action="store_true",
            default=False, help="Whether to use apache instead of lighttpd."),
        optparse.make_option("--time-out-ms",
            help="Set the timeout for each test"),
        # old-run-webkit-tests calls --randomize-order --random:
        optparse.make_option("--randomize-order", action="store_true",
            default=False, help=("Run tests in random order (useful "
                                "for tracking down corruption)")),
        optparse.make_option("--run-chunk",
            help=("Run a specified chunk (n:l), the nth of len l, "
                 "of the layout tests")),
        optparse.make_option("--run-part", help=("Run a specified part (n:m), "
                  "the nth of m parts, of the layout tests")),
        # old-run-webkit-tests calls --batch-size: --nthly n
        #   Restart DumpRenderTree every n tests (default: 1000)
        optparse.make_option("--batch-size",
            help=("Run a the tests in batches (n), after every n tests, "
                  "DumpRenderTree is relaunched."), type="int", default=0),
        # old-run-webkit-tests calls --run-singly: -1|--singly
        # Isolate each test case run (implies --nthly 1 --verbose)
        optparse.make_option("--run-singly", action="store_true",
            default=False, help="run a separate DumpRenderTree for each test"),
        optparse.make_option("--child-processes",
            help="Number of DumpRenderTrees to run in parallel."),
        # FIXME: Display default number of child processes that will run.
        optparse.make_option("--worker-model", action="store",
            default=None, help=("controls worker model. Valid values are 'old-inline', "
                                "'old-threads', 'inline', 'threads', and 'processes'.")),
        optparse.make_option("--experimental-fully-parallel",
            action="store_true", default=False,
            help="run all tests in parallel"),
        optparse.make_option("--exit-after-n-failures", type="int", nargs=1,
            help="Exit after the first N failures instead of running all "
            "tests"),
        optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
            nargs=1, help="Exit after the first N crashes instead of running "
            "all tests"),
        # FIXME: consider: --iterations n
        #      Number of times to run the set of tests (e.g. ABCABCABC)
        optparse.make_option("--print-last-failures", action="store_true",
            default=False, help="Print the tests in the last run that "
            "had unexpected failures (or passes) and then exit."),
        optparse.make_option("--retest-last-failures", action="store_true",
            default=False, help="re-test the tests in the last run that "
            "had unexpected failures (or passes)."),
        optparse.make_option("--retry-failures", action="store_true",
            default=True,
            help="Re-try any tests that produce unexpected results (default)"),
        optparse.make_option("--no-retry-failures", action="store_false",
            dest="retry_failures",
            help="Don't re-try any tests that produce unexpected results."),
    ]

    misc_options = [
        optparse.make_option("--lint-test-files", action="store_true",
        default=False, help=("Makes sure the test files parse for all "
                            "configurations. Does not run any tests.")),
    ]

    # FIXME: Move these into json_results_generator.py
    results_json_options = [
        optparse.make_option("--master-name", help="The name of the buildbot master."),
        optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
            help=("The name of the builder shown on the waterfall running "
                  "this script e.g. WebKit.")),
        optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
            help=("The name of the builder used in its path, e.g. "
                  "webkit-rel.")),
        optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
            help=("The build number of the builder running this script.")),
        optparse.make_option("--test-results-server", default="",
            help=("If specified, upload results json files to this appengine "
                  "server.")),
    ]

    option_list = (configuration_options + print_options +
                   chromium_options + results_options + test_options +
                   misc_options + results_json_options +
                   old_run_webkit_tests_compat)
    option_parser = optparse.OptionParser(option_list=option_list)

    return option_parser.parse_args(args)


def main():
    options, args = parse_args()
    port_obj = port.get(options.platform, options)
    return run(port_obj, options, args)


if '__main__' == __name__:
    try:
        sys.exit(main())
    except KeyboardInterrupt:
        # this mirrors what the shell normally does
        sys.exit(signal.SIGINT + 128)