diff options
Diffstat (limited to 'Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py')
| -rw-r--r-- | Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py | 77 |
1 files changed, 67 insertions, 10 deletions
diff --git a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py index 3fe7b14..7076ef2 100644 --- a/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py +++ b/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py @@ -55,6 +55,7 @@ from webkitpy.layout_tests import port from webkitpy.layout_tests import run_webkit_tests from webkitpy.layout_tests.layout_package import dump_render_tree_thread from webkitpy.layout_tests.port.test import TestPort, TestDriver +from webkitpy.layout_tests.port.test_files import is_reference_html_file from webkitpy.python24.versioning import compare_version from webkitpy.test.skip import skip_if @@ -72,8 +73,8 @@ def parse_args(extra_args=None, record_results=False, tests_included=False, args.extend(['--platform', 'test']) if not record_results: args.append('--no-record-results') - if not '--child-processes' in extra_args: - args.extend(['--worker-model', 'old-inline']) + if not '--child-processes' in extra_args and not '--worker-model' in extra_args: + args.extend(['--worker-model', 'inline']) args.extend(extra_args) if not tests_included: # We use the glob to test that globbing works. @@ -124,7 +125,8 @@ def run_and_capture(port_obj, options, parsed_args): return (res, buildbot_output, regular_output) -def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False, filesystem=None): +def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False, + filesystem=None, include_reference_html=False): extra_args = extra_args or [] if not tests_included: # Not including http tests since they get run out of order (that @@ -136,6 +138,7 @@ def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False, test_batches = [] + class RecordingTestDriver(TestDriver): def __init__(self, port, worker_number): TestDriver.__init__(self, port, worker_number) @@ -153,7 +156,11 @@ def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False, self._current_test_batch = [] test_batches.append(self._current_test_batch) test_name = self._port.relative_test_filename(test_input.filename) - self._current_test_batch.append(test_name) + # In case of reftest, one test calls the driver's run_test() twice. + # We should not add a reference html used by reftests to tests unless include_reference_html parameter + # is explicitly given. + if include_reference_html or not is_reference_html_file(test_input.filename): + self._current_test_batch.append(test_name) return TestDriver.run_test(self, test_input) class RecordingTestPort(TestPort): @@ -190,13 +197,13 @@ class MainTest(unittest.TestCase): def test_child_process_1(self): (res, buildbot_output, regular_output, user) = logging_run( - ['--print', 'config', '--child-processes', '1']) + ['--print', 'config', '--worker-model', 'threads', '--child-processes', '1']) self.assertTrue('Running one DumpRenderTree\n' in regular_output.get()) def test_child_processes_2(self): (res, buildbot_output, regular_output, user) = logging_run( - ['--print', 'config', '--child-processes', '2']) + ['--print', 'config', '--worker-model', 'threads', '--child-processes', '2']) self.assertTrue('Running 2 DumpRenderTrees in parallel\n' in regular_output.get()) @@ -352,7 +359,12 @@ class MainTest(unittest.TestCase): # Run tests including the unexpected failures. self._url_opened = None res, out, err, user = logging_run(tests_included=True) - self.assertEqual(res, 3) + + # Update this magic number if you add an unexpected test to webkitpy.layout_tests.port.test + # FIXME: It's nice to have a routine in port/test.py that returns this number. + unexpected_tests_count = 5 + + self.assertEqual(res, unexpected_tests_count) self.assertFalse(out.empty()) self.assertFalse(err.empty()) self.assertEqual(user.opened_urls, ['/tmp/layout-test-results/results.html']) @@ -458,6 +470,32 @@ class MainTest(unittest.TestCase): tests_included=True) self.assertEqual(user.opened_urls, ['/tmp/foo/results.html']) + # These next tests test that we run the tests in ascending alphabetical + # order per directory. HTTP tests are sharded separately from other tests, + # so we have to test both. + def assert_run_order(self, worker_model, child_processes='1'): + tests_run = get_tests_run(['--worker-model', worker_model, + '--child-processes', child_processes, 'passes'], + tests_included=True, flatten_batches=True) + self.assertEquals(tests_run, sorted(tests_run)) + + tests_run = get_tests_run(['--worker-model', worker_model, + '--child-processes', child_processes, 'http/tests/passes'], + tests_included=True, flatten_batches=True) + self.assertEquals(tests_run, sorted(tests_run)) + + def test_run_order__inline(self): + self.assert_run_order('inline') + + def test_run_order__old_inline(self): + self.assert_run_order('old-inline') + + def test_run_order__threads(self): + self.assert_run_order('old-inline', child_processes='2') + + def test_run_order__old_threads(self): + self.assert_run_order('old-threads', child_processes='2') + def test_tolerance(self): class ImageDiffTestPort(TestPort): def diff_image(self, expected_contents, actual_contents, @@ -487,11 +525,11 @@ class MainTest(unittest.TestCase): def test_worker_model__inline(self): self.assertTrue(passing_run(['--worker-model', 'inline'])) - def test_worker_model__old_inline_with_child_processes(self): - res, out, err, user = logging_run(['--worker-model', 'old-inline', + def test_worker_model__inline_with_child_processes(self): + res, out, err, user = logging_run(['--worker-model', 'inline', '--child-processes', '2']) self.assertEqual(res, 0) - self.assertTrue('--worker-model=old-inline overrides --child-processes\n' in err.get()) + self.assertTrue('--worker-model=inline overrides --child-processes\n' in err.get()) def test_worker_model__old_inline(self): self.assertTrue(passing_run(['--worker-model', 'old-inline'])) @@ -516,6 +554,25 @@ class MainTest(unittest.TestCase): self.assertRaises(ValueError, logging_run, ['--worker-model', 'unknown']) + def test_reftest_run(self): + tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True) + self.assertEquals(['passes/reftest.html'], tests_run) + + def test_reftest_expected_html_should_be_ignored(self): + tests_run = get_tests_run(['passes/reftest-expected.html'], tests_included=True, flatten_batches=True) + self.assertEquals([], tests_run) + + def test_reftest_driver_should_run_expected_html(self): + tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True, + include_reference_html=True) + self.assertEquals(['passes/reftest.html', 'passes/reftest-expected.html'], tests_run) + + def test_reftest_driver_should_run_expected_mismatch_html(self): + tests_run = get_tests_run(['passes/mismatch.html'], tests_included=True, flatten_batches=True, + include_reference_html=True) + self.assertEquals(['passes/mismatch.html', 'passes/mismatch-expected-mismatch.html'], tests_run) + + MainTest = skip_if(MainTest, sys.platform == 'cygwin' and compare_version(sys, '2.6')[0] < 0, 'new-run-webkit-tests tests hang on Cygwin Python 2.5.2') |