diff options
Diffstat (limited to 'Tools/Scripts/webkitpy/tool')
23 files changed, 391 insertions, 101 deletions
diff --git a/Tools/Scripts/webkitpy/tool/bot/botinfo.py b/Tools/Scripts/webkitpy/tool/bot/botinfo.py new file mode 100644 index 0000000..b9fd938 --- /dev/null +++ b/Tools/Scripts/webkitpy/tool/bot/botinfo.py @@ -0,0 +1,39 @@ +# Copyright (c) 2011 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# FIXME: We should consider hanging one of these off the tool object. +class BotInfo(object): + def __init__(self, tool): + self._tool = tool + + def summary_text(self): + # bot_id is also stored on the options dictionary on the tool. + bot_id = self._tool.status_server.bot_id + bot_id_string = "Bot: %s " % (bot_id) if bot_id else "" + return "%sPort: %s Platform: %s" % (bot_id_string, self._tool.port().name(), self._tool.platform.display_name()) diff --git a/Tools/Scripts/webkitpy/tool/bot/botinfo_unittest.py b/Tools/Scripts/webkitpy/tool/bot/botinfo_unittest.py new file mode 100644 index 0000000..054acfc --- /dev/null +++ b/Tools/Scripts/webkitpy/tool/bot/botinfo_unittest.py @@ -0,0 +1,40 @@ +# Copyright (c) 2011 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.tool.bot.botinfo import BotInfo +from webkitpy.tool.mocktool import MockTool, MockStatusServer + + +class BotInfoTest(unittest.TestCase): + + def test_summary_text(self): + tool = MockTool() + tool.status_server = MockStatusServer("MockBotId") + self.assertEqual(BotInfo(tool).summary_text(), "Bot: MockBotId Port: MockPort Platform: MockPlatform 1.0") diff --git a/Tools/Scripts/webkitpy/tool/bot/commitqueuetask.py b/Tools/Scripts/webkitpy/tool/bot/commitqueuetask.py index c5d9001..93cbcc8 100644 --- a/Tools/Scripts/webkitpy/tool/bot/commitqueuetask.py +++ b/Tools/Scripts/webkitpy/tool/bot/commitqueuetask.py @@ -28,6 +28,7 @@ from webkitpy.common.system.executive import ScriptError from webkitpy.common.net.layouttestresults import LayoutTestResults +from webkitpy.tool.bot.expectedfailures import ExpectedFailures class CommitQueueTaskDelegate(object): @@ -59,6 +60,8 @@ class CommitQueueTask(object): self._delegate = delegate self._patch = patch self._script_error = None + self._results_archive_from_patch_test_run = None + self._expected_failures = ExpectedFailures() def _validate(self): # Bugs might get closed, or patches might be obsoleted or r-'d while the @@ -132,7 +135,7 @@ class CommitQueueTask(object): "Unable to build without patch") def _test(self): - return self._run_command([ + success = self._run_command([ "build-and-test", "--no-clean", "--no-update", @@ -143,8 +146,11 @@ class CommitQueueTask(object): "Passed tests", "Patch does not pass tests") + self._expected_failures.shrink_expected_failures(self._delegate.layout_test_results(), success) + return success + def _build_and_test_without_patch(self): - return self._run_command([ + success = self._run_command([ "build-and-test", "--force-clean", "--no-update", @@ -155,11 +161,8 @@ class CommitQueueTask(object): "Able to pass tests without patch", "Unable to pass tests without patch (tree is red?)") - def _failing_results_from_last_run(self): - results = self._delegate.layout_test_results() - if not results: - return [] # Makes callers slighty cleaner to not have to deal with None - return results.failing_test_results() + self._expected_failures.shrink_expected_failures(self._delegate.layout_test_results(), success) + return success def _land(self): # Unclear if this should pass --quiet or not. If --parent-command always does the reporting, then it should. @@ -177,36 +180,59 @@ class CommitQueueTask(object): def _report_flaky_tests(self, flaky_test_results, results_archive): self._delegate.report_flaky_tests(self._patch, flaky_test_results, results_archive) + def _results_failed_different_tests(self, first, second): + first_failing_tests = [] if not first else first.failing_tests() + second_failing_tests = [] if not second else second.failing_tests() + return first_failing_tests != second_failing_tests + def _test_patch(self): if self._test(): return True - first_results = self._failing_results_from_last_run() - first_failing_tests = [result.filename for result in first_results] + # Note: archive_last_layout_test_results deletes the results directory, making these calls order-sensitve. + # We could remove this dependency by building the layout_test_results from the archive. + first_results = self._delegate.layout_test_results() first_results_archive = self._delegate.archive_last_layout_test_results(self._patch) + + if self._expected_failures.failures_were_expected(first_results): + return True + if self._test(): - # Only report flaky tests if we were successful at archiving results. - if first_results_archive: - self._report_flaky_tests(first_results, first_results_archive) + # Only report flaky tests if we were successful at parsing results.html and archiving results. + if first_results and first_results_archive: + self._report_flaky_tests(first_results.failing_test_results(), first_results_archive) return True - second_results = self._failing_results_from_last_run() - second_failing_tests = [result.filename for result in second_results] - if first_failing_tests != second_failing_tests: - # We could report flaky tests here, but since run-webkit-tests - # is run with --exit-after-N-failures=1, we would need to - # be careful not to report constant failures as flaky due to earlier - # flaky test making them not fail (no results) in one of the runs. + second_results = self._delegate.layout_test_results() + if self._results_failed_different_tests(first_results, second_results): + # We could report flaky tests here, but we would need to be careful + # to use similar checks to ExpectedFailures._can_trust_results + # to make sure we don't report constant failures as flakes when + # we happen to hit the --exit-after-N-failures limit. # See https://bugs.webkit.org/show_bug.cgi?id=51272 return False + # Archive (and remove) second results so layout_test_results() after + # build_and_test_without_patch won't use second results instead of the clean-tree results. + second_results_archive = self._delegate.archive_last_layout_test_results(self._patch) + if self._build_and_test_without_patch(): - return self.report_failure() # The error from the previous ._test() run is real, report it. - return False # Tree must be red, just retry later. + # The error from the previous ._test() run is real, report it. + return self.report_failure(first_results_archive) + + clean_tree_results = self._delegate.layout_test_results() + self._expected_failures.grow_expected_failures(clean_tree_results) + + return False # Tree must be redder than we expected, just retry later. + + def results_archive_from_patch_test_run(self, patch): + assert(self._patch.id() == patch.id()) # CommitQueueTask is not currently re-useable. + return self._results_archive_from_patch_test_run - def report_failure(self): + def report_failure(self, results_archive=None): if not self._validate(): return False + self._results_archive_from_patch_test_run = results_archive raise self._script_error def run(self): diff --git a/Tools/Scripts/webkitpy/tool/bot/commitqueuetask_unittest.py b/Tools/Scripts/webkitpy/tool/bot/commitqueuetask_unittest.py index 87d0ab5..7324d78 100644 --- a/Tools/Scripts/webkitpy/tool/bot/commitqueuetask_unittest.py +++ b/Tools/Scripts/webkitpy/tool/bot/commitqueuetask_unittest.py @@ -30,6 +30,7 @@ from datetime import datetime import unittest from webkitpy.common.net import bugzilla +from webkitpy.common.net.layouttestresults import LayoutTestResults from webkitpy.common.system.deprecated_logging import error, log from webkitpy.common.system.outputcapture import OutputCapture from webkitpy.layout_tests.layout_package import test_results @@ -77,9 +78,6 @@ class MockCommitQueue(CommitQueueTaskDelegate): class CommitQueueTaskTest(unittest.TestCase): - def _mock_test_result(self, testname): - return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) - def _run_through_task(self, commit_queue, expected_stderr, expected_exception=None, expect_retry=False): tool = MockTool(log_executive=True) patch = tool.bugs.fetch_attachment(197) @@ -190,6 +188,9 @@ command_failed: failure_message='Unable to build without patch' script_error='MO None, ScriptError("MOCK tests failure"), ]) + # CommitQueueTask will only report flaky tests if we successfully parsed + # results.html and returned a LayoutTestResults object, so we fake one. + commit_queue.layout_test_results = lambda: LayoutTestResults([]) expected_stderr = """run_webkit_patch: ['clean'] command_passed: success_message='Cleaned working directory' patch='197' run_webkit_patch: ['update'] @@ -217,6 +218,7 @@ command_passed: success_message='Landed patch' patch='197' None, ScriptError("MOCK tests failure"), ]) + commit_queue.layout_test_results = lambda: LayoutTestResults([]) # It's possible delegate to fail to archive layout tests, don't try to report # flaky tests when that happens. commit_queue.archive_last_layout_test_results = lambda patch: None @@ -237,10 +239,25 @@ command_passed: success_message='Landed patch' patch='197' """ self._run_through_task(commit_queue, expected_stderr) - _double_flaky_test_counter = 0 - def test_double_flaky_test_failure(self): - commit_queue = MockCommitQueue([ + class DoubleFlakyCommitQueue(MockCommitQueue): + def __init__(self, error_plan): + MockCommitQueue.__init__(self, error_plan) + self._double_flaky_test_counter = 0 + + def run_command(self, command): + self._double_flaky_test_counter += 1 + MockCommitQueue.run_command(self, command) + + def _mock_test_result(self, testname): + return test_results.TestResult(testname, [test_failures.FailureTextMismatch()]) + + def layout_test_results(self): + if self._double_flaky_test_counter % 2: + return LayoutTestResults([self._mock_test_result('foo.html')]) + return LayoutTestResults([self._mock_test_result('bar.html')]) + + commit_queue = DoubleFlakyCommitQueue([ None, None, None, @@ -268,15 +285,6 @@ command_failed: failure_message='Patch does not pass tests' script_error='MOCK t tool = MockTool(log_executive=True) patch = tool.bugs.fetch_attachment(197) task = CommitQueueTask(commit_queue, patch) - self._double_flaky_test_counter = 0 - - def mock_failing_results_from_last_run(): - CommitQueueTaskTest._double_flaky_test_counter += 1 - if CommitQueueTaskTest._double_flaky_test_counter % 2: - return [self._mock_test_result('foo.html')] - return [self._mock_test_result('bar.html')] - - task._failing_results_from_last_run = mock_failing_results_from_last_run success = OutputCapture().assert_outputs(self, task.run, expected_stderr=expected_stderr) self.assertEqual(success, False) @@ -302,6 +310,7 @@ command_failed: failure_message='Patch does not pass tests' script_error='MOCK t archive_last_layout_test_results: patch='197' run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='197' +archive_last_layout_test_results: patch='197' run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive'] command_passed: success_message='Able to pass tests without patch' patch='197' """ @@ -330,6 +339,7 @@ command_failed: failure_message='Patch does not pass tests' script_error='MOCK t archive_last_layout_test_results: patch='197' run_webkit_patch: ['build-and-test', '--no-clean', '--no-update', '--test', '--non-interactive'] command_failed: failure_message='Patch does not pass tests' script_error='MOCK test failure again' patch='197' +archive_last_layout_test_results: patch='197' run_webkit_patch: ['build-and-test', '--force-clean', '--no-update', '--build', '--test', '--non-interactive'] command_failed: failure_message='Unable to pass tests without patch (tree is red?)' script_error='MOCK clean test failure' patch='197' """ diff --git a/Tools/Scripts/webkitpy/tool/bot/expectedfailures.py b/Tools/Scripts/webkitpy/tool/bot/expectedfailures.py new file mode 100644 index 0000000..8736ac0 --- /dev/null +++ b/Tools/Scripts/webkitpy/tool/bot/expectedfailures.py @@ -0,0 +1,55 @@ +# Copyright (c) 2011 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +class ExpectedFailures(object): + def __init__(self): + self._failures = set() + + def _can_trust_results(self, results): + if not results or not results.failure_limit_count(): + return False + return len(results.failing_tests()) != 0 and len(results.failing_tests()) != results.failure_limit_count() + + def failures_were_expected(self, results): + if not self._can_trust_results(results): + return False + return set(results.failing_tests()) <= self._failures + + def shrink_expected_failures(self, results, run_success): + if run_success: + self._failures = set() + elif self._can_trust_results(results): + # Remove all expected failures which are not in the new failing results. + self._failures.intersection_update(set(results.failing_tests())) + + def grow_expected_failures(self, results): + if not self._can_trust_results(results): + return + self._failures.update(results.failing_tests()) + # FIXME: Should we assert() here that expected_failures never crosses a certain size? diff --git a/Tools/Scripts/webkitpy/tool/bot/expectedfailures_unittest.py b/Tools/Scripts/webkitpy/tool/bot/expectedfailures_unittest.py new file mode 100644 index 0000000..8a2702b --- /dev/null +++ b/Tools/Scripts/webkitpy/tool/bot/expectedfailures_unittest.py @@ -0,0 +1,73 @@ +# Copyright (c) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.tool.bot.expectedfailures import ExpectedFailures + + +class MockResults(object): + def __init__(self, failing_tests=[], failure_limit=10): + self._failing_tests = failing_tests + self._failure_limit_count = failure_limit + + def failure_limit_count(self): + return self._failure_limit_count + + def failing_tests(self): + return self._failing_tests + + +class ExpectedFailuresTest(unittest.TestCase): + def _assert_can_trust(self, results, can_trust): + self.assertEquals(ExpectedFailures()._can_trust_results(results), can_trust) + + def test_can_trust_results(self): + self._assert_can_trust(None, False) + self._assert_can_trust(MockResults(failing_tests=[], failure_limit=None), False) + self._assert_can_trust(MockResults(failing_tests=[], failure_limit=10), False) + self._assert_can_trust(MockResults(failing_tests=[1], failure_limit=None), False) + self._assert_can_trust(MockResults(failing_tests=[1], failure_limit=2), True) + self._assert_can_trust(MockResults(failing_tests=[1], failure_limit=1), False) + + def _assert_expected(self, expected_failures, failures, expected): + self.assertEqual(expected_failures.failures_were_expected(MockResults(failures)), expected) + + def test_failures_were_expected(self): + failures = ExpectedFailures() + failures.grow_expected_failures(MockResults(['foo.html'])) + self._assert_expected(failures, ['foo.html'], True) + self._assert_expected(failures, ['bar.html'], False) + failures.shrink_expected_failures(MockResults(['baz.html']), False) + self._assert_expected(failures, ['foo.html'], False) + self._assert_expected(failures, ['baz.html'], False) + + failures.grow_expected_failures(MockResults(['baz.html'])) + self._assert_expected(failures, ['baz.html'], True) + failures.shrink_expected_failures(MockResults(), True) + self._assert_expected(failures, ['baz.html'], False) diff --git a/Tools/Scripts/webkitpy/tool/bot/flakytestreporter.py b/Tools/Scripts/webkitpy/tool/bot/flakytestreporter.py index bec593b..68e1c94 100644 --- a/Tools/Scripts/webkitpy/tool/bot/flakytestreporter.py +++ b/Tools/Scripts/webkitpy/tool/bot/flakytestreporter.py @@ -33,6 +33,7 @@ import os.path from webkitpy.common.net.layouttestresults import path_for_layout_test, LayoutTestResults from webkitpy.common.config import urls +from webkitpy.tool.bot.botinfo import BotInfo from webkitpy.tool.grammar import plural, pluralize, join_with_separators _log = logging.getLogger(__name__) @@ -42,6 +43,7 @@ class FlakyTestReporter(object): def __init__(self, tool, bot_name): self._tool = tool self._bot_name = bot_name + self._bot_info = BotInfo(tool) def _author_emails_for_test(self, flaky_test): test_path = path_for_layout_test(flaky_test) @@ -121,15 +123,10 @@ If you would like to track this test fix with another bug, please close this bug authors_string = join_with_separators(sorted(author_emails)) return " (%s: %s)" % (heading_string, authors_string) - def _bot_information(self): - bot_id = self._tool.status_server.bot_id - bot_id_string = "Bot: %s " % (bot_id) if bot_id else "" - return "%sPort: %s Platform: %s" % (bot_id_string, self._tool.port().name(), self._tool.platform.display_name()) - def _latest_flake_message(self, flaky_result, patch): failure_messages = [failure.message() for failure in flaky_result.failures] flake_message = "The %s just saw %s flake (%s) while processing attachment %s on bug %s." % (self._bot_name, flaky_result.filename, ", ".join(failure_messages), patch.id(), patch.bug_id()) - return "%s\n%s" % (flake_message, self._bot_information()) + return "%s\n%s" % (flake_message, self._bot_info.summary_text()) def _results_diff_path_for_test(self, test_path): # FIXME: This is a big hack. We should get this path from results.json diff --git a/Tools/Scripts/webkitpy/tool/bot/flakytestreporter_unittest.py b/Tools/Scripts/webkitpy/tool/bot/flakytestreporter_unittest.py index 26c98c1..1e3f35a 100644 --- a/Tools/Scripts/webkitpy/tool/bot/flakytestreporter_unittest.py +++ b/Tools/Scripts/webkitpy/tool/bot/flakytestreporter_unittest.py @@ -97,12 +97,6 @@ blocked: 50856 bug = tool.bugs.fetch_bug(78) self.assertEqual(reporter._follow_duplicate_chain(bug).id(), 76) - def test_bot_information(self): - tool = MockTool() - tool.status_server = MockStatusServer("MockBotId") - reporter = FlakyTestReporter(tool, 'dummy-queue') - self.assertEqual(reporter._bot_information(), "Bot: MockBotId Port: MockPort Platform: MockPlatform 1.0") - def test_report_flaky_tests_creating_bug(self): tool = MockTool() tool.filesystem = MockFileSystem({"/mock/foo/bar-diffs.txt": "mock"}) diff --git a/Tools/Scripts/webkitpy/tool/commands/download_unittest.py b/Tools/Scripts/webkitpy/tool/commands/download_unittest.py index ced5b2f..c8e5fd6 100644 --- a/Tools/Scripts/webkitpy/tool/commands/download_unittest.py +++ b/Tools/Scripts/webkitpy/tool/commands/download_unittest.py @@ -123,8 +123,13 @@ class DownloadCommandsTest(CommandsTest): self.assert_execute_outputs(Land(), [42], options=self._default_options(), expected_stderr=expected_stderr, tool=mock_tool) def test_check_style(self): - expected_stderr = "Processing 1 patch from 1 bug.\nUpdating working directory\nProcessing patch 197 from bug 42.\nRunning check-webkit-style\n" - self.assert_execute_outputs(CheckStyle(), [197], options=self._default_options(), expected_stderr=expected_stderr) + expected_stderr = """Processing 1 patch from 1 bug. +Updating working directory +MOCK run_and_throw_if_fail: ['mock-update-webkit'] +Processing patch 197 from bug 42. +MOCK run_and_throw_if_fail: ['mock-check-webkit-style', '--git-commit', 'MOCK git commit', '--diff-files', 'MockFile1'] +""" + self.assert_execute_outputs(CheckStyle(), [197], options=self._default_options(), expected_stderr=expected_stderr, tool=MockTool(log_executive=True)) def test_build_attachment(self): expected_stderr = "Processing 1 patch from 1 bug.\nUpdating working directory\nProcessing patch 197 from bug 42.\nBuilding WebKit\n" @@ -171,7 +176,7 @@ Not closing bug 42 as attachment 197 has review=+. Assuming there are more patc self.assert_execute_outputs(LandFromBug(), [42], options=self._default_options(), expected_stderr=expected_stderr) def test_prepare_rollout(self): - expected_stderr = "Preparing rollout for bug 42.\nUpdating working directory\nRunning prepare-ChangeLog\n" + expected_stderr = "Preparing rollout for bug 42.\nUpdating working directory\n" self.assert_execute_outputs(PrepareRollout(), [852, "Reason"], options=self._default_options(), expected_stderr=expected_stderr) def test_create_rollout(self): @@ -184,7 +189,6 @@ Reason component: MOCK component cc: MOCK cc blocked: 42 -Running prepare-ChangeLog MOCK add_patch_to_bug: bug_id=78, description=ROLLOUT of r852, mark_for_review=False, mark_for_commit_queue=True, mark_for_landing=False -- Begin comment -- Any committer can land this patch automatically by marking it commit-queue+. The commit-queue will build and test the patch before landing to ensure that the rollout will be successful. This process takes approximately 15 minutes. @@ -202,7 +206,6 @@ where ATTACHMENT_ID is the ID of this attachment. def test_rollout(self): expected_stderr = """Preparing rollout for bug 42. Updating working directory -Running prepare-ChangeLog MOCK: user.open_url: file://... Was that diff correct? Building WebKit diff --git a/Tools/Scripts/webkitpy/tool/commands/queues.py b/Tools/Scripts/webkitpy/tool/commands/queues.py index 9e50dd4..4eadb0e 100644 --- a/Tools/Scripts/webkitpy/tool/commands/queues.py +++ b/Tools/Scripts/webkitpy/tool/commands/queues.py @@ -44,11 +44,13 @@ from webkitpy.common.net.layouttestresults import LayoutTestResults from webkitpy.common.net.statusserver import StatusServer from webkitpy.common.system.deprecated_logging import error, log from webkitpy.common.system.executive import ScriptError +from webkitpy.tool.bot.botinfo import BotInfo from webkitpy.tool.bot.commitqueuetask import CommitQueueTask, CommitQueueTaskDelegate from webkitpy.tool.bot.feeders import CommitQueueFeeder, EWSFeeder from webkitpy.tool.bot.queueengine import QueueEngine, QueueEngineDelegate from webkitpy.tool.bot.flakytestreporter import FlakyTestReporter from webkitpy.tool.commands.stepsequence import StepSequenceErrorHandler +from webkitpy.tool.steps.runtests import RunTests from webkitpy.tool.multicommandtool import Command, TryAgain @@ -258,6 +260,20 @@ class CommitQueue(AbstractPatchQueue, StepSequenceErrorHandler, CommitQueueTaskD self._update_status("Processing %s" % patch_text, patch) return True + # FIXME: This is not really specific to the commit-queue and could be shared. + def _upload_results_archive_for_patch(self, patch, results_archive_zip): + bot_id = self._tool.status_server.bot_id or "bot" + description = "Archive of layout-test-results from %s" % bot_id + # results_archive is a ZipFile object, grab the File object (.fp) to pass to Mechanize for uploading. + results_archive_file = results_archive_zip.fp + # Rewind the file object to start (since Mechanize won't do that automatically) + # See https://bugs.webkit.org/show_bug.cgi?id=54593 + results_archive_file.seek(0) + comment_text = "The attached test failures were seen while running run-webkit-tests on the %s.\n" % (self.name) + # FIXME: We could easily list the test failures from the archive here. + comment_text += BotInfo(self._tool).summary_text() + self._tool.bugs.add_attachment_to_bug(patch.bug_id(), results_archive_file, description, filename="layout-test-results.zip", comment_text=comment_text) + def process_work_item(self, patch): self._cc_watchers(patch.bug_id()) task = CommitQueueTask(self, patch) @@ -269,6 +285,9 @@ class CommitQueue(AbstractPatchQueue, StepSequenceErrorHandler, CommitQueueTaskD except ScriptError, e: validator = CommitterValidator(self._tool.bugs) validator.reject_patch_from_commit_queue(patch.id(), self._error_message_for_bug(task.failure_status_id, e)) + results_archive = task.results_archive_from_patch_test_run(patch) + if results_archive: + self._upload_results_archive_for_patch(patch, results_archive) self._did_fail(patch) def _error_message_for_bug(self, status_id, script_error): @@ -296,19 +315,28 @@ class CommitQueue(AbstractPatchQueue, StepSequenceErrorHandler, CommitQueueTaskD # tool.filesystem.read_text_file. They have different error handling at the moment. def _read_file_contents(self, path): try: - with codecs.open(path, "r", "utf-8") as open_file: - return open_file.read() - except OSError, e: # File does not exist or can't be read. + return self._tool.filesystem.read_text_file(path) + except IOError, e: # File does not exist or can't be read. return None - # FIXME: This may belong on the Port object. - def layout_test_results(self): + # FIXME: This logic should move to the port object. + def _create_layout_test_results(self): results_path = self._tool.port().layout_tests_results_path() results_html = self._read_file_contents(results_path) if not results_html: return None return LayoutTestResults.results_from_string(results_html) + def layout_test_results(self): + results = self._create_layout_test_results() + # FIXME: We should not have to set failure_limit_count, but we + # do until run-webkit-tests can be updated save off the value + # of --exit-after-N-failures in results.html/results.json. + # https://bugs.webkit.org/show_bug.cgi?id=58481 + if results: + results.set_failure_limit_count(RunTests.NON_INTERACTIVE_FAILURE_LIMIT_COUNT) + return results + def _results_directory(self): results_path = self._tool.port().layout_tests_results_path() # FIXME: This is wrong in two ways: diff --git a/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py b/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py index e2fb09f..d577baa 100644 --- a/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py +++ b/Tools/Scripts/webkitpy/tool/commands/queues_unittest.py @@ -253,6 +253,7 @@ MOCK: release_work_item: commit-queue 197 def test_rollout(self): tool = MockTool(log_executive=True) + tool.filesystem.write_text_file('/mock/results.html', '') # Otherwise the commit-queue will hit a KeyError trying to read the results from the MockFileSystem. tool.buildbot.light_tree_on_fire() expected_stderr = { "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root), @@ -321,6 +322,7 @@ MOCK: release_work_item: commit-queue 106 def test_manual_reject_during_processing(self): queue = SecondThoughtsCommitQueue() queue.bind_to_tool(MockTool()) + queue._tool.filesystem.write_text_file('/mock/results.html', '') # Otherwise the commit-queue will hit a KeyError trying to read the results from the MockFileSystem. queue._options = Mock() queue._options.port = None expected_stderr = """MOCK: update_status: commit-queue Cleaned working directory @@ -376,6 +378,17 @@ The commit-queue is continuing to process your patch. OutputCapture().assert_outputs(self, queue.report_flaky_tests, [QueuesTest.mock_work_item, test_results, MockZipFile()], expected_stderr=expected_stderr) + def test_missing_layout_test_results(self): + queue = CommitQueue() + tool = MockTool() + results_path = '/mock/results.html' + tool.filesystem = MockFileSystem({results_path: None}) + queue.bind_to_tool(tool) + # Make sure that our filesystem mock functions as we expect. + self.assertRaises(IOError, tool.filesystem.read_text_file, results_path) + # layout_test_results shouldn't raise even if the results.html file is missing. + self.assertEquals(queue.layout_test_results(), None) + def test_layout_test_results(self): queue = CommitQueue() queue.bind_to_tool(MockTool()) @@ -383,13 +396,30 @@ The commit-queue is continuing to process your patch. self.assertEquals(queue.layout_test_results(), None) queue._read_file_contents = lambda path: "" self.assertEquals(queue.layout_test_results(), None) + queue._create_layout_test_results = lambda: LayoutTestResults([]) + results = queue.layout_test_results() + self.assertNotEquals(results, None) + self.assertEquals(results.failure_limit_count(), 10) # This value matches RunTests.NON_INTERACTIVE_FAILURE_LIMIT_COUNT def test_archive_last_layout_test_results(self): queue = CommitQueue() queue.bind_to_tool(MockTool()) patch = queue._tool.bugs.fetch_attachment(128) + # This is just to test that the method doesn't raise. queue.archive_last_layout_test_results(patch) + def test_upload_results_archive_for_patch(self): + queue = CommitQueue() + queue.bind_to_tool(MockTool()) + patch = queue._tool.bugs.fetch_attachment(128) + expected_stderr = """MOCK add_attachment_to_bug: bug_id=42, description=Archive of layout-test-results from bot filename=layout-test-results.zip +-- Begin comment -- +The attached test failures were seen while running run-webkit-tests on the commit-queue. +Port: MockPort Platform: MockPlatform 1.0 +-- End comment -- +""" + OutputCapture().assert_outputs(self, queue._upload_results_archive_for_patch, [patch, Mock()], expected_stderr=expected_stderr) + class StyleQueueTest(QueuesTest): def test_style_queue(self): diff --git a/Tools/Scripts/webkitpy/tool/commands/queuestest.py b/Tools/Scripts/webkitpy/tool/commands/queuestest.py index 6455617..758832e 100644 --- a/Tools/Scripts/webkitpy/tool/commands/queuestest.py +++ b/Tools/Scripts/webkitpy/tool/commands/queuestest.py @@ -67,6 +67,9 @@ class QueuesTest(unittest.TestCase): def assert_queue_outputs(self, queue, args=None, work_item=None, expected_stdout=None, expected_stderr=None, expected_exceptions=None, options=None, tool=None): if not tool: tool = MockTool() + # This is a hack to make it easy for callers to not have to setup a custom MockFileSystem just to test the commit-queue + # the cq tries to read the layout test results, and will hit a KeyError in MockFileSystem if we don't do this. + tool.filesystem.write_text_file('/mock/results.html', "") if not expected_stdout: expected_stdout = {} if not expected_stderr: diff --git a/Tools/Scripts/webkitpy/tool/commands/roll_unittest.py b/Tools/Scripts/webkitpy/tool/commands/roll_unittest.py index b6f69ea..da5c635 100644 --- a/Tools/Scripts/webkitpy/tool/commands/roll_unittest.py +++ b/Tools/Scripts/webkitpy/tool/commands/roll_unittest.py @@ -36,7 +36,6 @@ class RollCommandsTest(CommandsTest): def test_update_chromium_deps(self): expected_stderr = """Updating Chromium DEPS to 6764 MOCK: MockDEPS.write_variable(chromium_rev, 6764) -Running prepare-ChangeLog MOCK: user.open_url: file://... Was that diff correct? Committed r49824: <http://trac.webkit.org/changeset/49824> diff --git a/Tools/Scripts/webkitpy/tool/commands/upload_unittest.py b/Tools/Scripts/webkitpy/tool/commands/upload_unittest.py index 4313df9..4870423 100644 --- a/Tools/Scripts/webkitpy/tool/commands/upload_unittest.py +++ b/Tools/Scripts/webkitpy/tool/commands/upload_unittest.py @@ -59,8 +59,7 @@ class UploadCommandsTest(CommandsTest): options.request_commit = False options.review = True options.suggest_reviewers = False - expected_stderr = """Running check-webkit-style -MOCK: user.open_url: file://... + expected_stderr = """MOCK: user.open_url: file://... Was that diff correct? Obsoleting 2 old patches on bug 42 MOCK add_patch_to_bug: bug_id=42, description=MOCK description, mark_for_review=True, mark_for_commit_queue=False, mark_for_landing=False @@ -107,8 +106,7 @@ extra comment options.request_commit = False options.review = True options.suggest_reviewers = False - expected_stderr = """Running check-webkit-style -MOCK: user.open_url: file://... + expected_stderr = """MOCK: user.open_url: file://... Was that diff correct? Obsoleting 2 old patches on bug 42 MOCK add_patch_to_bug: bug_id=42, description=MOCK description, mark_for_review=True, mark_for_commit_queue=False, mark_for_landing=False diff --git a/Tools/Scripts/webkitpy/tool/mocktool.py b/Tools/Scripts/webkitpy/tool/mocktool.py index 73f55a7..ad03244 100644 --- a/Tools/Scripts/webkitpy/tool/mocktool.py +++ b/Tools/Scripts/webkitpy/tool/mocktool.py @@ -458,13 +458,14 @@ class MockSCM(Mock): fake_checkout_root = os.path.realpath("/tmp") # realpath is needed to allow for Mac OS X's /private/tmp - def __init__(self): + def __init__(self, filesystem=None): Mock.__init__(self) # FIXME: We should probably use real checkout-root detection logic here. # os.getcwd() can't work here because other parts of the code assume that "checkout_root" # will actually be the root. Since getcwd() is wrong, use a globally fake root for now. self.checkout_root = self.fake_checkout_root self.added_paths = set() + self._filesystem = filesystem def add(self, destination_path, return_exit_code=False): self.added_paths.add(destination_path) @@ -502,6 +503,12 @@ class MockSCM(Mock): def svn_revision_from_commit_text(self, commit_text): return "49824" + def delete(self, path): + if not self._filesystem: + return + if self._filesystem.exists(path): + self._filesystem.remove(path) + class MockDEPS(object): def read_variable(self, name): @@ -686,6 +693,13 @@ class MockPort(Mock): def layout_tests_results_path(self): return "/mock/results.html" + def check_webkit_style_command(self): + return ["mock-check-webkit-style"] + + def update_webkit_command(self): + return ["mock-update-webkit"] + + class MockTestPort1(object): def skips_layout_test(self, test_name): diff --git a/Tools/Scripts/webkitpy/tool/steps/abstractstep.py b/Tools/Scripts/webkitpy/tool/steps/abstractstep.py index 2ba4291..59ea36a 100644 --- a/Tools/Scripts/webkitpy/tool/steps/abstractstep.py +++ b/Tools/Scripts/webkitpy/tool/steps/abstractstep.py @@ -26,7 +26,6 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from webkitpy.common.system.deprecated_logging import log from webkitpy.common.system.executive import ScriptError from webkitpy.common.config.ports import WebKitPort from webkitpy.tool.steps.options import Options @@ -37,14 +36,6 @@ class AbstractStep(object): self._tool = tool self._options = options - # FIXME: This should use tool.port() - def _run_script(self, script_name, args=None, quiet=False, port=WebKitPort): - log("Running %s" % script_name) - command = port.script_shell_command(script_name) - if args: - command.extend(args) - self._tool.executive.run_and_throw_if_fail(command, quiet) - def _changed_files(self, state): return self.cached_lookup(state, "changed_files") diff --git a/Tools/Scripts/webkitpy/tool/steps/checkstyle.py b/Tools/Scripts/webkitpy/tool/steps/checkstyle.py index af66c50..c2377e9 100644 --- a/Tools/Scripts/webkitpy/tool/steps/checkstyle.py +++ b/Tools/Scripts/webkitpy/tool/steps/checkstyle.py @@ -56,7 +56,7 @@ class CheckStyle(AbstractStep): args.extend(self._changed_files(state)) try: - self._run_script("check-webkit-style", args) + self._tool.executive.run_and_throw_if_fail(self._tool.port().check_webkit_style_command() + args) except ScriptError, e: if self._options.non_interactive: # We need to re-raise the exception here to have the diff --git a/Tools/Scripts/webkitpy/tool/steps/commit.py b/Tools/Scripts/webkitpy/tool/steps/commit.py index 5dc4efb..7a03528 100644 --- a/Tools/Scripts/webkitpy/tool/steps/commit.py +++ b/Tools/Scripts/webkitpy/tool/steps/commit.py @@ -50,6 +50,7 @@ class Commit(AbstractStep): self._state = state username = None + password = None force_squash = False num_tries = 0 @@ -58,7 +59,7 @@ class Commit(AbstractStep): try: scm = self._tool.scm() - commit_text = scm.commit_with_message(self._commit_message, git_commit=self._options.git_commit, username=username, force_squash=force_squash, changed_files=self._changed_files(state)) + commit_text = scm.commit_with_message(self._commit_message, git_commit=self._options.git_commit, username=username, password=password, force_squash=force_squash, changed_files=self._changed_files(state)) svn_revision = scm.svn_revision_from_commit_text(commit_text) log("Committed r%s: <%s>" % (svn_revision, urls.view_revision_url(svn_revision))) self._state["commit_text"] = commit_text @@ -72,4 +73,8 @@ class Commit(AbstractStep): except AuthenticationError, e: username = self._tool.user.prompt("%s login: " % e.server_host, repeat=5) if not username: - raise ScriptError("You need to specify the username on %s to perform the commit as." % self.svn_server_host) + raise ScriptError("You need to specify the username on %s to perform the commit as." % e.server_host) + if e.prompt_for_password: + password = self._tool.user.prompt_password("%s password for %s: " % (e.server_host, username), repeat=5) + if not password: + raise ScriptError("You need to specify the password for %s on %s to perform the commit." % (username, e.server_host)) diff --git a/Tools/Scripts/webkitpy/tool/steps/preparechangelog.py b/Tools/Scripts/webkitpy/tool/steps/preparechangelog.py index 4be40ca..b30dd2f 100644 --- a/Tools/Scripts/webkitpy/tool/steps/preparechangelog.py +++ b/Tools/Scripts/webkitpy/tool/steps/preparechangelog.py @@ -61,7 +61,7 @@ class PrepareChangeLog(AbstractStep): self._ensure_bug_url(state) return os.chdir(self._tool.scm().checkout_root) - args = self._tool.port().script_shell_command("prepare-ChangeLog") + args = self._tool.port().prepare_changelog_command() if state.get("bug_id"): args.append("--bug=%s" % state["bug_id"]) args.append("--description=%s" % self._tool.bugs.fetch_bug(state["bug_id"]).title()) diff --git a/Tools/Scripts/webkitpy/tool/steps/preparechangelogfordepsroll.py b/Tools/Scripts/webkitpy/tool/steps/preparechangelogfordepsroll.py index 39c9a9a..e636cb4 100644 --- a/Tools/Scripts/webkitpy/tool/steps/preparechangelogfordepsroll.py +++ b/Tools/Scripts/webkitpy/tool/steps/preparechangelogfordepsroll.py @@ -34,7 +34,7 @@ from webkitpy.tool.steps.abstractstep import AbstractStep class PrepareChangeLogForDEPSRoll(AbstractStep): def run(self, state): - self._run_script("prepare-ChangeLog") + self._tool.executive.run_and_throw_if_fail(self._tool.port().prepare_changelog_command()) changelog_paths = self._tool.checkout().modified_changelogs(git_commit=None) for changelog_path in changelog_paths: ChangeLog(changelog_path).update_with_unreviewed_message("Rolled DEPS.\n\n") diff --git a/Tools/Scripts/webkitpy/tool/steps/preparechangelogforrevert.py b/Tools/Scripts/webkitpy/tool/steps/preparechangelogforrevert.py index dcd4b93..0a47573 100644 --- a/Tools/Scripts/webkitpy/tool/steps/preparechangelogforrevert.py +++ b/Tools/Scripts/webkitpy/tool/steps/preparechangelogforrevert.py @@ -50,7 +50,7 @@ class PrepareChangeLogForRevert(AbstractStep): def run(self, state): # This could move to prepare-ChangeLog by adding a --revert= option. - self._run_script("prepare-ChangeLog") + self._tool.executive.run_and_throw_if_fail(self._tool.port().prepare_changelog_command()) changelog_paths = self._tool.checkout().modified_changelogs(git_commit=None) bug_url = self._tool.bugs.bug_url_for_bug_id(state["bug_id"]) if state["bug_id"] else None message = self._message_for_revert(state["revision_list"], state["reason"], bug_url) diff --git a/Tools/Scripts/webkitpy/tool/steps/runtests.py b/Tools/Scripts/webkitpy/tool/steps/runtests.py index 282e381..793a94b 100644 --- a/Tools/Scripts/webkitpy/tool/steps/runtests.py +++ b/Tools/Scripts/webkitpy/tool/steps/runtests.py @@ -31,6 +31,9 @@ from webkitpy.tool.steps.options import Options from webkitpy.common.system.deprecated_logging import log class RunTests(AbstractStep): + # FIXME: This knowledge really belongs in the commit-queue. + NON_INTERACTIVE_FAILURE_LIMIT_COUNT = 10 + @classmethod def options(cls): return AbstractStep.options() + [ @@ -59,21 +62,8 @@ class RunTests(AbstractStep): if self._options.non_interactive: args.append("--no-new-test-results") args.append("--no-launch-safari") - args.append("--exit-after-n-failures=1") + args.append("--exit-after-n-failures=%s" % self.NON_INTERACTIVE_FAILURE_LIMIT_COUNT) args.append("--wait-for-httpd") - # FIXME: Hack to work around https://bugs.webkit.org/show_bug.cgi?id=38912 - # when running the commit-queue on a mac leopard machine since compositing - # does not work reliably on Leopard due to various graphics driver/system bugs. - if self._tool.port().name() == "Mac" and self._tool.port().is_leopard(): - tests_to_ignore = [] - tests_to_ignore.append("compositing") - - # media tests are also broken on mac leopard due to - # a separate CoreVideo bug which causes random crashes/hangs - # https://bugs.webkit.org/show_bug.cgi?id=38912 - tests_to_ignore.append("media") - - args.extend(["--ignore-tests", ",".join(tests_to_ignore)]) if self._options.quiet: args.append("--quiet") diff --git a/Tools/Scripts/webkitpy/tool/steps/steps_unittest.py b/Tools/Scripts/webkitpy/tool/steps/steps_unittest.py index 783ae29..e1ace2c 100644 --- a/Tools/Scripts/webkitpy/tool/steps/steps_unittest.py +++ b/Tools/Scripts/webkitpy/tool/steps/steps_unittest.py @@ -66,17 +66,12 @@ class StepsTest(unittest.TestCase): tool.user.prompt = lambda message: 42 self._run_step(PromptForBugOrTitle, tool=tool) - def test_runtests_leopard_commit_queue_hack_step(self): - expected_stderr = "Running Python unit tests\nRunning Perl unit tests\nRunning JavaScriptCore tests\nRunning run-webkit-tests\n" - OutputCapture().assert_outputs(self, self._run_step, [RunTests], expected_stderr=expected_stderr) - - def test_runtests_leopard_commit_queue_hack_command(self): + def test_runtests_args(self): mock_options = self._step_options() step = RunTests(MockTool(log_executive=True), mock_options) # FIXME: We shouldn't use a real port-object here, but there is too much to mock at the moment. mock_port = WebKitPort() mock_port.name = lambda: "Mac" - mock_port.is_leopard = lambda: True tool = MockTool(log_executive=True) tool.port = lambda: mock_port step = RunTests(tool, mock_options) @@ -87,6 +82,6 @@ MOCK run_and_throw_if_fail: ['Tools/Scripts/test-webkitperl'] Running JavaScriptCore tests MOCK run_and_throw_if_fail: ['Tools/Scripts/run-javascriptcore-tests'] Running run-webkit-tests -MOCK run_and_throw_if_fail: ['Tools/Scripts/run-webkit-tests', '--no-new-test-results', '--no-launch-safari', '--exit-after-n-failures=1', '--wait-for-httpd', '--ignore-tests', 'compositing,media', '--quiet'] +MOCK run_and_throw_if_fail: ['Tools/Scripts/run-webkit-tests', '--no-new-test-results', '--no-launch-safari', '--exit-after-n-failures=10', '--wait-for-httpd', '--quiet'] """ OutputCapture().assert_outputs(self, step.run, [{}], expected_stderr=expected_stderr) |