diff options
Diffstat (limited to 'Tools/Scripts/webkitpy/tool/bot/flakytestreporter_unittest.py')
| -rw-r--r-- | Tools/Scripts/webkitpy/tool/bot/flakytestreporter_unittest.py | 145 |
1 files changed, 145 insertions, 0 deletions
diff --git a/Tools/Scripts/webkitpy/tool/bot/flakytestreporter_unittest.py b/Tools/Scripts/webkitpy/tool/bot/flakytestreporter_unittest.py new file mode 100644 index 0000000..f72fb28 --- /dev/null +++ b/Tools/Scripts/webkitpy/tool/bot/flakytestreporter_unittest.py @@ -0,0 +1,145 @@ +# Copyright (c) 2010 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest + +from webkitpy.common.config.committers import Committer +from webkitpy.common.system.filesystem_mock import MockFileSystem +from webkitpy.common.system.outputcapture import OutputCapture +from webkitpy.tool.bot.flakytestreporter import FlakyTestReporter +from webkitpy.tool.mocktool import MockTool, MockStatusServer + + +# Creating fake CommitInfos is a pain, so we use a mock one here. +class MockCommitInfo(object): + def __init__(self, author_email): + self._author_email = author_email + + def author(self): + # It's definitely possible to have commits with authors who + # are not in our committers.py list. + if not self._author_email: + return None + return Committer("Mock Committer", self._author_email) + + +class FlakyTestReporterTest(unittest.TestCase): + def _assert_emails_for_test(self, emails): + tool = MockTool() + reporter = FlakyTestReporter(tool, 'dummy-queue') + commit_infos = [MockCommitInfo(email) for email in emails] + tool.checkout().recent_commit_infos_for_files = lambda paths: set(commit_infos) + self.assertEqual(reporter._author_emails_for_test([]), set(emails)) + + def test_author_emails_for_test(self): + self._assert_emails_for_test([]) + self._assert_emails_for_test(["test1@test.com", "test1@test.com"]) + self._assert_emails_for_test(["test1@test.com", "test2@test.com"]) + + def test_create_bug_for_flaky_test(self): + reporter = FlakyTestReporter(MockTool(), 'dummy-queue') + expected_stderr = """MOCK create_bug +bug_title: Flaky Test: foo/bar.html +bug_description: This is an automatically generated bug from the dummy-queue. +foo/bar.html has been flaky on the dummy-queue. + +foo/bar.html was authored by test@test.com. +http://trac.webkit.org/browser/trunk/LayoutTests/foo/bar.html + +FLAKE_MESSAGE + +The bots will update this with information from each new failure. + +If you would like to track this test fix with another bug, please close this bug as a duplicate. + +component: Tools / Tests +cc: test@test.com +blocked: 50856 +""" + OutputCapture().assert_outputs(self, reporter._create_bug_for_flaky_test, ['foo/bar.html', ['test@test.com'], 'FLAKE_MESSAGE'], expected_stderr=expected_stderr) + + def test_follow_duplicate_chain(self): + tool = MockTool() + reporter = FlakyTestReporter(tool, 'dummy-queue') + bug = tool.bugs.fetch_bug(78) + self.assertEqual(reporter._follow_duplicate_chain(bug).id(), 76) + + def test_bot_information(self): + tool = MockTool() + tool.status_server = MockStatusServer("MockBotId") + reporter = FlakyTestReporter(tool, 'dummy-queue') + self.assertEqual(reporter._bot_information(), "Bot: MockBotId Port: MockPort Platform: MockPlatform 1.0") + + def test_report_flaky_tests_creating_bug(self): + tool = MockTool() + tool.filesystem = MockFileSystem({"/mock/foo/bar-diffs.txt": "mock"}) + tool.status_server = MockStatusServer(bot_id="mock-bot-id") + reporter = FlakyTestReporter(tool, 'dummy-queue') + reporter._lookup_bug_for_flaky_test = lambda bug_id: None + patch = tool.bugs.fetch_attachment(197) + expected_stderr = """MOCK create_bug +bug_title: Flaky Test: foo/bar.html +bug_description: This is an automatically generated bug from the dummy-queue. +foo/bar.html has been flaky on the dummy-queue. + +foo/bar.html was authored by abarth@webkit.org. +http://trac.webkit.org/browser/trunk/LayoutTests/foo/bar.html + +The dummy-queue just saw foo/bar.html flake while processing attachment 197 on bug 42. +Bot: mock-bot-id Port: MockPort Platform: MockPlatform 1.0 + +The bots will update this with information from each new failure. + +If you would like to track this test fix with another bug, please close this bug as a duplicate. + +component: Tools / Tests +cc: abarth@webkit.org +blocked: 50856 +MOCK add_attachment_to_bug: bug_id=78, description=Failure diff from mock-bot-id filename=failure.diff +MOCK bug comment: bug_id=42, cc=None +--- Begin comment --- +The dummy-queue encountered the following flaky tests while processing attachment 197: + +foo/bar.html bug 78 (author: abarth@webkit.org) +The dummy-queue is continuing to process your patch. +--- End comment --- + +""" + OutputCapture().assert_outputs(self, reporter.report_flaky_tests, [['foo/bar.html'], patch], expected_stderr=expected_stderr) + + def test_optional_author_string(self): + reporter = FlakyTestReporter(MockTool(), 'dummy-queue') + self.assertEqual(reporter._optional_author_string([]), "") + self.assertEqual(reporter._optional_author_string(["foo@bar.com"]), " (author: foo@bar.com)") + self.assertEqual(reporter._optional_author_string(["a@b.com", "b@b.com"]), " (authors: a@b.com and b@b.com)") + + def test_results_diff_path_for_test(self): + reporter = FlakyTestReporter(MockTool(), 'dummy-queue') + self.assertEqual(reporter._results_diff_path_for_test("test.html"), "/mock/test-diffs.txt") + + # report_flaky_tests is also tested by queues_unittest |