Merge WebKit at r67908: Initial merge by Git

Change-Id: I43a553e7b3299b28cb6ee8aa035ed70fe342b972

6 files changed, 175 insertions, 66 deletions

diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/download.py b/WebKitTools/Scripts/webkitpy/tool/commands/download.py
index d27ab0e..ed0e3d6 100644
--- a/WebKitTools/Scripts/webkitpy/tool/commands/download.py
+++ b/WebKitTools/Scripts/webkitpy/tool/commands/download.py
@@ -194,6 +194,19 @@ class BuildAttachment(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
     ]
 
 
+class BuildAndTestAttachment(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
+    name = "build-and-test-attachment"
+    help_text = "Apply, build, and test patches from bugzilla"
+    argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
+    main_steps = [
+        steps.CleanWorkingDirectory,
+        steps.Update,
+        steps.ApplyPatch,
+        steps.Build,
+        steps.RunTests,
+    ]
+
+
 class PostAttachmentToRietveld(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
     name = "post-attachment-to-rietveld"
     help_text = "Uploads a bugzilla attachment to rietveld"
diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py
index 1f04923..3b0ea47 100644
--- a/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py
+++ b/WebKitTools/Scripts/webkitpy/tool/commands/earlywarningsystem_unittest.py
@@ -42,16 +42,15 @@ class EarlyWarningSytemTest(QueuesTest):
     def _default_expected_stderr(self, ews):
         string_replacemnts = {
             "name": ews.name,
-            "checkout_dir": os.getcwd(),  # FIXME: Use of os.getcwd() is wrong, should be scm.checkout_root
             "port": ews.port_name,
             "watchers": ews.watchers,
         }
         expected_stderr = {
-            "begin_work_queue": "CAUTION: %(name)s will discard all local changes in \"%(checkout_dir)s\"\nRunning WebKit %(name)s.\n" % string_replacemnts,
+            "begin_work_queue": self._default_begin_work_queue_stderr(ews.name, os.getcwd()),  # FIXME: Use of os.getcwd() is wrong, should be scm.checkout_root
             "handle_unexpected_error": "Mock error message\n",
             "next_work_item": "MOCK: update_work_items: %(name)s [103]\n" % string_replacemnts,
             "process_work_item": "MOCK: update_status: %(name)s Pass\n" % string_replacemnts,
-            "handle_script_error": "MOCK: update_status: %(name)s ScriptError error message\nMOCK bug comment: bug_id=345, cc=%(watchers)s\n--- Begin comment ---\\Attachment 1234 did not build on %(port)s:\nBuild output: http://dummy_url\n--- End comment ---\n\n" % string_replacemnts,
+            "handle_script_error": "MOCK: update_status: %(name)s ScriptError error message\nMOCK bug comment: bug_id=142, cc=%(watchers)s\n--- Begin comment ---\\Attachment 197 did not build on %(port)s:\nBuild output: http://dummy_url\n--- End comment ---\n\n" % string_replacemnts,
         }
         return expected_stderr
 
diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/queues.py b/WebKitTools/Scripts/webkitpy/tool/commands/queues.py
index 4d2a9df..bc9ee42 100644
--- a/WebKitTools/Scripts/webkitpy/tool/commands/queues.py
+++ b/WebKitTools/Scripts/webkitpy/tool/commands/queues.py
@@ -95,6 +95,10 @@ class AbstractQueue(Command, QueueEngineDelegate):
             if (response != "yes"):
                 error("User declined.")
         log("Running WebKit %s." % self.name)
+        self.tool.status_server.update_status(self.name, "Starting Queue")
+
+    def stop_work_queue(self, reason):
+        self.tool.status_server.update_status(self.name, "Stopping Queue, reason: %s" % reason)
 
     def should_continue_work_queue(self):
         self._iteration_count += 1
@@ -115,8 +119,8 @@ class AbstractQueue(Command, QueueEngineDelegate):
     # Command methods
 
     def execute(self, options, args, tool, engine=QueueEngine):
-        self.options = options
-        self.tool = tool
+        self.options = options  # FIXME: This code is wrong.  Command.options is a list, this assumes an Options element!
+        self.tool = tool  # FIXME: This code is wrong too!  Command.bind_to_tool handles this!
         return engine(self.name, self, self.tool.wakeup_event).run()
 
     @classmethod
@@ -144,8 +148,16 @@ class AbstractPatchQueue(AbstractQueue):
     def _update_status(self, message, patch=None, results_file=None):
         self.tool.status_server.update_status(self.name, message, patch, results_file)
 
+    # Note, eventually this will be done by a separate "feeder" queue
+    # whose job it is to poll bugzilla and feed work items into the
+    # status server for other queues to munch on.
     def _update_work_items(self, patch_ids):
         self.tool.status_server.update_work_items(self.name, patch_ids)
+        if patch_ids:
+            self.log_progress(patch_ids)
+
+    def _fetch_next_work_item(self):
+        return self.tool.status_server.next_work_item(self.name)
 
     def _did_pass(self, patch):
         self._update_status(self._pass_status, patch)
@@ -189,18 +201,22 @@ class CommitQueue(AbstractPatchQueue, StepSequenceErrorHandler):
             return rollout_cmp
         return cmp(a.attach_date(), b.attach_date())
 
-    def next_work_item(self):
+    def _feed_work_items_to_server(self):
+        # Grab the set of patches from bugzilla, sort them, and update the status server.
+        # Eventually this will all be done by a separate feeder queue.
         patches = self._validate_patches_in_commit_queue()
         patches = sorted(patches, self._patch_cmp)
         self._update_work_items([patch.id() for patch in patches])
-        if not patches:
-            self._update_status("Empty queue")
+
+    def next_work_item(self):
+        self._feed_work_items_to_server()
+        # The grab the next patch to work on back from the status server.
+        patch_id = self._fetch_next_work_item()
+        if not patch_id:
             return None
-        # Only bother logging if we have patches in the queue.
-        self.log_progress([patch.id() for patch in patches])
-        return patches[0]
+        return self.tool.bugs.fetch_attachment(patch_id)
 
-    def _can_build_and_test(self):
+    def _can_build_and_test_without_patch(self):
         try:
             self.run_webkit_patch([
                 "build-and-test",
@@ -211,25 +227,24 @@ class CommitQueue(AbstractPatchQueue, StepSequenceErrorHandler):
                 "--no-update",
                 "--build-style=both",
                 "--quiet"])
+            return True
         except ScriptError, e:
             failure_log = self._log_from_script_error_for_upload(e)
-            self._update_status("Unable to successfully build and test", results_file=failure_log)
+            self._update_status("Unable to build and test without patch", results_file=failure_log)
             return False
-        return True
 
     def should_proceed_with_work_item(self, patch):
         patch_text = "rollout patch" if patch.is_rollout() else "patch"
         self._update_status("Landing %s" % patch_text, patch)
         return True
 
-    def _land(self, patch, first_run=False):
+    def _build_and_test_patch(self, patch, first_run=False):
         try:
             args = [
-                "land-attachment",
+                "build-and-test-attachment",
                 "--force-clean",
                 "--build",
                 "--non-interactive",
-                "--ignore-builders",
                 "--build-style=both",
                 "--quiet",
                 patch.id()
@@ -249,27 +264,68 @@ class CommitQueue(AbstractPatchQueue, StepSequenceErrorHandler):
                 # built and tested.
                 args.append("--no-update")
             self.run_webkit_patch(args)
-            self._did_pass(patch)
             return True
         except ScriptError, e:
+            failure_log = self._log_from_script_error_for_upload(e)
+            self._update_status("Unable to build and test patch", patch=patch, results_file=failure_log)
             if first_run:
                 return False
             self._did_fail(patch)
             raise
 
+    def _revalidate_patch(self, patch):
+        # Bugs might get closed, or patches might be obsoleted or r-'d while the
+        # commit-queue is processing.  Do one last minute check before landing.
+        patch = self.tool.bugs.fetch_attachment(patch.id())
+        if patch.is_obsolete():
+            return None
+        if patch.bug().is_closed():
+            return None
+        if not patch.committer():
+            return None
+        # Reviewer is not required.  Misisng reviewers will be caught during the ChangeLog check during landing.
+        return patch
+
+    def _land(self, patch):
+        try:
+            args = [
+                "land-attachment",
+                "--force-clean",
+                "--non-interactive",
+                "--ignore-builders",
+                "--quiet",
+                "--parent-command=commit-queue",
+                patch.id(),
+            ]
+            self.run_webkit_patch(args)
+            self._did_pass(patch)
+        except ScriptError, e:
+            failure_log = self._log_from_script_error_for_upload(e)
+            self._update_status("Unable to land patch", patch=patch, results_file=failure_log)
+            self._did_fail(patch)
+            raise
+
     def process_work_item(self, patch):
         self._cc_watchers(patch.bug_id())
-        if not self._land(patch, first_run=True):
-            # The patch failed to land, but the bots were green. It's possible
-            # that the bots were behind. To check that case, we try to build and
-            # test ourselves.
-            if not self._can_build_and_test():
+        if not self._build_and_test_patch(patch, first_run=True):
+            self._update_status("Building and testing without the patch as a sanity check", patch)
+            # The patch failed to build and test. It's possible that the
+            # tree is busted. To check that case, we try to build and test
+            # without the patch.
+            if not self._can_build_and_test_without_patch():
                 return False
+            self._update_status("Build and test succeeded, trying again with patch", patch)
             # Hum, looks like the patch is actually bad. Of course, we could
             # have been bitten by a flaky test the first time around.  We try
-            # to land again.  If it fails a second time, we're pretty sure its
-            # a bad test and re can reject it outright.
-            self._land(patch)
+            # to build and test again. If it fails a second time, we're pretty
+            # sure its a bad test and re can reject it outright.
+            self._build_and_test_patch(patch)
+        # Do one last check to catch any bug changes (cq-, closed, reviewer changed, etc.)
+        # This helps catch races between the bots if locks expire.
+        patch = self._revalidate_patch(patch)
+        if not patch:
+            return False
+        self._land(patch)
         return True
 
     def handle_unexpected_error(self, patch, message):
@@ -291,6 +347,8 @@ class CommitQueue(AbstractPatchQueue, StepSequenceErrorHandler):
 
     @classmethod
     def handle_checkout_needs_update(cls, tool, state, options, error):
+        message = "Tests passed, but commit failed (checkout out of date).  Updating, then landing without building or re-running tests."
+        tool.status_server.update_status(cls.name, message, state["patch"])
         # The only time when we find out that out checkout needs update is
         # when we were ready to actually pull the trigger and land the patch.
         # Rather than spinning in the master process, we retry without
@@ -377,7 +435,6 @@ class AbstractReviewQueue(AbstractPatchQueue, PersistentPatchCollectionDelegate,
         patch_id = self._patches.next()
         if patch_id:
             return self.tool.bugs.fetch_attachment(patch_id)
-        self._update_status("Empty queue")
 
     def should_proceed_with_work_item(self, patch):
         raise NotImplementedError, "subclasses must implement"
diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/queues_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/queues_unittest.py
index fd6543c..2deee76 100644
--- a/WebKitTools/Scripts/webkitpy/tool/commands/queues_unittest.py
+++ b/WebKitTools/Scripts/webkitpy/tool/commands/queues_unittest.py
@@ -34,9 +34,9 @@ from webkitpy.common.system.outputcapture import OutputCapture
 from webkitpy.thirdparty.mock import Mock
 from webkitpy.tool.commands.commandtest import CommandsTest
 from webkitpy.tool.commands.queues import *
-from webkitpy.tool.commands.queuestest import QueuesTest
+from webkitpy.tool.commands.queuestest import QueuesTest, MockPatch
 from webkitpy.tool.commands.stepsequence import StepSequence
-from webkitpy.tool.mocktool import MockTool, MockSCM
+from webkitpy.tool.mocktool import MockTool, MockSCM, MockStatusServer
 
 
 class TestQueue(AbstractPatchQueue):
@@ -47,16 +47,10 @@ class TestReviewQueue(AbstractReviewQueue):
     name = "test-review-queue"
 
 
-class MockPatch(object):
+class MockRolloutPatch(MockPatch):
     def is_rollout(self):
         return True
 
-    def bug_id(self):
-        return 12345
-
-    def id(self):
-        return 76543
-
 
 class AbstractQueueTest(CommandsTest):
     def _assert_log_progress_output(self, patch_ids, progress_output):
@@ -121,6 +115,16 @@ class AbstractQueueTest(CommandsTest):
         self._assert_log_message(script_error, expected_output)
 
 
+class AbstractPatchQueueTest(CommandsTest):
+    def test_fetch_next_work_item(self):
+        queue = AbstractPatchQueue()
+        tool = MockTool()
+        queue.bind_to_tool(tool)
+        self.assertEquals(queue._fetch_next_work_item(), None)
+        tool.status_server = MockStatusServer(work_items=[2, 1, 3])
+        self.assertEquals(queue._fetch_next_work_item(), 2)
+
+
 class AbstractReviewQueueTest(CommandsTest):
     def test_patch_collection_delegate_methods(self):
         queue = TestReviewQueue()
@@ -141,25 +145,47 @@ class NeedsUpdateSequence(StepSequence):
 
 
 class AlwaysCommitQueueTool(object):
+    def __init__(self):
+        self.status_server = MockStatusServer()
+
     def command_by_name(self, name):
         return CommitQueue
 
 
+class SecondThoughtsCommitQueue(CommitQueue):
+    def _build_and_test_patch(self, patch, first_run=True):
+        attachment_dictionary = {
+            "id": patch.id(),
+            "bug_id": patch.bug_id(),
+            "name": "Rejected",
+            "is_obsolete": True,
+            "is_patch": False,
+            "review": "-",
+            "reviewer_email": "foo@bar.com",
+            "commit-queue": "-",
+            "committer_email": "foo@bar.com",
+            "attacher_email": "Contributer1",
+        }
+        patch = Attachment(attachment_dictionary, None)
+        self.tool.bugs.set_override_patch(patch)
+        return True
+
+
 class CommitQueueTest(QueuesTest):
     def test_commit_queue(self):
         expected_stderr = {
-            "begin_work_queue" : "CAUTION: commit-queue will discard all local changes in \"%s\"\nRunning WebKit commit-queue.\n" % MockSCM.fake_checkout_root,
+            "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root),
             "should_proceed_with_work_item": "MOCK: update_status: commit-queue Landing patch\n",
             # FIXME: The commit-queue warns about bad committers twice.  This is due to the fact that we access Attachment.reviewer() twice and it logs each time.
-            "next_work_item" : """Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com)
+            "next_work_item": """Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com)
 Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com)
 MOCK setting flag 'commit-queue' to '-' on attachment '128' with comment 'Rejecting patch 128 from commit-queue.' and additional comment 'non-committer@example.com does not have committer permissions according to http://trac.webkit.org/browser/trunk/WebKitTools/Scripts/webkitpy/common/config/committers.py.\n\n- If you do not have committer rights please read http://webkit.org/coding/contributing.html for instructions on how to use bugzilla flags.\n\n- If you have committer rights please correct the error in WebKitTools/Scripts/webkitpy/common/config/committers.py by adding yourself to the file (no review needed).  The commit-queue restarts itself every 2 hours.  After restart the commit-queue will correctly respect your committer rights.'
 MOCK: update_work_items: commit-queue [106, 197]
 2 patches in commit-queue [106, 197]
 """,
-            "process_work_item" : "MOCK: update_status: commit-queue Pass\n",
-            "handle_unexpected_error" : "MOCK setting flag 'commit-queue' to '-' on attachment '1234' with comment 'Rejecting patch 1234 from commit-queue.' and additional comment 'Mock error message'\n",
-            "handle_script_error": "MOCK: update_status: commit-queue ScriptError error message\nMOCK setting flag 'commit-queue' to '-' on attachment '1234' with comment 'Rejecting patch 1234 from commit-queue.' and additional comment 'ScriptError error message'\n",
+            "process_work_item": "MOCK: update_status: commit-queue Pass\n",
+            "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting patch 197 from commit-queue.' and additional comment 'Mock error message'\n",
+            "handle_script_error": "MOCK: update_status: commit-queue ScriptError error message\nMOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting patch 197 from commit-queue.' and additional comment 'ScriptError error message'\n",
         }
         self.assert_queue_outputs(CommitQueue(), expected_stderr=expected_stderr)
 
@@ -167,10 +193,10 @@ MOCK: update_work_items: commit-queue [106, 197]
         tool = MockTool(log_executive=True)
         tool.buildbot.light_tree_on_fire()
         expected_stderr = {
-            "begin_work_queue" : "CAUTION: commit-queue will discard all local changes in \"%s\"\nRunning WebKit commit-queue.\n" % MockSCM.fake_checkout_root,
+            "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root),
             "should_proceed_with_work_item": "MOCK: update_status: commit-queue Landing patch\n",
             # FIXME: The commit-queue warns about bad committers twice.  This is due to the fact that we access Attachment.reviewer() twice and it logs each time.
-            "next_work_item" : """Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com)
+            "next_work_item": """Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com)
 Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com)
 MOCK setting flag 'commit-queue' to '-' on attachment '128' with comment 'Rejecting patch 128 from commit-queue.' and additional comment 'non-committer@example.com does not have committer permissions according to http://trac.webkit.org/browser/trunk/WebKitTools/Scripts/webkitpy/common/config/committers.py.
 
@@ -180,18 +206,18 @@ MOCK setting flag 'commit-queue' to '-' on attachment '128' with comment 'Reject
 MOCK: update_work_items: commit-queue [106, 197]
 2 patches in commit-queue [106, 197]
 """,
-            "process_work_item" : "MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'land-attachment', '--force-clean', '--build', '--non-interactive', '--ignore-builders', '--build-style=both', '--quiet', 1234, '--test']\nMOCK: update_status: commit-queue Pass\n",
-            "handle_unexpected_error" : "MOCK setting flag 'commit-queue' to '-' on attachment '1234' with comment 'Rejecting patch 1234 from commit-queue.' and additional comment 'Mock error message'\n",
-            "handle_script_error": "MOCK: update_status: commit-queue ScriptError error message\nMOCK setting flag 'commit-queue' to '-' on attachment '1234' with comment 'Rejecting patch 1234 from commit-queue.' and additional comment 'ScriptError error message'\n",
+            "process_work_item": "MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'build-and-test-attachment', '--force-clean', '--build', '--non-interactive', '--build-style=both', '--quiet', 197, '--test']\nMOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'land-attachment', '--force-clean', '--non-interactive', '--ignore-builders', '--quiet', '--parent-command=commit-queue', 197]\nMOCK: update_status: commit-queue Pass\n",
+            "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting patch 197 from commit-queue.' and additional comment 'Mock error message'\n",
+            "handle_script_error": "MOCK: update_status: commit-queue ScriptError error message\nMOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting patch 197 from commit-queue.' and additional comment 'ScriptError error message'\n",
         }
         self.assert_queue_outputs(CommitQueue(), tool=tool, expected_stderr=expected_stderr)
 
     def test_rollout_lands(self):
         tool = MockTool(log_executive=True)
         tool.buildbot.light_tree_on_fire()
-        rollout_patch = MockPatch()
+        rollout_patch = MockRolloutPatch()
         expected_stderr = {
-            "begin_work_queue": "CAUTION: commit-queue will discard all local changes in \"%s\"\nRunning WebKit commit-queue.\n" % MockSCM.fake_checkout_root,
+            "begin_work_queue": self._default_begin_work_queue_stderr("commit-queue", MockSCM.fake_checkout_root),
             "should_proceed_with_work_item": "MOCK: update_status: commit-queue Landing rollout patch\n",
             # FIXME: The commit-queue warns about bad committers twice.  This is due to the fact that we access Attachment.reviewer() twice and it logs each time.
             "next_work_item": """Warning, attachment 128 on bug 42 has invalid committer (non-committer@example.com)
@@ -204,9 +230,9 @@ MOCK setting flag 'commit-queue' to '-' on attachment '128' with comment 'Reject
 MOCK: update_work_items: commit-queue [106, 197]
 2 patches in commit-queue [106, 197]
 """,
-            "process_work_item": "MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'land-attachment', '--force-clean', '--build', '--non-interactive', '--ignore-builders', '--build-style=both', '--quiet', 76543]\nMOCK: update_status: commit-queue Pass\n",
-            "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '76543' with comment 'Rejecting patch 76543 from commit-queue.' and additional comment 'Mock error message'\n",
-            "handle_script_error": "MOCK: update_status: commit-queue ScriptError error message\nMOCK setting flag 'commit-queue' to '-' on attachment '1234' with comment 'Rejecting patch 1234 from commit-queue.' and additional comment 'ScriptError error message'\n",
+            "process_work_item": "MOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'build-and-test-attachment', '--force-clean', '--build', '--non-interactive', '--build-style=both', '--quiet', 197]\nMOCK run_and_throw_if_fail: ['echo', '--status-host=example.com', 'land-attachment', '--force-clean', '--non-interactive', '--ignore-builders', '--quiet', '--parent-command=commit-queue', 197]\nMOCK: update_status: commit-queue Pass\n",
+            "handle_unexpected_error": "MOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting patch 197 from commit-queue.' and additional comment 'Mock error message'\n",
+            "handle_script_error": "MOCK: update_status: commit-queue ScriptError error message\nMOCK setting flag 'commit-queue' to '-' on attachment '197' with comment 'Rejecting patch 197 from commit-queue.' and additional comment 'ScriptError error message'\n",
         }
         self.assert_queue_outputs(CommitQueue(), tool=tool, work_item=rollout_patch, expected_stderr=expected_stderr)
 
@@ -215,7 +241,7 @@ MOCK: update_work_items: commit-queue [106, 197]
         tool = MockTool()
         tool.executive = Mock()
         queue.bind_to_tool(tool)
-        self.assertTrue(queue._can_build_and_test())
+        self.assertTrue(queue._can_build_and_test_without_patch())
         expected_run_args = ["echo", "--status-host=example.com", "build-and-test", "--force-clean", "--build", "--test", "--non-interactive", "--no-update", "--build-style=both", "--quiet"]
         tool.executive.run_and_throw_if_fail.assert_called_with(expected_run_args)
 
@@ -245,22 +271,28 @@ MOCK: update_work_items: commit-queue [106, 197]
         tool = AlwaysCommitQueueTool()
         sequence = NeedsUpdateSequence(None)
 
-        expected_stderr = "Commit failed because the checkout is out of date.  Please update and try again.\n"
-        OutputCapture().assert_outputs(self, sequence.run_and_handle_errors, [tool, options], expected_exception=TryAgain, expected_stderr=expected_stderr)
+        expected_stderr = "Commit failed because the checkout is out of date.  Please update and try again.\nMOCK: update_status: commit-queue Tests passed, but commit failed (checkout out of date).  Updating, then landing without building or re-running tests.\n"
+        state = {'patch': None}
+        OutputCapture().assert_outputs(self, sequence.run_and_handle_errors, [tool, options, state], expected_exception=TryAgain, expected_stderr=expected_stderr)
 
         self.assertEquals(options.update, True)
         self.assertEquals(options.build, False)
         self.assertEquals(options.test, False)
 
+    def test_manual_reject_during_processing(self):
+        queue = SecondThoughtsCommitQueue()
+        queue.bind_to_tool(MockTool())
+        queue.process_work_item(MockPatch())
+
 
 class RietveldUploadQueueTest(QueuesTest):
     def test_rietveld_upload_queue(self):
         expected_stderr = {
-            "begin_work_queue": "CAUTION: rietveld-upload-queue will discard all local changes in \"%s\"\nRunning WebKit rietveld-upload-queue.\n" % MockSCM.fake_checkout_root,
+            "begin_work_queue": self._default_begin_work_queue_stderr("rietveld-upload-queue", MockSCM.fake_checkout_root),
             "should_proceed_with_work_item": "MOCK: update_status: rietveld-upload-queue Uploading patch\n",
             "process_work_item": "MOCK: update_status: rietveld-upload-queue Pass\n",
-            "handle_unexpected_error": "Mock error message\nMOCK setting flag 'in-rietveld' to '-' on attachment '1234' with comment 'None' and additional comment 'None'\n",
-            "handle_script_error": "ScriptError error message\nMOCK: update_status: rietveld-upload-queue ScriptError error message\nMOCK setting flag 'in-rietveld' to '-' on attachment '1234' with comment 'None' and additional comment 'None'\n",
+            "handle_unexpected_error": "Mock error message\nMOCK setting flag 'in-rietveld' to '-' on attachment '197' with comment 'None' and additional comment 'None'\n",
+            "handle_script_error": "ScriptError error message\nMOCK: update_status: rietveld-upload-queue ScriptError error message\nMOCK setting flag 'in-rietveld' to '-' on attachment '197' with comment 'None' and additional comment 'None'\n",
         }
         self.assert_queue_outputs(RietveldUploadQueue(), expected_stderr=expected_stderr)
 
@@ -268,12 +300,12 @@ class RietveldUploadQueueTest(QueuesTest):
 class StyleQueueTest(QueuesTest):
     def test_style_queue(self):
         expected_stderr = {
-            "begin_work_queue" : "CAUTION: style-queue will discard all local changes in \"%s\"\nRunning WebKit style-queue.\n" % MockSCM.fake_checkout_root,
+            "begin_work_queue": self._default_begin_work_queue_stderr("style-queue", MockSCM.fake_checkout_root),
             "next_work_item": "MOCK: update_work_items: style-queue [103]\n",
             "should_proceed_with_work_item": "MOCK: update_status: style-queue Checking style\n",
-            "process_work_item" : "MOCK: update_status: style-queue Pass\n",
-            "handle_unexpected_error" : "Mock error message\n",
-            "handle_script_error": "MOCK: update_status: style-queue ScriptError error message\nMOCK bug comment: bug_id=345, cc=[]\n--- Begin comment ---\\Attachment 1234 did not pass style-queue:\n\nScriptError error message\n\nIf any of these errors are false positives, please file a bug against check-webkit-style.\n--- End comment ---\n\n",
+            "process_work_item": "MOCK: update_status: style-queue Pass\n",
+            "handle_unexpected_error": "Mock error message\n",
+            "handle_script_error": "MOCK: update_status: style-queue ScriptError error message\nMOCK bug comment: bug_id=142, cc=[]\n--- Begin comment ---\\Attachment 197 did not pass style-queue:\n\nScriptError error message\n\nIf any of these errors are false positives, please file a bug against check-webkit-style.\n--- End comment ---\n\n",
         }
         expected_exceptions = {
             "handle_script_error": SystemExit,
diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/queuestest.py b/WebKitTools/Scripts/webkitpy/tool/commands/queuestest.py
index 9e17c5c..aa3cef4 100644
--- a/WebKitTools/Scripts/webkitpy/tool/commands/queuestest.py
+++ b/WebKitTools/Scripts/webkitpy/tool/commands/queuestest.py
@@ -45,16 +45,20 @@ class MockQueueEngine(object):
 
 class MockPatch():
     def id(self):
-        return 1234
+        return 197
 
     def bug_id(self):
-        return 345
+        return 142
+
+    def is_rollout(self):
+        return False
 
 
 class QueuesTest(unittest.TestCase):
+    # Ids match patch1 in mocktool.py
     mock_work_item = Attachment({
-        "id": 1234,
-        "bug_id": 345,
+        "id": 197,
+        "bug_id": 142,
         "name": "Patch",
         "attacher_email": "adam@example.com",
     }, None)
@@ -71,6 +75,10 @@ class QueuesTest(unittest.TestCase):
                 expected_stderr=expected_stderr.get(func_name, ""),
                 expected_exception=exception)
 
+    def _default_begin_work_queue_stderr(self, name, checkout_dir):
+        string_replacements = {"name": name, 'checkout_dir': checkout_dir}
+        return "CAUTION: %(name)s will discard all local changes in \"%(checkout_dir)s\"\nRunning WebKit %(name)s.\nMOCK: update_status: %(name)s Starting Queue\n" % string_replacements
+
     def assert_queue_outputs(self, queue, args=None, work_item=None, expected_stdout=None, expected_stderr=None, expected_exceptions=None, options=Mock(), tool=MockTool()):
         if not expected_stdout:
             expected_stdout = {}
diff --git a/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot_unittest.py b/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot_unittest.py
index 4b4b8b6..a63ec24 100644
--- a/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot_unittest.py
+++ b/WebKitTools/Scripts/webkitpy/tool/commands/sheriffbot_unittest.py
@@ -42,7 +42,7 @@ class SheriffBotTest(QueuesTest):
             29837: [self.builder1],
         }
         expected_stderr = {
-            "begin_work_queue": "CAUTION: sheriff-bot will discard all local changes in \"%s\"\nRunning WebKit sheriff-bot.\n" % os.getcwd(),
+            "begin_work_queue": self._default_begin_work_queue_stderr("sheriff-bot", os.getcwd()),
             "next_work_item": "",
             "process_work_item": "MOCK: irc.post: abarth, darin, eseidel: http://trac.webkit.org/changeset/29837 might have broken Builder1\nMOCK bug comment: bug_id=42, cc=['abarth@webkit.org', 'eric@webkit.org']\n--- Begin comment ---\\http://trac.webkit.org/changeset/29837 might have broken Builder1\n--- End comment ---\n\n",
             "handle_unexpected_error": "Mock error message\n"