#!/usr/bin/env python # Copyright (c) 2009, Google Inc. All rights reserved. # Copyright (c) 2009 Apple Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # A tool for automating dealing with bugzilla, posting patches, committing patches, etc. import os import re import StringIO # for add_patch_to_bug file wrappers import subprocess import sys import time from datetime import datetime, timedelta from optparse import OptionParser, IndentedHelpFormatter, SUPPRESS_USAGE, make_option # Import WebKit-specific modules. from modules.bugzilla import Bugzilla, parse_bug_id from modules.changelogs import ChangeLog from modules.comments import bug_comment_from_commit_text from modules.logging import error, log, tee from modules.scm import CommitMessage, detect_scm_system, ScriptError, CheckoutNeedsUpdate from modules.buildbot import BuildBot from modules.statusbot import StatusBot def plural(noun): # This is a dumb plural() implementation which was just enough for our uses. if re.search('h$', noun): return noun + 'es' else: return noun + 's' def pluralize(noun, count): if count != 1: noun = plural(noun) return "%d %s" % (count, noun) def commit_message_for_this_commit(scm): changelog_paths = scm.modified_changelogs() if not len(changelog_paths): raise ScriptError(message="Found no modified ChangeLogs, cannot create a commit message.\n" "All changes require a ChangeLog. See:\n" "http://webkit.org/coding/contributing.html") changelog_messages = [] for changelog_path in changelog_paths: log("Parsing ChangeLog: %s" % changelog_path) changelog_entry = ChangeLog(changelog_path).latest_entry() if not changelog_entry: raise ScriptError(message="Failed to parse ChangeLog: " + os.path.abspath(changelog_path)) changelog_messages.append(changelog_entry) # FIXME: We should sort and label the ChangeLog messages like commit-log-editor does. return CommitMessage(''.join(changelog_messages).splitlines()) class Command: def __init__(self, help_text, argument_names="", options=[], requires_local_commits=False): self.help_text = help_text self.argument_names = argument_names self.options = options self.option_parser = HelpPrintingOptionParser(usage=SUPPRESS_USAGE, add_help_option=False, option_list=self.options) self.requires_local_commits = requires_local_commits def name_with_arguments(self, command_name): usage_string = command_name if len(self.options) > 0: usage_string += " [options]" if self.argument_names: usage_string += " " + self.argument_names return usage_string def parse_args(self, args): return self.option_parser.parse_args(args) def execute(self, options, args, tool): raise NotImplementedError, "subclasses must implement" class BugsInCommitQueue(Command): def __init__(self): Command.__init__(self, 'Bugs in the commit queue') def execute(self, options, args, tool): bug_ids = tool.bugs.fetch_bug_ids_from_commit_queue() for bug_id in bug_ids: print "%s" % bug_id class PatchesInCommitQueue(Command): def __init__(self): Command.__init__(self, 'Patches in the commit queue') def execute(self, options, args, tool): patches = tool.bugs.fetch_patches_from_commit_queue() log("Patches in commit queue:") for patch in patches: print "%s" % patch['url'] class ReviewedPatchesOnBug(Command): def __init__(self): Command.__init__(self, 'r+\'d patches on a bug', 'BUGID') def execute(self, options, args, tool): bug_id = args[0] patches_to_land = tool.bugs.fetch_reviewed_patches_from_bug(bug_id) for patch in patches_to_land: print "%s" % patch['url'] class ApplyPatchesFromBug(Command): def __init__(self): options = [ make_option("--no-update", action="store_false", dest="update", default=True, help="Don't update the working directory before applying patches"), make_option("--local-commit", action="store_true", dest="local_commit", default=False, help="Make a local commit for each applied patch"), ] options += WebKitLandingScripts.cleaning_options() Command.__init__(self, 'Applies all patches on a bug to the local working directory without committing.', 'BUGID', options=options) @staticmethod def apply_patches(patches, scm, commit_each): for patch in patches: scm.apply_patch(patch) if commit_each: commit_message = commit_message_for_this_commit(scm) scm.commit_locally_with_message(commit_message.message() or patch['name']) def execute(self, options, args, tool): bug_id = args[0] patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id) os.chdir(tool.scm().checkout_root) if options.clean: tool.scm().ensure_clean_working_directory(options.force_clean) if options.update: tool.scm().update_webkit() if options.local_commit and not tool.scm().supports_local_commits(): error("--local-commit passed, but %s does not support local commits" % tool.scm().display_name()) self.apply_patches(patches, tool.scm(), options.local_commit) class WebKitLandingScripts: @staticmethod def cleaning_options(): return [ make_option("--force-clean", action="store_true", dest="force_clean", default=False, help="Clean working directory before applying patches (removes local changes and commits)"), make_option("--no-clean", action="store_false", dest="clean", default=True, help="Don't check if the working directory is clean before applying patches"), ] @staticmethod def land_options(): return [ make_option("--ignore-builders", action="store_false", dest="check_builders", default=True, help="Don't check to see if the build.webkit.org builders are green before landing."), make_option("--no-close", action="store_false", dest="close_bug", default=True, help="Leave bug open after landing."), make_option("--no-build", action="store_false", dest="build", default=True, help="Commit without building first, implies --no-test."), make_option("--no-test", action="store_false", dest="test", default=True, help="Commit without running run-webkit-tests."), make_option("--quiet", action="store_true", dest="quiet", default=False, help="Produce less console output."), make_option("--commit-queue", action="store_true", dest="commit_queue", default=False, help="Run in commit queue mode (no user interaction)."), ] @staticmethod def run_command_with_teed_output(args, teed_output): child_process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) # Use our own custom wait loop because Popen ignores a tee'd stderr/stdout. # FIXME: This could be improved not to flatten output to stdout. while True: output_line = child_process.stdout.readline() if output_line == '' and child_process.poll() != None: return child_process.poll() teed_output.write(output_line) @staticmethod def run_and_throw_if_fail(args, quiet=False): # Cache the child's output locally so it can be used for error reports. child_out_file = StringIO.StringIO() if quiet: dev_null = open(os.devnull, "w") child_stdout = tee(child_out_file, dev_null if quiet else sys.stdout) exit_code = WebKitLandingScripts.run_command_with_teed_output(args, child_stdout) if quiet: dev_null.close() child_output = child_out_file.getvalue() child_out_file.close() if exit_code: raise ScriptError(script_args=args, exit_code=exit_code, output=child_output) # We might need to pass scm into this function for scm.checkout_root @staticmethod def webkit_script_path(script_name): return os.path.join("WebKitTools", "Scripts", script_name) @classmethod def run_webkit_script(cls, script_name, quiet=False): log("Running %s" % script_name) cls.run_and_throw_if_fail(cls.webkit_script_path(script_name), quiet) @classmethod def build_webkit(cls, quiet=False): cls.run_webkit_script("build-webkit", quiet) @staticmethod def ensure_builders_are_green(buildbot, options): if not options.check_builders or buildbot.core_builders_are_green(): return error("Builders at %s are red, please do not commit. Pass --ignore-builders to bypass this check." % (buildbot.buildbot_host)) @classmethod def run_webkit_tests(cls, launch_safari, fail_fast=False, quiet=False): args = [cls.webkit_script_path("run-webkit-tests")] if not launch_safari: args.append("--no-launch-safari") if quiet: args.append("--quiet") if fail_fast: args.append("--exit-after-n-failures=1") cls.run_and_throw_if_fail(args) @staticmethod def setup_for_landing(scm, options): os.chdir(scm.checkout_root) scm.ensure_no_local_commits(options.force_clean) if options.clean: scm.ensure_clean_working_directory(options.force_clean) @classmethod def build_and_commit(cls, scm, options): if options.build: cls.build_webkit(quiet=options.quiet) if options.test: # When running the commit-queue we don't want to launch Safari and we want to exit after the first failure. cls.run_webkit_tests(launch_safari=not options.commit_queue, fail_fast=options.commit_queue, quiet=options.quiet) commit_message = commit_message_for_this_commit(scm) commit_log = scm.commit_with_message(commit_message.message()) return bug_comment_from_commit_text(scm, commit_log) class LandAndUpdateBug(Command): def __init__(self): options = [ make_option("-r", "--reviewer", action="store", type="string", dest="reviewer", help="Update ChangeLogs to say Reviewed by REVIEWER."), ] options += WebKitLandingScripts.land_options() Command.__init__(self, 'Lands the current working directory diff and updates the bug if provided.', '[BUGID]', options=options) def guess_reviewer_from_bug(self, bugs, bug_id): patches = bugs.fetch_reviewed_patches_from_bug(bug_id) if len(patches) != 1: log("%s on bug %s, cannot infer reviewer." % (pluralize("reviewed patch", len(patches)), bug_id)) return None patch = patches[0] reviewer = patch['reviewer'] log('Guessing "%s" as reviewer from attachment %s on bug %s.' % (reviewer, patch['id'], bug_id)) return reviewer def update_changelogs_with_reviewer(self, reviewer, bug_id, tool): if not reviewer: if not bug_id: log("No bug id provided and --reviewer= not provided. Not updating ChangeLogs with reviewer.") return reviewer = self.guess_reviewer_from_bug(tool.bugs, bug_id) if not reviewer: log("Failed to guess reviewer from bug %s and --reviewer= not provided. Not updating ChangeLogs with reviewer." % bug_id) return for changelog_path in tool.scm().modified_changelogs(): ChangeLog(changelog_path).set_reviewer(reviewer) def execute(self, options, args, tool): bug_id = args[0] if len(args) else None os.chdir(tool.scm().checkout_root) WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options) self.update_changelogs_with_reviewer(options.reviewer, bug_id, tool) comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options) if bug_id: log("Updating bug %s" % bug_id) if options.close_bug: tool.bugs.close_bug_as_fixed(bug_id, comment_text) else: # FIXME: We should a smart way to figure out if the patch is attached # to the bug, and if so obsolete it. tool.bugs.post_comment_to_bug(bug_id, comment_text) else: log(comment_text) log("No bug id provided.") class LandPatchesFromBugs(Command): def __init__(self): options = WebKitLandingScripts.cleaning_options() options += WebKitLandingScripts.land_options() Command.__init__(self, 'Lands all patches on a bug optionally testing them first', 'BUGID', options=options) @staticmethod def handled_error(error): log(error) exit(2) # Exit 2 insted of 1 to indicate to the commit-queue to indicate we handled the error, and that the queue should keep looping. @classmethod def land_patches(cls, bug_id, patches, options, tool): try: comment_text = "" for patch in patches: tool.scm().update_webkit() # Update before every patch in case the tree has changed log("Applying %s from bug %s." % (patch['id'], bug_id)) tool.scm().apply_patch(patch, force=options.commit_queue) # Make sure the tree is still green after updating, before building this patch. # The first patch ends up checking tree status twice, but that's OK. WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options) comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options) tool.bugs.clear_attachment_flags(patch['id'], comment_text) if options.close_bug: tool.bugs.close_bug_as_fixed(bug_id, "All reviewed patches have been landed. Closing bug.") except CheckoutNeedsUpdate, e: log("Commit was rejected because the checkout is out of date. Please update and try again.") log("You can pass --no-build to skip building/testing after update if you believe the new commits did not affect the results.") cls.handled_error(e) except ScriptError, e: # Mark the patch as commit-queue- and comment in the bug. tool.bugs.reject_patch_from_commit_queue(patch['id'], e.message_with_output()) cls.handled_error(e) @staticmethod def _fetch_list_of_patches_to_land(options, args, tool): bugs_to_patches = {} patch_count = 0 for bug_id in args: patches = [] if options.commit_queue: patches = tool.bugs.fetch_commit_queue_patches_from_bug(bug_id, reject_invalid_patches=True) else: patches = tool.bugs.fetch_reviewed_patches_from_bug(bug_id) patches_found = len(patches) log("%s found on bug %s." % (pluralize("reviewed patch", patches_found), bug_id)) patch_count += patches_found if patches_found: bugs_to_patches[bug_id] = patches log("Landing %s from %s." % (pluralize("patch", patch_count), pluralize("bug", len(args)))) return bugs_to_patches def execute(self, options, args, tool): if not len(args): error("bug-id(s) required") # Check the tree status here so we can fail early WebKitLandingScripts.ensure_builders_are_green(tool.buildbot, options) bugs_to_patches = self._fetch_list_of_patches_to_land(options, args, tool) WebKitLandingScripts.setup_for_landing(tool.scm(), options) for bug_id in bugs_to_patches.keys(): self.land_patches(bug_id, bugs_to_patches[bug_id], options, tool) class CommitMessageForCurrentDiff(Command): def __init__(self): Command.__init__(self, 'Prints a commit message suitable for the uncommitted changes.') def execute(self, options, args, tool): os.chdir(tool.scm().checkout_root) print "%s" % commit_message_for_this_commit(tool.scm()).message() class ObsoleteAttachmentsOnBug(Command): def __init__(self): Command.__init__(self, 'Marks all attachments on a bug as obsolete.', 'BUGID') def execute(self, options, args, tool): bug_id = args[0] attachments = tool.bugs.fetch_attachments_from_bug(bug_id) for attachment in attachments: if not attachment['is_obsolete']: tool.bugs.obsolete_attachment(attachment['id']) class PostDiffAsPatchToBug(Command): def __init__(self): options = [ make_option("-m", "--description", action="store", type="string", dest="description", help="Description string for the attachment (default: 'patch')"), ] options += self.posting_options() Command.__init__(self, 'Attaches the current working directory diff to a bug as a patch file.', '[BUGID]', options=options) @staticmethod def posting_options(): return [ make_option("--no-obsolete", action="store_false", dest="obsolete_patches", default=True, help="Do not obsolete old patches before posting this one."), make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."), make_option("--request-commit", action="store_true", dest="request_commit", default=False, help="Mark the patch as needing auto-commit after review."), ] @staticmethod def obsolete_patches_on_bug(bug_id, bugs): patches = bugs.fetch_patches_from_bug(bug_id) if len(patches): log("Obsoleting %s on bug %s" % (pluralize('old patch', len(patches)), bug_id)) for patch in patches: bugs.obsolete_attachment(patch['id']) def execute(self, options, args, tool): # Perfer a bug id passed as an argument over a bug url in the diff (i.e. ChangeLogs). bug_id = (args and args[0]) or parse_bug_id(tool.scm().create_patch()) if not bug_id: error("No bug id passed and no bug url found in diff, can't post.") if options.obsolete_patches: self.obsolete_patches_on_bug(bug_id, tool.bugs) diff = tool.scm().create_patch() diff_file = StringIO.StringIO(diff) # add_patch_to_bug expects a file-like object description = options.description or "Patch v1" tool.bugs.add_patch_to_bug(bug_id, diff_file, description, mark_for_review=options.review, mark_for_commit_queue=options.request_commit) class PostCommitsAsPatchesToBug(Command): def __init__(self): options = [ make_option("-b", "--bug-id", action="store", type="string", dest="bug_id", help="Specify bug id if no URL is provided in the commit log."), make_option("--add-log-as-comment", action="store_true", dest="add_log_as_comment", default=False, help="Add commit log message as a comment when uploading the patch."), make_option("-m", "--description", action="store", type="string", dest="description", help="Description string for the attachment (default: description from commit message)"), ] options += PostDiffAsPatchToBug.posting_options() Command.__init__(self, 'Attaches a range of local commits to bugs as patch files.', 'COMMITISH', options=options, requires_local_commits=True) def _comment_text_for_commit(self, options, commit_message, tool, commit_id): comment_text = None if (options.add_log_as_comment): comment_text = commit_message.body(lstrip=True) comment_text += "---\n" comment_text += tool.scm().files_changed_summary_for_commit(commit_id) return comment_text def _diff_file_for_commit(self, tool, commit_id): diff = tool.scm().create_patch_from_local_commit(commit_id) return StringIO.StringIO(diff) # add_patch_to_bug expects a file-like object def execute(self, options, args, tool): if not args: error("%s argument is required" % self.argument_names) commit_ids = tool.scm().commit_ids_from_commitish_arguments(args) if len(commit_ids) > 10: # We could lower this limit, 10 is too many for one bug as-is. error("bugzilla-tool does not support attaching %s at once. Are you sure you passed the right commit range?" % (pluralize('patch', len(commit_ids)))) have_obsoleted_patches = set() for commit_id in commit_ids: commit_message = tool.scm().commit_message_for_local_commit(commit_id) # Prefer --bug-id=, then a bug url in the commit message, then a bug url in the entire commit diff (i.e. ChangeLogs). bug_id = options.bug_id or parse_bug_id(commit_message.message()) or parse_bug_id(tool.scm().create_patch_from_local_commit(commit_id)) if not bug_id: log("Skipping %s: No bug id found in commit or specified with --bug-id." % commit_id) continue if options.obsolete_patches and bug_id not in have_obsoleted_patches: PostDiffAsPatchToBug.obsolete_patches_on_bug(bug_id, tool.bugs) have_obsoleted_patches.add(bug_id) diff_file = self._diff_file_for_commit(tool, commit_id) description = options.description or commit_message.description(lstrip=True, strip_url=True) comment_text = self._comment_text_for_commit(options, commit_message, tool, commit_id) tool.bugs.add_patch_to_bug(bug_id, diff_file, description, comment_text, mark_for_review=options.review, mark_for_commit_queue=options.request_commit) class RolloutCommit(Command): def __init__(self): options = WebKitLandingScripts.land_options() options += WebKitLandingScripts.cleaning_options() options.append(make_option("--complete-rollout", action="store_true", dest="complete_rollout", help="Experimental support for complete unsupervised rollouts, including re-opening the bug. Not recommended.")) Command.__init__(self, 'Reverts the given revision and commits the revert and re-opens the original bug.', 'REVISION [BUGID]', options=options) @staticmethod def _create_changelogs_for_revert(scm, revision): # First, discard the ChangeLog changes from the rollout. changelog_paths = scm.modified_changelogs() scm.revert_files(changelog_paths) # Second, make new ChangeLog entries for this rollout. # This could move to prepare-ChangeLog by adding a --revert= option. WebKitLandingScripts.run_webkit_script("prepare-ChangeLog") for changelog_path in changelog_paths: ChangeLog(changelog_path).update_for_revert(revision) @staticmethod def _parse_bug_id_from_revision_diff(tool, revision): original_diff = tool.scm().diff_for_revision(revision) return parse_bug_id(original_diff) @staticmethod def _reopen_bug_after_rollout(tool, bug_id, comment_text): if bug_id: tool.bugs.reopen_bug(bug_id, comment_text) else: log(comment_text) log("No bugs were updated or re-opened to reflect this rollout.") def execute(self, options, args, tool): if not args: error("REVISION is required, see --help.") revision = args[0] bug_id = self._parse_bug_id_from_revision_diff(tool, revision) if options.complete_rollout: if bug_id: log("Will re-open bug %s after rollout." % bug_id) else: log("Failed to parse bug number from diff. No bugs will be updated/reopened after the rollout.") WebKitLandingScripts.setup_for_landing(tool.scm(), options) tool.scm().update_webkit() tool.scm().apply_reverse_diff(revision) self._create_changelogs_for_revert(tool.scm(), revision) # FIXME: Fully automated rollout is not 100% idiot-proof yet, so for now just log with instructions on how to complete the rollout. # Once we trust rollout we will remove this option. if not options.complete_rollout: log("\nNOTE: Rollout support is experimental.\nPlease verify the rollout diff and use 'bugzilla-tool land-diff %s' to commit the rollout." % bug_id) else: comment_text = WebKitLandingScripts.build_and_commit(tool.scm(), options) self._reopen_bug_after_rollout(tool, bug_id, comment_text) class CreateBug(Command): def __init__(self): options = [ make_option("--cc", action="store", type="string", dest="cc", help="Comma-separated list of email addresses to carbon-copy."), make_option("--component", action="store", type="string", dest="component", help="Component for the new bug."), make_option("--no-prompt", action="store_false", dest="prompt", default=True, help="Do not prompt for bug title and comment; use commit log instead."), make_option("--no-review", action="store_false", dest="review", default=True, help="Do not mark the patch for review."), make_option("--request-commit", action="store_true", dest="request_commit", default=False, help="Mark the patch as needing auto-commit after review."), ] Command.__init__(self, 'Create a bug from local changes or local commits.', '[COMMITISH]', options=options) def create_bug_from_commit(self, options, args, tool): commit_ids = tool.scm().commit_ids_from_commitish_arguments(args) if len(commit_ids) > 3: error("Are you sure you want to create one bug with %s patches?" % len(commit_ids)) commit_id = commit_ids[0] bug_title = "" comment_text = "" if options.prompt: (bug_title, comment_text) = self.prompt_for_bug_title_and_comment() else: commit_message = tool.scm().commit_message_for_local_commit(commit_id) bug_title = commit_message.description(lstrip=True, strip_url=True) comment_text = commit_message.body(lstrip=True) comment_text += "---\n" comment_text += tool.scm().files_changed_summary_for_commit(commit_id) diff = tool.scm().create_patch_from_local_commit(commit_id) diff_file = StringIO.StringIO(diff) # create_bug_with_patch expects a file-like object bug_id = tool.bugs.create_bug_with_patch(bug_title, comment_text, options.component, diff_file, "Patch v1", cc=options.cc, mark_for_review=options.review, mark_for_commit_queue=options.request_commit) if bug_id and len(commit_ids) > 1: options.bug_id = bug_id options.obsolete_patches = False # FIXME: We should pass through --no-comment switch as well. PostCommitsAsPatchesToBug.execute(self, options, commit_ids[1:], tool) def create_bug_from_patch(self, options, args, tool): bug_title = "" comment_text = "" if options.prompt: (bug_title, comment_text) = self.prompt_for_bug_title_and_comment() else: commit_message = commit_message_for_this_commit(tool.scm()) bug_title = commit_message.description(lstrip=True, strip_url=True) comment_text = commit_message.body(lstrip=True) diff = tool.scm().create_patch() diff_file = StringIO.StringIO(diff) # create_bug_with_patch expects a file-like object bug_id = tool.bugs.create_bug_with_patch(bug_title, comment_text, options.component, diff_file, "Patch v1", cc=options.cc, mark_for_review=options.review, mark_for_commit_queue=options.request_commit) def prompt_for_bug_title_and_comment(self): bug_title = raw_input("Bug title: ") print "Bug comment (hit ^D on blank line to end):" lines = sys.stdin.readlines() try: sys.stdin.seek(0, os.SEEK_END) except IOError: # Cygwin raises an Illegal Seek (errno 29) exception when the above # seek() call is made. Ignoring it seems to cause no harm. # FIXME: Figure out a way to get avoid the exception in the first # place. pass else: raise comment_text = ''.join(lines) return (bug_title, comment_text) def execute(self, options, args, tool): if len(args): if (not tool.scm().supports_local_commits()): error("Extra arguments not supported; patch is taken from working directory.") self.create_bug_from_commit(options, args, tool) else: self.create_bug_from_patch(options, args, tool) class CheckTreeStatus(Command): def __init__(self): Command.__init__(self, 'Print out the status of the webkit builders.') def execute(self, options, args, tool): for builder in tool.buildbot.builder_statuses(): status_string = "ok" if builder['is_green'] else 'FAIL' print "%s : %s" % (status_string.ljust(4), builder['name']) class LandPatchesFromCommitQueue(Command): def __init__(self): options = [ make_option("--is-relaunch", action="store_true", dest="is_relaunch", default=False, help="Internal: Used by the queue to indicate that it's relaunching itself."), make_option("--no-confirm", action="store_false", dest="confirm", default=True, help="Do not ask the user for confirmation before running the queue. Dangerous!"), make_option("--status-host", action="store", type="string", dest="status_host", default=StatusBot.default_host, help="Do not ask the user for confirmation before running the queue. Dangerous!"), ] Command.__init__(self, 'Run the commit queue.', options=options) self._original_stdout = None self._original_stderr = None self._files_for_output = [] queue_log_path = 'commit_queue.log' bug_logs_directory = 'commit_queue_logs' log_date_format = "%Y-%m-%d %H:%M:%S" sleep_duration_text = "5 mins" seconds_to_sleep = 300 def _tee_outputs_to_files(self, files): if not self._original_stdout: self._original_stdout = sys.stdout self._original_stderr = sys.stderr if files and len(files): sys.stdout = tee(self._original_stdout, *files) sys.stderr = tee(self._original_stderr, *files) else: sys.stdout = self._original_stdout sys.stderr = self._original_stderr @classmethod def _sleep_message(cls, message): wake_time = datetime.now() + timedelta(seconds=cls.seconds_to_sleep) return "%s Sleeping until %s (%s)." % (message, wake_time.strftime(cls.log_date_format), cls.sleep_duration_text) def _sleep(self, message): log(self._sleep_message(message)) time.sleep(self.seconds_to_sleep) self._next_patch() def _update_status_and_sleep(self, message): status_message = self._sleep_message(message) self.status_bot.update_status(status_message) log(status_message) time.sleep(self.seconds_to_sleep) self._next_patch() def _next_patch(self): # Re-exec this script to catch any updates to the script. # Make sure that the re-execed commit-queue does not wait for the user. args = sys.argv[:] if args.count("--is-relaunch") == 0: args.append("--is-relaunch") os.execvp(sys.argv[0], args) @staticmethod def _open_log_file(log_path): (log_directory, log_name) = os.path.split(log_path) if log_directory and not os.path.exists(log_directory): os.makedirs(log_directory) return open(log_path, 'a+') def _add_log_to_output_tee(self, path): log_file = self._open_log_file(path) self._files_for_output.append(log_file) self._tee_outputs_to_files(self._files_for_output) return log_file def _remove_log_from_output_tee(self, log_file): self._files_for_output.remove(log_file) self._tee_outputs_to_files(self._files_for_output) log_file.close() def execute(self, options, args, tool): if not options.is_relaunch: log("CAUTION: commit-queue will discard all local changes in %s" % tool.scm().checkout_root) if options.confirm: response = raw_input("Are you sure? Type 'yes' to continue: ") if (response != 'yes'): error("User declined.") queue_log = self._add_log_to_output_tee(self.queue_log_path) if not options.is_relaunch: log("Running WebKit Commit Queue. %s" % datetime.now().strftime(self.log_date_format)) self.status_bot = StatusBot(host=options.status_host) # Either of these calls could throw URLError which shouldn't stop the queue. # We catch all exceptions just in case. try: # Fetch patches instead of just bug ids to that we validate reviewer/committer flags on every patch. patches = tool.bugs.fetch_patches_from_commit_queue(reject_invalid_patches=True) if not len(patches): self._update_status_and_sleep("Empty queue.") patch_ids = map(lambda patch: patch['id'], patches) first_bug_id = patches[0]['bug_id'] log("%s in commit queue [%s]" % (pluralize('patch', len(patches)), ", ".join(patch_ids))) red_builders_names = tool.buildbot.red_core_builders_names() if red_builders_names: red_builders_names = map(lambda name: '"%s"' % name, red_builders_names) # Add quotes around the names. self._update_status_and_sleep("Builders [%s] are red. See http://build.webkit.org." % ", ".join(red_builders_names)) self.status_bot.update_status("Landing patches from bug %s." % first_bug_id, bug_id=first_bug_id) except Exception, e: # Don't try tell the status bot, in case telling it causes an exception. self._sleep("Exception while checking queue and bots: %s." % e) # Try to land patches on the first bug in the queue before looping bug_log_path = os.path.join(self.bug_logs_directory, "%s.log" % first_bug_id) bug_log = self._add_log_to_output_tee(bug_log_path) bugzilla_tool_path = __file__ # re-execute this script bugzilla_tool_args = [bugzilla_tool_path, 'land-patches', '--force-clean', '--commit-queue', '--quiet', first_bug_id] try: WebKitLandingScripts.run_and_throw_if_fail(bugzilla_tool_args) except ScriptError, e: # Unexpected failure! Mark the patch as commit-queue- and comment in the bug. # exit(2) is a special exit code we use to indicate that the error was already handled by land-patches and we should keep looping anyway. if e.exit_code != 2: tool.bugs.reject_patch_from_commit_queue(patch['id'], "Unexpected failure when landing patch! Please file a bug against bugzilla-tool.\n%s" % e.message_with_output()) self._remove_log_from_output_tee(bug_log) # self._remove_log_from_output_tee(queue_log) # implicit in the exec() self._next_patch() class NonWrappingEpilogIndentedHelpFormatter(IndentedHelpFormatter): def __init__(self): IndentedHelpFormatter.__init__(self) # The standard IndentedHelpFormatter paragraph-wraps the epilog, killing our custom formatting. def format_epilog(self, epilog): if epilog: return "\n" + epilog + "\n" return "" class HelpPrintingOptionParser(OptionParser): def error(self, msg): self.print_usage(sys.stderr) error_message = "%s: error: %s\n" % (self.get_prog_name(), msg) error_message += "\nType '" + self.get_prog_name() + " --help' to see usage.\n" self.exit(2, error_message) class BugzillaTool: def __init__(self): self.cached_scm = None self.bugs = Bugzilla() self.buildbot = BuildBot() self.commands = [ { 'name' : 'bugs-to-commit', 'object' : BugsInCommitQueue() }, { 'name' : 'patches-to-commit', 'object' : PatchesInCommitQueue() }, { 'name' : 'reviewed-patches', 'object' : ReviewedPatchesOnBug() }, { 'name' : 'create-bug', 'object' : CreateBug() }, { 'name' : 'apply-patches', 'object' : ApplyPatchesFromBug() }, { 'name' : 'land-diff', 'object' : LandAndUpdateBug() }, { 'name' : 'land-patches', 'object' : LandPatchesFromBugs() }, { 'name' : 'commit-message', 'object' : CommitMessageForCurrentDiff() }, { 'name' : 'obsolete-attachments', 'object' : ObsoleteAttachmentsOnBug() }, { 'name' : 'post-diff', 'object' : PostDiffAsPatchToBug() }, { 'name' : 'post-commits', 'object' : PostCommitsAsPatchesToBug() }, { 'name' : 'tree-status', 'object' : CheckTreeStatus() }, { 'name' : 'commit-queue', 'object' : LandPatchesFromCommitQueue() }, { 'name' : 'rollout', 'object' : RolloutCommit() }, ] self.global_option_parser = HelpPrintingOptionParser(usage=self.usage_line(), formatter=NonWrappingEpilogIndentedHelpFormatter(), epilog=self.commands_usage()) self.global_option_parser.add_option("--dry-run", action="store_true", dest="dryrun", help="do not touch remote servers", default=False) def scm(self): # Lazily initialize SCM to not error-out before command line parsing (or when running non-scm commands). original_cwd = os.path.abspath('.') if not self.cached_scm: self.cached_scm = detect_scm_system(original_cwd) if not self.cached_scm: script_directory = os.path.abspath(sys.path[0]) webkit_directory = os.path.abspath(os.path.join(script_directory, "../..")) self.cached_scm = detect_scm_system(webkit_directory) if self.cached_scm: log("The current directory (%s) is not a WebKit checkout, using %s" % (original_cwd, webkit_directory)) else: error("FATAL: Failed to determine the SCM system for either %s or %s" % (original_cwd, webkit_directory)) return self.cached_scm @staticmethod def usage_line(): return "Usage: %prog [options] command [command-options] [command-arguments]" def commands_usage(self): commands_text = "Commands:\n" longest_name_length = 0 command_rows = [] scm_supports_local_commits = self.scm().supports_local_commits() for command in self.commands: command_object = command['object'] if command_object.requires_local_commits and not scm_supports_local_commits: continue command_name_and_args = command_object.name_with_arguments(command['name']) command_rows.append({ 'name-and-args': command_name_and_args, 'object': command_object }) longest_name_length = max([longest_name_length, len(command_name_and_args)]) # Use our own help formatter so as to indent enough. formatter = IndentedHelpFormatter() formatter.indent() formatter.indent() for row in command_rows: command_object = row['object'] commands_text += " " + row['name-and-args'].ljust(longest_name_length + 3) + command_object.help_text + "\n" commands_text += command_object.option_parser.format_option_help(formatter) return commands_text def handle_global_args(self, args): (options, args) = self.global_option_parser.parse_args(args) if len(args): # We'll never hit this because split_args splits at the first arg without a leading '-' self.global_option_parser.error("Extra arguments before command: " + args) if options.dryrun: self.scm().dryrun = True self.bugs.dryrun = True @staticmethod def split_args(args): # Assume the first argument which doesn't start with '-' is the command name. command_index = 0 for arg in args: if arg[0] != '-': break command_index += 1 else: return (args[:], None, []) global_args = args[:command_index] command = args[command_index] command_args = args[command_index + 1:] return (global_args, command, command_args) def command_by_name(self, command_name): for command in self.commands: if command_name == command['name']: return command return None def main(self): (global_args, command_name, args_after_command_name) = self.split_args(sys.argv[1:]) # Handle --help, etc: self.handle_global_args(global_args) if not command_name: self.global_option_parser.error("No command specified") command = self.command_by_name(command_name) if not command: self.global_option_parser.error(command_name + " is not a recognized command") command_object = command['object'] if command_object.requires_local_commits and not self.scm().supports_local_commits(): error(command_name + " requires local commits using %s in %s." % (self.scm().display_name(), self.scm().checkout_root)) (command_options, command_args) = command_object.parse_args(args_after_command_name) return command_object.execute(command_options, command_args, self) def main(): tool = BugzillaTool() return tool.main() if __name__ == "__main__": main()