path: root/Tools/Scripts/webkitpy/common/checkout/scm.py

# Copyright (c) 2009, Google Inc. All rights reserved.
# Copyright (c) 2009 Apple Inc. All rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
# 
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
# 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Python module for interacting with an SCM system (like SVN or Git)

import os
import re
import sys
import shutil

from webkitpy.common.system.executive import Executive, run_command, ScriptError
from webkitpy.common.system.deprecated_logging import error, log
import webkitpy.common.system.ospath as ospath
from webkitpy.common.memoized import memoized


def find_checkout_root():
    """Returns the current checkout root (as determined by default_scm().

    Returns the absolute path to the top of the WebKit checkout, or None
    if it cannot be determined.

    """
    scm_system = default_scm()
    if scm_system:
        return scm_system.checkout_root
    return None


def default_scm(patch_directories=None):
    """Return the default SCM object as determined by the CWD and running code.

    Returns the default SCM object for the current working directory; if the
    CWD is not in a checkout, then we attempt to figure out if the SCM module
    itself is part of a checkout, and return that one. If neither is part of
    a checkout, None is returned.

    """
    cwd = os.getcwd()
    scm_system = detect_scm_system(cwd, patch_directories)
    if not scm_system:
        script_directory = os.path.dirname(os.path.abspath(__file__))
        scm_system = detect_scm_system(script_directory, patch_directories)
        if scm_system:
            log("The current directory (%s) is not a WebKit checkout, using %s" % (cwd, scm_system.checkout_root))
        else:
            error("FATAL: Failed to determine the SCM system for either %s or %s" % (cwd, script_directory))
    return scm_system


def detect_scm_system(path, patch_directories=None):
    absolute_path = os.path.abspath(path)

    if patch_directories == []:
        patch_directories = None

    if SVN.in_working_directory(absolute_path):
        return SVN(cwd=absolute_path, patch_directories=patch_directories)
    
    if Git.in_working_directory(absolute_path):
        return Git(cwd=absolute_path)
    
    return None


def first_non_empty_line_after_index(lines, index=0):
    first_non_empty_line = index
    for line in lines[index:]:
        if re.match("^\s*$", line):
            first_non_empty_line += 1
        else:
            break
    return first_non_empty_line


class CommitMessage:
    def __init__(self, message):
        self.message_lines = message[first_non_empty_line_after_index(message, 0):]

    def body(self, lstrip=False):
        lines = self.message_lines[first_non_empty_line_after_index(self.message_lines, 1):]
        if lstrip:
            lines = [line.lstrip() for line in lines]
        return "\n".join(lines) + "\n"

    def description(self, lstrip=False, strip_url=False):
        line = self.message_lines[0]
        if lstrip:
            line = line.lstrip()
        if strip_url:
            line = re.sub("^(\s*)<.+> ", "\1", line)
        return line

    def message(self):
        return "\n".join(self.message_lines) + "\n"


class CheckoutNeedsUpdate(ScriptError):
    def __init__(self, script_args, exit_code, output, cwd):
        ScriptError.__init__(self, script_args=script_args, exit_code=exit_code, output=output, cwd=cwd)


def commit_error_handler(error):
    if re.search("resource out of date", error.output):
        raise CheckoutNeedsUpdate(script_args=error.script_args, exit_code=error.exit_code, output=error.output, cwd=error.cwd)
    Executive.default_error_handler(error)


class AuthenticationError(Exception):
    def __init__(self, server_host):
        self.server_host = server_host


class AmbiguousCommitError(Exception):
    def __init__(self, num_local_commits, working_directory_is_clean):
        self.num_local_commits = num_local_commits
        self.working_directory_is_clean = working_directory_is_clean


# SCM methods are expected to return paths relative to self.checkout_root.
class SCM:
    def __init__(self, cwd, executive=None):
        self.cwd = cwd
        self.checkout_root = self.find_checkout_root(self.cwd)
        self.dryrun = False
        self._executive = executive or Executive()

    # A wrapper used by subclasses to create processes.
    def run(self, args, cwd=None, input=None, error_handler=None, return_exit_code=False, return_stderr=True, decode_output=True):
        # FIXME: We should set cwd appropriately.
        return self._executive.run_command(args,
                           cwd=cwd,
                           input=input,
                           error_handler=error_handler,
                           return_exit_code=return_exit_code,
                           return_stderr=return_stderr,
                           decode_output=decode_output)

    # SCM always returns repository relative path, but sometimes we need
    # absolute paths to pass to rm, etc.
    def absolute_path(self, repository_relative_path):
        return os.path.join(self.checkout_root, repository_relative_path)

    # FIXME: This belongs in Checkout, not SCM.
    def scripts_directory(self):
        return os.path.join(self.checkout_root, "Tools", "Scripts")

    # FIXME: This belongs in Checkout, not SCM.
    def script_path(self, script_name):
        return os.path.join(self.scripts_directory(), script_name)

    def ensure_clean_working_directory(self, force_clean):
        if self.working_directory_is_clean():
            return
        if not force_clean:
            # FIXME: Shouldn't this use cwd=self.checkout_root?
            print self.run(self.status_command(), error_handler=Executive.ignore_error)
            raise ScriptError(message="Working directory has modifications, pass --force-clean or --no-clean to continue.")
        log("Cleaning working directory")
        self.clean_working_directory()

    def ensure_no_local_commits(self, force):
        if not self.supports_local_commits():
            return
        commits = self.local_commits()
        if not len(commits):
            return
        if not force:
            error("Working directory has local commits, pass --force-clean to continue.")
        self.discard_local_commits()

    def run_status_and_extract_filenames(self, status_command, status_regexp):
        filenames = []
        # We run with cwd=self.checkout_root so that returned-paths are root-relative.
        for line in self.run(status_command, cwd=self.checkout_root).splitlines():
            match = re.search(status_regexp, line)
            if not match:
                continue
            # status = match.group('status')
            filename = match.group('filename')
            filenames.append(filename)
        return filenames

    def strip_r_from_svn_revision(self, svn_revision):
        match = re.match("^r(?P<svn_revision>\d+)", unicode(svn_revision))
        if (match):
            return match.group('svn_revision')
        return svn_revision

    def svn_revision_from_commit_text(self, commit_text):
        match = re.search(self.commit_success_regexp(), commit_text, re.MULTILINE)
        return match.group('svn_revision')

    @staticmethod
    def _subclass_must_implement():
        raise NotImplementedError("subclasses must implement")

    @staticmethod
    def in_working_directory(path):
        SCM._subclass_must_implement()

    @staticmethod
    def find_checkout_root(path):
        SCM._subclass_must_implement()

    @staticmethod
    def commit_success_regexp():
        SCM._subclass_must_implement()

    def working_directory_is_clean(self):
        self._subclass_must_implement()

    def clean_working_directory(self):
        self._subclass_must_implement()

    def status_command(self):
        self._subclass_must_implement()

    def add(self, path, return_exit_code=False):
        self._subclass_must_implement()

    def delete(self, path):
        self._subclass_must_implement()

    def changed_files(self, git_commit=None):
        self._subclass_must_implement()

    def changed_files_for_revision(self, revision):
        self._subclass_must_implement()

    def revisions_changing_file(self, path, limit=5):
        self._subclass_must_implement()

    def added_files(self):
        self._subclass_must_implement()

    def conflicted_files(self):
        self._subclass_must_implement()

    def display_name(self):
        self._subclass_must_implement()

    def create_patch(self, git_commit=None, changed_files=None):
        self._subclass_must_implement()

    def committer_email_for_revision(self, revision):
        self._subclass_must_implement()

    def contents_at_revision(self, path, revision):
        self._subclass_must_implement()

    def diff_for_revision(self, revision):
        self._subclass_must_implement()

    def diff_for_file(self, path, log=None):
        self._subclass_must_implement()

    def show_head(self, path):
        self._subclass_must_implement()

    def apply_reverse_diff(self, revision):
        self._subclass_must_implement()

    def revert_files(self, file_paths):
        self._subclass_must_implement()

    def commit_with_message(self, message, username=None, git_commit=None, force_squash=False):
        self._subclass_must_implement()

    def svn_commit_log(self, svn_revision):
        self._subclass_must_implement()

    def last_svn_commit_log(self):
        self._subclass_must_implement()

    # Subclasses must indicate if they support local commits,
    # but the SCM baseclass will only call local_commits methods when this is true.
    @staticmethod
    def supports_local_commits():
        SCM._subclass_must_implement()

    def remote_merge_base():
        SCM._subclass_must_implement()

    def commit_locally_with_message(self, message):
        error("Your source control manager does not support local commits.")

    def discard_local_commits(self):
        pass

    def local_commits(self):
        return []


class SVN(SCM):
    # FIXME: We should move these values to a WebKit-specific config file.
    svn_server_host = "svn.webkit.org"
    svn_server_realm = "<http://svn.webkit.org:80> Mac OS Forge"

    def __init__(self, cwd, patch_directories, executive=None):
        SCM.__init__(self, cwd, executive)
        self._bogus_dir = None
        if patch_directories == []:
            # FIXME: ScriptError is for Executive, this should probably be a normal Exception.
            raise ScriptError(script_args=svn_info_args, message='Empty list of patch directories passed to SCM.__init__')
        elif patch_directories == None:
            self._patch_directories = [ospath.relpath(cwd, self.checkout_root)]
        else:
            self._patch_directories = patch_directories

    @staticmethod
    def in_working_directory(path):
        return os.path.isdir(os.path.join(path, '.svn'))
    
    @classmethod
    def find_uuid(cls, path):
        if not cls.in_working_directory(path):
            return None
        return cls.value_from_svn_info(path, 'Repository UUID')

    @classmethod
    def value_from_svn_info(cls, path, field_name):
        svn_info_args = ['svn', 'info', path]
        info_output = run_command(svn_info_args).rstrip()
        match = re.search("^%s: (?P<value>.+)$" % field_name, info_output, re.MULTILINE)
        if not match:
            raise ScriptError(script_args=svn_info_args, message='svn info did not contain a %s.' % field_name)
        return match.group('value')

    @staticmethod
    def find_checkout_root(path):
        uuid = SVN.find_uuid(path)
        # If |path| is not in a working directory, we're supposed to return |path|.
        if not uuid:
            return path
        # Search up the directory hierarchy until we find a different UUID.
        last_path = None
        while True:
            if uuid != SVN.find_uuid(path):
                return last_path
            last_path = path
            (path, last_component) = os.path.split(path)
            if last_path == path:
                return None

    @staticmethod
    def commit_success_regexp():
        return "^Committed revision (?P<svn_revision>\d+)\.$"

    def has_authorization_for_realm(self, realm=svn_server_realm, home_directory=os.getenv("HOME")):
        # Assumes find and grep are installed.
        if not os.path.isdir(os.path.join(home_directory, ".subversion")):
            return False
        find_args = ["find", ".subversion", "-type", "f", "-exec", "grep", "-q", realm, "{}", ";", "-print"];
        find_output = self.run(find_args, cwd=home_directory, error_handler=Executive.ignore_error).rstrip()
        return find_output and os.path.isfile(os.path.join(home_directory, find_output))

    @memoized
    def svn_version(self):
        return self.run(['svn', '--version', '--quiet'])

    def working_directory_is_clean(self):
        return self.run(["svn", "diff"], cwd=self.checkout_root, decode_output=False) == ""

    def clean_working_directory(self):
        # Make sure there are no locks lying around from a previously aborted svn invocation.
        # This is slightly dangerous, as it's possible the user is running another svn process
        # on this checkout at the same time.  However, it's much more likely that we're running
        # under windows and svn just sucks (or the user interrupted svn and it failed to clean up).
        self.run(["svn", "cleanup"], cwd=self.checkout_root)

        # svn revert -R is not as awesome as git reset --hard.
        # It will leave added files around, causing later svn update
        # calls to fail on the bots.  We make this mirror git reset --hard
        # by deleting any added files as well.
        added_files = reversed(sorted(self.added_files()))
        # added_files() returns directories for SVN, we walk the files in reverse path
        # length order so that we remove files before we try to remove the directories.
        self.run(["svn", "revert", "-R", "."], cwd=self.checkout_root)
        for path in added_files:
            # This is robust against cwd != self.checkout_root
            absolute_path = self.absolute_path(path)
            # Completely lame that there is no easy way to remove both types with one call.
            if os.path.isdir(path):
                os.rmdir(absolute_path)
            else:
                os.remove(absolute_path)

    def status_command(self):
        return ['svn', 'status']

    def _status_regexp(self, expected_types):
        field_count = 6 if self.svn_version() > "1.6" else 5
        return "^(?P<status>[%s]).{%s} (?P<filename>.+)$" % (expected_types, field_count)

    def _add_parent_directories(self, path):
        """Does 'svn add' to the path and its parents."""
        if self.in_working_directory(path):
            return
        dirname = os.path.dirname(path)
        # We have dirname directry - ensure it added.
        if dirname != path:
            self._add_parent_directories(dirname)
        self.add(path)

    def add(self, path, return_exit_code=False):
        self._add_parent_directories(os.path.dirname(os.path.abspath(path)))
        return self.run(["svn", "add", path], return_exit_code=return_exit_code)

    def delete(self, path):
        parent, base = os.path.split(os.path.abspath(path))
        return self.run(["svn", "delete", "--force", base], cwd=parent)

    def changed_files(self, git_commit=None):
        status_command = ["svn", "status"]
        status_command.extend(self._patch_directories)
        # ACDMR: Addded, Conflicted, Deleted, Modified or Replaced
        return self.run_status_and_extract_filenames(status_command, self._status_regexp("ACDMR"))

    def changed_files_for_revision(self, revision):
        # As far as I can tell svn diff --summarize output looks just like svn status output.
        # No file contents printed, thus utf-8 auto-decoding in self.run is fine.
        status_command = ["svn", "diff", "--summarize", "-c", revision]
        return self.run_status_and_extract_filenames(status_command, self._status_regexp("ACDMR"))

    def revisions_changing_file(self, path, limit=5):
        revisions = []
        # svn log will exit(1) (and thus self.run will raise) if the path does not exist.
        log_command = ['svn', 'log', '--quiet', '--limit=%s' % limit, path]
        for line in self.run(log_command, cwd=self.checkout_root).splitlines():
            match = re.search('^r(?P<revision>\d+) ', line)
            if not match:
                continue
            revisions.append(int(match.group('revision')))
        return revisions

    def conflicted_files(self):
        return self.run_status_and_extract_filenames(self.status_command(), self._status_regexp("C"))

    def added_files(self):
        return self.run_status_and_extract_filenames(self.status_command(), self._status_regexp("A"))

    def deleted_files(self):
        return self.run_status_and_extract_filenames(self.status_command(), self._status_regexp("D"))

    @staticmethod
    def supports_local_commits():
        return False

    def display_name(self):
        return "svn"

    # FIXME: This method should be on Checkout.
    def create_patch(self, git_commit=None, changed_files=None):
        """Returns a byte array (str()) representing the patch file.
        Patch files are effectively binary since they may contain
        files of multiple different encodings."""
        if changed_files == []:
            return ""
        elif changed_files == None:
            changed_files = []
        return self.run([self.script_path("svn-create-patch")] + changed_files,
            cwd=self.checkout_root, return_stderr=False,
            decode_output=False)

    def committer_email_for_revision(self, revision):
        return self.run(["svn", "propget", "svn:author", "--revprop", "-r", revision]).rstrip()

    def contents_at_revision(self, path, revision):
        """Returns a byte array (str()) containing the contents
        of path @ revision in the repository."""
        remote_path = "%s/%s" % (self._repository_url(), path)
        return self.run(["svn", "cat", "-r", revision, remote_path], decode_output=False)

    def diff_for_revision(self, revision):
        # FIXME: This should probably use cwd=self.checkout_root
        return self.run(['svn', 'diff', '-c', revision])

    def _bogus_dir_name(self):
        if sys.platform.startswith("win"):
            parent_dir = tempfile.gettempdir()
        else:
            parent_dir = sys.path[0]  # tempdir is not secure.
        return os.path.join(parent_dir, "temp_svn_config")

    def _setup_bogus_dir(self, log):
        self._bogus_dir = self._bogus_dir_name()
        if not os.path.exists(self._bogus_dir):
            os.mkdir(self._bogus_dir)
            self._delete_bogus_dir = True
        else:
            self._delete_bogus_dir = False
        if log:
            log.debug('  Html: temp config dir: "%s".', self._bogus_dir)

    def _teardown_bogus_dir(self, log):
        if self._delete_bogus_dir:
            shutil.rmtree(self._bogus_dir, True)
            if log:
                log.debug('  Html: removed temp config dir: "%s".', self._bogus_dir)
        self._bogus_dir = None

    def diff_for_file(self, path, log=None):
        self._setup_bogus_dir(log)
        try:
            args = ['svn', 'diff']
            if self._bogus_dir:
                args += ['--config-dir', self._bogus_dir]
            args.append(path)
            return self.run(args)
        finally:
            self._teardown_bogus_dir(log)

    def show_head(self, path):
        return self.run(['svn', 'cat', '-r', 'BASE', path], decode_output=False)

    def _repository_url(self):
        return self.value_from_svn_info(self.checkout_root, 'URL')

    def apply_reverse_diff(self, revision):
        # '-c -revision' applies the inverse diff of 'revision'
        svn_merge_args = ['svn', 'merge', '--non-interactive', '-c', '-%s' % revision, self._repository_url()]
        log("WARNING: svn merge has been known to take more than 10 minutes to complete.  It is recommended you use git for rollouts.")
        log("Running '%s'" % " ".join(svn_merge_args))
        # FIXME: Should this use cwd=self.checkout_root?
        self.run(svn_merge_args)

    def revert_files(self, file_paths):
        # FIXME: This should probably use cwd=self.checkout_root.
        self.run(['svn', 'revert'] + file_paths)

    def commit_with_message(self, message, username=None, git_commit=None, force_squash=False):
        # git-commit and force are not used by SVN.
        if self.dryrun:
            # Return a string which looks like a commit so that things which parse this output will succeed.
            return "Dry run, no commit.\nCommitted revision 0."

        svn_commit_args = ["svn", "commit"]

        if not username and not self.has_authorization_for_realm():
            raise AuthenticationError(self.svn_server_host)
        if username:
            svn_commit_args.extend(["--username", username])

        svn_commit_args.extend(["-m", message])
        # FIXME: Should this use cwd=self.checkout_root?
        return self.run(svn_commit_args, error_handler=commit_error_handler)

    def svn_commit_log(self, svn_revision):
        svn_revision = self.strip_r_from_svn_revision(svn_revision)
        return self.run(['svn', 'log', '--non-interactive', '--revision', svn_revision])

    def last_svn_commit_log(self):
        # BASE is the checkout revision, HEAD is the remote repository revision
        # http://svnbook.red-bean.com/en/1.0/ch03s03.html
        return self.svn_commit_log('BASE')

    def propset(self, pname, pvalue, path):
        dir, base = os.path.split(path)
        return self.run(['svn', 'pset', pname, pvalue, base], cwd=dir)

    def propget(self, pname, path):
        dir, base = os.path.split(path)
        return self.run(['svn', 'pget', pname, base], cwd=dir).encode('utf-8').rstrip("\n")


# All git-specific logic should go here.
class Git(SCM):
    def __init__(self, cwd, executive=None):
        SCM.__init__(self, cwd, executive)
        self._check_git_architecture()

    def _machine_is_64bit(self):
        import platform
        # This only is tested on Mac.
        if not platform.mac_ver()[0]:
            return False

        # platform.architecture()[0] can be '64bit' even if the machine is 32bit:
        # http://mail.python.org/pipermail/pythonmac-sig/2009-September/021648.html
        # Use the sysctl command to find out what the processor actually supports.
        return self.run(['sysctl', '-n', 'hw.cpu64bit_capable']).rstrip() == '1'

    def _executable_is_64bit(self, path):
        # Again, platform.architecture() fails us.  On my machine
        # git_bits = platform.architecture(executable=git_path, bits='default')[0]
        # git_bits is just 'default', meaning the call failed.
        file_output = self.run(['file', path])
        return re.search('x86_64', file_output)

    def _check_git_architecture(self):
        if not self._machine_is_64bit():
            return

        # We could path-search entirely in python or with
        # which.py (http://code.google.com/p/which), but this is easier:
        git_path = self.run(['which', 'git']).rstrip()
        if self._executable_is_64bit(git_path):
            return

        webkit_dev_thead_url = "https://lists.webkit.org/pipermail/webkit-dev/2010-December/015249.html"
        log("Warning: This machine is 64-bit, but the git binary (%s) does not support 64-bit.\nInstall a 64-bit git for better performance, see:\n%s\n" % (git_path, webkit_dev_thead_url))

    @classmethod
    def in_working_directory(cls, path):
        return run_command(['git', 'rev-parse', '--is-inside-work-tree'], cwd=path, error_handler=Executive.ignore_error).rstrip() == "true"

    @classmethod
    def find_checkout_root(cls, path):
        # "git rev-parse --show-cdup" would be another way to get to the root
        (checkout_root, dot_git) = os.path.split(run_command(['git', 'rev-parse', '--git-dir'], cwd=(path or "./")))
        # If we were using 2.6 # checkout_root = os.path.relpath(checkout_root, path)
        if not os.path.isabs(checkout_root): # Sometimes git returns relative paths
            checkout_root = os.path.join(path, checkout_root)
        return checkout_root

    @classmethod
    def to_object_name(cls, filepath):
        root_end_with_slash = os.path.join(cls.find_checkout_root(os.path.dirname(filepath)), '')
        return filepath.replace(root_end_with_slash, '')

    @classmethod
    def read_git_config(cls, key):
        # FIXME: This should probably use cwd=self.checkout_root.
        # Pass --get-all for cases where the config has multiple values
        return run_command(["git", "config", "--get-all", key],
            error_handler=Executive.ignore_error).rstrip('\n')

    @staticmethod
    def commit_success_regexp():
        return "^Committed r(?P<svn_revision>\d+)$"

    def discard_local_commits(self):
        # FIXME: This should probably use cwd=self.checkout_root
        self.run(['git', 'reset', '--hard', self.remote_branch_ref()])
    
    def local_commits(self):
        # FIXME: This should probably use cwd=self.checkout_root
        return self.run(['git', 'log', '--pretty=oneline', 'HEAD...' + self.remote_branch_ref()]).splitlines()

    def rebase_in_progress(self):
        return os.path.exists(os.path.join(self.checkout_root, '.git/rebase-apply'))

    def working_directory_is_clean(self):
        # FIXME: This should probably use cwd=self.checkout_root
        return self.run(['git', 'diff', 'HEAD', '--name-only']) == ""

    def clean_working_directory(self):
        # FIXME: These should probably use cwd=self.checkout_root.
        # Could run git clean here too, but that wouldn't match working_directory_is_clean
        self.run(['git', 'reset', '--hard', 'HEAD'])
        # Aborting rebase even though this does not match working_directory_is_clean
        if self.rebase_in_progress():
            self.run(['git', 'rebase', '--abort'])

    def status_command(self):
        # git status returns non-zero when there are changes, so we use git diff name --name-status HEAD instead.
        # No file contents printed, thus utf-8 autodecoding in self.run is fine.
        return ["git", "diff", "--name-status", "HEAD"]

    def _status_regexp(self, expected_types):
        return '^(?P<status>[%s])\t(?P<filename>.+)$' % expected_types

    def add(self, path, return_exit_code=False):
        return self.run(["git", "add", path], return_exit_code=return_exit_code)

    def delete(self, path):
        return self.run(["git", "rm", "-f", path])

    def merge_base(self, git_commit):
        if git_commit:
            # Special-case HEAD.. to mean working-copy changes only.
            if git_commit.upper() == 'HEAD..':
                return 'HEAD'

            if '..' not in git_commit:
                git_commit = git_commit + "^.." + git_commit
            return git_commit

        return self.remote_merge_base()

    def changed_files(self, git_commit=None):
        # FIXME: --diff-filter could be used to avoid the "extract_filenames" step.
        status_command = ['git', 'diff', '-r', '--name-status', '-C', '-M', "--no-ext-diff", "--full-index", self.merge_base(git_commit)]
        # FIXME: I'm not sure we're returning the same set of files that SVN.changed_files is.
        # Added (A), Copied (C), Deleted (D), Modified (M), Renamed (R)
        return self.run_status_and_extract_filenames(status_command, self._status_regexp("ADM"))

    def _changes_files_for_commit(self, git_commit):
        # --pretty="format:" makes git show not print the commit log header,
        changed_files = self.run(["git", "show", "--pretty=format:", "--name-only", git_commit]).splitlines()
        # instead it just prints a blank line at the top, so we skip the blank line:
        return changed_files[1:]

    def changed_files_for_revision(self, revision):
        commit_id = self.git_commit_from_svn_revision(revision)
        return self._changes_files_for_commit(commit_id)

    def revisions_changing_file(self, path, limit=5):
        # git rev-list head --remove-empty --limit=5 -- path would be equivalent.
        commit_ids = self.run(["git", "log", "--remove-empty", "--pretty=format:%H", "-%s" % limit, "--", path]).splitlines()
        return filter(lambda revision: revision, map(self.svn_revision_from_git_commit, commit_ids))

    def conflicted_files(self):
        # We do not need to pass decode_output for this diff command
        # as we're passing --name-status which does not output any data.
        status_command = ['git', 'diff', '--name-status', '-C', '-M', '--diff-filter=U']
        return self.run_status_and_extract_filenames(status_command, self._status_regexp("U"))

    def added_files(self):
        return self.run_status_and_extract_filenames(self.status_command(), self._status_regexp("A"))

    def deleted_files(self):
        return self.run_status_and_extract_filenames(self.status_command(), self._status_regexp("D"))

    @staticmethod
    def supports_local_commits():
        return True

    def display_name(self):
        return "git"

    def create_patch(self, git_commit=None, changed_files=None):
        """Returns a byte array (str()) representing the patch file.
        Patch files are effectively binary since they may contain
        files of multiple different encodings."""
        command = ['git', 'diff', '--binary', "--no-ext-diff", "--full-index", "-M", self.merge_base(git_commit), "--"]
        if changed_files:
            command += changed_files
        return self.run(command, decode_output=False, cwd=self.checkout_root)

    def _run_git_svn_find_rev(self, arg):
        # git svn find-rev always exits 0, even when the revision or commit is not found.
        return self.run(['git', 'svn', 'find-rev', arg], cwd=self.checkout_root).rstrip()

    def _string_to_int_or_none(self, string):
        try:
            return int(string)
        except ValueError, e:
            return None

    @memoized
    def git_commit_from_svn_revision(self, svn_revision):
        git_commit = self._run_git_svn_find_rev('r%s' % svn_revision)
        if not git_commit:
            # FIXME: Alternatively we could offer to update the checkout? Or return None?
            raise ScriptError(message='Failed to find git commit for revision %s, your checkout likely needs an update.' % svn_revision)
        return git_commit

    @memoized
    def svn_revision_from_git_commit(self, git_commit):
        svn_revision = self._run_git_svn_find_rev(git_commit)
        return self._string_to_int_or_none(svn_revision)

    def contents_at_revision(self, path, revision):
        """Returns a byte array (str()) containing the contents
        of path @ revision in the repository."""
        return self.run(["git", "show", "%s:%s" % (self.git_commit_from_svn_revision(revision), path)], decode_output=False)

    def diff_for_revision(self, revision):
        git_commit = self.git_commit_from_svn_revision(revision)
        return self.create_patch(git_commit)

    def diff_for_file(self, path, log=None):
        return self.run(['git', 'diff', 'HEAD', '--', path])

    def show_head(self, path):
        return self.run(['git', 'show', 'HEAD:' + self.to_object_name(path)], decode_output=False)

    def committer_email_for_revision(self, revision):
        git_commit = self.git_commit_from_svn_revision(revision)
        committer_email = self.run(["git", "log", "-1", "--pretty=format:%ce", git_commit])
        # Git adds an extra @repository_hash to the end of every committer email, remove it:
        return committer_email.rsplit("@", 1)[0]

    def apply_reverse_diff(self, revision):
        # Assume the revision is an svn revision.
        git_commit = self.git_commit_from_svn_revision(revision)
        # I think this will always fail due to ChangeLogs.
        self.run(['git', 'revert', '--no-commit', git_commit], error_handler=Executive.ignore_error)

    def revert_files(self, file_paths):
        self.run(['git', 'checkout', 'HEAD'] + file_paths)

    def _assert_can_squash(self, working_directory_is_clean):
        squash = Git.read_git_config('webkit-patch.commit-should-always-squash')
        should_squash = squash and squash.lower() == "true"

        if not should_squash:
            # Only warn if there are actually multiple commits to squash.
            num_local_commits = len(self.local_commits())
            if num_local_commits > 1 or (num_local_commits > 0 and not working_directory_is_clean):
                raise AmbiguousCommitError(num_local_commits, working_directory_is_clean)

    def commit_with_message(self, message, username=None, git_commit=None, force_squash=False):
        # Username is ignored during Git commits.
        working_directory_is_clean = self.working_directory_is_clean()

        if git_commit:
            # Special-case HEAD.. to mean working-copy changes only.
            if git_commit.upper() == 'HEAD..':
                if working_directory_is_clean:
                    raise ScriptError(message="The working copy is not modified. --git-commit=HEAD.. only commits working copy changes.")
                self.commit_locally_with_message(message)
                return self._commit_on_branch(message, 'HEAD')

            # Need working directory changes to be committed so we can checkout the merge branch.
            if not working_directory_is_clean:
                # FIXME: webkit-patch land will modify the ChangeLogs to correct the reviewer.
                # That will modify the working-copy and cause us to hit this error.
                # The ChangeLog modification could be made to modify the existing local commit.
                raise ScriptError(message="Working copy is modified. Cannot commit individual git_commits.")
            return self._commit_on_branch(message, git_commit)

        if not force_squash:
            self._assert_can_squash(working_directory_is_clean)
        self.run(['git', 'reset', '--soft', self.remote_merge_base()])
        self.commit_locally_with_message(message)
        return self.push_local_commits_to_server()

    def _commit_on_branch(self, message, git_commit):
        branch_ref = self.run(['git', 'symbolic-ref', 'HEAD']).strip()
        branch_name = branch_ref.replace('refs/heads/', '')
        commit_ids = self.commit_ids_from_commitish_arguments([git_commit])

        # We want to squash all this branch's commits into one commit with the proper description.
        # We do this by doing a "merge --squash" into a new commit branch, then dcommitting that.
        MERGE_BRANCH_NAME = 'webkit-patch-land'
        self.delete_branch(MERGE_BRANCH_NAME)

        # We might be in a directory that's present in this branch but not in the
        # trunk.  Move up to the top of the tree so that git commands that expect a
        # valid CWD won't fail after we check out the merge branch.
        os.chdir(self.checkout_root)

        # Stuff our change into the merge branch.
        # We wrap in a try...finally block so if anything goes wrong, we clean up the branches.
        commit_succeeded = True
        try:
            self.run(['git', 'checkout', '-q', '-b', MERGE_BRANCH_NAME, self.remote_branch_ref()])

            for commit in commit_ids:
                # We're on a different branch now, so convert "head" to the branch name.
                commit = re.sub(r'(?i)head', branch_name, commit)
                # FIXME: Once changed_files and create_patch are modified to separately handle each
                # commit in a commit range, commit each cherry pick so they'll get dcommitted separately.
                self.run(['git', 'cherry-pick', '--no-commit', commit])

            self.run(['git', 'commit', '-m', message])
            output = self.push_local_commits_to_server()
        except Exception, e:
            log("COMMIT FAILED: " + str(e))
            output = "Commit failed."
            commit_succeeded = False
        finally:
            # And then swap back to the original branch and clean up.
            self.clean_working_directory()
            self.run(['git', 'checkout', '-q', branch_name])
            self.delete_branch(MERGE_BRANCH_NAME)

        return output

    def svn_commit_log(self, svn_revision):
        svn_revision = self.strip_r_from_svn_revision(svn_revision)
        return self.run(['git', 'svn', 'log', '-r', svn_revision])

    def last_svn_commit_log(self):
        return self.run(['git', 'svn', 'log', '--limit=1'])

    # Git-specific methods:
    def _branch_ref_exists(self, branch_ref):
        return self.run(['git', 'show-ref', '--quiet', '--verify', branch_ref], return_exit_code=True) == 0

    def delete_branch(self, branch_name):
        if self._branch_ref_exists('refs/heads/' + branch_name):
            self.run(['git', 'branch', '-D', branch_name])

    def remote_merge_base(self):
        return self.run(['git', 'merge-base', self.remote_branch_ref(), 'HEAD']).strip()

    def remote_branch_ref(self):
        # Use references so that we can avoid collisions, e.g. we don't want to operate on refs/heads/trunk if it exists.
        remote_branch_refs = Git.read_git_config('svn-remote.svn.fetch')
        if not remote_branch_refs:
            remote_master_ref = 'refs/remotes/origin/master'
            if not self._branch_ref_exists(remote_master_ref):
                raise ScriptError(message="Can't find a branch to diff against. svn-remote.svn.fetch is not in the git config and %s does not exist" % remote_master_ref)
            return remote_master_ref

        # FIXME: What's the right behavior when there are multiple svn-remotes listed?
        # For now, just use the first one.
        first_remote_branch_ref = remote_branch_refs.split('\n')[0]
        return first_remote_branch_ref.split(':')[1]

    def commit_locally_with_message(self, message):
        self.run(['git', 'commit', '--all', '-F', '-'], input=message)

    def push_local_commits_to_server(self):
        dcommit_command = ['git', 'svn', 'dcommit']
        if self.dryrun:
            dcommit_command.append('--dry-run')
        output = self.run(dcommit_command, error_handler=commit_error_handler)
        # Return a string which looks like a commit so that things which parse this output will succeed.
        if self.dryrun:
            output += "\nCommitted r0"
        return output

    # This function supports the following argument formats:
    # no args : rev-list trunk..HEAD
    # A..B    : rev-list A..B
    # A...B   : error!
    # A B     : [A, B]  (different from git diff, which would use "rev-list A..B")
    def commit_ids_from_commitish_arguments(self, args):
        if not len(args):
            args.append('%s..HEAD' % self.remote_branch_ref())

        commit_ids = []
        for commitish in args:
            if '...' in commitish:
                raise ScriptError(message="'...' is not supported (found in '%s'). Did you mean '..'?" % commitish)
            elif '..' in commitish:
                commit_ids += reversed(self.run(['git', 'rev-list', commitish]).splitlines())
            else:
                # Turn single commits or branch or tag names into commit ids.
                commit_ids += self.run(['git', 'rev-parse', '--revs-only', commitish]).splitlines()
        return commit_ids

    def commit_message_for_local_commit(self, commit_id):
        commit_lines = self.run(['git', 'cat-file', 'commit', commit_id]).splitlines()

        # Skip the git headers.
        first_line_after_headers = 0
        for line in commit_lines:
            first_line_after_headers += 1
            if line == "":
                break
        return CommitMessage(commit_lines[first_line_after_headers:])

    def files_changed_summary_for_commit(self, commit_id):
        return self.run(['git', 'diff-tree', '--shortstat', '--no-commit-id', commit_id])