First cut at a tool to perform automated syncs from Harmony to Dalvik.

Much of the work of the tool is managing three versions of our code:
 - the current Harmony code
 - the current Dalvik code
 - the common ancestor, an older version of Harmony

The tool calls out to the command line to perform the actual filesystem
work. Eventually I'd like to support all of our modules, and also
gathering diffs to send upstream to Harmony.

9 files changed, 580 insertions, 0 deletions

diff --git a/tools/integrate/Android.mk b/tools/integrate/Android.mk
new file mode 100644
index 0000000..629a5fd
--- /dev/null
+++ b/tools/integrate/Android.mk
@@ -0,0 +1,19 @@
+LOCAL_PATH:= $(call my-dir)
+
+include $(CLEAR_VARS)
+
+LOCAL_SRC_FILES := \
+	Command.java  \
+	Filesystem.java \
+	Git.java \
+	Module.java \
+	Modules.java \
+	MappedDirectory.java \
+	PullHarmonyCode.java \
+	Svn.java
+
+LOCAL_MODULE:= integrate
+
+include $(BUILD_HOST_JAVA_LIBRARY)
+
+include $(call all-subdir-makefiles)
diff --git a/tools/integrate/Command.java b/tools/integrate/Command.java
new file mode 100644
index 0000000..5e7796f
--- /dev/null
+++ b/tools/integrate/Command.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * An out of process executable.
+ */
+class Command {
+
+    private final List<String> args;
+    private final boolean permitNonZeroExitStatus;
+
+    Command(String... args) {
+        this(Arrays.asList(args));
+    }
+
+    Command(List<String> args) {
+        this.args = new ArrayList<String>(args);
+        this.permitNonZeroExitStatus = false;
+    }
+
+    private Command(Builder builder) {
+        this.args = new ArrayList<String>(builder.args);
+        this.permitNonZeroExitStatus = builder.permitNonZeroExitStatus;
+    }
+
+    static class Builder {
+        private final List<String> args = new ArrayList<String>();
+        private boolean permitNonZeroExitStatus = false;
+
+        public Builder args(String... args) {
+            return args(Arrays.asList(args));
+        }
+
+        public Builder args(Collection<String> args) {
+            this.args.addAll(args);
+            return this;
+        }
+
+        public Builder permitNonZeroExitStatus() {
+            permitNonZeroExitStatus = true;
+            return this;
+        }
+
+        public Command build() {
+            return new Command(this);
+        }
+
+        public List<String> execute() {
+            return build().execute();
+        }
+    }
+
+    public List<String> execute() {
+        try {
+            Process process = new ProcessBuilder()
+                    .command(args)
+                    .redirectErrorStream(true)
+                    .start();
+
+            BufferedReader in = new BufferedReader(new InputStreamReader(process.getInputStream()));
+            List<String> outputLines = new ArrayList<String>();
+            String outputLine;
+            while ((outputLine = in.readLine()) != null) {
+                outputLines.add(outputLine);
+            }
+
+            if (process.waitFor() != 0 && !permitNonZeroExitStatus) {
+                StringBuilder message = new StringBuilder();
+                for (String line : outputLines) {
+                    message.append("\n").append(line);
+                }
+                throw new RuntimeException("Process failed: " + args + message);
+            }
+
+            return outputLines;
+        } catch (IOException e) {
+            throw new RuntimeException("Process failed: " + args, e);
+        } catch (InterruptedException e) {
+            throw new RuntimeException("Process failed: " + args, e);
+        }
+    }
+
+}
diff --git a/tools/integrate/Filesystem.java b/tools/integrate/Filesystem.java
new file mode 100644
index 0000000..a9c1789
--- /dev/null
+++ b/tools/integrate/Filesystem.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Factory for filesystem commands.
+ */
+class Filesystem {
+
+    public void move(String source, String target) {
+        new Command("mv", source, target).execute();
+    }
+
+    /**
+     * Moves all of the files in {@code source} to {@code target}, one at a
+     * time. Unlike {@code move}, this approach works even if the target
+     * directory is nonempty.
+     */
+    public int moveContents(String source, String target) {
+        List<String> files = new Command("find", source, "-type", "f") .execute();
+        for (String file : files) {
+            String targetFile = target + "/" + file.substring(source.length());
+            mkdir(parent(targetFile));
+            new Command("mv", "-i", file, targetFile).execute();
+        }
+        return files.size();
+    }
+
+    private String parent(String file) {
+        return file.substring(0, file.lastIndexOf('/'));
+    }
+
+    public void mkdir(String dir) {
+        new Command("mkdir", "-p", dir).execute();
+    }
+
+    public List<String> find(String where, String name) {
+        return new Command("find", where, "-name", name).execute();
+    }
+
+    public void rm(Collection<String> files) {
+        new Command.Builder().args("rm", "-r").args(files).execute();
+    }
+
+    public void rm(String file) {
+        new Command("rm", "-r", file).execute();
+    }
+}
diff --git a/tools/integrate/Git.java b/tools/integrate/Git.java
new file mode 100644
index 0000000..da7dcfa
--- /dev/null
+++ b/tools/integrate/Git.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Factory for git commands.
+ */
+class Git {
+
+    private static final Pattern STATUS_DELETED
+            = Pattern.compile("#\\tdeleted:    (.*)");
+
+    public void branch(String newBranch) {
+        branch(newBranch, "HEAD");
+    }
+
+    /**
+     * @param base another branch, or a revision identifier like {@code HEAD}.
+     */
+    public void branch(String newBranch, String base) {
+        // -b is used by git to branch from another checkout
+        new Command("git", "checkout", "-b", newBranch, base).execute();
+    }
+
+    public void commit(String message) {
+        new Command("git", "commit", "-m", message).execute();
+    }
+
+    public void add(String path) {
+        new Command("git", "add", path).execute();
+    }
+
+    public void remove(Collection<String> paths) {
+        new Command.Builder().args("git", "rm").args(paths).execute();
+    }
+
+    public List<String> merge(String otherBranch) {
+        return new Command.Builder()
+                .args("git", "merge", "-s", "recursive", otherBranch)
+                .permitNonZeroExitStatus()
+                .execute();
+    }
+
+    /**
+     * Returns the files that have been deleted from the filesystem, but that
+     * don't exist in the active git change.
+     */
+    public List<String> listDeleted() {
+        List<String> statusLines = new Command.Builder()
+                .args("git", "status")
+                .permitNonZeroExitStatus()
+                .execute();
+
+        List<String> deletedFiles = new ArrayList<String>();
+        Matcher matcher = STATUS_DELETED.matcher("");
+        for (String line : statusLines) {
+            matcher.reset(line);
+            if (matcher.matches()) {
+                deletedFiles.add(matcher.group(1));
+            }
+        }
+        return deletedFiles;
+    }
+
+    public void rm(List<String> files) {
+        new Command.Builder()
+                .args("git", "rm").args(files)
+                .permitNonZeroExitStatus()
+                .build()
+                .execute();
+    }
+}
diff --git a/tools/integrate/MappedDirectory.java b/tools/integrate/MappedDirectory.java
new file mode 100644
index 0000000..8e28d29
--- /dev/null
+++ b/tools/integrate/MappedDirectory.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A logical directory that has a different location in Harmony and Dalvik.
+ */
+class MappedDirectory {
+
+    private final String svnPath;
+    private final String gitPath;
+
+    public MappedDirectory(String svnPath, String gitPath) {
+        this.svnPath = svnPath;
+        this.gitPath = gitPath;
+    }
+
+    public String svnPath() {
+        return svnPath;
+    }
+
+    public String gitPath() {
+        return gitPath;
+    }
+
+    @Override public String toString() {
+        return "svn:" + svnPath + " -> git:" + gitPath;
+    }
+}
diff --git a/tools/integrate/Module.java b/tools/integrate/Module.java
new file mode 100644
index 0000000..5cb7035
--- /dev/null
+++ b/tools/integrate/Module.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+/**
+ * A logical unit of code shared between Apache Harmony and Dalvik.
+ */
+class Module {
+
+    private final String svnBaseUrl;
+    private final String path;
+    private final Set<MappedDirectory> mappedDirectories;
+
+    public String getSvnBaseUrl() {
+        return svnBaseUrl;
+    }
+
+    public String path() {
+        return path;
+    }
+
+    public Set<MappedDirectory> getMappedDirectories() {
+        return mappedDirectories;
+    }
+
+    private Module(Builder builder) {
+        this.svnBaseUrl = builder.svnBaseUrl;
+        this.path = builder.path;
+        this.mappedDirectories = new LinkedHashSet<MappedDirectory>(builder.mappedDirectories);
+    }
+
+    public static class Builder {
+        private final String svnBaseUrl;
+        private final String path;
+        private final Set<MappedDirectory> mappedDirectories
+                = new LinkedHashSet<MappedDirectory>();
+
+        public Builder(String svnBaseUrl, String path) {
+            this.svnBaseUrl = svnBaseUrl;
+            this.path = path;
+        }
+
+        public Builder mapDirectory(String svnPath, String gitPath) {
+            mappedDirectories.add(new MappedDirectory(svnPath, gitPath));
+            return this;
+        }
+
+        public Module build() {
+            return new Module(this);
+        }
+    }
+}
diff --git a/tools/integrate/Modules.java b/tools/integrate/Modules.java
new file mode 100644
index 0000000..2475852
--- /dev/null
+++ b/tools/integrate/Modules.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Constants that define modules shared by Harmony and Dalvik.
+ */
+public class Modules {
+
+    private static final String SVN_ROOT
+            = "http://svn.apache.org/repos/asf/harmony/enhanced/classlib/trunk/modules";
+
+    public static final Module ARCHIVE = new Module.Builder(SVN_ROOT, "archive")
+            .mapDirectory("archive/src/main/native/archive/shared",
+                    "archive/src/main/native")
+            .mapDirectory("archive/src/main/native/zip/shared",
+                    "archive/src/main/native")
+            .build();
+
+    public static final Module CRYPTO = new Module.Builder(SVN_ROOT, "crypto")
+            .mapDirectory("crypto/src/test/api/java.injected/javax",
+                    "crypto/src/test/java/org/apache/harmony/crypto/tests/javax")
+            .mapDirectory("crypto/src/test/api/java",
+                    "crypto/src/test/java")
+            .mapDirectory("crypto/src/test/resources/serialization",
+                    "crypto/src/test/java/serialization")
+            .mapDirectory("crypto/src/test/support/common/java",
+                    "crypto/src/test/java")
+            .build();
+
+    public static final Module REGEX
+            = new Module.Builder(SVN_ROOT, "regex").build();
+
+    public static final Module SECURITY = new Module.Builder(SVN_ROOT, "security")
+            .mapDirectory("security/src/main/java/common",
+                    "security/src/main/java")
+            .mapDirectory("security/src/main/java/unix/org",
+                    "security/src/main/java/org")
+            .mapDirectory("security/src/test/api/java",
+                    "security/src/test/java")
+            .build();
+
+    public static final Module TEXT
+            = new Module.Builder(SVN_ROOT, "text").build();
+
+    public static final Module X_NET
+            = new Module.Builder(SVN_ROOT, "x-net").build();
+
+    // TODO: add the other modules
+}
diff --git a/tools/integrate/PullHarmonyCode.java b/tools/integrate/PullHarmonyCode.java
new file mode 100644
index 0000000..6710801
--- /dev/null
+++ b/tools/integrate/PullHarmonyCode.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.List;
+import java.util.UUID;
+
+/**
+ * Download two versions of Apache Harmony from their SVN version, and use it
+ * to perform a three-way merge with Dalvik.
+ */
+public class PullHarmonyCode {
+
+    private final int currentVersion;
+    private final int targetVersion;
+
+    public PullHarmonyCode(int currentVersion, int targetVersion) {
+        this.currentVersion = currentVersion;
+        this.targetVersion = targetVersion;
+    }
+
+    public void pull(Module module) {
+        String path = module.path();
+        String svnOldBranch = path + "_" + currentVersion;
+        String svnNewBranch = path + "_" + targetVersion;
+        String dalvikBranch = path + "_dalvik";
+
+        Git git = new Git();
+        Filesystem filesystem = new Filesystem();
+        Svn svn = new Svn();
+
+        // Assume we're starting with the current Dalvik code. Tuck this away
+        // somewhere while we rewrite history.
+        String temp = "/tmp/" + UUID.randomUUID();
+        filesystem.mkdir(temp);
+
+        // To prepare a three-way-merge, we need a common starting point: the
+        // time at which Dalvik and Harmony were most the same. We'll use the
+        // previous Harmony SVN code as this starting point. We grab the old
+        // code from their repository, and commit it as a git branch.
+        System.out.print("Creating branch " + svnOldBranch + "...");
+        git.branch(svnOldBranch);
+        filesystem.move(path, temp + "/" + path);
+        svn.checkOut(currentVersion, module.getSvnBaseUrl() + "/" + path);
+        filesystem.rm(filesystem.find(path, ".svn"));
+        for (MappedDirectory mappedDirectory : module.getMappedDirectories()) {
+            filesystem.moveContents(mappedDirectory.svnPath(), mappedDirectory.gitPath());
+        }
+        git.rm(git.listDeleted());
+        git.add(path);
+        git.commit(svnOldBranch);
+        System.out.println("done");
+
+        // Create a branch that's derived from the starting point. It will
+        // contain all of the changes Harmony has made from then until now.
+        System.out.print("Creating branch " + svnNewBranch + "...");
+        git.branch(svnNewBranch, svnOldBranch);
+        filesystem.rm(path);
+        svn.checkOut(targetVersion, module.getSvnBaseUrl() + "/" + path);
+        filesystem.rm(filesystem.find(path, ".svn"));
+        for (MappedDirectory mappedDirectory : module.getMappedDirectories()) {
+            filesystem.moveContents(mappedDirectory.svnPath(), mappedDirectory.gitPath());
+        }
+        git.rm(git.listDeleted());
+        git.add(path);
+        git.commit(svnNewBranch);
+        System.out.println("done");
+
+        // Create another branch that's derived from the starting point. It will
+        // contain all of the changes Dalvik has made from then until now.
+        System.out.print("Creating branch " + dalvikBranch + "...");
+        git.branch(dalvikBranch, svnOldBranch);
+        filesystem.rm(path);
+        filesystem.move(temp + "/" + path, path);
+        git.rm(git.listDeleted());
+        git.add(path);
+        git.commit(dalvikBranch);
+        System.out.println("done");
+
+        // Merge the two sets of changes together: Harmony's and Dalvik's. By
+        // initializing a common starting point, git can make better decisions
+        // when the two new versions differ. For example, if today's Dalvik has
+        // a method that today's Harmony does not, it may be because Dalvik
+        // added it, or because Harmony deleted it!
+        System.out.println("Merging " + svnNewBranch + " into " + dalvikBranch + ":");
+        List<String> mergeResults = git.merge(svnNewBranch);
+        for (String mergeResult : mergeResults) {
+            System.out.print("  ");
+            System.out.println(mergeResult);
+        }
+    }
+
+    public static void main(String[] args) {
+//        new PullHarmonyCode(527399, 802921).pull(Modules.CRYPTO);
+        new PullHarmonyCode(772995, 802921).pull(Modules.ARCHIVE);
+    }
+}
diff --git a/tools/integrate/Svn.java b/tools/integrate/Svn.java
new file mode 100644
index 0000000..dc9be35
--- /dev/null
+++ b/tools/integrate/Svn.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Factory for Subversion commands.
+ */
+class Svn {
+
+    public void checkOut(int version, String url) {
+        new Command("svn", "co", "-r", "" + version, url).execute();
+    }
+}