Emitting JUnit+Ant style XML from the jtreg test runner.

This change adds some unfortunate code duplication between
the jtreg runner and the CoreTestSuite runner. I'd like to
eventually loop back and create a unified test runner for
either type of test.

4 files changed, 257 insertions, 35 deletions

diff --git a/tools/dalvik_jtreg/java/dalvik/jtreg/Adb.java b/tools/dalvik_jtreg/java/dalvik/jtreg/Adb.java
index b7813d8..ef8508e 100644
--- a/tools/dalvik_jtreg/java/dalvik/jtreg/Adb.java
+++ b/tools/dalvik_jtreg/java/dalvik/jtreg/Adb.java
@@ -24,8 +24,11 @@ import java.io.File;
 final class Adb {
 
     public void mkdir(File name) {
-        new Command("adb", "shell", "mkdir", name.toString())
-                .execute();
+        new Command("adb", "shell", "mkdir", name.toString()).execute();
+    }
+
+    public void rm(File name) {
+        new Command("adb", "shell", "rm", "-r", name.toString()).execute();
     }
 
     public void push(File local, File remote) {
diff --git a/tools/dalvik_jtreg/java/dalvik/jtreg/JtregRunner.java b/tools/dalvik_jtreg/java/dalvik/jtreg/JtregRunner.java
index 0ef24b2..bfe2908 100644
--- a/tools/dalvik_jtreg/java/dalvik/jtreg/JtregRunner.java
+++ b/tools/dalvik_jtreg/java/dalvik/jtreg/JtregRunner.java
@@ -20,6 +20,7 @@ import com.sun.javatest.TestDescription;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -57,6 +58,7 @@ public final class JtregRunner {
     private Integer debugPort;
     private Set<File> expectationDirs = new LinkedHashSet<File>();
     private long timeoutSeconds = 10 * 60; // default is ten minutes
+    private File xmlReportsDirectory;
 
     private File deviceTestRunner;
 
@@ -78,17 +80,14 @@ public final class JtregRunner {
         // threads helps for packages that contain many unsupported tests
         ExecutorService builders = Executors.newFixedThreadPool(8);
         for (final TestDescription testDescription : tests) {
-            builders.submit(new Runnable() {
-                public void run() {
-                    try {
-                        String qualifiedName = TestDescriptions.qualifiedName(testDescription);
-                        ExpectedResult expectedResult = ExpectedResult.forRun(expectationDirs, qualifiedName);
-                        TestRun testRun = new TestRun(qualifiedName, testDescription, expectedResult);
-                        buildAndInstall(testRun);
-                        readyToRun.put(testRun);
-                    } catch (Exception e) {
-                        throw new RuntimeException(e);
-                    }
+            builders.submit(new Callable<Void>() {
+                public Void call() throws Exception {
+                    String qualifiedName = TestDescriptions.qualifiedName(testDescription);
+                    ExpectedResult expectedResult = ExpectedResult.forRun(expectationDirs, qualifiedName);
+                    TestRun testRun = new TestRun(qualifiedName, testDescription, expectedResult);
+                    buildAndInstall(testRun);
+                    readyToRun.put(testRun);
+                    return null;
                 }
             });
         }
@@ -98,8 +97,10 @@ public final class JtregRunner {
 
         int unsupportedTests = 0;
 
+        List<TestRun> runs = new ArrayList<TestRun>(tests.size());
         while (!builders.isTerminated() || !readyToRun.isEmpty()) {
             TestRun testRun = readyToRun.take();
+            runs.add(testRun);
 
             if (testRun.getResult() == Result.UNSUPPORTED) {
                 logger.fine("skipping " + testRun.getQualifiedName());
@@ -117,6 +118,12 @@ public final class JtregRunner {
         if (unsupportedTests > 0) {
             logger.info("Skipped " + unsupportedTests + " unsupported tests.");
         }
+
+        if (xmlReportsDirectory != null) {
+            logger.info("Printing XML Reports... ");
+            int numFiles = new XmlReportPrinter().generateReports(xmlReportsDirectory, runs);
+            logger.info(numFiles + " XML files written.");
+        }
     }
 
     /**
@@ -253,16 +260,21 @@ public final class JtregRunner {
         }
     }
 
+    private void shutdown() {
+        adb.rm(deviceTemp);
+        outputReaders.shutdown();
+    }
+
     public static void main(String[] args) throws Exception {
         if (args.length < 2) {
-            System.out.println("Usage: JTRegRunner [options]... <android_jar> <directoryWithTests>");
+            System.out.println("Usage: JTRegRunner [options]... <android jar> <tests directory>");
             System.out.println();
-            System.out.println("  android_jar: the API jar file to compile against. Usually");
+            System.out.println("  <android jar>: the API jar file to compile against. Usually");
             System.out.println("      this is <SDK>/platforms/android-<X.X>/android.jar where");
             System.out.println("      <SDK> is the path to an Android SDK path and <X.X> is a");
             System.out.println("      release version like 1.5.");
             System.out.println();
-            System.out.println("  directoryWithTests: a directory to scan for test cases;");
+            System.out.println("  <tests directory>: a directory to scan for test cases;");
             System.out.println("      typically this is 'platform_v6/jdk/test' if 'platform_v6'");
             System.out.println("      contains the sources of a platform implementation.");
             System.out.println();
@@ -276,8 +288,11 @@ public final class JtregRunner {
             System.out.println("      looking for test expectations. The directory should include");
             System.out.println("      <test>.expected files describing expected results.");
             System.out.println();
-            System.out.println("  --timeoutSeconds <seconds>: maximum execution time of each test");
-            System.out.println("      before the runner aborts it.");
+            System.out.println("  --timeout-seconds <seconds>: maximum execution time of each");
+            System.out.println("      test before the runner aborts it.");
+            System.out.println();
+            System.out.println("  --xml-reports-directory <path>: directory to emit JUnit-style");
+            System.out.println("      XML test results.");
             System.out.println();
             System.out.println("  --verbose: turn on verbose output");
             System.out.println();
@@ -302,12 +317,6 @@ public final class JtregRunner {
             if ("--debug".equals(args[i])) {
                 jtregRunner.debugPort = Integer.valueOf(args[++i]);
 
-            } else if ("--timeoutSeconds".equals(args[i])) {
-                jtregRunner.timeoutSeconds = Long.valueOf(args[++i]);
-
-            } else if ("--verbose".equals(args[i])) {
-                Logger.getLogger("dalvik.jtreg").setLevel(Level.FINE);
-
             } else if ("--expectations".equals(args[i])) {
                 File expectationDir = new File(args[++i]);
                 if (!expectationDir.isDirectory()) {
@@ -315,12 +324,26 @@ public final class JtregRunner {
                 }
                 jtregRunner.expectationDirs.add(expectationDir);
 
+            } else if ("--timeout-seconds".equals(args[i])) {
+                jtregRunner.timeoutSeconds = Long.valueOf(args[++i]);
+
+            } else if ("--verbose".equals(args[i])) {
+                Logger.getLogger("dalvik.jtreg").setLevel(Level.FINE);
+
+            } else if ("--xml-reports-directory".equals(args[i])) {
+                jtregRunner.xmlReportsDirectory = new File(args[++i]);
+                if (!jtregRunner.xmlReportsDirectory.isDirectory()) {
+                    throw new RuntimeException("Invalid XML reports directory: "
+                            + jtregRunner.xmlReportsDirectory);
+                }
+
             } else {
                 throw new RuntimeException("Unrecognized option: " + args[i]);
             }
         }
 
         jtregRunner.buildAndRunAllTests();
+        jtregRunner.shutdown();
     }
 
     private static void prepareLogging() {
diff --git a/tools/dalvik_jtreg/java/dalvik/jtreg/TestDescriptions.java b/tools/dalvik_jtreg/java/dalvik/jtreg/TestDescriptions.java
index d0be8d2..c7ffe7f 100644
--- a/tools/dalvik_jtreg/java/dalvik/jtreg/TestDescriptions.java
+++ b/tools/dalvik_jtreg/java/dalvik/jtreg/TestDescriptions.java
@@ -44,23 +44,22 @@ class TestDescriptions {
 
     /**
      * Returns a fully qualified name of the form {@code
-     * java_lang_Math_PowTests} from the given test description. The returned
+     * java.lang.Math.PowTests} from the given test description. The returned
      * name is appropriate for use in a filename.
      */
     static String qualifiedName(TestDescription testDescription) {
-        StringBuilder result = new StringBuilder();
+        return className(testDescription) + "." + escape(testDescription.getName());
+    }
 
+    /**
+     * Returns the name of the class under test, such as {@code java.lang.Math}.
+     */
+    static String className(TestDescription testDescription) {
         String dir = testDescription.getDir().toString();
         int separatorIndex = dir.indexOf(TEST_ROOT);
-        if (separatorIndex != -1) {
-            result.append(escape(dir.substring(separatorIndex + TEST_ROOT.length())));
-        } else {
-            result.append(escape(dir));
-        }
-
-        result.append(".");
-        result.append(escape(testDescription.getName()));
-        return result.toString();
+        return separatorIndex != -1
+                ? escape(dir.substring(separatorIndex + TEST_ROOT.length()))
+                : escape(dir);
     }
 
     /**
diff --git a/tools/dalvik_jtreg/java/dalvik/jtreg/XmlReportPrinter.java b/tools/dalvik_jtreg/java/dalvik/jtreg/XmlReportPrinter.java
new file mode 100644
index 0000000..bc3c507
--- /dev/null
+++ b/tools/dalvik_jtreg/java/dalvik/jtreg/XmlReportPrinter.java
@@ -0,0 +1,197 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dalvik.jtreg;
+
+import org.kxml2.io.KXmlSerializer;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.EnumSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+
+
+/**
+ * Writes JUnit results to a series of XML files in a format consistent with
+ * Ant's XMLJUnitResultFormatter.
+ *
+ * <p>Unlike Ant's formatter, this class does not report the execution time of
+ * tests.
+ *
+ * TODO: unify this and com.google.coretests.XmlReportPrinter
+ */
+public class XmlReportPrinter {
+
+    /**
+     * Test results of these types are omitted from the report, as if the tests
+     * never existed. Equivalent to the core test runner's @BrokenTest.
+     */
+    private static final EnumSet<Result> IGNORED_RESULTS
+            = EnumSet.of(Result.COMPILE_FAILED);
+
+    private static final EnumSet<Result> FAILED_RESULTS
+            = EnumSet.of(Result.EXEC_FAILED);
+    private static final EnumSet<Result> ERROR_RESULTS
+            = EnumSet.complementOf(EnumSet.of(Result.EXEC_FAILED, Result.SUCCESS));
+
+    private static final String TESTSUITE = "testsuite";
+    private static final String TESTCASE = "testcase";
+    private static final String ERROR = "error";
+    private static final String FAILURE = "failure";
+    private static final String ATTR_NAME = "name";
+    private static final String ATTR_TIME = "time";
+    private static final String ATTR_ERRORS = "errors";
+    private static final String ATTR_FAILURES = "failures";
+    private static final String ATTR_TESTS = "tests";
+    private static final String ATTR_TYPE = "type";
+    private static final String ATTR_MESSAGE = "message";
+    private static final String PROPERTIES = "properties";
+    private static final String ATTR_CLASSNAME = "classname";
+    private static final String TIMESTAMP = "timestamp";
+    private static final String HOSTNAME = "hostname";
+
+    /** the XML namespace */
+    private static final String ns = null;
+
+    /**
+     * Populates the directory with the report data from the completed tests.
+     */
+    public int generateReports(File directory, Collection<TestRun> results) {
+        Map<String, Suite> suites = testsToSuites(results);
+
+        SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
+        TimeZone gmt = TimeZone.getTimeZone("GMT");
+        dateFormat.setTimeZone(gmt);
+        dateFormat.setLenient(true);
+        String timestamp = dateFormat.format(new Date());
+
+        for (Suite suite : suites.values()) {
+            FileOutputStream stream = null;
+            try {
+                stream = new FileOutputStream(new File(directory, "TEST-" + suite.name + ".xml"));
+
+                KXmlSerializer serializer = new KXmlSerializer();
+                serializer.setOutput(stream, "UTF-8");
+                serializer.startDocument("UTF-8", null);
+                serializer.setFeature(
+                        "http://xmlpull.org/v1/doc/features.html#indent-output", true);
+                suite.print(serializer, timestamp);
+                serializer.endDocument();
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            } finally {
+                if (stream != null) {
+                    try {
+                        stream.close();
+                    } catch (IOException ignored) {
+                    }
+                }
+            }
+        }
+
+        return suites.size();
+    }
+
+    private Map<String, Suite> testsToSuites(Collection<TestRun> testRuns) {
+        Map<String, Suite> result = new LinkedHashMap<String, Suite>();
+        for (TestRun testRun : testRuns) {
+            if (IGNORED_RESULTS.contains(testRun.getResult())) {
+                continue;
+            }
+
+            String suiteName = suiteName(testRun);
+            Suite suite = result.get(suiteName);
+            if (suite == null) {
+                suite = new Suite(suiteName);
+                result.put(suiteName, suite);
+            }
+
+            suite.tests.add(testRun);
+            if (FAILED_RESULTS.contains(testRun.getResult())) {
+                suite.failuresCount++;
+            } else if (ERROR_RESULTS.contains(testRun.getResult())) {
+                suite.errorsCount++;
+            }
+        }
+        return result;
+    }
+
+    private static String suiteName(TestRun testRun) {
+        return TestDescriptions.className(testRun.getTestDescription());
+    }
+
+    static class Suite {
+        private final String name;
+        private final List<TestRun> tests = new ArrayList<TestRun>();
+        private int failuresCount;
+        private int errorsCount;
+
+        Suite(String name) {
+            this.name = name;
+        }
+
+        void print(KXmlSerializer serializer, String timestamp) throws IOException {
+            serializer.startTag(ns, TESTSUITE);
+            serializer.attribute(ns, ATTR_NAME, name);
+            serializer.attribute(ns, ATTR_TESTS, Integer.toString(tests.size()));
+            serializer.attribute(ns, ATTR_FAILURES, Integer.toString(failuresCount));
+            serializer.attribute(ns, ATTR_ERRORS, Integer.toString(errorsCount));
+            serializer.attribute(ns, ATTR_TIME, "0");
+            serializer.attribute(ns, TIMESTAMP, timestamp);
+            serializer.attribute(ns, HOSTNAME, "localhost");
+            serializer.startTag(ns, PROPERTIES);
+            serializer.endTag(ns, PROPERTIES);
+
+            for (TestRun testRun : tests) {
+                print(serializer, testRun);
+            }
+
+            serializer.endTag(ns, TESTSUITE);
+        }
+
+        void print(KXmlSerializer serializer, TestRun testRun) throws IOException {
+            serializer.startTag(ns, TESTCASE);
+            serializer.attribute(ns, ATTR_NAME, testRun.getTestDescription().getName());
+            serializer.attribute(ns, ATTR_CLASSNAME, suiteName(testRun));
+            serializer.attribute(ns, ATTR_TIME, "0");
+
+            String result = ERROR_RESULTS.contains(testRun.getResult()) ? ERROR
+                    : FAILED_RESULTS.contains(testRun.getResult()) ? FAILURE
+                    : null;
+
+            if (result != null) {
+                serializer.startTag(ns, result);
+                String title = testRun.getTestDescription().getTitle();
+                if (title != null && title.length() > 0) {
+                    serializer.attribute(ns, ATTR_MESSAGE, title);
+                }
+                serializer.attribute(ns, ATTR_TYPE, testRun.getResult().toString());
+                serializer.text(Strings.join(testRun.getOutputLines(), "\n"));
+                serializer.endTag(ns, result);
+            }
+
+            serializer.endTag(ns, TESTCASE);
+        }
+    }
+}
\ No newline at end of file