Support running caliper with the jtreg runner. This is an early first look;

we'll need to fix a few things before the UI for this is nice.

13 files changed, 217 insertions, 40 deletions

diff --git a/JavaLibrary.mk b/JavaLibrary.mk
index d1015f5..b12b47d 100644
--- a/JavaLibrary.mk
+++ b/JavaLibrary.mk
@@ -75,7 +75,7 @@ LOCAL_SRC_FILES := $(test_src_files)
 LOCAL_JAVA_RESOURCE_DIRS := $(test_resource_dirs)
 
 LOCAL_NO_STANDARD_LIBRARIES := true
-LOCAL_JAVA_LIBRARIES := core
+LOCAL_JAVA_LIBRARIES := core caliper
 LOCAL_DX_FLAGS := --core-library
 
 LOCAL_MODULE_TAGS := tests
diff --git a/luni/src/test/java/org/apache/harmony/luni/tests/java/util/NullBenchmarkSuite.java b/luni/src/test/java/org/apache/harmony/luni/tests/java/util/NullBenchmarkSuite.java
new file mode 100644
index 0000000..0d739c2
--- /dev/null
+++ b/luni/src/test/java/org/apache/harmony/luni/tests/java/util/NullBenchmarkSuite.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.harmony.luni.tests.java.util;
+
+import com.google.caliper.Benchmark;
+import com.google.caliper.DefaultBenchmarkSuite;
+
+/**
+ * This class exists only to force a dependency from our libraries on Caliper,
+ * our micro benchmarking framework. 
+ */
+public class NullBenchmarkSuite extends DefaultBenchmarkSuite {
+
+    class NullBenchmark extends Benchmark {
+        @Override public Object run(int trials) throws Exception {
+            for (int i = 0; i < trials; i++) {
+                // code under test goes here!
+            }
+            return null;
+        }
+    }
+}
diff --git a/tools/dalvik_jtreg/Android.mk b/tools/dalvik_jtreg/Android.mk
index 5dbe9e6..fd6fe73 100644
--- a/tools/dalvik_jtreg/Android.mk
+++ b/tools/dalvik_jtreg/Android.mk
@@ -4,6 +4,8 @@ include $(CLEAR_VARS)
 
 LOCAL_SRC_FILES := \
         java/dalvik/jtreg/Adb.java \
+        java/dalvik/jtreg/CaliperFinder.java \
+        java/dalvik/jtreg/CaliperRunner.java \
         java/dalvik/jtreg/Classpath.java \
         java/dalvik/jtreg/Command.java \
         java/dalvik/jtreg/CommandFailedException.java \
@@ -12,21 +14,22 @@ LOCAL_SRC_FILES := \
         java/dalvik/jtreg/Dx.java \
         java/dalvik/jtreg/ExpectedResult.java \
         java/dalvik/jtreg/Harness.java \
-        java/dalvik/jtreg/JUnit.java \
+        java/dalvik/jtreg/JUnitFinder.java \
         java/dalvik/jtreg/JUnitRunner.java \
         java/dalvik/jtreg/JavaVm.java \
         java/dalvik/jtreg/Javac.java \
-        java/dalvik/jtreg/Jtreg.java \
+        java/dalvik/jtreg/JtregFinder.java \
         java/dalvik/jtreg/JtregRunner.java \
         java/dalvik/jtreg/Result.java \
         java/dalvik/jtreg/Strings.java \
         java/dalvik/jtreg/TestRun.java \
+        java/dalvik/jtreg/TestFinder.java \
         java/dalvik/jtreg/TestRunner.java \
         java/dalvik/jtreg/Vm.java \
         java/dalvik/jtreg/XmlReportPrinter.java \
 
 LOCAL_MODULE:= dalvik_jtreg
-LOCAL_STATIC_JAVA_LIBRARIES := javatest jh jtreg kxml2-2.3.0
+LOCAL_STATIC_JAVA_LIBRARIES := javatest jh jtreg kxml2-2.3.0 caliper
 
 # TODO this only works when junit is already built...
 LOCAL_JAVA_LIBRARIES := junit
diff --git a/tools/dalvik_jtreg/java/dalvik/jtreg/CaliperFinder.java b/tools/dalvik_jtreg/java/dalvik/jtreg/CaliperFinder.java
new file mode 100644
index 0000000..0fa173d
--- /dev/null
+++ b/tools/dalvik_jtreg/java/dalvik/jtreg/CaliperFinder.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dalvik.jtreg;
+
+import java.io.File;
+
+/**
+ * Create {@link TestRun}s for {@code .java} files with Caliper benchmarks in
+ * them.
+ */
+class CaliperFinder extends TestFinder {
+
+    @Override protected boolean matches(File file) {
+        return file.getName().endsWith("BenchmarkSuite.java");
+    }
+
+    @Override protected String testName(File file) {
+        return "caliper";
+    }
+
+    @Override protected Class<? extends TestRunner> runnerClass() {
+        return CaliperRunner.class;
+    }
+}
diff --git a/tools/dalvik_jtreg/java/dalvik/jtreg/CaliperRunner.java b/tools/dalvik_jtreg/java/dalvik/jtreg/CaliperRunner.java
new file mode 100644
index 0000000..ace92f0
--- /dev/null
+++ b/tools/dalvik_jtreg/java/dalvik/jtreg/CaliperRunner.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dalvik.jtreg;
+
+import com.google.caliper.Runner;
+
+/**
+ * Runs a <a href="http://code.google.com/p/caliper/">Caliper</a> benchmark.
+ */
+public final class CaliperRunner extends TestRunner {
+
+    @Override public boolean test() {
+        Runner.main(className);
+        return false; // always print benchmarking results
+    }
+
+    public static void main(String[] args) throws Exception {
+        new CaliperRunner().run();
+    }
+}
diff --git a/tools/dalvik_jtreg/java/dalvik/jtreg/DeviceDalvikVm.java b/tools/dalvik_jtreg/java/dalvik/jtreg/DeviceDalvikVm.java
index f574cff..73a64c9 100644
--- a/tools/dalvik_jtreg/java/dalvik/jtreg/DeviceDalvikVm.java
+++ b/tools/dalvik_jtreg/java/dalvik/jtreg/DeviceDalvikVm.java
@@ -25,8 +25,11 @@ import java.util.logging.Logger;
  */
 final class DeviceDalvikVm extends Vm {
 
-    private static final File DEVICE_SUPPORT_JAR
-            = new File("/system/framework/core-tests.jar");
+    private static final Classpath RUNTIME_SUPPORT_CLASSPATH = Classpath.of(
+            new File("/system/framework/core-tests.jar"),
+            new File("/system/framework/caliper.jar"),
+            new File("/system/framework/guava.jar"),
+            new File("/system/framework/jsr305.jar"));
 
     private static final Logger logger = Logger.getLogger(DeviceDalvikVm.class.getName());
     private final File deviceTemp = new File("/data/jtreg" + UUID.randomUUID());
@@ -86,7 +89,7 @@ final class DeviceDalvikVm extends Vm {
                 .temp(testTemp);
     }
 
-    @Override protected Classpath getRuntimeSupportClasses() {
-        return Classpath.of(DEVICE_SUPPORT_JAR);
+    @Override protected Classpath getRuntimeSupportClasspath() {
+        return RUNTIME_SUPPORT_CLASSPATH;
     }
 }
diff --git a/tools/dalvik_jtreg/java/dalvik/jtreg/Driver.java b/tools/dalvik_jtreg/java/dalvik/jtreg/Driver.java
index 89382ca..0f23d4d 100644
--- a/tools/dalvik_jtreg/java/dalvik/jtreg/Driver.java
+++ b/tools/dalvik_jtreg/java/dalvik/jtreg/Driver.java
@@ -40,19 +40,22 @@ final class Driver {
 
     private final File localTemp;
     private final Set<File> expectationDirs;
-    private final Jtreg jtreg;
-    private final JUnit junit;
+    private final JtregFinder jtregFinder;
+    private final JUnitFinder junitFinder;
+    private final CaliperFinder caliperFinder;
     private final Vm vm;
     private final File xmlReportsDirectory;
 
     public Driver(File localTemp, Vm vm, Set<File> expectationDirs,
-            File xmlReportsDirectory, Jtreg jtreg, JUnit junit) {
+            File xmlReportsDirectory, JtregFinder jtregFinder,
+            JUnitFinder junit, CaliperFinder caliperFinder) {
         this.localTemp = localTemp;
         this.expectationDirs = expectationDirs;
         this.vm = vm;
         this.xmlReportsDirectory = xmlReportsDirectory;
-        this.jtreg = jtreg;
-        this.junit = junit;
+        this.jtregFinder = jtregFinder;
+        this.junitFinder = junit;
+        this.caliperFinder = caliperFinder;
     }
 
     /**
@@ -67,15 +70,18 @@ final class Driver {
         for (File testFile : testFiles) {
             Set<TestRun> testsForFile = Collections.emptySet();
 
-            // Look for Jtreg tests. If we don't find any, look for JUnit tests.
             if (testFile.isDirectory()) {
-                testsForFile = jtreg.findTests(testFile);
+                testsForFile = jtregFinder.findTests(testFile);
                 logger.fine("found " + testsForFile.size() + " jtreg tests for " + testFile);
             }
             if (testsForFile.isEmpty()) {
-                testsForFile = junit.findTests(testFile);
+                testsForFile = junitFinder.findTests(testFile);
                 logger.fine("found " + testsForFile.size() + " JUnit tests for " + testFile);
             }
+            if (testsForFile.isEmpty()) {
+                testsForFile = caliperFinder.findTests(testFile);
+                logger.fine("found " + testsForFile.size() + " Caliper benchmarks for " + testFile);
+            }
             tests.addAll(testsForFile);
         }
 
diff --git a/tools/dalvik_jtreg/java/dalvik/jtreg/Harness.java b/tools/dalvik_jtreg/java/dalvik/jtreg/Harness.java
index 5347a37..fa4959d 100644
--- a/tools/dalvik_jtreg/java/dalvik/jtreg/Harness.java
+++ b/tools/dalvik_jtreg/java/dalvik/jtreg/Harness.java
@@ -162,10 +162,12 @@ public final class Harness {
         Vm vm = javaHome != null
                 ? new JavaVm(debugPort, timeoutSeconds, sdkJar, localTemp, javaHome)
                 : new DeviceDalvikVm(debugPort, timeoutSeconds, sdkJar, localTemp);
-        Jtreg jtreg = new Jtreg(localTemp);
-        JUnit jUnit = new JUnit();
+        JtregFinder jtregFinder = new JtregFinder(localTemp);
+        JUnitFinder jUnitFinder = new JUnitFinder();
+        CaliperFinder caliperFinder = new CaliperFinder();
         Driver driver = new Driver(localTemp,
-                vm, expectationDirs, xmlReportsDirectory, jtreg, jUnit);
+                vm, expectationDirs, xmlReportsDirectory, jtregFinder,
+                jUnitFinder, caliperFinder);
         driver.buildAndRunAllTests(testFiles);
         vm.shutdown();
     }
diff --git a/tools/dalvik_jtreg/java/dalvik/jtreg/JUnitFinder.java b/tools/dalvik_jtreg/java/dalvik/jtreg/JUnitFinder.java
new file mode 100644
index 0000000..0f801cc
--- /dev/null
+++ b/tools/dalvik_jtreg/java/dalvik/jtreg/JUnitFinder.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2009 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dalvik.jtreg;
+
+import java.io.File;
+
+/**
+ * Create {@link TestRun}s for {@code .java} files with JUnit tests in them.
+ */
+class JUnitFinder extends TestFinder {
+
+    @Override protected boolean matches(File file) {
+        return file.getName().endsWith("Test.java");
+    }
+
+    // TODO: try to get names for each method?
+    @Override protected String testName(File file) {
+        return "junit";
+    }
+
+    @Override protected Class<? extends TestRunner> runnerClass() {
+        return JUnitRunner.class;
+    }
+}
diff --git a/tools/dalvik_jtreg/java/dalvik/jtreg/Jtreg.java b/tools/dalvik_jtreg/java/dalvik/jtreg/JtregFinder.java
index 3fdab78..1bbf1f1 100644
--- a/tools/dalvik_jtreg/java/dalvik/jtreg/Jtreg.java
+++ b/tools/dalvik_jtreg/java/dalvik/jtreg/JtregFinder.java
@@ -33,9 +33,9 @@ import java.util.logging.Logger;
 /**
  * Create {@link TestRun}s for {@code .java} files with jtreg tests in them.
  */
-class Jtreg {
+class JtregFinder {
 
-    private static final Logger logger = Logger.getLogger(Jtreg.class.getName());
+    private static final Logger logger = Logger.getLogger(JtregFinder.class.getName());
 
     /**
      * The subpath of a platform implementation under which tests live. Used to
@@ -46,7 +46,7 @@ class Jtreg {
 
     private final File localTemp;
 
-    Jtreg(File localTemp) {
+    JtregFinder(File localTemp) {
         this.localTemp = localTemp;
     }
 
diff --git a/tools/dalvik_jtreg/java/dalvik/jtreg/JUnit.java b/tools/dalvik_jtreg/java/dalvik/jtreg/TestFinder.java
index fbcf9c0..03c24be 100644
--- a/tools/dalvik_jtreg/java/dalvik/jtreg/JUnit.java
+++ b/tools/dalvik_jtreg/java/dalvik/jtreg/TestFinder.java
@@ -19,14 +19,11 @@ package dalvik.jtreg;
 import java.io.File;
 import java.util.LinkedHashSet;
 import java.util.Set;
-import java.util.logging.Logger;
 
 /**
- * Create {@link TestRun}s for {@code .java} files with JUnit tests in them.
+ * A pluggable strategy for converting files into test runs.
  */
-class JUnit {
-
-    private static final Logger logger = Logger.getLogger(JUnit.class.getName());
+abstract class TestFinder {
 
     public Set<TestRun> findTests(File testDirectory) {
         Set<TestRun> result = new LinkedHashSet<TestRun>();
@@ -34,21 +31,35 @@ class JUnit {
         return result;
     }
 
+    /**
+     * Returns true if {@code file} contains a test class of this type.
+     */
+    protected boolean matches(File file) {
+        return file.getName().endsWith(".java");
+    }
+
+    protected abstract String testName(File file);
+
+    protected abstract Class<? extends TestRunner> runnerClass();
+
     private void findTestsRecursive(Set<TestRun> sink, File file) {
         if (file.isDirectory()) {
             for (File child : file.listFiles()) {
                 findTestsRecursive(sink, child);
             }
-        } else if (file.getName().endsWith(".java")) {
-            String className = fileToClass(file);
-            File testDirectory = file.getParentFile();
-            String testName = "junit"; // TODO: try to get names for each method?
-            String testDescription = null;
-            sink.add(new TestRun(testDirectory, file, className, className,
-                    testName, className, testDescription, JUnitRunner.class));
-        } else {
-            logger.fine("skipping " + file);
+            return;
         }
+
+        if (!matches(file)) {
+            return;
+        }
+
+        String className = fileToClass(file);
+        File testDirectory = file.getParentFile();
+        String testName = testName(file);
+        String testDescription = null;
+        sink.add(new TestRun(testDirectory, file, className, className,
+                testName, className, testDescription, runnerClass()));
     }
 
     /**
diff --git a/tools/dalvik_jtreg/java/dalvik/jtreg/TestRunner.java b/tools/dalvik_jtreg/java/dalvik/jtreg/TestRunner.java
index 31df7ed..fce8aa8 100644
--- a/tools/dalvik_jtreg/java/dalvik/jtreg/TestRunner.java
+++ b/tools/dalvik_jtreg/java/dalvik/jtreg/TestRunner.java
@@ -64,7 +64,8 @@ public abstract class TestRunner {
         return properties;
     }
 
-    public abstract void prepareTest();
+    public void prepareTest() {}
+
     public abstract boolean test();
 
     public void run() {
diff --git a/tools/dalvik_jtreg/java/dalvik/jtreg/Vm.java b/tools/dalvik_jtreg/java/dalvik/jtreg/Vm.java
index e85eae1..09a75de 100644
--- a/tools/dalvik_jtreg/java/dalvik/jtreg/Vm.java
+++ b/tools/dalvik_jtreg/java/dalvik/jtreg/Vm.java
@@ -41,6 +41,7 @@ public abstract class Vm {
     static final String DALVIK_JTREG_HOME = "dalvik/libcore/tools/dalvik_jtreg";
 
     static final Set<File> TEST_RUNNER_JAVA_FILES = new HashSet<File>(Arrays.asList(
+            new File(DALVIK_JTREG_HOME + "/java/dalvik/jtreg/CaliperRunner.java"),
             new File(DALVIK_JTREG_HOME + "/java/dalvik/jtreg/JUnitRunner.java"),
             new File(DALVIK_JTREG_HOME + "/java/dalvik/jtreg/JtregRunner.java"),
             new File(DALVIK_JTREG_HOME + "/java/dalvik/jtreg/TestRunner.java")));
@@ -48,7 +49,10 @@ public abstract class Vm {
     private final Pattern JAVA_TEST_PATTERN = Pattern.compile("\\/(\\w)+\\.java$");
     static final Classpath COMPILATION_CLASSPATH = Classpath.of(
             new File("out/target/common/obj/JAVA_LIBRARIES/core_intermediates/classes.jar"),
-            new File("out/host/common/core-tests.jar"));
+            new File("out/target/common/obj/JAVA_LIBRARIES/core-tests_intermediates/classes.jar"),
+            new File("out/target/common/obj/JAVA_LIBRARIES/jsr305_intermediates/classes.jar"),
+            new File("out/target/common/obj/JAVA_LIBRARIES/guava_intermediates/classes.jar"),
+            new File("out/target/common/obj/JAVA_LIBRARIES/caliper_intermediates/classes.jar"));
 
     private static final Logger logger = Logger.getLogger(Vm.class.getName());
 
@@ -86,6 +90,7 @@ public abstract class Vm {
                 .classpath(COMPILATION_CLASSPATH)
                 .destination(base)
                 .compile(TEST_RUNNER_JAVA_FILES);
+
         return postCompile("testrunner", Classpath.of(base));
     }
 
@@ -174,7 +179,7 @@ public abstract class Vm {
         final Command command = newVmCommandBuilder()
                 .classpath(testRun.getTestClasses())
                 .classpath(testRunnerClasses)
-                .classpath(getRuntimeSupportClasses())
+                .classpath(getRuntimeSupportClasspath())
                 .userDir(testRun.getUserDir())
                 .debugPort(debugPort)
                 .mainClass(testRun.getTestRunner().getName())
@@ -225,7 +230,7 @@ public abstract class Vm {
      * Returns the classpath containing JUnit and the dalvik annotations
      * required for test execution.
      */
-    protected Classpath getRuntimeSupportClasses() {
+    protected Classpath getRuntimeSupportClasspath() {
         return COMPILATION_CLASSPATH;
     }