diff options
Diffstat (limited to 'tools/runner/java')
-rw-r--r-- | tools/runner/java/dalvik/runner/Driver.java | 234 | ||||
-rw-r--r-- | tools/runner/java/dalvik/runner/JUnitRunner.java | 57 | ||||
-rw-r--r-- | tools/runner/java/dalvik/runner/Mode.java | 268 | ||||
-rw-r--r-- | tools/runner/java/dalvik/runner/TestRun.java | 242 | ||||
-rw-r--r-- | tools/runner/java/vogar/Action.java | 113 | ||||
-rw-r--r-- | tools/runner/java/vogar/ActivityMode.java (renamed from tools/runner/java/dalvik/runner/ActivityMode.java) | 138 | ||||
-rw-r--r-- | tools/runner/java/vogar/CaliperFinder.java (renamed from tools/runner/java/dalvik/runner/CaliperFinder.java) | 15 | ||||
-rw-r--r-- | tools/runner/java/vogar/Classpath.java (renamed from tools/runner/java/dalvik/runner/Classpath.java) | 4 | ||||
-rw-r--r-- | tools/runner/java/vogar/CodeFinder.java (renamed from tools/runner/java/dalvik/runner/CodeFinder.java) | 8 | ||||
-rw-r--r-- | tools/runner/java/vogar/Console.java | 216 | ||||
-rw-r--r-- | tools/runner/java/vogar/DeviceDalvikVm.java (renamed from tools/runner/java/dalvik/runner/DeviceDalvikVm.java) | 37 | ||||
-rw-r--r-- | tools/runner/java/vogar/Driver.java | 265 | ||||
-rw-r--r-- | tools/runner/java/vogar/Environment.java (renamed from tools/runner/java/dalvik/runner/Environment.java) | 48 | ||||
-rw-r--r-- | tools/runner/java/vogar/EnvironmentDevice.java (renamed from tools/runner/java/dalvik/runner/EnvironmentDevice.java) | 38 | ||||
-rw-r--r-- | tools/runner/java/vogar/EnvironmentHost.java (renamed from tools/runner/java/dalvik/runner/EnvironmentHost.java) | 19 | ||||
-rw-r--r-- | tools/runner/java/vogar/Expectation.java | 101 | ||||
-rw-r--r-- | tools/runner/java/vogar/ExpectationStore.java (renamed from tools/runner/java/dalvik/runner/ExpectedResult.java) | 83 | ||||
-rw-r--r-- | tools/runner/java/vogar/HostMonitor.java | 200 | ||||
-rw-r--r-- | tools/runner/java/vogar/JUnitFinder.java (renamed from tools/runner/java/dalvik/runner/JUnitFinder.java) | 18 | ||||
-rw-r--r-- | tools/runner/java/vogar/JavaVm.java (renamed from tools/runner/java/dalvik/runner/JavaVm.java) | 22 | ||||
-rw-r--r-- | tools/runner/java/vogar/Javac.java (renamed from tools/runner/java/dalvik/runner/Javac.java) | 11 | ||||
-rw-r--r-- | tools/runner/java/vogar/JtregFinder.java (renamed from tools/runner/java/dalvik/runner/JtregFinder.java) | 18 | ||||
-rw-r--r-- | tools/runner/java/vogar/MainFinder.java (renamed from tools/runner/java/dalvik/runner/MainFinder.java) | 17 | ||||
-rw-r--r-- | tools/runner/java/vogar/Md5Cache.java (renamed from tools/runner/java/dalvik/runner/Md5Cache.java) | 5 | ||||
-rw-r--r-- | tools/runner/java/vogar/Mode.java | 228 | ||||
-rw-r--r-- | tools/runner/java/vogar/NamingPatternCodeFinder.java (renamed from tools/runner/java/dalvik/runner/NamingPatternCodeFinder.java) | 27 | ||||
-rw-r--r-- | tools/runner/java/vogar/Option.java (renamed from tools/runner/java/dalvik/runner/Option.java) | 2 | ||||
-rw-r--r-- | tools/runner/java/vogar/OptionParser.java (renamed from tools/runner/java/dalvik/runner/OptionParser.java) | 2 | ||||
-rw-r--r-- | tools/runner/java/vogar/Outcome.java | 97 | ||||
-rw-r--r-- | tools/runner/java/vogar/Result.java (renamed from tools/runner/java/dalvik/runner/Result.java) | 4 | ||||
-rw-r--r-- | tools/runner/java/vogar/Strings.java (renamed from tools/runner/java/dalvik/runner/Strings.java) | 12 | ||||
-rw-r--r-- | tools/runner/java/vogar/TestProperties.java (renamed from tools/runner/java/dalvik/runner/TestProperties.java) | 9 | ||||
-rw-r--r-- | tools/runner/java/vogar/Threads.java (renamed from tools/runner/java/dalvik/runner/Threads.java) | 4 | ||||
-rw-r--r-- | tools/runner/java/vogar/Vm.java (renamed from tools/runner/java/dalvik/runner/Vm.java) | 39 | ||||
-rw-r--r-- | tools/runner/java/vogar/Vogar.java (renamed from tools/runner/java/dalvik/runner/DalvikRunner.java) | 162 | ||||
-rw-r--r-- | tools/runner/java/vogar/XmlReportPrinter.java (renamed from tools/runner/java/dalvik/runner/XmlReportPrinter.java) | 59 | ||||
-rw-r--r-- | tools/runner/java/vogar/commands/Aapt.java (renamed from tools/runner/java/dalvik/runner/Aapt.java) | 4 | ||||
-rw-r--r-- | tools/runner/java/vogar/commands/Adb.java (renamed from tools/runner/java/dalvik/runner/Adb.java) | 4 | ||||
-rw-r--r-- | tools/runner/java/vogar/commands/Command.java (renamed from tools/runner/java/dalvik/runner/Command.java) | 85 | ||||
-rw-r--r-- | tools/runner/java/vogar/commands/CommandFailedException.java (renamed from tools/runner/java/dalvik/runner/CommandFailedException.java) | 4 | ||||
-rw-r--r-- | tools/runner/java/vogar/commands/Dx.java (renamed from tools/runner/java/dalvik/runner/Dx.java) | 8 | ||||
-rw-r--r-- | tools/runner/java/vogar/commands/Mkdir.java (renamed from tools/runner/java/dalvik/runner/Mkdir.java) | 4 | ||||
-rw-r--r-- | tools/runner/java/vogar/commands/Rm.java (renamed from tools/runner/java/dalvik/runner/Rm.java) | 4 | ||||
-rw-r--r-- | tools/runner/java/vogar/target/CaliperRunner.java (renamed from tools/runner/java/dalvik/runner/CaliperRunner.java) | 17 | ||||
-rw-r--r-- | tools/runner/java/vogar/target/JUnitRunner.java | 131 | ||||
-rw-r--r-- | tools/runner/java/vogar/target/JtregRunner.java (renamed from tools/runner/java/dalvik/runner/JtregRunner.java) | 16 | ||||
-rw-r--r-- | tools/runner/java/vogar/target/MainRunner.java (renamed from tools/runner/java/dalvik/runner/MainRunner.java) | 14 | ||||
-rw-r--r-- | tools/runner/java/vogar/target/Runner.java (renamed from tools/runner/java/dalvik/runner/Runner.java) | 7 | ||||
-rw-r--r-- | tools/runner/java/vogar/target/TargetMonitor.java | 107 | ||||
-rw-r--r-- | tools/runner/java/vogar/target/TestRunner.java (renamed from tools/runner/java/dalvik/runner/TestRunner.java) | 38 |
50 files changed, 1991 insertions, 1274 deletions
diff --git a/tools/runner/java/dalvik/runner/Driver.java b/tools/runner/java/dalvik/runner/Driver.java deleted file mode 100644 index 574c8cb..0000000 --- a/tools/runner/java/dalvik/runner/Driver.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright (C) 2009 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dalvik.runner; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.logging.Logger; - -/** - * Compiles, installs, runs and reports tests. - */ -final class Driver { - - private static final Logger logger = Logger.getLogger(Driver.class.getName()); - - private final File localTemp; - private final Set<File> expectationFiles; - private final List<CodeFinder> codeFinders; - private final Mode mode; - private final File xmlReportsDirectory; - private final Map<String, ExpectedResult> expectedResults = new HashMap<String, ExpectedResult>(); - - /** - * The number of tests that weren't run because they aren't supported by - * this runner. - */ - private int unsupportedTests = 0; - - public Driver(File localTemp, Mode mode, Set<File> expectationFiles, - File xmlReportsDirectory, List<CodeFinder> codeFinders) { - this.localTemp = localTemp; - this.expectationFiles = expectationFiles; - this.mode = mode; - this.xmlReportsDirectory = xmlReportsDirectory; - this.codeFinders = codeFinders; - } - - public void loadExpectations() throws IOException { - for (File f : expectationFiles) { - if (f.exists()) { - expectedResults.putAll(ExpectedResult.parse(f)); - } - } - } - - /** - * Builds and executes all tests in the test directory. - */ - public void buildAndRunAllTests(Collection<File> testFiles) { - new Mkdir().mkdirs(localTemp); - - Set<TestRun> tests = new LinkedHashSet<TestRun>(); - for (File testFile : testFiles) { - Set<TestRun> testsForFile = Collections.emptySet(); - - for (CodeFinder codeFinder : codeFinders) { - testsForFile = codeFinder.findTests(testFile); - - // break as soon as we find any match. We don't need multiple - // matches for the same file, since that would run it twice. - if (!testsForFile.isEmpty()) { - break; - } - } - - tests.addAll(testsForFile); - } - - // compute TestRunner java and classpath to pass to mode.prepare - Set<File> testRunnerJava = new HashSet<File>(); - Classpath testRunnerClasspath = new Classpath(); - for (final TestRun testRun : tests) { - testRunnerJava.add(testRun.getRunnerJava()); - testRunnerClasspath.addAll(testRun.getRunnerClasspath()); - } - - // mode.prepare before mode.buildAndInstall to ensure test - // runner is built. packaging of activity APK files needs the - // test runner along with the test specific files. - mode.prepare(testRunnerJava, testRunnerClasspath); - - logger.info("Running " + tests.size() + " tests."); - - // build and install tests in a background thread. Using lots of - // threads helps for packages that contain many unsupported tests - final BlockingQueue<TestRun> readyToRun = new ArrayBlockingQueue<TestRun>(4); - - ExecutorService builders = Threads.threadPerCpuExecutor(); - int t = 0; - for (final TestRun testRun : tests) { - final int runIndex = t++; - builders.submit(new Runnable() { - public void run() { - try { - ExpectedResult expectedResult = lookupExpectedResult(testRun); - testRun.setExpectedResult(expectedResult); - - if (expectedResult.getResult() == Result.UNSUPPORTED) { - testRun.setResult(Result.UNSUPPORTED, Collections.<String>emptyList()); - logger.fine("skipping test " + testRun - + " because the expectations file says it is unsupported."); - - } else { - mode.buildAndInstall(testRun); - logger.fine("installed test " + runIndex + "; " - + readyToRun.size() + " are ready to run"); - } - - readyToRun.put(testRun); - } catch (Throwable throwable) { - testRun.setResult(Result.ERROR, throwable); - } - } - }); - } - builders.shutdown(); - - List<TestRun> runs = new ArrayList<TestRun>(tests.size()); - for (int i = 0; i < tests.size(); i++) { - logger.fine("executing test " + i + "; " - + readyToRun.size() + " are ready to run"); - - // if it takes 5 minutes for build and install, something is broken - TestRun testRun; - try { - testRun = readyToRun.poll(5 * 60, TimeUnit.SECONDS); - } catch (InterruptedException e) { - throw new RuntimeException("Unexpected interruption waiting for build and install", e); - } - - if (testRun == null) { - throw new IllegalStateException("Expected " + tests.size() + " tests but found only " + i); - } - - runs.add(testRun); - execute(testRun); - mode.cleanup(testRun); - } - - if (unsupportedTests > 0) { - logger.info("Skipped " + unsupportedTests + " unsupported tests."); - } - - if (xmlReportsDirectory != null) { - logger.info("Printing XML Reports... "); - int numFiles = new XmlReportPrinter().generateReports(xmlReportsDirectory, runs); - logger.info(numFiles + " XML files written."); - } - } - - /** - * Finds the expected result for the specified test run. This strips off - * parts of the test's qualified name until it either finds a match or runs - * out of name. - */ - private ExpectedResult lookupExpectedResult(TestRun testRun) { - String name = testRun.getQualifiedName(); - - while (true) { - ExpectedResult expectedResult = expectedResults.get(name); - if (expectedResult != null) { - return expectedResult; - } - - int dot = name.lastIndexOf('.'); - if (dot == -1) { - return ExpectedResult.SUCCESS; - } - - name = name.substring(0, dot); - } - } - - /** - * Executes a single test and then prints the result. - */ - private void execute(TestRun testRun) { - if (testRun.getResult() == Result.UNSUPPORTED) { - logger.fine("skipping " + testRun.getQualifiedName()); - unsupportedTests++; - return; - } - - if (testRun.isRunnable()) { - mode.runTest(testRun); - } - - printResult(testRun); - } - - private void printResult(TestRun testRun) { - if (testRun.isExpectedResult()) { - logger.info("OK " + testRun.getQualifiedName() + " (" + testRun.getResult() + ")"); - // In --verbose mode, show the output even on success. - logger.fine(" " + testRun.getFailureMessage().replace("\n", "\n ")); - return; - } - - logger.info("FAIL " + testRun.getQualifiedName() + " (" + testRun.getResult() + ")"); - String description = testRun.getDescription(); - if (description != null) { - logger.info(" \"" + description + "\""); - } - - logger.info(" " + testRun.getFailureMessage().replace("\n", "\n ")); - } -} diff --git a/tools/runner/java/dalvik/runner/JUnitRunner.java b/tools/runner/java/dalvik/runner/JUnitRunner.java deleted file mode 100644 index 4891448..0000000 --- a/tools/runner/java/dalvik/runner/JUnitRunner.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2009 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dalvik.runner; - -import junit.framework.Test; -import junit.framework.TestResult; -import junit.runner.TestSuiteLoader; - -/** - * Runs a JUnit test. - */ -public final class JUnitRunner implements Runner { - - private final junit.textui.TestRunner testRunner; - private Test junitTest; - - public JUnitRunner() { - final TestSuiteLoader testSuiteLoader = new TestSuiteLoader() { - public Class load(String suiteClassName) throws ClassNotFoundException { - return JUnitRunner.class.getClassLoader().loadClass(suiteClassName); - } - - public Class reload(Class c) { - return c; - } - }; - - testRunner = new junit.textui.TestRunner() { - @Override public TestSuiteLoader getLoader() { - return testSuiteLoader; - } - }; - } - - public void prepareTest(Class<?> testClass) { - junitTest = testRunner.getTest(testClass.getName()); - } - - public boolean test(Class<?> testClass) { - TestResult result = testRunner.doRun(junitTest); - return result.wasSuccessful(); - } -} diff --git a/tools/runner/java/dalvik/runner/Mode.java b/tools/runner/java/dalvik/runner/Mode.java deleted file mode 100644 index 0ad7172..0000000 --- a/tools/runner/java/dalvik/runner/Mode.java +++ /dev/null @@ -1,268 +0,0 @@ -/* - * Copyright (C) 2010 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dalvik.runner; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.FilenameFilter; -import java.io.IOException; -import java.io.PrintStream; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.TimeoutException; -import java.util.logging.Logger; -import java.util.regex.Pattern; - -/** - * A Mode for running tests. Examples including running in a virtual - * machine either on the host or a device or within a specific context - * such as within an Activity. - */ -abstract class Mode { - - private static final Pattern JAVA_TEST_PATTERN = Pattern.compile("\\/(\\w)+\\.java$"); - - private static final Logger logger = Logger.getLogger(Mode.class.getName()); - - protected final Environment environment; - protected final long timeoutSeconds; - protected final File sdkJar; - protected final PrintStream tee; - - /** - * Set of Java files needed to built to tun the currently selected - * set of tests. We build a subset rather than all the files all - * the time to reduce dex packaging costs in the activity mode - * case. - */ - protected final Set<File> testRunnerJava = new HashSet<File>(); - - /** - * Classpath of testRunner on the host side including any - * supporting libraries for testRunnerJava. Useful for compiling - * testRunnerJava as well as executing it on the host. Execution - * on the device requires further packaging typically done by - * postCompileTestRunner. - */ - protected final Classpath testRunnerClasspath = new Classpath(); - - // TODO: this should be an immutable collection. - protected final Classpath testClasspath = Classpath.of( - new File("dalvik/libcore/tools/runner/lib/jsr305.jar"), - new File("dalvik/libcore/tools/runner/lib/guava.jar"), - new File("dalvik/libcore/tools/runner/lib/caliper.jar"), - // TODO: we should be able to work with a shipping SDK, not depend on out/... - // dalvik/libcore/**/test/ for junit - // TODO: jar up just the junit classes and drop the jar in our lib/ directory. - new File("out/target/common/obj/JAVA_LIBRARIES/core-tests_intermediates/classes.jar").getAbsoluteFile()); - - Mode(Environment environment, long timeoutSeconds, File sdkJar, PrintStream tee) { - this.environment = environment; - this.timeoutSeconds = timeoutSeconds; - this.sdkJar = sdkJar; - this.tee = tee; - } - - /** - * Initializes the temporary directories and test harness necessary to run - * tests. - */ - protected void prepare(Set<File> testRunnerJava, Classpath testRunnerClasspath) { - this.testRunnerJava.add(new File(DalvikRunner.HOME_JAVA, "dalvik/runner/TestRunner.java")); - this.testRunnerJava.addAll(dalvikAnnotationSourceFiles()); - this.testRunnerJava.addAll(testRunnerJava); - this.testRunnerClasspath.addAll(testRunnerClasspath); - environment.prepare(); - compileTestRunner(); - } - - private List<File> dalvikAnnotationSourceFiles() { - // Hopefully one day we'll strip the dalvik annotations out, but until then we need to make - // them available to javac(1). - File sourceDir = new File("dalvik/libcore/dalvik/src/main/java/dalvik/annotation"); - File[] javaSourceFiles = sourceDir.listFiles(new FilenameFilter() { - public boolean accept(File dir, String filename) { - return filename.endsWith(".java"); - } - }); - return Arrays.asList(javaSourceFiles); - } - - private void compileTestRunner() { - logger.fine("build testrunner"); - - Classpath classpath = new Classpath(); - classpath.addAll(testClasspath); - classpath.addAll(testRunnerClasspath); - - File base = environment.testRunnerClassesDir(); - new Mkdir().mkdirs(base); - new Javac() - .bootClasspath(sdkJar) - .classpath(classpath) - .sourcepath(DalvikRunner.HOME_JAVA) - .destination(base) - .compile(testRunnerJava); - postCompileTestRunner(); - } - - /** - * Hook method called after TestRunner compilation. - */ - abstract protected void postCompileTestRunner(); - - /** - * Compiles classes for the given test and makes them ready for execution. - * If the test could not be compiled successfully, it will be updated with - * the appropriate test result. - */ - public void buildAndInstall(TestRun testRun) { - logger.fine("build " + testRun.getQualifiedName()); - - boolean testCompiled; - try { - testCompiled = compileTest(testRun); - if (!testCompiled) { - testRun.setResult(Result.UNSUPPORTED, Collections.<String>emptyList()); - return; - } - } catch (CommandFailedException e) { - testRun.setResult(Result.COMPILE_FAILED, e.getOutputLines()); - return; - } catch (IOException e) { - testRun.setResult(Result.ERROR, e); - return; - } - testRun.setTestCompiled(testCompiled); - environment.prepareUserDir(testRun); - } - - /** - * Compiles the classes for the described test. - * - * @return the path to the compiled classes (directory or jar), or {@code - * null} if the test could not be compiled. - * @throws CommandFailedException if javac fails - */ - private boolean compileTest(TestRun testRun) throws IOException { - if (!JAVA_TEST_PATTERN.matcher(testRun.getTestJava().toString()).find()) { - return false; - } - - String qualifiedName = testRun.getQualifiedName(); - File testClassesDir = environment.testClassesDir(testRun); - new Mkdir().mkdirs(testClassesDir); - FileOutputStream propertiesOut = new FileOutputStream( - new File(testClassesDir, TestProperties.FILE)); - Properties properties = new Properties(); - fillInProperties(properties, testRun); - properties.store(propertiesOut, "generated by " + Mode.class.getName()); - propertiesOut.close(); - - Classpath classpath = new Classpath(); - classpath.addAll(testClasspath); - classpath.addAll(testRun.getRunnerClasspath()); - - Set<File> sourceFiles = new HashSet<File>(); - sourceFiles.add(testRun.getTestJava()); - sourceFiles.addAll(dalvikAnnotationSourceFiles()); - - // compile the test case - new Javac() - .bootClasspath(sdkJar) - .classpath(classpath) - .sourcepath(testRun.getTestDirectory()) - .destination(testClassesDir) - .compile(sourceFiles); - postCompileTest(testRun); - return true; - } - - /** - * Hook method called after test compilation. - * - * @param testRun The test being compiled - */ - abstract protected void postCompileTest(TestRun testRun); - - - /** - * Fill in properties for running in this mode - */ - protected void fillInProperties(Properties properties, TestRun testRun) { - properties.setProperty(TestProperties.TEST_CLASS, testRun.getTestClass()); - properties.setProperty(TestProperties.QUALIFIED_NAME, testRun.getQualifiedName()); - properties.setProperty(TestProperties.RUNNER_CLASS, testRun.getRunnerClass().getName()); - } - - /** - * Runs the test, and updates its test result. - */ - void runTest(TestRun testRun) { - if (!testRun.isRunnable()) { - throw new IllegalArgumentException(); - } - - List<String> output; - try { - output = runTestCommand(testRun); - } catch (TimeoutException e) { - testRun.setResult(Result.EXEC_TIMEOUT, - Collections.singletonList("Exceeded timeout! (" + timeoutSeconds + "s)")); - return; - } catch (Exception e) { - testRun.setResult(Result.ERROR, e); - return; - } - // we only look at the output of the last command - if (output.isEmpty()) { - testRun.setResult(Result.ERROR, - Collections.singletonList("No output returned!")); - return; - } - - Result result = TestProperties.RESULT_SUCCESS.equals(output.get(output.size() - 1)) - ? Result.SUCCESS - : Result.EXEC_FAILED; - testRun.setResult(result, output.subList(0, output.size() - 1)); - } - - /** - * Run the actual test to gather output - */ - protected abstract List<String> runTestCommand(TestRun testRun) - throws TimeoutException; - - /** - * Deletes files and releases any resources required for the execution of - * the given test. - */ - void cleanup(TestRun testRun) { - environment.cleanup(testRun); - } - - /** - * Cleans up after all test runs have completed. - */ - void shutdown() { - environment.shutdown(); - } -} diff --git a/tools/runner/java/dalvik/runner/TestRun.java b/tools/runner/java/dalvik/runner/TestRun.java deleted file mode 100644 index c610b25..0000000 --- a/tools/runner/java/dalvik/runner/TestRun.java +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Copyright (C) 2009 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dalvik.runner; - -import java.io.File; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.Arrays; -import java.util.List; - -/** - * A test run and its outcome. This class tracks the complete lifecycle of a - * single test run: - * <ol> - * <li>the test source code (test directory, java file, test class) - * <li>the test identity (suite name, test name, qualified name) - * <li>the code to execute (test classes, user dir) - * <li>the result of execution (expected result, result, output lines) - * </ol> - */ -public final class TestRun { - - private final File testDirectory; - private final File testJava; - private final String testClass; - private final Class<? extends Runner> runnerClass; - private final File runnerJava; - private final Classpath runnerClasspath; - - private final String suiteName; - private final String testName; - private final String qualifiedName; - private final String description; - - private boolean testCompiled; - private File userDir = new File(System.getProperty("user.dir")); - - private ExpectedResult expectedResult = ExpectedResult.SUCCESS; - private Result result; - private List<String> outputLines; - - public TestRun(File testDirectory, File testJava, String testClass, - String suiteName, String testName, String qualifiedName, - String description, Class<? extends Runner> runnerClass, - File runnerJava, Classpath runnerClasspath) { - this.qualifiedName = qualifiedName; - this.suiteName = suiteName; - this.testName = testName; - this.testDirectory = testDirectory; - this.testJava = testJava; - this.description = description; - this.testClass = testClass; - this.runnerClass = runnerClass; - this.runnerJava = runnerJava; - this.runnerClasspath = runnerClasspath; - } - - /** - * Returns the local directory containing this test's java file. - */ - public File getTestDirectory() { - return testDirectory; - } - - public File getTestJava() { - return testJava; - } - - /** - * Returns the executable test's classname, such as java.lang.IntegerTest - * or BitTwiddle. - */ - public String getTestClass() { - return testClass; - } - - /** - * Returns the test suite name, such as java.lang.Integer or - * java.lang.IntegerTest. - */ - public String getSuiteName() { - return suiteName; - } - - /** - * Returns the specific test name, such as BitTwiddle or testBitTwiddle. - */ - public String getTestName() { - return testName; - } - - /** - * Returns a unique identifier for this test. - */ - public String getQualifiedName() { - return qualifiedName; - } - - /** - * Returns an English description of this test, or null if no such - * description is known. - */ - public String getDescription() { - return description; - } - - public void setExpectedResult(ExpectedResult expectedResult) { - this.expectedResult = expectedResult; - } - - /** - * Set when the test is successfully compiled. - */ - public void setTestCompiled(boolean testCompiled) { - this.testCompiled = testCompiled; - } - - public boolean getTestCompiled() { - return testCompiled; - } - - /** - * Initializes the directory from which local files can be read by the test. - */ - public void setUserDir(File base) { - this.userDir = base; - } - - public File getUserDir() { - return userDir; - } - - /** - * Returns true if this test is ready for execution. Such tests have their - * classpath prepared and have not yet been assigned a result. - */ - public boolean isRunnable() { - return testCompiled && result == null; - } - - public void setResult(Result result, Throwable e) { - setResult(result, throwableToLines(e)); - } - - public void setResult(Result result, List<String> outputLines) { - if (this.result != null) { - throw new IllegalStateException("result already set"); - } - - this.result = result; - this.outputLines = outputLines; - } - - private static List<String> throwableToLines(Throwable t) { - StringWriter writer = new StringWriter(); - PrintWriter out = new PrintWriter(writer); - t.printStackTrace(out); - return Arrays.asList(writer.toString().split("\\n")); - } - - public Result getResult() { - return result; - } - - public List<String> getOutputLines() { - return outputLines; - } - - public Class<? extends Runner> getRunnerClass() { - return runnerClass; - } - - public File getRunnerJava() { - return runnerJava; - } - - public Classpath getRunnerClasspath() { - return runnerClasspath; - } - - /** - * Returns true if the outcome of this run matches what was expected. - */ - public boolean isExpectedResult() { - return result == expectedResult.getResult() && matchesExpectedPattern(); - } - - /** - * Returns true if the test's output matches the expected output. - */ - private boolean matchesExpectedPattern() { - return expectedResult.getPattern() - .matcher(Strings.join(outputLines, "\n")) - .matches(); - } - - /** - * Returns the failure message for this failed test run. This message is - * intended to help to diagnose why the test result didn't match what was - * expected. - */ - public String getFailureMessage() { - StringBuilder builder = new StringBuilder(); - - if (expectedResult.getResult() != Result.SUCCESS - && expectedResult.getResult() != result) { - builder.append("Expected result: ") - .append(expectedResult.getResult()) - .append("\n"); - } - - if (!matchesExpectedPattern()) { - builder.append("Expected output to match \"") - .append(expectedResult.getPattern().pattern()) - .append("\"\n"); - } - - for (String output : outputLines) { - builder.append(output).append("\n"); - } - - return builder.toString(); - } - - @Override public String toString() { - return qualifiedName; - } -} diff --git a/tools/runner/java/vogar/Action.java b/tools/runner/java/vogar/Action.java new file mode 100644 index 0000000..1e3de3e --- /dev/null +++ b/tools/runner/java/vogar/Action.java @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package vogar; + +import vogar.target.Runner; + +import java.io.File; + +/** + * A named job such as a test or benchmark run. This class tracks the resource + * files and classes for compiling and running a Java source file. + */ +public final class Action { + + private final String name; + private final String actionClass; + private final File actionDirectory; + private final File actionJava; + private final String description; + private final Class<? extends Runner> runnerClass; + private final File runnerJava; + private final Classpath runnerClasspath; + private File userDir = new File(System.getProperty("user.dir")); + + public Action(String name, String actionClass, File actionDirectory, + File actionJava, String description, Class<? extends Runner> runnerClass, + File runnerJava, Classpath runnerClasspath) { + this.name = name; + this.actionClass = actionClass; + this.actionDirectory = actionDirectory; + this.actionJava = actionJava; + this.description = description; + this.runnerClass = runnerClass; + this.runnerJava = runnerJava; + this.runnerClasspath = runnerClasspath; + } + + /** + * Returns the local directory containing this action's java file. + */ + public File getJavaDirectory() { + return actionDirectory; + } + + public File getJavaFile() { + return actionJava; + } + + /** + * Returns the executable classname, such as java.lang.IntegerTest + * or BitTwiddle. + */ + public String getTargetClass() { + return actionClass; + } + + /** + * Returns a unique identifier for this action. + */ + public String getName() { + return name; + } + + /** + * Returns an English description of this action, or null if no such + * description is known. + */ + public String getDescription() { + return description; + } + + /** + * Initializes the directory from which local files can be read by the + * action. + */ + public void setUserDir(File base) { + this.userDir = base; + } + + public File getUserDir() { + return userDir; + } + + public Class<? extends Runner> getRunnerClass() { + return runnerClass; + } + + public File getRunnerJava() { + return runnerJava; + } + + public Classpath getRunnerClasspath() { + return runnerClasspath; + } + + @Override public String toString() { + return name; + } +} diff --git a/tools/runner/java/dalvik/runner/ActivityMode.java b/tools/runner/java/vogar/ActivityMode.java index 163c72a..4572b1d 100644 --- a/tools/runner/java/dalvik/runner/ActivityMode.java +++ b/tools/runner/java/vogar/ActivityMode.java @@ -14,35 +14,40 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; + +import vogar.commands.Aapt; +import vogar.commands.Command; +import vogar.commands.Dx; +import vogar.commands.Mkdir; +import vogar.commands.Rm; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.io.PrintStream; import java.util.ArrayList; import java.util.List; import java.util.Properties; import java.util.Set; -import java.util.concurrent.TimeoutException; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.logging.Logger; /** - * Runs a test in the context of an android.app.Activity on a device + * Runs an action in the context of an android.app.Activity on a device */ final class ActivityMode extends Mode { private static final Logger logger = Logger.getLogger(ActivityMode.class.getName()); - private static final String TEST_ACTIVITY_CLASS = "dalvik.runner.TestActivity"; + private static final String TEST_ACTIVITY_CLASS = "vogar.target.TestActivity"; - ActivityMode(Integer debugPort, long timeoutSeconds, File sdkJar, PrintStream tee, File localTemp, - boolean cleanBefore, boolean cleanAfter, File deviceRunnerDir) { + ActivityMode(Integer debugPort, File sdkJar, List<String> javacArgs, + int monitorPort, File localTemp, boolean cleanBefore, boolean cleanAfter, + File deviceRunnerDir) { super(new EnvironmentDevice(cleanBefore, cleanAfter, - debugPort, localTemp, deviceRunnerDir), - timeoutSeconds, sdkJar, tee); + debugPort, monitorPort, localTemp, deviceRunnerDir), + sdkJar, javacArgs, monitorPort); } private EnvironmentDevice getEnvironmentDevice() { @@ -54,11 +59,11 @@ final class ActivityMode extends Mode { super.prepare(testRunnerJava, testRunnerClasspath); } - @Override protected void postCompileTestRunner() { + @Override protected void postCompileRunner() { } - @Override protected void postCompileTest(TestRun testRun) { - logger.fine("aapt and push " + testRun.getQualifiedName()); + @Override protected void postCompile(Action action) { + logger.fine("aapt and push " + action.getName()); // Some things of note: // 1. we can't put multiple dex files in one apk @@ -76,49 +81,49 @@ final class ActivityMode extends Mode { // 7. aapt the dex to create apk // 8. sign the apk // 9. install the apk - File packagingDir = makePackagingDirectory(testRun); - addTestRunnerClasses(packagingDir); + File packagingDir = makePackagingDirectory(action); + addRunnerClasses(packagingDir); List<File> found = new ArrayList<File>(); - File originalTestJar = findOriginalTestJar(testRun); - if (originalTestJar != null) { - found.add(originalTestJar); + File originalJar = findOriginalJar(action); + if (originalJar != null) { + found.add(originalJar); } - found.addAll(testRun.getRunnerClasspath().getElements()); + found.addAll(action.getRunnerClasspath().getElements()); extractJars(packagingDir, found); - addTestClasses(testRun, packagingDir); - File dex = createDex(testRun, packagingDir); - File apkUnsigned = createApk(testRun, dex); - File apkSigned = signApk(testRun, apkUnsigned); - installApk(testRun, apkSigned); + addActionClasses(action, packagingDir); + File dex = createDex(action, packagingDir); + File apkUnsigned = createApk(action, dex); + File apkSigned = signApk(action, apkUnsigned); + installApk(action, apkSigned); } - private File makePackagingDirectory(TestRun testRun) { - File packagingDir = new File(environment.testCompilationDir(testRun), "packaging"); + private File makePackagingDirectory(Action action) { + File packagingDir = new File(environment.actionCompilationDir(action), "packaging"); new Rm().directoryTree(packagingDir); new Mkdir().mkdirs(packagingDir); return packagingDir; } - private void addTestRunnerClasses(File packagingDir) { + private void addRunnerClasses(File packagingDir) { new Command("rsync", "-a", - environment.testRunnerClassesDir() + "/", + environment.runnerClassesDir() + "/", packagingDir + "/").execute(); } - private File findOriginalTestJar(TestRun testRun) { - String testClass = testRun.getTestClass(); - String testFile = testClass.replace('.', '/') + ".class"; - for (File element : testClasspath.getElements()) { + private File findOriginalJar(Action action) { + String targetClass = action.getTargetClass(); + String targetClassFile = targetClass.replace('.', '/') + ".class"; + for (File element : classpath.getElements()) { try { JarFile jar = new JarFile(element); - JarEntry jarEntry = jar.getJarEntry(testFile); + JarEntry jarEntry = jar.getJarEntry(targetClassFile); if (jarEntry != null) { return element; } } catch (IOException e) { throw new RuntimeException( "Could not find element " + element + - " of test class path " + testClasspath, e); + " of class path " + classpath, e); } } return null; @@ -137,15 +142,15 @@ final class ActivityMode extends Mode { new Rm().directoryTree(new File(packagingDir, "META-INF")); } - private void addTestClasses(TestRun testRun, File packagingDir) { - File testClassesDir = environment.testClassesDir(testRun); + private void addActionClasses(Action action, File packagingDir) { + File classesDir = environment.classesDir(action); new Command("rsync", "-a", - testClassesDir + "/", + classesDir + "/", packagingDir + "/").execute(); } - private File createDex (TestRun testRun, File packagingDir) { - File testClassesDir = environment.testClassesDir(testRun); - File dex = new File(testClassesDir + ".dex"); + private File createDex(Action action, File packagingDir) { + File classesDir = environment.classesDir(action); + File dex = new File(classesDir + ".dex"); new Dx().dex(dex, Classpath.of(packagingDir)); return dex; } @@ -156,15 +161,15 @@ final class ActivityMode extends Mode { * may not contain a dot, we prefix containing one to ensure we * are compliant. */ - private static String packageName(TestRun testRun) { - return "DalvikRunner." + testRun.getQualifiedName(); + private static String packageName(Action action) { + return "vogar.test." + action.getName(); } - private File createApk (TestRun testRun, File dex) { + private File createApk (Action action, File dex) { String androidManifest = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + "<manifest xmlns:android=\"http://schemas.android.com/apk/res/android\"\n" + - " package=\"" + packageName(testRun) + "\">\n" + + " package=\"" + packageName(action) + "\">\n" + " <uses-permission android:name=\"android.permission.INTERNET\" />\n" + " <application>\n" + " <activity android:name=\"" + TEST_ACTIVITY_CLASS + "\">\n" + @@ -176,7 +181,7 @@ final class ActivityMode extends Mode { " </application>\n" + "</manifest>\n"; File androidManifestFile = - new File(environment.testCompilationDir(testRun), + new File(environment.actionCompilationDir(action), "AndroidManifest.xml"); try { FileOutputStream androidManifestOut = @@ -187,17 +192,17 @@ final class ActivityMode extends Mode { throw new RuntimeException("Problem writing " + androidManifestFile, e); } - File testClassesDir = environment.testClassesDir(testRun); - File apkUnsigned = new File(testClassesDir + ".apk.unsigned"); + File classesDir = environment.classesDir(action); + File apkUnsigned = new File(classesDir + ".apk.unsigned"); new Aapt().apk(apkUnsigned, androidManifestFile); new Aapt().add(apkUnsigned, dex); - new Aapt().add(apkUnsigned, new File(testClassesDir, TestProperties.FILE)); + new Aapt().add(apkUnsigned, new File(classesDir , TestProperties.FILE)); return apkUnsigned; } - private File signApk(TestRun testRun, File apkUnsigned) { - File testClassesDir = environment.testClassesDir(testRun); - File apkSigned = new File(testClassesDir, testRun.getQualifiedName() + ".apk"); + private File signApk(Action action, File apkUnsigned) { + File classesDir = environment.classesDir(action); + File apkSigned = new File(classesDir, action.getName() + ".apk"); // TODO: we should be able to work with a shipping SDK, not depend on out/... // TODO: we should be able to work without hardwired keys, not depend on build/... new Command.Builder() @@ -212,37 +217,28 @@ final class ActivityMode extends Mode { return apkSigned; } - private void installApk(TestRun testRun, File apkSigned) { + private void installApk(Action action, File apkSigned) { // install the local apk ona the device - getEnvironmentDevice().adb.uninstall(packageName(testRun)); + getEnvironmentDevice().adb.uninstall(packageName(action)); getEnvironmentDevice().adb.install(apkSigned); } - @Override protected void fillInProperties(Properties properties, TestRun testRun) { - super.fillInProperties(properties, testRun); + @Override protected void fillInProperties(Properties properties, Action action) { + super.fillInProperties(properties, action); properties.setProperty(TestProperties.DEVICE_RUNNER_DIR, getEnvironmentDevice().runnerDir.getPath()); } - @Override protected List<String> runTestCommand(TestRun testRun) - throws TimeoutException { - new Command( - "adb", "shell", "am", "start", - "-a","android.intent.action.MAIN", - "-n", (packageName(testRun) + "/" + TEST_ACTIVITY_CLASS)).executeWithTimeout(timeoutSeconds); - - File resultDir = new File(getEnvironmentDevice().runnerDir, testRun.getQualifiedName()); - File resultFile = new File(resultDir, TestProperties.RESULT_FILE); - getEnvironmentDevice().adb.waitForFile(resultFile, timeoutSeconds); - return new Command.Builder() - .args("adb", "shell", "cat", resultFile.getPath()) - .tee(tee) - .build().executeWithTimeout(timeoutSeconds); + @Override protected Command createActionCommand(Action action) { + return new Command( + "adb", "shell", "am", "start", "-W", + "-a", "android.intent.action.MAIN", + "-n", (packageName(action) + "/" + TEST_ACTIVITY_CLASS)); } - @Override void cleanup(TestRun testRun) { - super.cleanup(testRun); + @Override void cleanup(Action action) { + super.cleanup(action); if (environment.cleanAfter) { - getEnvironmentDevice().adb.uninstall(testRun.getQualifiedName()); + getEnvironmentDevice().adb.uninstall(action.getName()); } } } diff --git a/tools/runner/java/dalvik/runner/CaliperFinder.java b/tools/runner/java/vogar/CaliperFinder.java index 3609471..d277ea3 100644 --- a/tools/runner/java/dalvik/runner/CaliperFinder.java +++ b/tools/runner/java/vogar/CaliperFinder.java @@ -14,22 +14,21 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; + +import vogar.target.CaliperRunner; +import vogar.target.Runner; import java.io.File; /** - * Create {@link TestRun}s for {@code .java} files with Caliper benchmarks in + * Create {@link Action}s for {@code .java} files with Caliper benchmarks in * them. */ class CaliperFinder extends NamingPatternCodeFinder { @Override protected boolean matches(File file) { - return file.getName().endsWith("Benchmark.java"); - } - - @Override protected String testName(File file) { - return "caliper"; + return super.matches(file) && file.getName().endsWith("Benchmark.java"); } public Class<? extends Runner> getRunnerClass() { @@ -37,7 +36,7 @@ class CaliperFinder extends NamingPatternCodeFinder { } public File getRunnerJava() { - return new File(DalvikRunner.HOME_JAVA, "dalvik/runner/CaliperRunner.java"); + return new File(Vogar.HOME_JAVA, "vogar/target/CaliperRunner.java"); } public Classpath getRunnerClasspath() { diff --git a/tools/runner/java/dalvik/runner/Classpath.java b/tools/runner/java/vogar/Classpath.java index 607615a..cd83409 100644 --- a/tools/runner/java/dalvik/runner/Classpath.java +++ b/tools/runner/java/vogar/Classpath.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; import java.io.File; import java.util.ArrayList; @@ -25,7 +25,7 @@ import java.util.List; /** * A list of jar files and directories. */ -final class Classpath { +public final class Classpath { private final List<File> elements = new ArrayList<File>(); diff --git a/tools/runner/java/dalvik/runner/CodeFinder.java b/tools/runner/java/vogar/CodeFinder.java index f770fa4..7e357ab 100644 --- a/tools/runner/java/dalvik/runner/CodeFinder.java +++ b/tools/runner/java/vogar/CodeFinder.java @@ -14,7 +14,9 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; + +import vogar.target.Runner; import java.io.File; import java.util.Set; @@ -25,10 +27,10 @@ import java.util.Set; public interface CodeFinder { /** - * Returns all test runs in the given file or directory. If the returned set + * Returns all actions in the given file or directory. If the returned set * is empty, no executable code of this kind were found. */ - public Set<TestRun> findTests(File file); + public Set<Action> findActions(File file); /** * Return the class for the TestRunner diff --git a/tools/runner/java/vogar/Console.java b/tools/runner/java/vogar/Console.java new file mode 100644 index 0000000..953c660 --- /dev/null +++ b/tools/runner/java/vogar/Console.java @@ -0,0 +1,216 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package vogar; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.logging.ConsoleHandler; +import java.util.logging.Formatter; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +/** + * Controls, formats and emits output to the command line. Command line output + * can be generated both by java.util.logging and by direct calls to this class. + */ +public class Console { + + private final boolean stream; + private final boolean color; + private final String indent; + + private String currentName; + private CurrentLine currentLine = CurrentLine.NEW; + private final StringBuilder bufferedOutput = new StringBuilder(); + + public Console(boolean stream, String indent, boolean color) { + this.stream = stream; + this.indent = indent; + this.color = color; + } + + public void configureJavaLogging(boolean verbose) { + ConsoleHandler handler = new ConsoleHandler(); + handler.setLevel(Level.ALL); + handler.setFormatter(new Formatter() { + @Override public String format(LogRecord r) { + return logRecordToString(r); + } + }); + + Logger logger = Logger.getLogger("vogar"); + logger.setLevel(verbose ? Level.FINE : Level.INFO); + logger.addHandler(handler); + logger.setUseParentHandlers(false); + } + + /** + * Formats an alternating sequence of regular log messages and messages + * streamed from a foreign process. + */ + private String logRecordToString(LogRecord logRecord) { + String message = logRecord.getMessage(); + + if (logRecord.getThrown() != null) { + StringWriter writer = new StringWriter(); + writer.write(message); + writer.write("\n"); + logRecord.getThrown().printStackTrace(new PrintWriter(writer)); + message = writer.toString(); + } + + newLine(); + return message + "\n"; + } + + public void action(String name) { + newLine(); + System.out.print("Action " + name); + currentName = name; + currentLine = CurrentLine.NAME; + } + + /** + * Prints the beginning of the named outcome. + */ + public void outcome(String name) { + // if the outcome and action names are the same, omit the outcome name + if (name.equals(currentName)) { + return; + } + + currentName = name; + newLine(); + System.out.print(indent + name); + currentLine = CurrentLine.NAME; + } + + /** + * Appends the action output immediately to the stream when streaming is on, + * or to a buffer when streaming is off. Buffered output will be held and + * printed only if the outcome is unsuccessful. + */ + public void streamOutput(String output) { + if (stream) { + printOutput(output); + } else { + bufferedOutput.append(output); + } + } + + /** + * Writes the action's outcome. + */ + public void printResult(Result result, boolean ok) { + if (ok) { + String prefix = (currentLine == CurrentLine.NAME) ? " " : "\n" + indent; + System.out.println(prefix + green("OK (" + result + ")")); + + } else { + if (bufferedOutput.length() > 0) { + printOutput(bufferedOutput.toString()); + bufferedOutput.delete(0, bufferedOutput.length()); + } + + newLine(); + System.out.println(indent + red("FAIL (" + result + ")")); + } + + currentName = null; + currentLine = CurrentLine.NEW; + } + + /** + * Prints the action output with appropriate indentation. + */ + private void printOutput(String streamedOutput) { + String[] lines = messageToLines(streamedOutput); + + if (currentLine != CurrentLine.STREAMED_OUTPUT) { + newLine(); + System.out.print(indent); + System.out.print(indent); + } + System.out.print(lines[0]); + currentLine = CurrentLine.STREAMED_OUTPUT; + + for (int i = 1; i < lines.length; i++) { + newLine(); + + if (lines[i].length() > 0) { + System.out.print(indent); + System.out.print(indent); + System.out.print(lines[i]); + currentLine = CurrentLine.STREAMED_OUTPUT; + } + } + } + + /** + * Inserts a linebreak if necessary. + */ + private void newLine() { + if (currentLine == CurrentLine.NEW) { + return; + } + + System.out.println(); + currentLine = CurrentLine.NEW; + } + + /** + * Status of a currently-in-progress line of output. + */ + enum CurrentLine { + + /** + * The line is blank. + */ + NEW, + + /** + * The line contains streamed application output. Additional streamed + * output may be appended without additional line separators or + * indentation. + */ + STREAMED_OUTPUT, + + /** + * The line contains the name of an action or outcome. The outcome's + * result (such as "OK") can be appended without additional line + * separators or indentation. + */ + NAME, + } + + /** + * Returns an array containing the lines of the given text. + */ + private String[] messageToLines(String message) { + // pass Integer.MAX_VALUE so split doesn't trim trailing empty strings. + return message.split("\r\n|\r|\n", Integer.MAX_VALUE); + } + + private String green(String message) { + return color ? ("\u001b[32;1m" + message + "\u001b[0m") : message; + } + + private String red(String message) { + return color ? ("\u001b[31;1m" + message + "\u001b[0m") : message; + } +} diff --git a/tools/runner/java/dalvik/runner/DeviceDalvikVm.java b/tools/runner/java/vogar/DeviceDalvikVm.java index 061e374..2f98793 100644 --- a/tools/runner/java/dalvik/runner/DeviceDalvikVm.java +++ b/tools/runner/java/vogar/DeviceDalvikVm.java @@ -14,41 +14,42 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; + +import vogar.commands.Dx; import java.io.File; -import java.io.PrintStream; import java.util.List; import java.util.logging.Logger; /** - * Execute tests on a Dalvik VM using an Android device or emulator. + * Execute actions on a Dalvik VM using an Android device or emulator. */ final class DeviceDalvikVm extends Vm { private static final Logger logger = Logger.getLogger(DeviceDalvikVm.class.getName()); - DeviceDalvikVm(Integer debugPort, long timeoutSeconds, File sdkJar, PrintStream tee, - File localTemp, List<String> additionalVmArgs, + DeviceDalvikVm(Integer debugPort, File sdkJar, List<String> javacArgs, + int monitorPort, File localTemp, List<String> additionalVmArgs, boolean cleanBefore, boolean cleanAfter, File runnerDir) { - super(new EnvironmentDevice(cleanBefore, cleanAfter, debugPort, localTemp, runnerDir), - timeoutSeconds, sdkJar, tee, additionalVmArgs); + super(new EnvironmentDevice(cleanBefore, cleanAfter, debugPort, monitorPort, localTemp, + runnerDir), sdkJar, javacArgs, additionalVmArgs, monitorPort); } private EnvironmentDevice getEnvironmentDevice() { return (EnvironmentDevice) environment; } - @Override protected void postCompileTestRunner() { + @Override protected void postCompileRunner() { // TODO: does this really need to be a special case? - postCompile("testrunner", environment.testRunnerClassesDir()); + postCompile("testrunner", environment.runnerClassesDir()); // dex everything on the classpath and push it to the device. - for (File classpathElement : testClasspath.getElements()) { + for (File classpathElement : classpath.getElements()) { String name = basenameOfJar(classpathElement); logger.fine("dex and push " + name); // make the local dex (inside a jar) // TODO: this is *really* expensive. we need a cache! - File outputFile = getEnvironmentDevice().testDir(name + ".jar"); + File outputFile = getEnvironmentDevice().actionDir(name + ".jar"); new Dx().dex(outputFile, Classpath.of(classpathElement)); // push the local dex to the device getEnvironmentDevice().adb.push(outputFile, deviceDexFile(name)); @@ -59,8 +60,8 @@ final class DeviceDalvikVm extends Vm { return jarFile.getName().replaceAll("\\.jar$", ""); } - @Override protected void postCompileTest(TestRun testRun) { - postCompile(testRun.getQualifiedName(), environment.testClassesDir(testRun)); + @Override protected void postCompile(Action action) { + postCompile(action.getName(), environment.classesDir(action)); } private void postCompile(String name, File dir) { @@ -93,15 +94,15 @@ final class DeviceDalvikVm extends Vm { .vmArgs("-Duser.language=en") .vmArgs("-Duser.region=US") .vmArgs("-Djavax.net.ssl.trustStore=/system/etc/security/cacerts.bks") - .temp(getEnvironmentDevice().testTemp); + .temp(getEnvironmentDevice().vogarTemp); } - @Override protected Classpath getRuntimeSupportClasspath(TestRun testRun) { + @Override protected Classpath getRuntimeSupportClasspath(Action action) { Classpath classpath = new Classpath(); - classpath.addAll(deviceDexFile(testRun.getQualifiedName())); + classpath.addAll(deviceDexFile(action.getName())); classpath.addAll(deviceDexFile("testrunner")); - for (File testClasspathElement : testClasspath.getElements()) { - classpath.addAll(deviceDexFile(basenameOfJar(testClasspathElement))); + for (File classpathElement : this.classpath.getElements()) { + classpath.addAll(deviceDexFile(basenameOfJar(classpathElement))); } return classpath; } diff --git a/tools/runner/java/vogar/Driver.java b/tools/runner/java/vogar/Driver.java new file mode 100644 index 0000000..f646fe8 --- /dev/null +++ b/tools/runner/java/vogar/Driver.java @@ -0,0 +1,265 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package vogar; + +import vogar.commands.Command; +import vogar.commands.Mkdir; + +import java.io.File; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Logger; + +/** + * Compiles, installs, runs and reports on actions. + */ +final class Driver implements HostMonitor.Handler { + + private static final Logger logger = Logger.getLogger(Driver.class.getName()); + + private final File localTemp; + private final ExpectationStore expectationStore; + private final List<CodeFinder> codeFinders; + private final Mode mode; + private final XmlReportPrinter reportPrinter; + private final Console console; + private final int monitorPort; + private final HostMonitor monitor; + private final long timeoutSeconds; + private int successes = 0; + private int failures = 0; + + private Timer actionTimeoutTimer = new Timer("action timeout", true); + + private final Map<String, Action> actions = Collections.synchronizedMap( + new LinkedHashMap<String, Action>()); + private final Map<String, Outcome> outcomes = Collections.synchronizedMap( + new LinkedHashMap<String, Outcome>()); + + /** + * The number of tests that weren't run because they aren't supported by + * this runner. + */ + private int unsupportedActions = 0; + + public Driver(File localTemp, Mode mode, ExpectationStore expectationStore, + List<CodeFinder> codeFinders, XmlReportPrinter reportPrinter, + Console console, HostMonitor monitor, int monitorPort, long timeoutSeconds) { + this.localTemp = localTemp; + this.expectationStore = expectationStore; + this.mode = mode; + this.console = console; + this.codeFinders = codeFinders; + this.reportPrinter = reportPrinter; + this.monitor = monitor; + this.monitorPort = monitorPort; + this.timeoutSeconds = timeoutSeconds; + } + + /** + * Builds and executes the actions in the given files. + */ + public void buildAndRunAllActions(Collection<File> files) { + if (!actions.isEmpty()) { + throw new IllegalStateException("Drivers are not reusable"); + } + + new Mkdir().mkdirs(localTemp); + for (File file : files) { + Set<Action> actionsForFile = Collections.emptySet(); + + for (CodeFinder codeFinder : codeFinders) { + actionsForFile = codeFinder.findActions(file); + + // break as soon as we find any match. We don't need multiple + // matches for the same file, since that would run it twice. + if (!actionsForFile.isEmpty()) { + break; + } + } + + for (Action action : actionsForFile) { + actions.put(action.getName(), action); + } + } + + // compute TestRunner java and classpath to pass to mode.prepare + Set<File> runnerJava = new HashSet<File>(); + Classpath runnerClasspath = new Classpath(); + for (final Action action : actions.values()) { + runnerJava.add(action.getRunnerJava()); + runnerClasspath.addAll(action.getRunnerClasspath()); + } + + // mode.prepare before mode.buildAndInstall to ensure the runner is + // built. packaging of activity APK files needs the runner along with + // the action-specific files. + mode.prepare(runnerJava, runnerClasspath); + + logger.info("Actions: " + actions.size()); + + // build and install actions in a background thread. Using lots of + // threads helps for packages that contain many unsupported actions + final BlockingQueue<Action> readyToRun = new ArrayBlockingQueue<Action>(4); + + ExecutorService builders = Threads.threadPerCpuExecutor(); + int t = 0; + + for (final Action action : actions.values()) { + final String name = action.getName(); + final int runIndex = t++; + builders.submit(new Runnable() { + public void run() { + try { + logger.fine("installing action " + runIndex + "; " + + readyToRun.size() + " are runnable"); + + if (expectationStore.get(name).getResult() == Result.UNSUPPORTED) { + outcomes.put(name, new Outcome(name, Result.UNSUPPORTED, + "Unsupported according to expectations file")); + + } else { + Outcome outcome = mode.buildAndInstall(action); + if (outcome != null) { + outcomes.put(name, outcome); + } + } + + readyToRun.put(action); + } catch (InterruptedException e) { + outcomes.put(name, new Outcome(name, Result.ERROR, e)); + } + } + }); + } + builders.shutdown(); + + for (int i = 0; i < actions.size(); i++) { + logger.fine("executing action " + i + "; " + + readyToRun.size() + " are ready to run"); + + // if it takes 5 minutes for build and install, something is broken + Action action; + try { + action = readyToRun.poll(5 * 60, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException("Unexpected interruption waiting for build and install", e); + } + + if (action == null) { + throw new IllegalStateException("Expected " + actions.size() + + " actions but found only " + i); + } + + execute(action); + mode.cleanup(action); + } + + if (reportPrinter != null) { + logger.info("Printing XML Reports... "); + int numFiles = reportPrinter.generateReports(outcomes.values()); + logger.info(numFiles + " XML files written."); + } + + if (failures > 0 || unsupportedActions > 0) { + logger.info(String.format("Outcomes: %s. Passed: %d, Failed: %d, Skipped: %d", + (successes + failures), successes, failures, unsupportedActions)); + } else { + logger.info(String.format("Outcomes: %s. All successful.", + (successes + failures))); + } + } + + /** + * Executes a single action and then prints the result. + */ + private void execute(final Action action) { + console.action(action.getName()); + + Outcome earlyFailure = outcomes.get(action.getName()); + if (earlyFailure == null) { + final Command command = mode.createActionCommand(action); + Future<List<String>> consoleOut = command.executeLater(); + final AtomicBoolean done = new AtomicBoolean(); + + actionTimeoutTimer.schedule(new TimerTask() { + @Override public void run() { + if (!done.get()) { + // TODO: set a "timout" bit somewhere so we know why this failed. + // currently we report ERROR for all timeouts. + logger.fine("killing " + action.getName() + " because it " + + "timed out after " + timeoutSeconds + " seconds"); + } + command.destroy(); + } + }, timeoutSeconds * 1000); + + boolean success = monitor.monitor(monitorPort, this); + done.set(true); + if (success) { + return; + } + + try { + earlyFailure = new Outcome(action.getName(), action.getName(), + Result.ERROR, consoleOut.get()); + } catch (Exception e) { + earlyFailure = new Outcome(action.getName(), Result.ERROR, e); + } + } + + if (earlyFailure.getResult() == Result.UNSUPPORTED) { + logger.fine("skipping " + action.getName()); + unsupportedActions++; + } else { + for (String line : earlyFailure.getOutputLines()) { + console.streamOutput(line + "\n"); + } + outcome(earlyFailure); + } + } + + public void outcome(Outcome outcome) { + Expectation expectation = expectationStore.get(outcome.getName()); + boolean ok = expectation.matches(outcome); + if (ok) { + successes++; + } else { + failures++; + } + console.outcome(outcome.getName()); + console.printResult(outcome.getResult(), ok); + } + + public void output(String outcomeName, String output) { + console.outcome(outcomeName); + console.streamOutput(output); + } +} diff --git a/tools/runner/java/dalvik/runner/Environment.java b/tools/runner/java/vogar/Environment.java index c284a37..8ccfc7b 100644 --- a/tools/runner/java/dalvik/runner/Environment.java +++ b/tools/runner/java/vogar/Environment.java @@ -14,7 +14,9 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; + +import vogar.commands.Rm; import java.io.File; import java.util.logging.Logger; @@ -38,53 +40,53 @@ abstract class Environment { } /** - * Initializes the temporary directories and test harness necessary to run - * tests. + * Initializes the temporary directories and harness necessary to run + * actions. */ abstract void prepare(); /** - * Prepares the directory from which the test will be executed. Some tests - * expect to read data files from the current working directory; this step - * should ensure such files are available. + * Prepares the directory from which the action will be executed. Some + * actions expect to read data files from the current working directory; + * this step should ensure such files are available. */ - abstract void prepareUserDir(TestRun testRun); + abstract void prepareUserDir(Action action); /** * Deletes files and releases any resources required for the execution of - * the given test. + * the given action. */ - void cleanup(TestRun testRun) { + void cleanup(Action action) { if (cleanAfter) { - logger.fine("clean " + testRun.getQualifiedName()); - new Rm().directoryTree(testCompilationDir(testRun)); - new Rm().directoryTree(testUserDir(testRun)); + logger.fine("clean " + action.getName()); + new Rm().directoryTree(actionCompilationDir(action)); + new Rm().directoryTree(actionUserDir(action)); } } - final File testDir(String name) { + final File actionDir(String name) { return new File(localTemp, name); } - final File testRunnerDir(String name) { - return new File(testDir("testrunner"), name); + final File runnerDir(String name) { + return new File(actionDir("testrunner"), name); } - final File testRunnerClassesDir() { - return testRunnerDir("classes"); + final File runnerClassesDir() { + return runnerDir("classes"); } - final File testCompilationDir(TestRun testRun) { - return new File(localTemp, testRun.getQualifiedName()); + final File actionCompilationDir(Action action) { + return new File(localTemp, action.getName()); } - final File testClassesDir(TestRun testRun) { - return new File(testCompilationDir(testRun), "classes"); + final File classesDir(Action action) { + return new File(actionCompilationDir(action), "classes"); } - final File testUserDir(TestRun testRun) { + final File actionUserDir(Action action) { File testTemp = new File(localTemp, "userDir"); - return new File(testTemp, testRun.getQualifiedName()); + return new File(testTemp, action.getName()); } abstract void shutdown(); diff --git a/tools/runner/java/dalvik/runner/EnvironmentDevice.java b/tools/runner/java/vogar/EnvironmentDevice.java index 9ac1c64..c49bbd9 100644 --- a/tools/runner/java/dalvik/runner/EnvironmentDevice.java +++ b/tools/runner/java/vogar/EnvironmentDevice.java @@ -14,23 +14,24 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; + +import vogar.commands.Adb; import java.io.File; -import java.util.logging.Logger; class EnvironmentDevice extends Environment { - private static final Logger logger = Logger.getLogger(EnvironmentDevice.class.getName()); - final Adb adb = new Adb(); final File runnerDir; - final File testTemp; + final File vogarTemp; + final int monitorPort; EnvironmentDevice (boolean cleanBefore, boolean cleanAfter, - Integer debugPort, File localTemp, File runnerDir) { + Integer debugPort, int monitorPort, File localTemp, File runnerDir) { super(cleanBefore, cleanAfter, debugPort, localTemp); this.runnerDir = runnerDir; - this.testTemp = new File(runnerDir, "/tests.tmp"); + this.vogarTemp = new File(runnerDir, "/vogar.tmp"); + this.monitorPort = monitorPort; } @Override void prepare() { @@ -40,28 +41,29 @@ class EnvironmentDevice extends Environment { adb.rm(runnerDir); } adb.mkdir(runnerDir); - adb.mkdir(testTemp); + adb.mkdir(vogarTemp); adb.mkdir(new File("/sdcard/dalvik-cache")); // TODO: only necessary on production devices. + adb.forwardTcp(monitorPort, monitorPort); if (debugPort != null) { adb.forwardTcp(debugPort, debugPort); } } - @Override protected void prepareUserDir(TestRun testRun) { - File testClassesDirOnDevice = testClassesDirOnDevice(testRun); - adb.mkdir(testClassesDirOnDevice); - adb.push(testRun.getTestDirectory(), testClassesDirOnDevice); - testRun.setUserDir(testClassesDirOnDevice); + @Override protected void prepareUserDir(Action action) { + File actionClassesDirOnDevice = actionClassesDirOnDevice(action); + adb.mkdir(actionClassesDirOnDevice); + adb.push(action.getJavaDirectory(), actionClassesDirOnDevice); + action.setUserDir(actionClassesDirOnDevice); } - private File testClassesDirOnDevice(TestRun testRun) { - return new File(runnerDir, testRun.getQualifiedName()); + private File actionClassesDirOnDevice(Action action) { + return new File(runnerDir, action.getName()); } - @Override void cleanup(TestRun testRun) { - super.cleanup(testRun); + @Override void cleanup(Action action) { + super.cleanup(action); if (cleanAfter) { - adb.rm(testClassesDirOnDevice(testRun)); + adb.rm(actionClassesDirOnDevice(action)); } } diff --git a/tools/runner/java/dalvik/runner/EnvironmentHost.java b/tools/runner/java/vogar/EnvironmentHost.java index d02ea55..e7ad4db 100644 --- a/tools/runner/java/dalvik/runner/EnvironmentHost.java +++ b/tools/runner/java/vogar/EnvironmentHost.java @@ -14,7 +14,10 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; + +import vogar.commands.Command; +import vogar.commands.Mkdir; import java.io.File; @@ -27,18 +30,18 @@ class EnvironmentHost extends Environment { @Override void prepare() {} - @Override protected void prepareUserDir(TestRun testRun) { - File testUserDir = testUserDir(testRun); + @Override protected void prepareUserDir(Action action) { + File actionUserDir = actionUserDir(action); // if the user dir exists, cp would copy the files to the wrong place - if (testUserDir.exists()) { + if (actionUserDir.exists()) { throw new IllegalStateException(); } - new Mkdir().mkdirs(testUserDir.getParentFile()); - new Command("cp", "-r", testRun.getTestDirectory().toString(), - testUserDir.toString()).execute(); - testRun.setUserDir(testUserDir); + new Mkdir().mkdirs(actionUserDir.getParentFile()); + new Command("cp", "-r", action.getJavaDirectory().toString(), + actionUserDir.toString()).execute(); + action.setUserDir(actionUserDir); } @Override void shutdown() {} diff --git a/tools/runner/java/vogar/Expectation.java b/tools/runner/java/vogar/Expectation.java new file mode 100644 index 0000000..b52b5a3 --- /dev/null +++ b/tools/runner/java/vogar/Expectation.java @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package vogar; + +import java.util.regex.Pattern; + +/** + * The expected result of an action execution. This is typically encoded in the + * expectations text file, which has the following format: + * <pre> + * test java.io.StreamTokenizer.Reset + * result UNSUPPORTED + * pattern .*should get token \[, but get -1.* + * + * # should we fix this? + * test java.util.Arrays.CopyMethods + * result COMPILE_FAILED + * pattern .*cannot find symbol.* + * </pre> + */ +final class Expectation { + + /** The pattern to use when no expected output is specified */ + private static final Pattern MATCH_ALL_PATTERN + = Pattern.compile(".*", Pattern.MULTILINE | Pattern.DOTALL); + + /** The expectation of a general successful run. */ + static final Expectation SUCCESS = new Expectation(Result.SUCCESS, null); + + /** The action's expected result, such as {@code EXEC_FAILED}. */ + private final Result result; + + /** The pattern the expected output will match. */ + private final Pattern pattern; + + public Expectation(Result result, String pattern) { + if (result == null) { + throw new IllegalArgumentException(); + } + + this.result = result; + this.pattern = pattern != null + ? Pattern.compile(pattern, Pattern.MULTILINE | Pattern.DOTALL) + : MATCH_ALL_PATTERN; + } + + public Result getResult() { + return result; + } + + /** + * Returns true if {@code outcome} matches this expectation. + */ + public boolean matches(Outcome outcome) { + return result == outcome.getResult() && patternMatches(outcome); + } + + /** + * Returns the failure message for this failed run. This message is intended + * to help to diagnose why the run result didn't match what was expected. + */ + public String getFailureMessage(Outcome outcome) { + StringBuilder builder = new StringBuilder(); + + if (result != Result.SUCCESS && result != outcome.getResult()) { + builder.append("Expected result: ") + .append(result) + .append("\n"); + } + + if (!patternMatches(outcome)) { + builder.append("Expected output to match \"") + .append(pattern.pattern()) + .append("\"\n"); + } + + for (String output : outcome.getOutputLines()) { + builder.append(output).append("\n"); + } + + return builder.toString(); + } + + private boolean patternMatches(Outcome outcome) { + return pattern.matcher(Strings.join(outcome.getOutputLines(), "\n")).matches(); + } +} diff --git a/tools/runner/java/dalvik/runner/ExpectedResult.java b/tools/runner/java/vogar/ExpectationStore.java index a0244ce..f10ae22 100644 --- a/tools/runner/java/dalvik/runner/ExpectedResult.java +++ b/tools/runner/java/vogar/ExpectationStore.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2009 The Android Open Source Project + * Copyright (C) 2010 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; import java.io.BufferedReader; import java.io.File; @@ -22,70 +22,65 @@ import java.io.FileReader; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Set; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; /** - * The expected outcome of a test execution. This is typically encoded in the - * expectations text file, which has the following format: - * <pre> - * test java.io.StreamTokenizer.Reset - * result UNSUPPORTED - * pattern .*should get token \[, but get -1.* - * - * # should we fix this? - * test java.util.Arrays.CopyMethods - * result COMPILE_FAILED - * pattern .*cannot find symbol.* - * </pre> + * A database of expected outcomes. */ -class ExpectedResult { +final class ExpectationStore { - private static final Logger logger = Logger.getLogger(ExpectedResult.class.getName()); + private static final Logger logger = Logger.getLogger(ExpectationStore.class.getName()); /** Matches lines in the file containing a key and value pair. */ private static final Pattern KEY_VALUE_PAIR_PATTERN = Pattern.compile("(\\w+)\\s+(.+)"); - /** The pattern to use when no expected output is specified */ - private static final Pattern MATCH_ALL_PATTERN - = Pattern.compile(".*", Pattern.MULTILINE | Pattern.DOTALL); + private final Map<String, Expectation> expectedResults; - /** The expectation of a general successful test run. */ - static final ExpectedResult SUCCESS = new ExpectedResult(Result.SUCCESS, null); + private ExpectationStore(Map<String, Expectation> expectedResults) { + this.expectedResults = expectedResults; + } - /** The test's expected result, such as {@code EXEC_FAILED}. */ - private final Result result; + /** + * Finds the expected result for the specified action or outcome. This + * returns a value for all names, even if no explicit expectation was set. + */ + public Expectation get(String name) { + while (true) { + Expectation expectation = expectedResults.get(name); + if (expectation != null) { + return expectation; + } - /** The pattern the expected output will match. */ - private final Pattern pattern; + int dot = name.lastIndexOf('.'); + if (dot == -1) { + return Expectation.SUCCESS; + } - private ExpectedResult(Result result, String pattern) { - if (result == null) { - throw new IllegalArgumentException(); + name = name.substring(0, dot); } - - this.result = result; - this.pattern = pattern != null - ? Pattern.compile(pattern, Pattern.MULTILINE | Pattern.DOTALL) - : MATCH_ALL_PATTERN; } - public Result getResult() { - return result; + public static ExpectationStore parse(Set<File> expectationFiles) throws IOException { + Map<String, Expectation> expectedResults = new HashMap<String, Expectation>(); + for (File f : expectationFiles) { + if (f.exists()) { + expectedResults.putAll(parse(f)); + } + } + return new ExpectationStore(expectedResults); } - public Pattern getPattern() { - return pattern; - } - public static Map<String, ExpectedResult> parse(File expectationsFile) + public static Map<String, Expectation> parse(File expectationsFile) throws IOException { logger.fine("loading expectations file " + expectationsFile); BufferedReader reader = new BufferedReader(new FileReader(expectationsFile)); try { - Map<String, ExpectedResult> results = new HashMap<String, ExpectedResult>(); + Map<String, Expectation> results = new HashMap<String, Expectation>(); Matcher keyValuePairMatcher = KEY_VALUE_PAIR_PATTERN.matcher(""); // the fields of interest for the current element @@ -119,8 +114,8 @@ class ExpectedResult { // when we encounter a new qualified name, the previous // element is complete. Add it to the results. if (qualifiedName != null) { - ExpectedResult expectation = new ExpectedResult(result, pattern); - ExpectedResult previous = results.put(qualifiedName, expectation); + Expectation expectation = new Expectation(result, pattern); + Expectation previous = results.put(qualifiedName, expectation); if (previous != null) { throw new IllegalArgumentException( "Duplicate expectations for " + qualifiedName); @@ -140,8 +135,8 @@ class ExpectedResult { // add the last element in the file if (qualifiedName != null) { - ExpectedResult expectation = new ExpectedResult(result, pattern); - ExpectedResult previous = results.put(qualifiedName, expectation); + Expectation expectation = new Expectation(result, pattern); + Expectation previous = results.put(qualifiedName, expectation); if (previous != null) { throw new IllegalArgumentException( "Duplicate expectations for " + qualifiedName); diff --git a/tools/runner/java/vogar/HostMonitor.java b/tools/runner/java/vogar/HostMonitor.java new file mode 100644 index 0000000..6c30d8d --- /dev/null +++ b/tools/runner/java/vogar/HostMonitor.java @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package vogar; + +import org.xml.sax.Attributes; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.Socket; +import java.util.Arrays; +import java.util.Collections; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Connects to a target process to monitor its action. + */ +class HostMonitor { + + private static final Logger logger = Logger.getLogger(HostMonitor.class.getName()); + + private final int MAX_CONNECT_ATTEMPTS = 10; + private final int CONNECTION_ATTEMPT_DELAY_MILLIS = 1000; + + /** + * Connect to the target process on the given port, read all of its + * outcomes into {@code handler}, and disconnect. + */ + public boolean monitor(int port, Handler handler) { + Socket socket; + InputStream in; + try { + int attempt = 0; + do { + socket = new Socket("localhost", port); + in = new BufferedInputStream(socket.getInputStream()); + if (checkStream(in)) { + logger.fine("action monitor connected to " + socket.getRemoteSocketAddress()); + break; + } + + if (attempt++ == MAX_CONNECT_ATTEMPTS) { + throw new IOException("Exceeded max connection attempts!"); + } + logger.fine("connection " + attempt + " to localhost:" + port + " is dead; retrying..."); + in.close(); + socket.close(); + try { + Thread.sleep(CONNECTION_ATTEMPT_DELAY_MILLIS); + } catch (InterruptedException e) { + } + } while (true); + } catch (IOException e) { + logger.log(Level.WARNING, "Failed to connect to localhost:" + port, e); + return false; + } + + try { + SAXParser parser = SAXParserFactory.newInstance().newSAXParser(); + InputSource inputSource = new InputSource(in); + parser.parse(inputSource, new ClientXmlHandler(handler)); + } catch (ParserConfigurationException e) { + throw new RuntimeException(e); + } catch (IOException e) { + logger.log(Level.WARNING, "Connection error from localhost:" + port, e); + return false; + } catch (SAXException e) { + logger.log(Level.WARNING, "Received bad XML from localhost:" + port, e); + return false; + } + + try { + socket.close(); + } catch (IOException ignored) { + } + + return true; + } + + /** + * Somewhere between the host and client process, broken socket connections + * are being accepted. Before we try to do any work on such a connection, + * check it to make sure it's not dead! + * + * TODO: file a bug (against adb?) for this + */ + private boolean checkStream(InputStream in) throws IOException { + in.mark(1); + if (in.read() == -1) { + return false; + } else { + in.reset(); + return true; + } + } + + /** + * Handles updates on the outcomes of a target process. + */ + public interface Handler { + + /** + * Receive a completed outcome. + */ + void outcome(Outcome outcome); + + /** + * Receive partial output from an action being executed. + */ + void output(String outcomeName, String output); + } + + class ClientXmlHandler extends DefaultHandler { + private final Handler handler; + + private String currentOutcomeName; + private String currentActionName; + private Result currentResult; + private StringBuilder output = new StringBuilder(); + + ClientXmlHandler(Handler handler) { + this.handler = handler; + } + + /* + * Our XML wire format looks like this: + * + * <?xml version='1.0' encoding='UTF-8' ?> + * <vogar> + * <outcome name="java.util.FormatterTest" action="java.util.FormatterTest"> + * test output + * more test output + * <result value="SUCCESS" /> + * </outcome> + * </vogar> + */ + + @Override public void startElement(String uri, String localName, + String qName, Attributes attributes) throws SAXException { + if (qName.equals("outcome")) { + if (currentOutcomeName != null) { + throw new IllegalStateException(); + } + + currentOutcomeName = attributes.getValue("name"); + currentActionName = attributes.getValue("action"); + return; + + } else if (qName.equals("result")) { + currentResult = Result.valueOf(attributes.getValue("value")); + return; + + } else if (!qName.equals("vogar")) { + throw new IllegalArgumentException("Unrecognized: " + qName); + } + } + + @Override public void characters(char[] ch, int start, int length) + throws SAXException { + if (currentOutcomeName != null) { + String text = new String(ch, start, length); + output.append(text); + handler.output(currentOutcomeName, text); + } + } + + @Override public void endElement(String uri, String localName, String qName) + throws SAXException { + if (qName.equals("outcome")) { + handler.outcome(new Outcome(currentOutcomeName, currentActionName, + currentResult, Collections.singletonList(output.toString()))); + currentOutcomeName = null; + currentActionName = null; + currentResult = null; + output.delete(0, output.length()); + } + } + } +} diff --git a/tools/runner/java/dalvik/runner/JUnitFinder.java b/tools/runner/java/vogar/JUnitFinder.java index 131a8cf..4d98f86 100644 --- a/tools/runner/java/dalvik/runner/JUnitFinder.java +++ b/tools/runner/java/vogar/JUnitFinder.java @@ -14,22 +14,22 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; + +import vogar.target.JUnitRunner; +import vogar.target.Runner; import java.io.File; /** - * Create {@link TestRun}s for {@code .java} files with JUnit tests in them. + * Create {@link Action}s for {@code .java} files with JUnit tests in them. */ class JUnitFinder extends NamingPatternCodeFinder { @Override protected boolean matches(File file) { - return file.getName().endsWith("Test.java"); - } - - // TODO: try to get names for each method? - @Override protected String testName(File file) { - return "junit"; + String filename = file.getName(); + return super.matches(file) + && (filename.endsWith("Test.java") || filename.endsWith("TestSuite.java")); } public Class<? extends Runner> getRunnerClass() { @@ -37,7 +37,7 @@ class JUnitFinder extends NamingPatternCodeFinder { } public File getRunnerJava() { - return new File(DalvikRunner.HOME_JAVA, "dalvik/runner/JUnitRunner.java"); + return new File(Vogar.HOME_JAVA, "vogar/target/JUnitRunner.java"); } public Classpath getRunnerClasspath() { diff --git a/tools/runner/java/dalvik/runner/JavaVm.java b/tools/runner/java/vogar/JavaVm.java index 38e0386..9c4e175 100644 --- a/tools/runner/java/dalvik/runner/JavaVm.java +++ b/tools/runner/java/vogar/JavaVm.java @@ -14,12 +14,10 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; import java.io.File; -import java.io.PrintStream; import java.util.List; -import java.util.Set; /** * A local Java virtual machine like Harmony or the RI. @@ -28,18 +26,18 @@ final class JavaVm extends Vm { private final File javaHome; - JavaVm(Integer debugPort, long timeoutSeconds, File sdkJar, PrintStream tee, + JavaVm(Integer debugPort, File sdkJar, List<String> javacArgs, int monitorPort, File localTemp, File javaHome, List<String> additionalVmArgs, boolean cleanBefore, boolean cleanAfter) { super(new EnvironmentHost(cleanBefore, cleanAfter, debugPort, localTemp), - timeoutSeconds, sdkJar, tee, additionalVmArgs); + sdkJar, javacArgs, additionalVmArgs, monitorPort); this.javaHome = javaHome; } - @Override protected void postCompileTestRunner() { + @Override protected void postCompileRunner() { } - @Override protected void postCompileTest(TestRun testRun) { + @Override protected void postCompile(Action action) { } @Override protected VmCommandBuilder newVmCommandBuilder( @@ -49,12 +47,12 @@ final class JavaVm extends Vm { .vmCommand(java) .workingDir(workingDirectory); } - @Override protected Classpath getRuntimeSupportClasspath(TestRun testRun) { + @Override protected Classpath getRuntimeSupportClasspath(Action action) { Classpath classpath = new Classpath(); - classpath.addAll(environment.testClassesDir(testRun)); - classpath.addAll(testClasspath); - classpath.addAll(environment.testRunnerClassesDir()); - classpath.addAll(testRunnerClasspath); + classpath.addAll(environment.classesDir(action)); + classpath.addAll(this.classpath); + classpath.addAll(environment.runnerClassesDir()); + classpath.addAll(runnerClasspath); return classpath; } } diff --git a/tools/runner/java/dalvik/runner/Javac.java b/tools/runner/java/vogar/Javac.java index 26e8bb9..c10a428 100644 --- a/tools/runner/java/dalvik/runner/Javac.java +++ b/tools/runner/java/vogar/Javac.java @@ -14,7 +14,9 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; + +import vogar.commands.Command; import java.io.File; import java.util.Arrays; @@ -29,7 +31,7 @@ final class Javac { private final Command.Builder builder = new Command.Builder(); Javac() { - builder.args("javac", "-Xmaxerrs", "1"); + builder.args("javac"); } public Javac bootClasspath(File... path) { @@ -56,6 +58,11 @@ final class Javac { return this; } + public Javac extra(List<String> extra) { + builder.args(extra); + return this; + } + public List<String> compile(Collection<File> files) { return builder.args(Strings.objectsToStrings(files)) .execute(); diff --git a/tools/runner/java/dalvik/runner/JtregFinder.java b/tools/runner/java/vogar/JtregFinder.java index d846ae2..7319b6b 100644 --- a/tools/runner/java/dalvik/runner/JtregFinder.java +++ b/tools/runner/java/vogar/JtregFinder.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; import com.sun.javatest.TestDescription; import com.sun.javatest.TestResult; @@ -22,6 +22,9 @@ import com.sun.javatest.TestResultTable; import com.sun.javatest.TestSuite; import com.sun.javatest.WorkDirectory; import com.sun.javatest.regtest.RegressionTestSuite; +import vogar.commands.Mkdir; +import vogar.target.JtregRunner; +import vogar.target.Runner; import java.io.File; import java.util.Collections; @@ -31,7 +34,7 @@ import java.util.Set; import java.util.logging.Logger; /** - * Create {@link TestRun}s for {@code .java} files with jtreg tests in them. + * Create {@link Action}s for {@code .java} files with jtreg tests in them. */ class JtregFinder implements CodeFinder { @@ -56,7 +59,7 @@ class JtregFinder implements CodeFinder { /** * Returns the tests in {@code directoryToScan}. */ - public Set<TestRun> findTests(File directoryToScan) { + public Set<Action> findActions(File directoryToScan) { // for now, jtreg doesn't know how to scan anything but directories if (!directoryToScan.isDirectory()) { return Collections.emptySet(); @@ -77,16 +80,13 @@ class JtregFinder implements CodeFinder { WorkDirectory wd = WorkDirectory.convert(workDirectory, testSuite); TestResultTable resultTable = wd.getTestResultTable(); - Set<TestRun> result = new LinkedHashSet<TestRun>(); + Set<Action> result = new LinkedHashSet<Action>(); for (Iterator i = resultTable.getIterator(); i.hasNext(); ) { TestResult testResult = (TestResult) i.next(); TestDescription description = testResult.getDescription(); String qualifiedName = qualifiedName(description); - String suiteName = suiteName(description); - String testName = description.getName(); String testClass = description.getName(); - result.add(new TestRun(description.getDir(), description.getFile(), - testClass, suiteName, testName, qualifiedName, + result.add(new Action(qualifiedName, testClass, description.getDir(), description.getFile(), description.getTitle(), getRunnerClass(), getRunnerJava(), getRunnerClasspath())); } @@ -130,7 +130,7 @@ class JtregFinder implements CodeFinder { } public File getRunnerJava() { - return new File(DalvikRunner.HOME_JAVA, "dalvik/runner/JtregRunner.java"); + return new File(Vogar.HOME_JAVA, "vogar/target/JtregRunner.java"); } public Classpath getRunnerClasspath() { diff --git a/tools/runner/java/dalvik/runner/MainFinder.java b/tools/runner/java/vogar/MainFinder.java index 282969f..e98098a 100644 --- a/tools/runner/java/dalvik/runner/MainFinder.java +++ b/tools/runner/java/vogar/MainFinder.java @@ -14,29 +14,24 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; + +import vogar.target.MainRunner; +import vogar.target.Runner; import java.io.File; /** - * Create {@link TestRun}s for {@code .java} files with main methods in them. + * Create {@link Action}s for {@code .java} files with main methods in them. */ class MainFinder extends NamingPatternCodeFinder { - @Override protected boolean matches(File file) { - return file.getName().endsWith(".java"); - } - - @Override protected String testName(File file) { - return "main"; - } - public Class<? extends Runner> getRunnerClass() { return MainRunner.class; } public File getRunnerJava() { - return new File(DalvikRunner.HOME_JAVA, "dalvik/runner/MainRunner.java"); + return new File(Vogar.HOME_JAVA, "vogar/target/MainRunner.java"); } public Classpath getRunnerClasspath() { diff --git a/tools/runner/java/dalvik/runner/Md5Cache.java b/tools/runner/java/vogar/Md5Cache.java index f6ba85d..b1844b8 100644 --- a/tools/runner/java/dalvik/runner/Md5Cache.java +++ b/tools/runner/java/vogar/Md5Cache.java @@ -14,7 +14,10 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; + +import vogar.commands.Command; +import vogar.commands.Mkdir; import java.io.File; import java.io.FileInputStream; diff --git a/tools/runner/java/vogar/Mode.java b/tools/runner/java/vogar/Mode.java new file mode 100644 index 0000000..d9032a0 --- /dev/null +++ b/tools/runner/java/vogar/Mode.java @@ -0,0 +1,228 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package vogar; + +import vogar.commands.Command; +import vogar.commands.CommandFailedException; +import vogar.commands.Mkdir; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +/** + * A Mode for running actions. Examples including running in a virtual machine + * either on the host or a device or within a specific context such as within an + * Activity. + */ +abstract class Mode { + + private static final Pattern JAVA_SOURCE_PATTERN = Pattern.compile("\\/(\\w)+\\.java$"); + + private static final Logger logger = Logger.getLogger(Mode.class.getName()); + + protected final Environment environment; + protected final File sdkJar; + protected final List<String> javacArgs; + protected final int monitorPort; + + /** + * Set of Java files needed to built to tun the currently selected set of + * actions. We build a subset rather than all the files all the time to + * reduce dex packaging costs in the activity mode case. + */ + protected final Set<File> runnerJava = new HashSet<File>(); + + /** + * Classpath of runner on the host side including any supporting libraries + * for runnerJava. Useful for compiling runnerJava as well as executing it + * on the host. Execution on the device requires further packaging typically + * done by postCompile. + */ + protected final Classpath runnerClasspath = new Classpath(); + + // TODO: this should be an immutable collection. + protected final Classpath classpath = Classpath.of( + new File("dalvik/libcore/tools/runner/lib/jsr305.jar"), + new File("dalvik/libcore/tools/runner/lib/guava.jar"), + new File("dalvik/libcore/tools/runner/lib/caliper.jar"), + // TODO: we should be able to work with a shipping SDK, not depend on out/... + // dalvik/libcore/**/test/ for junit + // TODO: jar up just the junit classes and drop the jar in our lib/ directory. + new File("out/target/common/obj/JAVA_LIBRARIES/core-tests-luni_intermediates/classes.jar").getAbsoluteFile()); + + Mode(Environment environment, File sdkJar, List<String> javacArgs, int monitorPort) { + this.environment = environment; + this.sdkJar = sdkJar; + this.javacArgs = javacArgs; + this.monitorPort = monitorPort; + } + + /** + * Initializes the temporary directories and harness necessary to run + * actions. + */ + protected void prepare(Set<File> runnerJava, Classpath runnerClasspath) { + this.runnerJava.add(new File(Vogar.HOME_JAVA, "vogar/target/TestRunner.java")); + this.runnerJava.addAll(dalvikAnnotationSourceFiles()); + this.runnerJava.addAll(runnerJava); + this.runnerClasspath.addAll(runnerClasspath); + environment.prepare(); + compileRunner(); + } + + private List<File> dalvikAnnotationSourceFiles() { + // Hopefully one day we'll strip the dalvik annotations out, but until then we need to make + // them available to javac(1). + File sourceDir = new File("dalvik/libcore/dalvik/src/main/java/dalvik/annotation"); + File[] javaSourceFiles = sourceDir.listFiles(new FilenameFilter() { + public boolean accept(File dir, String filename) { + return filename.endsWith(".java"); + } + }); + return Arrays.asList(javaSourceFiles); + } + + private void compileRunner() { + logger.fine("build runner"); + + Classpath classpath = new Classpath(); + classpath.addAll(this.classpath); + classpath.addAll(runnerClasspath); + + File base = environment.runnerClassesDir(); + new Mkdir().mkdirs(base); + new Javac() + .bootClasspath(sdkJar) + .classpath(classpath) + .sourcepath(Vogar.HOME_JAVA) + .destination(base) + .extra(javacArgs) + .compile(runnerJava); + postCompileRunner(); + } + + /** + * Hook method called after runner compilation. + */ + abstract protected void postCompileRunner(); + + /** + * Compiles classes for the given action and makes them ready for execution. + * + * @return null if the compilation succeeded, or an outcome describing the + * failure otherwise. + */ + public Outcome buildAndInstall(Action action) { + logger.fine("build " + action.getName()); + + try { + compile(action); + } catch (CommandFailedException e) { + return new Outcome(action.getName(), action.getName(), + Result.COMPILE_FAILED, e.getOutputLines()); + } catch (IOException e) { + return new Outcome(action.getName(), Result.ERROR, e); + } + environment.prepareUserDir(action); + return null; + } + + /** + * Compiles the classes for the described action. + * + * @throws CommandFailedException if javac fails + */ + private void compile(Action action) throws IOException { + if (!JAVA_SOURCE_PATTERN.matcher(action.getJavaFile().toString()).find()) { + throw new CommandFailedException(Collections.<String>emptyList(), + Collections.singletonList("Cannot compile: " + action.getJavaFile())); + } + + File classesDir = environment.classesDir(action); + new Mkdir().mkdirs(classesDir); + FileOutputStream propertiesOut = new FileOutputStream( + new File(classesDir, TestProperties.FILE)); + Properties properties = new Properties(); + fillInProperties(properties, action); + properties.store(propertiesOut, "generated by " + Mode.class.getName()); + propertiesOut.close(); + + Classpath classpath = new Classpath(); + classpath.addAll(this.classpath); + classpath.addAll(action.getRunnerClasspath()); + + Set<File> sourceFiles = new HashSet<File>(); + sourceFiles.add(action.getJavaFile()); + sourceFiles.addAll(dalvikAnnotationSourceFiles()); + + // compile the action case + new Javac() + .bootClasspath(sdkJar) + .classpath(classpath) + .sourcepath(action.getJavaDirectory()) + .destination(classesDir) + .extra(javacArgs) + .compile(sourceFiles); + postCompile(action); + } + + /** + * Hook method called after action compilation. + */ + abstract protected void postCompile(Action action); + + + /** + * Fill in properties for running in this mode + */ + protected void fillInProperties(Properties properties, Action action) { + properties.setProperty(TestProperties.TEST_CLASS, action.getTargetClass()); + properties.setProperty(TestProperties.QUALIFIED_NAME, action.getName()); + properties.setProperty(TestProperties.RUNNER_CLASS, action.getRunnerClass().getName()); + properties.setProperty(TestProperties.MONITOR_PORT, String.valueOf(monitorPort)); + } + + /** + * Create the command that executes the action. + */ + protected abstract Command createActionCommand(Action action); + + /** + * Deletes files and releases any resources required for the execution of + * the given action. + */ + void cleanup(Action action) { + environment.cleanup(action); + } + + /** + * Cleans up after all actions have completed. + */ + void shutdown() { + environment.shutdown(); + } +} diff --git a/tools/runner/java/dalvik/runner/NamingPatternCodeFinder.java b/tools/runner/java/vogar/NamingPatternCodeFinder.java index 19c9df2..d87a35f 100644 --- a/tools/runner/java/dalvik/runner/NamingPatternCodeFinder.java +++ b/tools/runner/java/vogar/NamingPatternCodeFinder.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; import java.io.File; import java.io.IOException; @@ -34,25 +34,24 @@ abstract class NamingPatternCodeFinder implements CodeFinder { private final String TYPE_DECLARATION_PATTERN = "(?m)\\b(?:public|private)\\s+(?:final\\s+)?(?:interface|class|enum)\\b"; - public Set<TestRun> findTests(File testDirectory) { - Set<TestRun> result = new LinkedHashSet<TestRun>(); - findTestsRecursive(result, testDirectory); + public Set<Action> findActions(File searchDirectory) { + Set<Action> result = new LinkedHashSet<Action>(); + findActionsRecursive(result, searchDirectory); return result; } /** - * Returns true if {@code file} contains a test class of this type. + * Returns true if {@code file} contains a action class of this type. */ protected boolean matches(File file) { - return file.getName().endsWith(".java"); + return (!file.getName().startsWith(".") + && file.getName().endsWith(".java")); } - protected abstract String testName(File file); - - private void findTestsRecursive(Set<TestRun> sink, File file) { + private void findActionsRecursive(Set<Action> sink, File file) { if (file.isDirectory()) { for (File child : file.listFiles()) { - findTestsRecursive(sink, child); + findActionsRecursive(sink, child); } return; } @@ -62,11 +61,9 @@ abstract class NamingPatternCodeFinder implements CodeFinder { } String className = fileToClass(file); - File testDirectory = file.getParentFile(); - String testName = testName(file); - String testDescription = null; - sink.add(new TestRun(testDirectory, file, className, className, - testName, className, testDescription, + File directory = file.getParentFile(); + String description = null; + sink.add(new Action(className, className, directory, file, description, getRunnerClass(), getRunnerJava(), getRunnerClasspath())); } diff --git a/tools/runner/java/dalvik/runner/Option.java b/tools/runner/java/vogar/Option.java index 779aa63..a73fbbf 100644 --- a/tools/runner/java/dalvik/runner/Option.java +++ b/tools/runner/java/vogar/Option.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/tools/runner/java/dalvik/runner/OptionParser.java b/tools/runner/java/vogar/OptionParser.java index 3516264..d031316 100644 --- a/tools/runner/java/dalvik/runner/OptionParser.java +++ b/tools/runner/java/vogar/OptionParser.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; import java.io.File; import java.lang.reflect.Field; diff --git a/tools/runner/java/vogar/Outcome.java b/tools/runner/java/vogar/Outcome.java new file mode 100644 index 0000000..253a3cc --- /dev/null +++ b/tools/runner/java/vogar/Outcome.java @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package vogar; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * An outcome of an action. Some actions may have multiple outcomes. For + * example, JUnit tests have one outcome for each test method. + */ +final class Outcome { + + private final String outcomeName; + private final String actionName; + private final Result result; + private final List<String> outputLines; + + public Outcome(String outcomeName, String actionName, Result result, + List<String> outputLines) { + this.outcomeName = outcomeName; + this.actionName = actionName; + this.result = result; + this.outputLines = outputLines; + } + + public Outcome(String actionName, Result result, String outputLine) { + this.outcomeName = actionName; + this.actionName = actionName; + this.result = result; + this.outputLines = Collections.singletonList(outputLine); + } + + public Outcome(String actionName, Result result, Throwable throwable) { + this.outcomeName = actionName; + this.actionName = actionName; + this.result = result; + this.outputLines = throwableToLines(throwable); + } + + public String getName() { + return outcomeName; + } + + public String getActionName() { + return actionName; + } + + public Result getResult() { + return result; + } + + public List<String> getOutputLines() { + return outputLines; + } + + private static List<String> throwableToLines(Throwable t) { + StringWriter writer = new StringWriter(); + PrintWriter out = new PrintWriter(writer); + t.printStackTrace(out); + return Arrays.asList(writer.toString().split("\\n")); + } + + /** + * Returns the action's suite name, such as java.lang.Integer or + * java.lang.IntegerTest. + */ + public String getSuiteName() { + int lastDot = outcomeName.lastIndexOf('.'); + return lastDot == -1 ? "defaultpackage" : outcomeName.substring(0, lastDot); + } + + /** + * Returns the specific action name, such as BitTwiddle or testBitTwiddle. + */ + public String getTestName() { + int lastDot = outcomeName.lastIndexOf('.'); + return lastDot == -1 ? outcomeName : outcomeName.substring(lastDot + 1); + } +} diff --git a/tools/runner/java/dalvik/runner/Result.java b/tools/runner/java/vogar/Result.java index 461f102..45c88ce 100644 --- a/tools/runner/java/dalvik/runner/Result.java +++ b/tools/runner/java/vogar/Result.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; /** * The result of a test or benchmark execution. @@ -22,7 +22,7 @@ package dalvik.runner; public enum Result { /** - * A test that cannot be run by this harness, such as a shell script. + * An action that cannot be run by this harness, such as a shell script. */ UNSUPPORTED, diff --git a/tools/runner/java/dalvik/runner/Strings.java b/tools/runner/java/vogar/Strings.java index e696841..d46d860 100644 --- a/tools/runner/java/dalvik/runner/Strings.java +++ b/tools/runner/java/vogar/Strings.java @@ -15,7 +15,7 @@ */ -package dalvik.runner; +package vogar; import java.io.BufferedReader; import java.io.File; @@ -31,7 +31,7 @@ import java.util.Iterator; */ public class Strings { - static String readFile(File f) throws IOException { + public static String readFile(File f) throws IOException { StringBuilder result = new StringBuilder(); BufferedReader in = new BufferedReader(new InputStreamReader(new FileInputStream(f), "UTF-8")); @@ -44,11 +44,11 @@ public class Strings { return result.toString(); } - static String join(Object[] objects, String delimiter) { + public static String join(Object[] objects, String delimiter) { return join(Arrays.asList(objects), delimiter); } - static String join(Iterable<?> objects, String delimiter) { + public static String join(Iterable<?> objects, String delimiter) { Iterator<?> i = objects.iterator(); if (!i.hasNext()) { return ""; @@ -62,7 +62,7 @@ public class Strings { return result.toString(); } - static String[] objectsToStrings(Object[] objects) { + public static String[] objectsToStrings(Object[] objects) { String[] result = new String[objects.length]; int i = 0; for (Object o : objects) { @@ -71,7 +71,7 @@ public class Strings { return result; } - static String[] objectsToStrings(Collection<?> objects) { + public static String[] objectsToStrings(Collection<?> objects) { return objectsToStrings(objects.toArray()); } } diff --git a/tools/runner/java/dalvik/runner/TestProperties.java b/tools/runner/java/vogar/TestProperties.java index 1e90799..d35f349 100644 --- a/tools/runner/java/dalvik/runner/TestProperties.java +++ b/tools/runner/java/vogar/TestProperties.java @@ -14,11 +14,11 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; /** * TestProperties is a common class of constants shared between the - * DalvikRunner on the host and TestRunner classes potentially running + * Vogar on the host and TestRunner classes potentially running * on other devices. */ final public class TestProperties { @@ -52,11 +52,10 @@ final public class TestProperties { */ public static final String DEVICE_RUNNER_DIR = "deviceRunnerDir"; - /** - * The output file written by TestActivity + * Port to accept monitor connections on. */ - public static final String RESULT_FILE = "result.txt"; + public static final String MONITOR_PORT = "monitorPort"; /** * Result value for successful test diff --git a/tools/runner/java/dalvik/runner/Threads.java b/tools/runner/java/vogar/Threads.java index 58b075b..35cc3ab 100644 --- a/tools/runner/java/dalvik/runner/Threads.java +++ b/tools/runner/java/vogar/Threads.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -23,7 +23,7 @@ import java.util.concurrent.ThreadFactory; /** * Utility methods for working with threads. */ -class Threads { +public class Threads { public static ThreadFactory daemonThreadFactory() { return new ThreadFactory() { diff --git a/tools/runner/java/dalvik/runner/Vm.java b/tools/runner/java/vogar/Vm.java index 8ff5858..b035266 100644 --- a/tools/runner/java/dalvik/runner/Vm.java +++ b/tools/runner/java/vogar/Vm.java @@ -14,64 +14,55 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; + +import vogar.commands.Command; +import vogar.target.TestRunner; import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.TimeoutException; -import java.util.logging.Logger; /** * A Java-like virtual machine for compiling and running tests. */ public abstract class Vm extends Mode { - private static final Logger logger = Logger.getLogger(Vm.class.getName()); - protected final List<String> additionalVmArgs; - Vm(Environment environment, long timeoutSeconds, File sdkJar, - PrintStream tee, List<String> additionalVmArgs) { - super(environment, timeoutSeconds, sdkJar, tee); + Vm(Environment environment, File sdkJar, List<String> javacArgs, + List<String> additionalVmArgs, int monitorPort) { + super(environment, sdkJar, javacArgs, monitorPort); this.additionalVmArgs = additionalVmArgs; } /** - * Returns a VM for test execution. + * Returns a VM for action execution. */ - @Override protected List<String> runTestCommand(TestRun testRun) - throws TimeoutException { - Command command = newVmCommandBuilder(testRun.getUserDir()) - .classpath(getRuntimeSupportClasspath(testRun)) - .userDir(testRun.getUserDir()) + @Override protected Command createActionCommand(Action action) { + return newVmCommandBuilder(action.getUserDir()) + .classpath(getRuntimeSupportClasspath(action)) + .userDir(action.getUserDir()) .debugPort(environment.debugPort) .vmArgs(additionalVmArgs) .mainClass(TestRunner.class.getName()) - .output(tee) .build(); - return command.executeWithTimeout(timeoutSeconds); } /** - * Returns a VM for test execution. + * Returns a VM for action execution. */ protected abstract VmCommandBuilder newVmCommandBuilder(File workingDirectory); /** * Returns the classpath containing JUnit and the dalvik annotations - * required for test execution. + * required for action execution. */ - protected abstract Classpath getRuntimeSupportClasspath(TestRun testRun); + protected abstract Classpath getRuntimeSupportClasspath(Action action); /** * Builds a virtual machine command. diff --git a/tools/runner/java/dalvik/runner/DalvikRunner.java b/tools/runner/java/vogar/Vogar.java index c78866e..c12c2d5 100644 --- a/tools/runner/java/dalvik/runner/DalvikRunner.java +++ b/tools/runner/java/vogar/Vogar.java @@ -14,37 +14,29 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; -import java.io.BufferedOutputStream; import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.PrintStream; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashSet; import java.util.List; +import java.util.Random; import java.util.Set; import java.util.UUID; -import java.util.logging.ConsoleHandler; -import java.util.logging.Formatter; -import java.util.logging.Level; -import java.util.logging.LogRecord; -import java.util.logging.Logger; /** * Command line interface for running benchmarks and tests on dalvik. */ -public final class DalvikRunner { +public final class Vogar { static final File HOME = new File("dalvik/libcore/tools/runner"); static final File HOME_JAVA = new File(HOME, "java"); private static class Options { - private final List<File> testFiles = new ArrayList<File>(); + private final List<File> actionFiles = new ArrayList<File>(); @Option(names = { "--expectations" }) private Set<File> expectationFiles = new LinkedHashSet<File>(); @@ -73,12 +65,17 @@ public final class DalvikRunner { @Option(names = { "--xml-reports-directory" }) private File xmlReportsDirectory; + @Option(names = { "--indent" }) + private String indent = " "; + @Option(names = { "--verbose" }) private boolean verbose; - @Option(names = { "--tee" }) - private String teeName; - private PrintStream tee; + @Option(names = { "--stream" }) + private boolean stream; + + @Option(names = { "--color" }) + private boolean color = true; @Option(names = { "--debug" }) private Integer debugPort; @@ -92,24 +89,27 @@ public final class DalvikRunner { @Option(names = { "--java-home" }) private File javaHome; + @Option(names = { "--javac-arg" }) + private List<String> javacArgs = new ArrayList<String>(); + @Option(names = { "--sdk" }) private File sdkJar = new File("/home/dalvik-prebuild/android-sdk-linux/platforms/android-2.0/android.jar"); private void printUsage() { - System.out.println("Usage: DalvikRunner [options]... <tests>..."); + System.out.println("Usage: Vogar [options]... <actions>..."); System.out.println(); - System.out.println(" <tests>: a .java file containing a jtreg test, JUnit test,"); - System.out.println(" Caliper benchmark, or a directory of such tests."); + System.out.println(" <actions>: .java files containing a jtreg tests, JUnit tests,"); + System.out.println(" Caliper benchmarks, or a directory of such tests."); System.out.println(); System.out.println("GENERAL OPTIONS"); System.out.println(); System.out.println(" --expectations <file>: include the specified file when looking for"); - System.out.println(" test expectations. The file should include qualified test names"); + System.out.println(" action expectations. The file should include qualified action names"); System.out.println(" and the corresponding expected output."); System.out.println(" Default is: " + expectationFiles); System.out.println(); System.out.println(" --mode <device|host|activity>: specify which environment to run the"); - System.out.println(" tests in. Options are on the device VM, on the host VM, and on"); + System.out.println(" actions in. Options are on the device VM, on the host VM, and on"); System.out.println(" device within an android.app.Activity."); System.out.println(" Default is: " + mode); System.out.println(); @@ -124,23 +124,29 @@ public final class DalvikRunner { System.out.println(" --clean: synonym for --clean-before and --clean-after (default)."); System.out.println(" Disable with --no-clean if you want no files removed."); System.out.println(); - System.out.println(" --tee <file>: emit test output to file during execution."); - System.out.println(" Specify '-' for stdout."); + System.out.println(" --color: format output in technicolor."); + System.out.println(); + System.out.println(" --stream: stream output as it is emitted."); System.out.println(); System.out.println(" --timeout-seconds <seconds>: maximum execution time of each"); - System.out.println(" test before the runner aborts it."); + System.out.println(" action before the runner aborts it. Specifying zero seconds"); + System.out.println(" or using --debug will disable the execution timeout"); System.out.println(" Default is: " + timeoutSeconds); System.out.println(); System.out.println(" --xml-reports-directory <path>: directory to emit JUnit-style"); System.out.println(" XML test results."); System.out.println(); + System.out.println(" --ident: amount to indent action result output. Can be set to ''"); + System.out.println(" (aka empty string) to simplify output parsing."); + System.out.println(" Default is: '" + indent + "'"); + System.out.println(); System.out.println(" --verbose: turn on verbose output"); System.out.println(); System.out.println("DEVICE OPTIONS"); System.out.println(); System.out.println(" --debug <port>: enable Java debugging on the specified port."); System.out.println(" This port must be free both on the device and on the local"); - System.out.println(" system."); + System.out.println(" system. Disables the timeout specified by --timeout-seconds."); System.out.println(); System.out.println(" --device-runner-dir <directory>: use the specified directory for"); System.out.println(" on-device temporary files and code."); @@ -153,7 +159,7 @@ public final class DalvikRunner { System.out.println(); System.out.println("HOST VM OPTIONS"); System.out.println(); - System.out.println(" --java-home <java_home>: execute the tests on the local workstation"); + System.out.println(" --java-home <java_home>: execute the actions on the local workstation"); System.out.println(" using the specified java home directory. This does not impact"); System.out.println(" which javac gets used. When unset, java is used from the PATH."); System.out.println(); @@ -165,12 +171,15 @@ public final class DalvikRunner { System.out.println(" a release version like 1.5."); System.out.println(" Default is: " + sdkJar); System.out.println(); + System.out.println(" --javac-arg <argument>: include the specified argument when invoking"); + System.out.println(" javac. Examples: --javac-arg -Xmaxerrs --javac-arg 1"); + System.out.println(); } private boolean parseArgs(String[] args) { - final List<String> testFilenames; + final List<String> actionFilenames; try { - testFilenames = new OptionParser(this).parse(args); + actionFilenames = new OptionParser(this).parse(args); } catch (RuntimeException e) { System.out.println(e.getMessage()); return false; @@ -228,8 +237,8 @@ public final class DalvikRunner { return false; } - if (testFilenames.isEmpty()) { - System.out.println("No tests provided."); + if (actionFilenames.isEmpty()) { + System.out.println("No actions provided."); return false; } @@ -242,25 +251,13 @@ public final class DalvikRunner { // Post-processing arguments // - for (String testFilename : testFilenames) { - testFiles.add(new File(testFilename)); + // disable timeout when debugging + if (debugPort != null) { + timeoutSeconds = 0; } - if (teeName != null) { - if (teeName.equals("-")) { - tee = System.out; - } else { - try { - tee = new PrintStream(new BufferedOutputStream(new FileOutputStream(teeName))); - } catch (FileNotFoundException e) { - System.out.println("Could not open file teeName: " + e); - return false; - } - } - } - - if (verbose) { - Logger.getLogger("dalvik.runner").setLevel(Level.FINE); + for (String actionFilename : actionFilenames) { + actionFiles.add(new File(actionFilename)); } return true; @@ -271,29 +268,20 @@ public final class DalvikRunner { private final Options options = new Options(); private final File localTemp = new File("/tmp/dalvikrunner/" + UUID.randomUUID()); - private DalvikRunner() {} - - private void prepareLogging() { - ConsoleHandler handler = new ConsoleHandler(); - handler.setLevel(Level.ALL); - handler.setFormatter(new Formatter() { - @Override public String format(LogRecord r) { - return r.getMessage() + "\n"; - } - }); - Logger logger = Logger.getLogger("dalvik.runner"); - logger.addHandler(handler); - logger.setUseParentHandlers(false); - } + private Vogar() {} private void run() { + Console console = new Console(options.stream, options.indent, options.color); + console.configureJavaLogging(options.verbose); + + int monitorPort = 8787; Mode mode; if (options.mode.equals(Options.MODE_DEVICE)) { mode = new DeviceDalvikVm( options.debugPort, - options.timeoutSeconds, options.sdkJar, - options.tee, + options.javacArgs, + monitorPort, localTemp, options.vmArgs, options.cleanBefore, @@ -302,20 +290,21 @@ public final class DalvikRunner { } else if (options.mode.equals(Options.MODE_HOST)) { mode = new JavaVm( options.debugPort, - options.timeoutSeconds, options.sdkJar, - options.tee, + options.javacArgs, + monitorPort, localTemp, options.javaHome, options.vmArgs, options.cleanBefore, - options.cleanAfter); + options.cleanAfter + ); } else if (options.mode.equals(Options.MODE_ACTIVITY)) { mode = new ActivityMode( options.debugPort, - options.timeoutSeconds, options.sdkJar, - options.tee, + options.javacArgs, + monitorPort, localTemp, options.cleanBefore, options.cleanAfter, @@ -325,35 +314,48 @@ public final class DalvikRunner { return; } + HostMonitor monitor = new HostMonitor(); + List<CodeFinder> codeFinders = Arrays.asList( new JtregFinder(localTemp), new JUnitFinder(), new CaliperFinder(), new MainFinder()); - Driver driver = new Driver( - localTemp, - mode, - options.expectationFiles, - options.xmlReportsDirectory, - codeFinders); + + ExpectationStore expectationStore; try { - driver.loadExpectations(); + expectationStore = ExpectationStore.parse(options.expectationFiles); } catch (IOException e) { System.out.println("Problem loading expectations: " + e); return; } - driver.buildAndRunAllTests(options.testFiles); + XmlReportPrinter xmlReportPrinter = options.xmlReportsDirectory != null + ? new XmlReportPrinter(options.xmlReportsDirectory, expectationStore) + : null; + + Driver driver = new Driver( + localTemp, + mode, + expectationStore, + codeFinders, + xmlReportPrinter, + console, + monitor, + monitorPort, + options.timeoutSeconds); + + driver.buildAndRunAllActions(options.actionFiles); + mode.shutdown(); } public static void main(String[] args) { - DalvikRunner dalvikRunner = new DalvikRunner(); - if (!dalvikRunner.options.parseArgs(args)) { - dalvikRunner.options.printUsage(); + Vogar vogar = new Vogar(); + if (!vogar.options.parseArgs(args)) { + vogar.options.printUsage(); return; } - dalvikRunner.prepareLogging(); - dalvikRunner.run(); + vogar.run(); } } diff --git a/tools/runner/java/dalvik/runner/XmlReportPrinter.java b/tools/runner/java/vogar/XmlReportPrinter.java index 669a26c..2cd8c66 100644 --- a/tools/runner/java/dalvik/runner/XmlReportPrinter.java +++ b/tools/runner/java/vogar/XmlReportPrinter.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package dalvik.runner; +package vogar; import org.kxml2.io.KXmlSerializer; @@ -52,7 +52,6 @@ public class XmlReportPrinter { private static final String ATTR_FAILURES = "failures"; private static final String ATTR_TESTS = "tests"; private static final String ATTR_TYPE = "type"; - private static final String ATTR_MESSAGE = "message"; private static final String PROPERTIES = "properties"; private static final String ATTR_CLASSNAME = "classname"; private static final String TIMESTAMP = "timestamp"; @@ -61,10 +60,18 @@ public class XmlReportPrinter { /** the XML namespace */ private static final String ns = null; + private final File directory; + private final ExpectationStore expectationStore; + + public XmlReportPrinter(File directory, ExpectationStore expectationStore) { + this.directory = directory; + this.expectationStore = expectationStore; + } + /** * Populates the directory with the report data from the completed tests. */ - public int generateReports(File directory, Collection<TestRun> results) { + public int generateReports(Collection<Outcome> results) { Map<String, Suite> suites = testsToSuites(results); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); @@ -100,24 +107,25 @@ public class XmlReportPrinter { return suites.size(); } - private Map<String, Suite> testsToSuites(Collection<TestRun> testRuns) { + private Map<String, Suite> testsToSuites(Collection<Outcome> outcomes) { Map<String, Suite> result = new LinkedHashMap<String, Suite>(); - for (TestRun testRun : testRuns) { - if (testRun.getResult() == Result.UNSUPPORTED) { + for (Outcome outcome : outcomes) { + if (outcome.getResult() == Result.UNSUPPORTED) { continue; } - String suiteName = testRun.getSuiteName(); + String suiteName = outcome.getSuiteName(); Suite suite = result.get(suiteName); if (suite == null) { suite = new Suite(suiteName); result.put(suiteName, suite); } - suite.tests.add(testRun); + suite.outcomes.add(outcome); - if (!testRun.isExpectedResult()) { - if (testRun.getResult() == Result.EXEC_FAILED) { + Expectation expectation = expectationStore.get(outcome.getName()); + if (!expectation.matches(outcome)) { + if (outcome.getResult() == Result.EXEC_FAILED) { suite.failuresCount++; } else { suite.errorsCount++; @@ -127,9 +135,9 @@ public class XmlReportPrinter { return result; } - static class Suite { + class Suite { private final String name; - private final List<TestRun> tests = new ArrayList<TestRun>(); + private final List<Outcome> outcomes = new ArrayList<Outcome>(); private int failuresCount; private int errorsCount; @@ -140,7 +148,7 @@ public class XmlReportPrinter { void print(KXmlSerializer serializer, String timestamp) throws IOException { serializer.startTag(ns, TESTSUITE); serializer.attribute(ns, ATTR_NAME, name); - serializer.attribute(ns, ATTR_TESTS, Integer.toString(tests.size())); + serializer.attribute(ns, ATTR_TESTS, Integer.toString(outcomes.size())); serializer.attribute(ns, ATTR_FAILURES, Integer.toString(failuresCount)); serializer.attribute(ns, ATTR_ERRORS, Integer.toString(errorsCount)); serializer.attribute(ns, ATTR_TIME, "0"); @@ -149,28 +157,25 @@ public class XmlReportPrinter { serializer.startTag(ns, PROPERTIES); serializer.endTag(ns, PROPERTIES); - for (TestRun testRun : tests) { - print(serializer, testRun); + for (Outcome outcome : outcomes) { + print(serializer, outcome); } serializer.endTag(ns, TESTSUITE); } - void print(KXmlSerializer serializer, TestRun testRun) throws IOException { + void print(KXmlSerializer serializer, Outcome outcome) throws IOException { serializer.startTag(ns, TESTCASE); - serializer.attribute(ns, ATTR_NAME, testRun.getTestName()); - serializer.attribute(ns, ATTR_CLASSNAME, testRun.getSuiteName()); + serializer.attribute(ns, ATTR_NAME, outcome.getTestName()); + serializer.attribute(ns, ATTR_CLASSNAME, outcome.getSuiteName()); serializer.attribute(ns, ATTR_TIME, "0"); - if (!testRun.isExpectedResult()) { - String result = testRun.getResult() == Result.EXEC_FAILED ? FAILURE : ERROR; + Expectation expectation = expectationStore.get(outcome.getName()); + if (!expectation.matches(outcome)) { + String result = outcome.getResult() == Result.EXEC_FAILED ? FAILURE : ERROR; serializer.startTag(ns, result); - String title = testRun.getDescription(); - if (title != null && title.length() > 0) { - serializer.attribute(ns, ATTR_MESSAGE, title); - } - serializer.attribute(ns, ATTR_TYPE, testRun.getResult().toString()); - String text = sanitize(Strings.join(testRun.getOutputLines(), "\n")); + serializer.attribute(ns, ATTR_TYPE, outcome.getResult().toString()); + String text = sanitize(Strings.join(outcome.getOutputLines(), "\n")); serializer.text(text); serializer.endTag(ns, result); } @@ -185,4 +190,4 @@ public class XmlReportPrinter { return text.replace("\0", "<\\0>"); } } -}
\ No newline at end of file +} diff --git a/tools/runner/java/dalvik/runner/Aapt.java b/tools/runner/java/vogar/commands/Aapt.java index 4d1a873..3778586 100644 --- a/tools/runner/java/dalvik/runner/Aapt.java +++ b/tools/runner/java/vogar/commands/Aapt.java @@ -14,14 +14,14 @@ * limitations under the License. */ -package dalvik.runner; +package vogar.commands; import java.io.File; /** * An aapt (Android Asset Packaging Tool) command. */ -final class Aapt { +public final class Aapt { public void apk(File apk, File manifest) { diff --git a/tools/runner/java/dalvik/runner/Adb.java b/tools/runner/java/vogar/commands/Adb.java index c982058..fd746fa 100644 --- a/tools/runner/java/dalvik/runner/Adb.java +++ b/tools/runner/java/vogar/commands/Adb.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package dalvik.runner; +package vogar.commands; import java.io.File; import java.util.List; @@ -23,7 +23,7 @@ import java.util.concurrent.TimeoutException; /** * An adb command. */ -final class Adb { +public final class Adb { public void mkdir(File name) { new Command("adb", "shell", "mkdir", name.getPath()).execute(); diff --git a/tools/runner/java/dalvik/runner/Command.java b/tools/runner/java/vogar/commands/Command.java index 88ba38e..8a014b5 100644 --- a/tools/runner/java/dalvik/runner/Command.java +++ b/tools/runner/java/vogar/commands/Command.java @@ -14,7 +14,10 @@ * limitations under the License. */ -package dalvik.runner; +package vogar.commands; + +import vogar.Strings; +import vogar.Threads; import java.io.BufferedReader; import java.io.File; @@ -38,7 +41,7 @@ import java.util.logging.Logger; /** * An out of process executable. */ -final class Command { +public final class Command { private final Logger logger = Logger.getLogger(Command.class.getName()); @@ -46,13 +49,13 @@ final class Command { private final File workingDirectory; private final boolean permitNonZeroExitStatus; private final PrintStream tee; - private Process process; + private volatile Process process; - Command(String... args) { + public Command(String... args) { this(Arrays.asList(args)); } - Command(List<String> args) { + public Command(List<String> args) { this.args = new ArrayList<String>(args); this.workingDirectory = null; this.permitNonZeroExitStatus = false; @@ -70,7 +73,7 @@ final class Command { return Collections.unmodifiableList(args); } - public synchronized void start() throws IOException { + public void start() throws IOException { if (isStarted()) { throw new IllegalStateException("Already started!"); } @@ -91,15 +94,7 @@ final class Command { return process != null; } - public Process getProcess() { - if (!isStarted()) { - throw new IllegalStateException("Not started!"); - } - - return process; - } - - public synchronized List<String> gatherOutput() + public List<String> gatherOutput() throws IOException, InterruptedException { if (!isStarted()) { throw new IllegalStateException("Not started!"); @@ -127,7 +122,7 @@ final class Command { return outputLines; } - public synchronized List<String> execute() { + public List<String> execute() { try { start(); return gatherOutput(); @@ -139,38 +134,58 @@ final class Command { } /** - * Executes a command with a specified timeout. Output is returned - * if the command succeeds. If Otherwise null is returned if the - * command timed out. + * Executes a command with a specified timeout. If the process does not + * complete normally before the timeout has elapsed, it will be destroyed. + * + * @param timeoutSeconds how long to wait, or 0 to wait indefinitely + * @return the command's output, or null if the command timed out */ public List<String> executeWithTimeout(long timeoutSeconds) throws TimeoutException { - ExecutorService outputReader - = Executors.newFixedThreadPool(1, Threads.daemonThreadFactory()); + if (timeoutSeconds == 0) { + return execute(); + } + try { - start(); - // run on a different thread to allow a timeout - Future<List<String>> future = outputReader.submit(new Callable<List<String>>() { - public List<String> call() throws Exception { - return gatherOutput(); - } - }); - return future.get(timeoutSeconds, TimeUnit.SECONDS); - } catch (IOException e) { - throw new RuntimeException("Failed to execute process: " + args, e); + return executeLater().get(timeoutSeconds, TimeUnit.SECONDS); } catch (InterruptedException e) { throw new RuntimeException("Interrupted while executing process: " + args, e); } catch (ExecutionException e) { throw new RuntimeException(e); } finally { - if (isStarted()) { - getProcess().destroy(); // to release the output reader + destroy(); + } + } + + /** + * Executes the command on a new background thread. This method returns + * immediately. + * + * @return a future to retrieve the command's output. + */ + public Future<List<String>> executeLater() { + ExecutorService executor = Executors.newFixedThreadPool( + 1, Threads.daemonThreadFactory()); + Future<List<String>> result = executor.submit(new Callable<List<String>>() { + public List<String> call() throws Exception { + start(); + return gatherOutput(); } - outputReader.shutdown(); + }); + executor.shutdown(); + return result; + } + + /** + * Destroys the underlying process and closes its associated streams. + */ + public void destroy() { + if (process != null) { + process.destroy(); } } - static class Builder { + public static class Builder { private final List<String> args = new ArrayList<String>(); private File workingDirectory; private boolean permitNonZeroExitStatus = false; diff --git a/tools/runner/java/dalvik/runner/CommandFailedException.java b/tools/runner/java/vogar/commands/CommandFailedException.java index d16a279..8d1fa33 100644 --- a/tools/runner/java/dalvik/runner/CommandFailedException.java +++ b/tools/runner/java/vogar/commands/CommandFailedException.java @@ -14,14 +14,14 @@ * limitations under the License. */ -package dalvik.runner; +package vogar.commands; import java.util.List; /** * Thrown when an out of process executable does not return normally. */ -class CommandFailedException extends RuntimeException { +public class CommandFailedException extends RuntimeException { private final List<String> args; private final List<String> outputLines; diff --git a/tools/runner/java/dalvik/runner/Dx.java b/tools/runner/java/vogar/commands/Dx.java index 393b70d..678a294 100644 --- a/tools/runner/java/dalvik/runner/Dx.java +++ b/tools/runner/java/vogar/commands/Dx.java @@ -14,7 +14,11 @@ * limitations under the License. */ -package dalvik.runner; +package vogar.commands; + +import vogar.Classpath; +import vogar.Md5Cache; +import vogar.Strings; import java.io.File; import java.util.logging.Logger; @@ -22,7 +26,7 @@ import java.util.logging.Logger; /** * A dx command. */ -final class Dx { +public final class Dx { private static final Logger logger = Logger.getLogger(Dx.class.getName()); private static final Md5Cache DEX_CACHE = new Md5Cache("dex"); diff --git a/tools/runner/java/dalvik/runner/Mkdir.java b/tools/runner/java/vogar/commands/Mkdir.java index 46dcf08..fc08f1b 100644 --- a/tools/runner/java/dalvik/runner/Mkdir.java +++ b/tools/runner/java/vogar/commands/Mkdir.java @@ -14,14 +14,14 @@ * limitations under the License. */ -package dalvik.runner; +package vogar.commands; import java.io.File; /** * A mkdir command. */ -final class Mkdir { +public final class Mkdir { public void mkdirs(File directory) { new Command("mkdir", "-p", directory.getPath()).execute(); diff --git a/tools/runner/java/dalvik/runner/Rm.java b/tools/runner/java/vogar/commands/Rm.java index 1fc11d9..425bb5d 100644 --- a/tools/runner/java/dalvik/runner/Rm.java +++ b/tools/runner/java/vogar/commands/Rm.java @@ -14,14 +14,14 @@ * limitations under the License. */ -package dalvik.runner; +package vogar.commands; import java.io.File; /** * A rm command. */ -final class Rm { +public final class Rm { public void file(File file) { new Command.Builder() diff --git a/tools/runner/java/dalvik/runner/CaliperRunner.java b/tools/runner/java/vogar/target/CaliperRunner.java index b30644b..5b424c8 100644 --- a/tools/runner/java/dalvik/runner/CaliperRunner.java +++ b/tools/runner/java/vogar/target/CaliperRunner.java @@ -14,24 +14,31 @@ * limitations under the License. */ -package dalvik.runner; +package vogar.target; import com.google.caliper.Benchmark; import com.google.caliper.Runner; +import vogar.Result; /** * Runs a <a href="http://code.google.com/p/caliper/">Caliper</a> benchmark. */ -public final class CaliperRunner implements dalvik.runner.Runner { +public final class CaliperRunner implements vogar.target.Runner { - public void prepareTest(Class<?> testClass) {} + private TargetMonitor monitor; - public boolean test(Class<?> testClass) { + public void init(TargetMonitor monitor, String actionName, + Class<?> testClass) { + this.monitor = monitor; + } + + public void run(String actionName, Class<?> testClass) { + monitor.outcomeStarted(actionName, actionName); try { Runner.main(testClass.asSubclass(Benchmark.class), new String[0]); } catch (Exception ex) { ex.printStackTrace(); } - return false; // always print benchmarking results + monitor.outcomeFinished(Result.SUCCESS); } } diff --git a/tools/runner/java/vogar/target/JUnitRunner.java b/tools/runner/java/vogar/target/JUnitRunner.java new file mode 100644 index 0000000..767d80d --- /dev/null +++ b/tools/runner/java/vogar/target/JUnitRunner.java @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2009 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package vogar.target; + +import junit.framework.AssertionFailedError; +import junit.framework.Test; +import junit.framework.TestResult; +import junit.runner.BaseTestRunner; +import junit.runner.TestSuiteLoader; +import junit.textui.ResultPrinter; +import vogar.Result; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Adapts a JUnit test for use by vogar. + */ +public final class JUnitRunner implements Runner { + + private static final Pattern NAME_THEN_TEST_CLASS = Pattern.compile("(.*)\\(([\\w\\.$]+)\\)"); + + private junit.textui.TestRunner testRunner; + private Test junitTest; + + public void init(TargetMonitor monitor, String actionName, Class<?> testClass) { + final TestSuiteLoader testSuiteLoader = new TestSuiteLoader() { + public Class load(String suiteClassName) throws ClassNotFoundException { + return JUnitRunner.class.getClassLoader().loadClass(suiteClassName); + } + + public Class reload(Class c) { + return c; + } + }; + + testRunner = new junit.textui.TestRunner( + new MonitoringResultPrinter(monitor, actionName)) { + @Override public TestSuiteLoader getLoader() { + return testSuiteLoader; + } + }; + + this.junitTest = testRunner.getTest(testClass.getName()); + } + + public void run(String actionName, Class<?> testClass) { + testRunner.doRun(junitTest); + } + + /** + * Returns the vogar name like {@code tests.xml.DomTest#testFoo} for a test + * with a JUnit name like {@code testFoo(tests.xml.DomTest)}. + */ + private String getOutcomeName(Test test) { + String testToString = test.toString(); + + Matcher matcher = NAME_THEN_TEST_CLASS.matcher(testToString); + if (matcher.matches()) { + return matcher.group(2) + "#" + matcher.group(1); + } + + return testToString; + } + + /** + * This result printer posts test names, output and exceptions to the + * hosting process. + */ + private class MonitoringResultPrinter extends ResultPrinter { + private final TargetMonitor monitor; + private final String actionName; + + private Test current; + private Throwable failure; + + public MonitoringResultPrinter(TargetMonitor monitor, + String actionName) { + super(System.out); + this.monitor = monitor; + this.actionName = actionName; + } + + @Override public void addError(Test test, Throwable t) { + System.out.println(BaseTestRunner.getFilteredTrace(t)); + failure = t; + } + + @Override public void addFailure(Test test, AssertionFailedError t) { + System.out.println(BaseTestRunner.getFilteredTrace(t)); + failure = t; + } + + @Override public void endTest(Test test) { + if (current == null) { + throw new IllegalStateException(); + } + monitor.outcomeFinished( + failure == null ? Result.SUCCESS : Result.EXEC_FAILED); + current = null; + failure = null; + } + + @Override public void startTest(Test test) { + if (current != null) { + throw new IllegalStateException(); + } + current = test; + monitor.outcomeStarted(getOutcomeName(test), actionName); + } + + @Override protected void printHeader(long runTime) {} + @Override protected void printErrors(TestResult result) {} + @Override protected void printFailures(TestResult result) {} + @Override protected void printFooter(TestResult result) {} + } +} diff --git a/tools/runner/java/dalvik/runner/JtregRunner.java b/tools/runner/java/vogar/target/JtregRunner.java index 633a529..a6c7f4f 100644 --- a/tools/runner/java/dalvik/runner/JtregRunner.java +++ b/tools/runner/java/vogar/target/JtregRunner.java @@ -14,7 +14,9 @@ * limitations under the License. */ -package dalvik.runner; +package vogar.target; + +import vogar.Result; import java.lang.reflect.Method; @@ -24,8 +26,11 @@ import java.lang.reflect.Method; public final class JtregRunner implements Runner { private Method main; + private TargetMonitor monitor; - public void prepareTest(Class<?> testClass) { + public void init(TargetMonitor monitor, String actionName, + Class<?> testClass) { + this.monitor = monitor; try { main = testClass.getMethod("main", String[].class); } catch (Exception e) { @@ -33,13 +38,14 @@ public final class JtregRunner implements Runner { } } - public boolean test(Class<?> testClass) { + public void run(String actionName, Class<?> testClass) { + monitor.outcomeStarted(actionName, actionName); try { main.invoke(null, new Object[] { new String[0] }); - return true; + monitor.outcomeFinished(Result.SUCCESS); } catch (Throwable failure) { failure.printStackTrace(); - return false; + monitor.outcomeFinished(Result.EXEC_FAILED); } } } diff --git a/tools/runner/java/dalvik/runner/MainRunner.java b/tools/runner/java/vogar/target/MainRunner.java index 34a4a47..c091795 100644 --- a/tools/runner/java/dalvik/runner/MainRunner.java +++ b/tools/runner/java/vogar/target/MainRunner.java @@ -14,7 +14,9 @@ * limitations under the License. */ -package dalvik.runner; +package vogar.target; + +import vogar.Result; import java.lang.reflect.Method; @@ -23,9 +25,12 @@ import java.lang.reflect.Method; */ public final class MainRunner implements Runner { + private TargetMonitor monitor; private Method main; - public void prepareTest(Class<?> testClass) { + public void init(TargetMonitor monitor, String actionName, + Class<?> testClass) { + this.monitor = monitor; try { main = testClass.getMethod("main", String[].class); } catch (Exception e) { @@ -33,12 +38,13 @@ public final class MainRunner implements Runner { } } - public boolean test(Class<?> testClass) { + public void run(String actionName, Class<?> testClass) { + monitor.outcomeStarted(actionName, actionName); try { main.invoke(null, new Object[] { new String[0] }); } catch (Throwable ex) { ex.printStackTrace(); } - return false; // always print main method output + monitor.outcomeFinished(Result.SUCCESS); } } diff --git a/tools/runner/java/dalvik/runner/Runner.java b/tools/runner/java/vogar/target/Runner.java index 7d7b0ee..af98a00 100644 --- a/tools/runner/java/dalvik/runner/Runner.java +++ b/tools/runner/java/vogar/target/Runner.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package dalvik.runner; +package vogar.target; /** * Interface between the generic TestRunner and the more specific @@ -22,7 +22,8 @@ package dalvik.runner; */ public interface Runner { - public void prepareTest(Class<?> testClass); + public void init(TargetMonitor monitor, String actionName, + Class<?> testClass); - public boolean test(Class<?> testClass); + public void run(String actionName, Class<?> testClass); } diff --git a/tools/runner/java/vogar/target/TargetMonitor.java b/tools/runner/java/vogar/target/TargetMonitor.java new file mode 100644 index 0000000..c14c09f --- /dev/null +++ b/tools/runner/java/vogar/target/TargetMonitor.java @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package vogar.target; + +import org.xmlpull.v1.XmlPullParserException; +import org.xmlpull.v1.XmlPullParserFactory; +import org.xmlpull.v1.XmlSerializer; +import vogar.Result; + +import java.io.IOException; +import java.net.ServerSocket; +import java.net.Socket; + +/** + * Accepts a connection for a host process to monitor this action. + */ +class TargetMonitor { + + private static final int ACCEPT_TIMEOUT_MILLIS = 10 * 1000; + + private static final String ns = null; // no namespaces + ServerSocket serverSocket; + private Socket socket; + private XmlSerializer serializer; + + public void await(int port) { + if (socket != null) { + throw new IllegalStateException(); + } + + try { + serverSocket = new ServerSocket(port); + serverSocket.setSoTimeout(ACCEPT_TIMEOUT_MILLIS); + serverSocket.setReuseAddress(true); + socket = serverSocket.accept(); + + serializer = XmlPullParserFactory.newInstance().newSerializer(); + serializer.setOutput(socket.getOutputStream(), "UTF-8"); + serializer.startDocument("UTF-8", null); + serializer.startTag(ns, "vogar"); + } catch (IOException e) { + throw new RuntimeException("Failed to accept a monitor on localhost:" + port, e); + } catch (XmlPullParserException e) { + throw new RuntimeException(e); + } + } + + public void outcomeStarted(String outcomeName, String actionName) { + try { + serializer.startTag(ns, "outcome"); + serializer.attribute(ns, "name", outcomeName); + serializer.attribute(ns, "action", actionName); + serializer.flush(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public void output(String text) { + try { + serializer.text(text); + serializer.flush(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public void outcomeFinished(Result result) { + try { + serializer.startTag(ns, "result"); + serializer.attribute(ns, "value", result.name()); + serializer.endTag(ns, "result"); + serializer.endTag(ns, "outcome"); + serializer.flush(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public void close() { + try { + serializer.endTag(ns, "vogar"); + serializer.endDocument(); + socket.close(); + serverSocket.close(); + } catch (IOException e) { + throw new RuntimeException(e); + } + socket = null; + serverSocket = null; + serializer = null; + } +} diff --git a/tools/runner/java/dalvik/runner/TestRunner.java b/tools/runner/java/vogar/target/TestRunner.java index a706d40..0eb082c 100644 --- a/tools/runner/java/dalvik/runner/TestRunner.java +++ b/tools/runner/java/vogar/target/TestRunner.java @@ -14,14 +14,17 @@ * limitations under the License. */ -package dalvik.runner; +package vogar.target; + +import vogar.TestProperties; import java.io.IOException; import java.io.InputStream; +import java.io.PrintStream; import java.util.Properties; /** - * Runs a test. + * Runs an action, in process on the target. */ public class TestRunner { @@ -30,12 +33,14 @@ public class TestRunner { protected final String qualifiedName; protected final Class<?> testClass; protected final Class<?> runnerClass; + protected final int monitorPort; - protected TestRunner () { + protected TestRunner() { properties = loadProperties(); qualifiedName = properties.getProperty(TestProperties.QUALIFIED_NAME); testClass = classProperty(TestProperties.TEST_CLASS, Object.class); runnerClass = classProperty(TestProperties.RUNNER_CLASS, Runner.class); + monitorPort = Integer.parseInt(properties.getProperty(TestProperties.MONITOR_PORT)); } protected static Properties loadProperties() { @@ -71,7 +76,19 @@ public class TestRunner { } } - public boolean run() { + public void run() { + final TargetMonitor monitor = new TargetMonitor(); + monitor.await(monitorPort); + + PrintStream monitorPrintStream = new PrintStream(System.out) { + @Override public void print(String str) { + super.print(str); + monitor.output(str); + } + }; + System.setOut(monitorPrintStream); + System.setErr(monitorPrintStream); + Runner runner; try { runner = (Runner) runnerClass.newInstance(); @@ -80,11 +97,18 @@ public class TestRunner { } catch (IllegalAccessException e) { throw new RuntimeException(e); } - runner.prepareTest(testClass); - return runner.test(testClass); + runner.init(monitor, qualifiedName, testClass); + runner.run(qualifiedName, testClass); + + monitor.close(); } + + public static void main(String[] args) { - System.out.println(TestProperties.result(new TestRunner().run())); + if (args.length != 0) { + throw new RuntimeException("TestRunner doesn't take arguments"); + } + new TestRunner().run(); } } |