diff options
22 files changed, 710 insertions, 694 deletions
diff --git a/tools/runner/java/vogar/Action.java b/tools/runner/java/vogar/Action.java new file mode 100644 index 0000000..3580ef4 --- /dev/null +++ b/tools/runner/java/vogar/Action.java @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package vogar; + +import vogar.target.Runner; + +import java.io.File; + +/** + * A named job such as a test or benchmark run. This class tracks the resource + * files and classes for compiling and running a Java source file. + */ +public final class Action { + + private final String name; + private final String actionClass; + private final File actionDirectory; + private final File actionJava; + private final String description; + private final Class<? extends Runner> runnerClass; + private final File runnerJava; + private final Classpath runnerClasspath; + private File userDir = new File(System.getProperty("user.dir")); + + public Action(String name, String actionClass, File actionDirectory, + File actionJava, String description, Class<? extends Runner> runnerClass, + File runnerJava, Classpath runnerClasspath) { + this.name = name; + this.actionClass = actionClass; + this.actionDirectory = actionDirectory; + this.actionJava = actionJava; + this.description = description; + this.runnerClass = runnerClass; + this.runnerJava = runnerJava; + this.runnerClasspath = runnerClasspath; + } + + /** + * Returns the local directory containing this test's java file. + */ + public File getJavaDirectory() { + return actionDirectory; + } + + public File getJavaFile() { + return actionJava; + } + + /** + * Returns the executable classname, such as java.lang.IntegerTest + * or BitTwiddle. + */ + public String getTargetClass() { + return actionClass; + } + + /** + * Returns a unique identifier for this test. + */ + public String getName() { + return name; + } + + /** + * Returns an English description of this test, or null if no such + * description is known. + */ + public String getDescription() { + return description; + } + + /** + * Initializes the directory from which local files can be read by the test. + */ + public void setUserDir(File base) { + this.userDir = base; + } + + public File getUserDir() { + return userDir; + } + + public Class<? extends Runner> getRunnerClass() { + return runnerClass; + } + + public File getRunnerJava() { + return runnerJava; + } + + public Classpath getRunnerClasspath() { + return runnerClasspath; + } + + @Override public String toString() { + return name; + } +} diff --git a/tools/runner/java/vogar/ActivityMode.java b/tools/runner/java/vogar/ActivityMode.java index 4e5773f..94adbd7 100644 --- a/tools/runner/java/vogar/ActivityMode.java +++ b/tools/runner/java/vogar/ActivityMode.java @@ -61,11 +61,11 @@ final class ActivityMode extends Mode { super.prepare(testRunnerJava, testRunnerClasspath); } - @Override protected void postCompileTestRunner() { + @Override protected void postCompileRunner() { } - @Override protected void postCompileTest(TestRun testRun) { - logger.fine("aapt and push " + testRun.getQualifiedName()); + @Override protected void postCompile(Action action) { + logger.fine("aapt and push " + action.getName()); // Some things of note: // 1. we can't put multiple dex files in one apk @@ -83,49 +83,49 @@ final class ActivityMode extends Mode { // 7. aapt the dex to create apk // 8. sign the apk // 9. install the apk - File packagingDir = makePackagingDirectory(testRun); - addTestRunnerClasses(packagingDir); + File packagingDir = makePackagingDirectory(action); + addRunnerClasses(packagingDir); List<File> found = new ArrayList<File>(); - File originalTestJar = findOriginalTestJar(testRun); - if (originalTestJar != null) { - found.add(originalTestJar); + File originalJar = findOriginalJar(action); + if (originalJar != null) { + found.add(originalJar); } - found.addAll(testRun.getRunnerClasspath().getElements()); + found.addAll(action.getRunnerClasspath().getElements()); extractJars(packagingDir, found); - addTestClasses(testRun, packagingDir); - File dex = createDex(testRun, packagingDir); - File apkUnsigned = createApk(testRun, dex); - File apkSigned = signApk(testRun, apkUnsigned); - installApk(testRun, apkSigned); + addActionClasses(action, packagingDir); + File dex = createDex(action, packagingDir); + File apkUnsigned = createApk(action, dex); + File apkSigned = signApk(action, apkUnsigned); + installApk(action, apkSigned); } - private File makePackagingDirectory(TestRun testRun) { - File packagingDir = new File(environment.testCompilationDir(testRun), "packaging"); + private File makePackagingDirectory(Action action) { + File packagingDir = new File(environment.actionCompilationDir(action), "packaging"); new Rm().directoryTree(packagingDir); new Mkdir().mkdirs(packagingDir); return packagingDir; } - private void addTestRunnerClasses(File packagingDir) { + private void addRunnerClasses(File packagingDir) { new Command("rsync", "-a", - environment.testRunnerClassesDir() + "/", + environment.runnerClassesDir() + "/", packagingDir + "/").execute(); } - private File findOriginalTestJar(TestRun testRun) { - String testClass = testRun.getTestClass(); - String testFile = testClass.replace('.', '/') + ".class"; - for (File element : testClasspath.getElements()) { + private File findOriginalJar(Action action) { + String targetClass = action.getTargetClass(); + String targetClassFile = targetClass.replace('.', '/') + ".class"; + for (File element : classpath.getElements()) { try { JarFile jar = new JarFile(element); - JarEntry jarEntry = jar.getJarEntry(testFile); + JarEntry jarEntry = jar.getJarEntry(targetClassFile); if (jarEntry != null) { return element; } } catch (IOException e) { throw new RuntimeException( "Could not find element " + element + - " of test class path " + testClasspath, e); + " of class path " + classpath, e); } } return null; @@ -144,15 +144,15 @@ final class ActivityMode extends Mode { new Rm().directoryTree(new File(packagingDir, "META-INF")); } - private void addTestClasses(TestRun testRun, File packagingDir) { - File testClassesDir = environment.testClassesDir(testRun); + private void addActionClasses(Action action, File packagingDir) { + File classesDir = environment.classesDir(action); new Command("rsync", "-a", - testClassesDir + "/", + classesDir + "/", packagingDir + "/").execute(); } - private File createDex (TestRun testRun, File packagingDir) { - File testClassesDir = environment.testClassesDir(testRun); - File dex = new File(testClassesDir + ".dex"); + private File createDex(Action action, File packagingDir) { + File classesDir = environment.classesDir(action); + File dex = new File(classesDir + ".dex"); new Dx().dex(dex, Classpath.of(packagingDir)); return dex; } @@ -163,15 +163,15 @@ final class ActivityMode extends Mode { * may not contain a dot, we prefix containing one to ensure we * are compliant. */ - private static String packageName(TestRun testRun) { - return "vogar.test." + testRun.getQualifiedName(); + private static String packageName(Action action) { + return "vogar.test." + action.getName(); } - private File createApk (TestRun testRun, File dex) { + private File createApk (Action action, File dex) { String androidManifest = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + "<manifest xmlns:android=\"http://schemas.android.com/apk/res/android\"\n" + - " package=\"" + packageName(testRun) + "\">\n" + + " package=\"" + packageName(action) + "\">\n" + " <uses-permission android:name=\"android.permission.INTERNET\" />\n" + " <application>\n" + " <activity android:name=\"" + TEST_ACTIVITY_CLASS + "\">\n" + @@ -183,7 +183,7 @@ final class ActivityMode extends Mode { " </application>\n" + "</manifest>\n"; File androidManifestFile = - new File(environment.testCompilationDir(testRun), + new File(environment.actionCompilationDir(action), "AndroidManifest.xml"); try { FileOutputStream androidManifestOut = @@ -194,17 +194,17 @@ final class ActivityMode extends Mode { throw new RuntimeException("Problem writing " + androidManifestFile, e); } - File testClassesDir = environment.testClassesDir(testRun); - File apkUnsigned = new File(testClassesDir + ".apk.unsigned"); + File classesDir = environment.classesDir(action); + File apkUnsigned = new File(classesDir + ".apk.unsigned"); new Aapt().apk(apkUnsigned, androidManifestFile); new Aapt().add(apkUnsigned, dex); - new Aapt().add(apkUnsigned, new File(testClassesDir, TestProperties.FILE)); + new Aapt().add(apkUnsigned, new File(classesDir , TestProperties.FILE)); return apkUnsigned; } - private File signApk(TestRun testRun, File apkUnsigned) { - File testClassesDir = environment.testClassesDir(testRun); - File apkSigned = new File(testClassesDir, testRun.getQualifiedName() + ".apk"); + private File signApk(Action action, File apkUnsigned) { + File classesDir = environment.classesDir(action); + File apkSigned = new File(classesDir, action.getName() + ".apk"); // TODO: we should be able to work with a shipping SDK, not depend on out/... // TODO: we should be able to work without hardwired keys, not depend on build/... new Command.Builder() @@ -219,25 +219,25 @@ final class ActivityMode extends Mode { return apkSigned; } - private void installApk(TestRun testRun, File apkSigned) { + private void installApk(Action action, File apkSigned) { // install the local apk ona the device - getEnvironmentDevice().adb.uninstall(packageName(testRun)); + getEnvironmentDevice().adb.uninstall(packageName(action)); getEnvironmentDevice().adb.install(apkSigned); } - @Override protected void fillInProperties(Properties properties, TestRun testRun) { - super.fillInProperties(properties, testRun); + @Override protected void fillInProperties(Properties properties, Action action) { + super.fillInProperties(properties, action); properties.setProperty(TestProperties.DEVICE_RUNNER_DIR, getEnvironmentDevice().runnerDir.getPath()); } - @Override protected List<String> runTestCommand(TestRun testRun) + @Override protected List<String> executeAction(Action action) throws TimeoutException { new Command( "adb", "shell", "am", "start", "-a","android.intent.action.MAIN", - "-n", (packageName(testRun) + "/" + TEST_ACTIVITY_CLASS)).executeWithTimeout(timeoutSeconds); + "-n", (packageName(action) + "/" + TEST_ACTIVITY_CLASS)).executeWithTimeout(timeoutSeconds); - File resultDir = new File(getEnvironmentDevice().runnerDir, testRun.getQualifiedName()); + File resultDir = new File(getEnvironmentDevice().runnerDir, action.getName()); File resultFile = new File(resultDir, TestProperties.RESULT_FILE); getEnvironmentDevice().adb.waitForFile(resultFile, timeoutSeconds); return new Command.Builder() @@ -246,10 +246,10 @@ final class ActivityMode extends Mode { .build().executeWithTimeout(timeoutSeconds); } - @Override void cleanup(TestRun testRun) { - super.cleanup(testRun); + @Override void cleanup(Action action) { + super.cleanup(action); if (environment.cleanAfter) { - getEnvironmentDevice().adb.uninstall(testRun.getQualifiedName()); + getEnvironmentDevice().adb.uninstall(action.getName()); } } } diff --git a/tools/runner/java/vogar/CaliperFinder.java b/tools/runner/java/vogar/CaliperFinder.java index cabdd46..d277ea3 100644 --- a/tools/runner/java/vogar/CaliperFinder.java +++ b/tools/runner/java/vogar/CaliperFinder.java @@ -22,7 +22,7 @@ import vogar.target.Runner; import java.io.File; /** - * Create {@link TestRun}s for {@code .java} files with Caliper benchmarks in + * Create {@link Action}s for {@code .java} files with Caliper benchmarks in * them. */ class CaliperFinder extends NamingPatternCodeFinder { @@ -31,10 +31,6 @@ class CaliperFinder extends NamingPatternCodeFinder { return super.matches(file) && file.getName().endsWith("Benchmark.java"); } - @Override protected String testName(File file) { - return "caliper"; - } - public Class<? extends Runner> getRunnerClass() { return CaliperRunner.class; } diff --git a/tools/runner/java/vogar/CodeFinder.java b/tools/runner/java/vogar/CodeFinder.java index 1a0d043..4e4aee7 100644 --- a/tools/runner/java/vogar/CodeFinder.java +++ b/tools/runner/java/vogar/CodeFinder.java @@ -30,7 +30,7 @@ public interface CodeFinder { * Returns all test runs in the given file or directory. If the returned set * is empty, no executable code of this kind were found. */ - public Set<TestRun> findTests(File file); + public Set<Action> findActions(File file); /** * Return the class for the TestRunner diff --git a/tools/runner/java/vogar/DeviceDalvikVm.java b/tools/runner/java/vogar/DeviceDalvikVm.java index ac2da70..b94b7f8 100644 --- a/tools/runner/java/vogar/DeviceDalvikVm.java +++ b/tools/runner/java/vogar/DeviceDalvikVm.java @@ -24,7 +24,7 @@ import java.util.List; import java.util.logging.Logger; /** - * Execute tests on a Dalvik VM using an Android device or emulator. + * Execute actions on a Dalvik VM using an Android device or emulator. */ final class DeviceDalvikVm extends Vm { private static final Logger logger = Logger.getLogger(DeviceDalvikVm.class.getName()); @@ -40,17 +40,17 @@ final class DeviceDalvikVm extends Vm { return (EnvironmentDevice) environment; } - @Override protected void postCompileTestRunner() { + @Override protected void postCompileRunner() { // TODO: does this really need to be a special case? - postCompile("testrunner", environment.testRunnerClassesDir()); + postCompile("testrunner", environment.runnerClassesDir()); // dex everything on the classpath and push it to the device. - for (File classpathElement : testClasspath.getElements()) { + for (File classpathElement : classpath.getElements()) { String name = basenameOfJar(classpathElement); logger.fine("dex and push " + name); // make the local dex (inside a jar) // TODO: this is *really* expensive. we need a cache! - File outputFile = getEnvironmentDevice().testDir(name + ".jar"); + File outputFile = getEnvironmentDevice().actionDir(name + ".jar"); new Dx().dex(outputFile, Classpath.of(classpathElement)); // push the local dex to the device getEnvironmentDevice().adb.push(outputFile, deviceDexFile(name)); @@ -61,8 +61,8 @@ final class DeviceDalvikVm extends Vm { return jarFile.getName().replaceAll("\\.jar$", ""); } - @Override protected void postCompileTest(TestRun testRun) { - postCompile(testRun.getQualifiedName(), environment.testClassesDir(testRun)); + @Override protected void postCompile(Action action) { + postCompile(action.getName(), environment.classesDir(action)); } private void postCompile(String name, File dir) { @@ -95,15 +95,15 @@ final class DeviceDalvikVm extends Vm { .vmArgs("-Duser.language=en") .vmArgs("-Duser.region=US") .vmArgs("-Djavax.net.ssl.trustStore=/system/etc/security/cacerts.bks") - .temp(getEnvironmentDevice().testTemp); + .temp(getEnvironmentDevice().vogarTemp); } - @Override protected Classpath getRuntimeSupportClasspath(TestRun testRun) { + @Override protected Classpath getRuntimeSupportClasspath(Action action) { Classpath classpath = new Classpath(); - classpath.addAll(deviceDexFile(testRun.getQualifiedName())); + classpath.addAll(deviceDexFile(action.getName())); classpath.addAll(deviceDexFile("testrunner")); - for (File testClasspathElement : testClasspath.getElements()) { - classpath.addAll(deviceDexFile(basenameOfJar(testClasspathElement))); + for (File classpathElement : this.classpath.getElements()) { + classpath.addAll(deviceDexFile(basenameOfJar(classpathElement))); } return classpath; } diff --git a/tools/runner/java/vogar/Driver.java b/tools/runner/java/vogar/Driver.java index 96153a2..4b94909 100644 --- a/tools/runner/java/vogar/Driver.java +++ b/tools/runner/java/vogar/Driver.java @@ -19,13 +19,10 @@ package vogar; import vogar.commands.Mkdir; import java.io.File; -import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -36,205 +33,190 @@ import java.util.concurrent.TimeUnit; import java.util.logging.Logger; /** - * Compiles, installs, runs and reports tests. + * Compiles, installs, runs and reports on actions. */ final class Driver { private static final Logger logger = Logger.getLogger(Driver.class.getName()); private final File localTemp; - private final Set<File> expectationFiles; + private final ExpectationStore expectationStore; private final List<CodeFinder> codeFinders; private final Mode mode; - private final File xmlReportsDirectory; private final String indent; - private final Map<String, ExpectedResult> expectedResults = new HashMap<String, ExpectedResult>(); + private final XmlReportPrinter reportPrinter; + + private final Map<String, Action> actions = Collections.synchronizedMap( + new LinkedHashMap<String, Action>()); + private final Map<String, Outcome> outcomes = Collections.synchronizedMap( + new LinkedHashMap<String, Outcome>()); /** * The number of tests that weren't run because they aren't supported by * this runner. */ - private int unsupportedTests = 0; + private int unsupportedActions = 0; - public Driver(File localTemp, Mode mode, Set<File> expectationFiles, - File xmlReportsDirectory, String indent, List<CodeFinder> codeFinders) { + public Driver(File localTemp, Mode mode, ExpectationStore expectationStore, + String indent, List<CodeFinder> codeFinders, XmlReportPrinter reportPrinter) { this.localTemp = localTemp; - this.expectationFiles = expectationFiles; + this.expectationStore = expectationStore; this.mode = mode; - this.xmlReportsDirectory = xmlReportsDirectory; this.indent = indent; this.codeFinders = codeFinders; - } - - public void loadExpectations() throws IOException { - for (File f : expectationFiles) { - if (f.exists()) { - expectedResults.putAll(ExpectedResult.parse(f)); - } - } + this.reportPrinter = reportPrinter; } /** * Builds and executes all tests in the test directory. */ - public void buildAndRunAllTests(Collection<File> testFiles) { - new Mkdir().mkdirs(localTemp); + public void buildAndRunAllActions(Collection<File> files) { + if (!actions.isEmpty()) { + throw new IllegalStateException("Drivers are not reusable"); + } - Set<TestRun> tests = new LinkedHashSet<TestRun>(); - for (File testFile : testFiles) { - Set<TestRun> testsForFile = Collections.emptySet(); + new Mkdir().mkdirs(localTemp); + for (File file : files) { + Set<Action> actionsForFile = Collections.emptySet(); for (CodeFinder codeFinder : codeFinders) { - testsForFile = codeFinder.findTests(testFile); + actionsForFile = codeFinder.findActions(file); // break as soon as we find any match. We don't need multiple // matches for the same file, since that would run it twice. - if (!testsForFile.isEmpty()) { + if (!actionsForFile.isEmpty()) { break; } } - tests.addAll(testsForFile); + for (Action action : actionsForFile) { + actions.put(action.getName(), action); + } } // compute TestRunner java and classpath to pass to mode.prepare - Set<File> testRunnerJava = new HashSet<File>(); - Classpath testRunnerClasspath = new Classpath(); - for (final TestRun testRun : tests) { - testRunnerJava.add(testRun.getRunnerJava()); - testRunnerClasspath.addAll(testRun.getRunnerClasspath()); + Set<File> runnerJava = new HashSet<File>(); + Classpath runnerClasspath = new Classpath(); + for (final Action action : actions.values()) { + runnerJava.add(action.getRunnerJava()); + runnerClasspath.addAll(action.getRunnerClasspath()); } - // mode.prepare before mode.buildAndInstall to ensure test - // runner is built. packaging of activity APK files needs the - // test runner along with the test specific files. - mode.prepare(testRunnerJava, testRunnerClasspath); + // mode.prepare before mode.buildAndInstall to ensure the runner is + // built. packaging of activity APK files needs the runner along with + // the action-specific files. + mode.prepare(runnerJava, runnerClasspath); - logger.info("Running " + tests.size() + " tests."); + logger.info("Running " + actions.size() + " actions."); - // build and install tests in a background thread. Using lots of - // threads helps for packages that contain many unsupported tests - final BlockingQueue<TestRun> readyToRun = new ArrayBlockingQueue<TestRun>(4); + // build and install actions in a background thread. Using lots of + // threads helps for packages that contain many unsupported actions + final BlockingQueue<Action> readyToRun = new ArrayBlockingQueue<Action>(4); ExecutorService builders = Threads.threadPerCpuExecutor(); int t = 0; - for (final TestRun testRun : tests) { + + for (final Action action : actions.values()) { + final String name = action.getName(); final int runIndex = t++; builders.submit(new Runnable() { public void run() { try { - ExpectedResult expectedResult = lookupExpectedResult(testRun); - testRun.setExpectedResult(expectedResult); + logger.fine("installing action " + runIndex + "; " + + readyToRun.size() + " are runnable"); - if (expectedResult.getResult() == Result.UNSUPPORTED) { - testRun.setResult(Result.UNSUPPORTED, Collections.<String>emptyList()); - logger.fine("skipping test " + testRun - + " because the expectations file says it is unsupported."); + if (expectationStore.get(name).getResult() == Result.UNSUPPORTED) { + outcomes.put(name, new Outcome(name, Result.UNSUPPORTED, + "Unsupported according to expectations file")); } else { - mode.buildAndInstall(testRun); - logger.fine("installed test " + runIndex + "; " - + readyToRun.size() + " are ready to run"); + Outcome outcome = mode.buildAndInstall(action); + if (outcome != null) { + outcomes.put(name, outcome); + } } - readyToRun.put(testRun); - } catch (Throwable throwable) { - testRun.setResult(Result.ERROR, throwable); + readyToRun.put(action); + } catch (InterruptedException e) { + outcomes.put(name, new Outcome(name, Result.ERROR, e)); } } }); } builders.shutdown(); - List<TestRun> runs = new ArrayList<TestRun>(tests.size()); - for (int i = 0; i < tests.size(); i++) { - logger.fine("executing test " + i + "; " + for (int i = 0; i < actions.size(); i++) { + logger.fine("executing action " + i + "; " + readyToRun.size() + " are ready to run"); // if it takes 5 minutes for build and install, something is broken - TestRun testRun; + Action action; try { - testRun = readyToRun.poll(5 * 60, TimeUnit.SECONDS); + action = readyToRun.poll(5 * 60, TimeUnit.SECONDS); } catch (InterruptedException e) { throw new RuntimeException("Unexpected interruption waiting for build and install", e); } - if (testRun == null) { - throw new IllegalStateException("Expected " + tests.size() + " tests but found only " + i); + if (action == null) { + throw new IllegalStateException("Expected " + actions.size() + + " actions but found only " + i); } - runs.add(testRun); - execute(testRun); - mode.cleanup(testRun); + execute(action); + mode.cleanup(action); } - if (unsupportedTests > 0) { - logger.info("Skipped " + unsupportedTests + " unsupported tests."); + if (unsupportedActions > 0) { + logger.info("Skipped " + unsupportedActions + " unsupported actions."); } - if (xmlReportsDirectory != null) { + if (reportPrinter != null) { logger.info("Printing XML Reports... "); - int numFiles = new XmlReportPrinter().generateReports(xmlReportsDirectory, runs); + int numFiles = reportPrinter.generateReports(outcomes.values()); logger.info(numFiles + " XML files written."); } } /** - * Finds the expected result for the specified test run. This strips off - * parts of the test's qualified name until it either finds a match or runs - * out of name. + * Executes a single action and then prints the result. */ - private ExpectedResult lookupExpectedResult(TestRun testRun) { - String name = testRun.getQualifiedName(); - - while (true) { - ExpectedResult expectedResult = expectedResults.get(name); - if (expectedResult != null) { - return expectedResult; + private void execute(Action action) { + Outcome earlyFailure = outcomes.get(action.getName()); + if (earlyFailure != null) { + if (earlyFailure.getResult() == Result.UNSUPPORTED) { + logger.fine("skipping " + action.getName()); + unsupportedActions++; + } else { + printResult(earlyFailure); } - - int dot = name.lastIndexOf('.'); - if (dot == -1) { - return ExpectedResult.SUCCESS; - } - - name = name.substring(0, dot); - } - } - - /** - * Executes a single test and then prints the result. - */ - private void execute(TestRun testRun) { - if (testRun.getResult() == Result.UNSUPPORTED) { - logger.fine("skipping " + testRun.getQualifiedName()); - unsupportedTests++; return; } - if (testRun.isRunnable()) { - mode.runTest(testRun); + Set<Outcome> outcomes = mode.run(action); + for (Outcome outcome : outcomes) { + printResult(outcome); } - - printResult(testRun); } - private void printResult(TestRun testRun) { - if (testRun.isExpectedResult()) { - logger.info("OK " + testRun.getQualifiedName() + " (" + testRun.getResult() + ")"); + private void printResult(Outcome outcome) { + Expectation expected = expectationStore.get(outcome.getName()); + Action action = actions.get(outcome.getActionName()); + + if (expected.matches(outcome)) { + logger.info("OK " + outcome.getName() + " (" + outcome.getResult() + ")"); // In --verbose mode, show the output even on success. - logger.fine(indent + testRun.getFailureMessage().replace("\n", "\n" + indent)); + logger.fine(indent + expected.getFailureMessage(outcome).replace("\n", "\n" + indent)); return; } - logger.info("FAIL " + testRun.getQualifiedName() + " (" + testRun.getResult() + ")"); - String description = testRun.getDescription(); + logger.info("FAIL " + outcome.getName() + " (" + outcome.getResult() + ")"); + String description = action.getDescription(); if (description != null) { logger.info(indent + "\"" + description + "\""); } // Don't mess with compiler error output for tools (such as // Emacs) that are trying to parse it with regexps - logger.info(indent + testRun.getFailureMessage().replace("\n", "\n" + indent)); + logger.info(indent + expected.getFailureMessage(outcome).replace("\n", "\n" + indent)); } } diff --git a/tools/runner/java/vogar/Environment.java b/tools/runner/java/vogar/Environment.java index 25b1768..5d7f2fb 100644 --- a/tools/runner/java/vogar/Environment.java +++ b/tools/runner/java/vogar/Environment.java @@ -50,43 +50,43 @@ abstract class Environment { * expect to read data files from the current working directory; this step * should ensure such files are available. */ - abstract void prepareUserDir(TestRun testRun); + abstract void prepareUserDir(Action action); /** * Deletes files and releases any resources required for the execution of * the given test. */ - void cleanup(TestRun testRun) { + void cleanup(Action action) { if (cleanAfter) { - logger.fine("clean " + testRun.getQualifiedName()); - new Rm().directoryTree(testCompilationDir(testRun)); - new Rm().directoryTree(testUserDir(testRun)); + logger.fine("clean " + action.getName()); + new Rm().directoryTree(actionCompilationDir(action)); + new Rm().directoryTree(actionUserDir(action)); } } - final File testDir(String name) { + final File actionDir(String name) { return new File(localTemp, name); } - final File testRunnerDir(String name) { - return new File(testDir("testrunner"), name); + final File runnerDir(String name) { + return new File(actionDir("testrunner"), name); } - final File testRunnerClassesDir() { - return testRunnerDir("classes"); + final File runnerClassesDir() { + return runnerDir("classes"); } - final File testCompilationDir(TestRun testRun) { - return new File(localTemp, testRun.getQualifiedName()); + final File actionCompilationDir(Action action) { + return new File(localTemp, action.getName()); } - final File testClassesDir(TestRun testRun) { - return new File(testCompilationDir(testRun), "classes"); + final File classesDir(Action action) { + return new File(actionCompilationDir(action), "classes"); } - final File testUserDir(TestRun testRun) { + final File actionUserDir(Action action) { File testTemp = new File(localTemp, "userDir"); - return new File(testTemp, testRun.getQualifiedName()); + return new File(testTemp, action.getName()); } abstract void shutdown(); diff --git a/tools/runner/java/vogar/EnvironmentDevice.java b/tools/runner/java/vogar/EnvironmentDevice.java index 90e74cd..91508f7 100644 --- a/tools/runner/java/vogar/EnvironmentDevice.java +++ b/tools/runner/java/vogar/EnvironmentDevice.java @@ -23,13 +23,13 @@ import java.io.File; class EnvironmentDevice extends Environment { final Adb adb = new Adb(); final File runnerDir; - final File testTemp; + final File vogarTemp; EnvironmentDevice (boolean cleanBefore, boolean cleanAfter, Integer debugPort, File localTemp, File runnerDir) { super(cleanBefore, cleanAfter, debugPort, localTemp); this.runnerDir = runnerDir; - this.testTemp = new File(runnerDir, "/tests.tmp"); + this.vogarTemp = new File(runnerDir, "/vogar.tmp"); } @Override void prepare() { @@ -39,28 +39,28 @@ class EnvironmentDevice extends Environment { adb.rm(runnerDir); } adb.mkdir(runnerDir); - adb.mkdir(testTemp); + adb.mkdir(vogarTemp); adb.mkdir(new File("/sdcard/dalvik-cache")); // TODO: only necessary on production devices. if (debugPort != null) { adb.forwardTcp(debugPort, debugPort); } } - @Override protected void prepareUserDir(TestRun testRun) { - File testClassesDirOnDevice = testClassesDirOnDevice(testRun); - adb.mkdir(testClassesDirOnDevice); - adb.push(testRun.getTestDirectory(), testClassesDirOnDevice); - testRun.setUserDir(testClassesDirOnDevice); + @Override protected void prepareUserDir(Action action) { + File actionClassesDirOnDevice = actionClassesDirOnDevice(action); + adb.mkdir(actionClassesDirOnDevice); + adb.push(action.getJavaDirectory(), actionClassesDirOnDevice); + action.setUserDir(actionClassesDirOnDevice); } - private File testClassesDirOnDevice(TestRun testRun) { - return new File(runnerDir, testRun.getQualifiedName()); + private File actionClassesDirOnDevice(Action action) { + return new File(runnerDir, action.getName()); } - @Override void cleanup(TestRun testRun) { - super.cleanup(testRun); + @Override void cleanup(Action action) { + super.cleanup(action); if (cleanAfter) { - adb.rm(testClassesDirOnDevice(testRun)); + adb.rm(actionClassesDirOnDevice(action)); } } diff --git a/tools/runner/java/vogar/EnvironmentHost.java b/tools/runner/java/vogar/EnvironmentHost.java index 9522348..e7ad4db 100644 --- a/tools/runner/java/vogar/EnvironmentHost.java +++ b/tools/runner/java/vogar/EnvironmentHost.java @@ -30,18 +30,18 @@ class EnvironmentHost extends Environment { @Override void prepare() {} - @Override protected void prepareUserDir(TestRun testRun) { - File testUserDir = testUserDir(testRun); + @Override protected void prepareUserDir(Action action) { + File actionUserDir = actionUserDir(action); // if the user dir exists, cp would copy the files to the wrong place - if (testUserDir.exists()) { + if (actionUserDir.exists()) { throw new IllegalStateException(); } - new Mkdir().mkdirs(testUserDir.getParentFile()); - new Command("cp", "-r", testRun.getTestDirectory().toString(), - testUserDir.toString()).execute(); - testRun.setUserDir(testUserDir); + new Mkdir().mkdirs(actionUserDir.getParentFile()); + new Command("cp", "-r", action.getJavaDirectory().toString(), + actionUserDir.toString()).execute(); + action.setUserDir(actionUserDir); } @Override void shutdown() {} diff --git a/tools/runner/java/vogar/Expectation.java b/tools/runner/java/vogar/Expectation.java new file mode 100644 index 0000000..b52b5a3 --- /dev/null +++ b/tools/runner/java/vogar/Expectation.java @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package vogar; + +import java.util.regex.Pattern; + +/** + * The expected result of an action execution. This is typically encoded in the + * expectations text file, which has the following format: + * <pre> + * test java.io.StreamTokenizer.Reset + * result UNSUPPORTED + * pattern .*should get token \[, but get -1.* + * + * # should we fix this? + * test java.util.Arrays.CopyMethods + * result COMPILE_FAILED + * pattern .*cannot find symbol.* + * </pre> + */ +final class Expectation { + + /** The pattern to use when no expected output is specified */ + private static final Pattern MATCH_ALL_PATTERN + = Pattern.compile(".*", Pattern.MULTILINE | Pattern.DOTALL); + + /** The expectation of a general successful run. */ + static final Expectation SUCCESS = new Expectation(Result.SUCCESS, null); + + /** The action's expected result, such as {@code EXEC_FAILED}. */ + private final Result result; + + /** The pattern the expected output will match. */ + private final Pattern pattern; + + public Expectation(Result result, String pattern) { + if (result == null) { + throw new IllegalArgumentException(); + } + + this.result = result; + this.pattern = pattern != null + ? Pattern.compile(pattern, Pattern.MULTILINE | Pattern.DOTALL) + : MATCH_ALL_PATTERN; + } + + public Result getResult() { + return result; + } + + /** + * Returns true if {@code outcome} matches this expectation. + */ + public boolean matches(Outcome outcome) { + return result == outcome.getResult() && patternMatches(outcome); + } + + /** + * Returns the failure message for this failed run. This message is intended + * to help to diagnose why the run result didn't match what was expected. + */ + public String getFailureMessage(Outcome outcome) { + StringBuilder builder = new StringBuilder(); + + if (result != Result.SUCCESS && result != outcome.getResult()) { + builder.append("Expected result: ") + .append(result) + .append("\n"); + } + + if (!patternMatches(outcome)) { + builder.append("Expected output to match \"") + .append(pattern.pattern()) + .append("\"\n"); + } + + for (String output : outcome.getOutputLines()) { + builder.append(output).append("\n"); + } + + return builder.toString(); + } + + private boolean patternMatches(Outcome outcome) { + return pattern.matcher(Strings.join(outcome.getOutputLines(), "\n")).matches(); + } +} diff --git a/tools/runner/java/vogar/ExpectedResult.java b/tools/runner/java/vogar/ExpectationStore.java index a3a61da..f10ae22 100644 --- a/tools/runner/java/vogar/ExpectedResult.java +++ b/tools/runner/java/vogar/ExpectationStore.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2009 The Android Open Source Project + * Copyright (C) 2010 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,70 +22,65 @@ import java.io.FileReader; import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.Set; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; /** - * The expected outcome of a test execution. This is typically encoded in the - * expectations text file, which has the following format: - * <pre> - * test java.io.StreamTokenizer.Reset - * result UNSUPPORTED - * pattern .*should get token \[, but get -1.* - * - * # should we fix this? - * test java.util.Arrays.CopyMethods - * result COMPILE_FAILED - * pattern .*cannot find symbol.* - * </pre> + * A database of expected outcomes. */ -class ExpectedResult { +final class ExpectationStore { - private static final Logger logger = Logger.getLogger(ExpectedResult.class.getName()); + private static final Logger logger = Logger.getLogger(ExpectationStore.class.getName()); /** Matches lines in the file containing a key and value pair. */ private static final Pattern KEY_VALUE_PAIR_PATTERN = Pattern.compile("(\\w+)\\s+(.+)"); - /** The pattern to use when no expected output is specified */ - private static final Pattern MATCH_ALL_PATTERN - = Pattern.compile(".*", Pattern.MULTILINE | Pattern.DOTALL); + private final Map<String, Expectation> expectedResults; - /** The expectation of a general successful test run. */ - static final ExpectedResult SUCCESS = new ExpectedResult(Result.SUCCESS, null); + private ExpectationStore(Map<String, Expectation> expectedResults) { + this.expectedResults = expectedResults; + } - /** The test's expected result, such as {@code EXEC_FAILED}. */ - private final Result result; + /** + * Finds the expected result for the specified action or outcome. This + * returns a value for all names, even if no explicit expectation was set. + */ + public Expectation get(String name) { + while (true) { + Expectation expectation = expectedResults.get(name); + if (expectation != null) { + return expectation; + } - /** The pattern the expected output will match. */ - private final Pattern pattern; + int dot = name.lastIndexOf('.'); + if (dot == -1) { + return Expectation.SUCCESS; + } - private ExpectedResult(Result result, String pattern) { - if (result == null) { - throw new IllegalArgumentException(); + name = name.substring(0, dot); } - - this.result = result; - this.pattern = pattern != null - ? Pattern.compile(pattern, Pattern.MULTILINE | Pattern.DOTALL) - : MATCH_ALL_PATTERN; } - public Result getResult() { - return result; + public static ExpectationStore parse(Set<File> expectationFiles) throws IOException { + Map<String, Expectation> expectedResults = new HashMap<String, Expectation>(); + for (File f : expectationFiles) { + if (f.exists()) { + expectedResults.putAll(parse(f)); + } + } + return new ExpectationStore(expectedResults); } - public Pattern getPattern() { - return pattern; - } - public static Map<String, ExpectedResult> parse(File expectationsFile) + public static Map<String, Expectation> parse(File expectationsFile) throws IOException { logger.fine("loading expectations file " + expectationsFile); BufferedReader reader = new BufferedReader(new FileReader(expectationsFile)); try { - Map<String, ExpectedResult> results = new HashMap<String, ExpectedResult>(); + Map<String, Expectation> results = new HashMap<String, Expectation>(); Matcher keyValuePairMatcher = KEY_VALUE_PAIR_PATTERN.matcher(""); // the fields of interest for the current element @@ -119,8 +114,8 @@ class ExpectedResult { // when we encounter a new qualified name, the previous // element is complete. Add it to the results. if (qualifiedName != null) { - ExpectedResult expectation = new ExpectedResult(result, pattern); - ExpectedResult previous = results.put(qualifiedName, expectation); + Expectation expectation = new Expectation(result, pattern); + Expectation previous = results.put(qualifiedName, expectation); if (previous != null) { throw new IllegalArgumentException( "Duplicate expectations for " + qualifiedName); @@ -140,8 +135,8 @@ class ExpectedResult { // add the last element in the file if (qualifiedName != null) { - ExpectedResult expectation = new ExpectedResult(result, pattern); - ExpectedResult previous = results.put(qualifiedName, expectation); + Expectation expectation = new Expectation(result, pattern); + Expectation previous = results.put(qualifiedName, expectation); if (previous != null) { throw new IllegalArgumentException( "Duplicate expectations for " + qualifiedName); diff --git a/tools/runner/java/vogar/JUnitFinder.java b/tools/runner/java/vogar/JUnitFinder.java index 77a63a2..4d98f86 100644 --- a/tools/runner/java/vogar/JUnitFinder.java +++ b/tools/runner/java/vogar/JUnitFinder.java @@ -22,7 +22,7 @@ import vogar.target.Runner; import java.io.File; /** - * Create {@link TestRun}s for {@code .java} files with JUnit tests in them. + * Create {@link Action}s for {@code .java} files with JUnit tests in them. */ class JUnitFinder extends NamingPatternCodeFinder { @@ -32,11 +32,6 @@ class JUnitFinder extends NamingPatternCodeFinder { && (filename.endsWith("Test.java") || filename.endsWith("TestSuite.java")); } - // TODO: try to get names for each method? - @Override protected String testName(File file) { - return "junit"; - } - public Class<? extends Runner> getRunnerClass() { return JUnitRunner.class; } diff --git a/tools/runner/java/vogar/JavaVm.java b/tools/runner/java/vogar/JavaVm.java index 3d592ba..e6dc684 100644 --- a/tools/runner/java/vogar/JavaVm.java +++ b/tools/runner/java/vogar/JavaVm.java @@ -35,10 +35,10 @@ final class JavaVm extends Vm { this.javaHome = javaHome; } - @Override protected void postCompileTestRunner() { + @Override protected void postCompileRunner() { } - @Override protected void postCompileTest(TestRun testRun) { + @Override protected void postCompile(Action action) { } @Override protected VmCommandBuilder newVmCommandBuilder( @@ -48,12 +48,12 @@ final class JavaVm extends Vm { .vmCommand(java) .workingDir(workingDirectory); } - @Override protected Classpath getRuntimeSupportClasspath(TestRun testRun) { + @Override protected Classpath getRuntimeSupportClasspath(Action action) { Classpath classpath = new Classpath(); - classpath.addAll(environment.testClassesDir(testRun)); - classpath.addAll(testClasspath); - classpath.addAll(environment.testRunnerClassesDir()); - classpath.addAll(testRunnerClasspath); + classpath.addAll(environment.classesDir(action)); + classpath.addAll(this.classpath); + classpath.addAll(environment.runnerClassesDir()); + classpath.addAll(runnerClasspath); return classpath; } } diff --git a/tools/runner/java/vogar/JtregFinder.java b/tools/runner/java/vogar/JtregFinder.java index dae6e59..7319b6b 100644 --- a/tools/runner/java/vogar/JtregFinder.java +++ b/tools/runner/java/vogar/JtregFinder.java @@ -34,7 +34,7 @@ import java.util.Set; import java.util.logging.Logger; /** - * Create {@link TestRun}s for {@code .java} files with jtreg tests in them. + * Create {@link Action}s for {@code .java} files with jtreg tests in them. */ class JtregFinder implements CodeFinder { @@ -59,7 +59,7 @@ class JtregFinder implements CodeFinder { /** * Returns the tests in {@code directoryToScan}. */ - public Set<TestRun> findTests(File directoryToScan) { + public Set<Action> findActions(File directoryToScan) { // for now, jtreg doesn't know how to scan anything but directories if (!directoryToScan.isDirectory()) { return Collections.emptySet(); @@ -80,16 +80,13 @@ class JtregFinder implements CodeFinder { WorkDirectory wd = WorkDirectory.convert(workDirectory, testSuite); TestResultTable resultTable = wd.getTestResultTable(); - Set<TestRun> result = new LinkedHashSet<TestRun>(); + Set<Action> result = new LinkedHashSet<Action>(); for (Iterator i = resultTable.getIterator(); i.hasNext(); ) { TestResult testResult = (TestResult) i.next(); TestDescription description = testResult.getDescription(); String qualifiedName = qualifiedName(description); - String suiteName = suiteName(description); - String testName = description.getName(); String testClass = description.getName(); - result.add(new TestRun(description.getDir(), description.getFile(), - testClass, suiteName, testName, qualifiedName, + result.add(new Action(qualifiedName, testClass, description.getDir(), description.getFile(), description.getTitle(), getRunnerClass(), getRunnerJava(), getRunnerClasspath())); } diff --git a/tools/runner/java/vogar/MainFinder.java b/tools/runner/java/vogar/MainFinder.java index bdd3b7e..e98098a 100644 --- a/tools/runner/java/vogar/MainFinder.java +++ b/tools/runner/java/vogar/MainFinder.java @@ -22,14 +22,10 @@ import vogar.target.Runner; import java.io.File; /** - * Create {@link TestRun}s for {@code .java} files with main methods in them. + * Create {@link Action}s for {@code .java} files with main methods in them. */ class MainFinder extends NamingPatternCodeFinder { - @Override protected String testName(File file) { - return "main"; - } - public Class<? extends Runner> getRunnerClass() { return MainRunner.class; } diff --git a/tools/runner/java/vogar/Mode.java b/tools/runner/java/vogar/Mode.java index ae76f35..708e580 100644 --- a/tools/runner/java/vogar/Mode.java +++ b/tools/runner/java/vogar/Mode.java @@ -35,13 +35,13 @@ import java.util.logging.Logger; import java.util.regex.Pattern; /** - * A Mode for running tests. Examples including running in a virtual - * machine either on the host or a device or within a specific context - * such as within an Activity. + * A Mode for running actions. Examples including running in a virtual machine + * either on the host or a device or within a specific context such as within an + * Activity. */ abstract class Mode { - private static final Pattern JAVA_TEST_PATTERN = Pattern.compile("\\/(\\w)+\\.java$"); + private static final Pattern JAVA_SOURCE_PATTERN = Pattern.compile("\\/(\\w)+\\.java$"); private static final Logger logger = Logger.getLogger(Mode.class.getName()); @@ -52,24 +52,22 @@ abstract class Mode { protected final PrintStream tee; /** - * Set of Java files needed to built to tun the currently selected - * set of tests. We build a subset rather than all the files all - * the time to reduce dex packaging costs in the activity mode - * case. + * Set of Java files needed to built to tun the currently selected set of + * actions. We build a subset rather than all the files all the time to + * reduce dex packaging costs in the activity mode case. */ - protected final Set<File> testRunnerJava = new HashSet<File>(); + protected final Set<File> runnerJava = new HashSet<File>(); /** - * Classpath of testRunner on the host side including any - * supporting libraries for testRunnerJava. Useful for compiling - * testRunnerJava as well as executing it on the host. Execution - * on the device requires further packaging typically done by - * postCompileTestRunner. + * Classpath of runner on the host side including any supporting libraries + * for runnerJava. Useful for compiling runnerJava as well as executing it + * on the host. Execution on the device requires further packaging typically + * done by postCompile. */ - protected final Classpath testRunnerClasspath = new Classpath(); + protected final Classpath runnerClasspath = new Classpath(); // TODO: this should be an immutable collection. - protected final Classpath testClasspath = Classpath.of( + protected final Classpath classpath = Classpath.of( new File("dalvik/libcore/tools/runner/lib/jsr305.jar"), new File("dalvik/libcore/tools/runner/lib/guava.jar"), new File("dalvik/libcore/tools/runner/lib/caliper.jar"), @@ -87,16 +85,16 @@ abstract class Mode { } /** - * Initializes the temporary directories and test harness necessary to run - * tests. + * Initializes the temporary directories and harness necessary to run + * actions. */ - protected void prepare(Set<File> testRunnerJava, Classpath testRunnerClasspath) { - this.testRunnerJava.add(new File(Vogar.HOME_JAVA, "vogar/target/TestRunner.java")); - this.testRunnerJava.addAll(dalvikAnnotationSourceFiles()); - this.testRunnerJava.addAll(testRunnerJava); - this.testRunnerClasspath.addAll(testRunnerClasspath); + protected void prepare(Set<File> runnerJava, Classpath runnerClasspath) { + this.runnerJava.add(new File(Vogar.HOME_JAVA, "vogar/target/TestRunner.java")); + this.runnerJava.addAll(dalvikAnnotationSourceFiles()); + this.runnerJava.addAll(runnerJava); + this.runnerClasspath.addAll(runnerClasspath); environment.prepare(); - compileTestRunner(); + compileRunner(); } private List<File> dalvikAnnotationSourceFiles() { @@ -111,14 +109,14 @@ abstract class Mode { return Arrays.asList(javaSourceFiles); } - private void compileTestRunner() { - logger.fine("build testrunner"); + private void compileRunner() { + logger.fine("build runner"); Classpath classpath = new Classpath(); - classpath.addAll(testClasspath); - classpath.addAll(testRunnerClasspath); + classpath.addAll(this.classpath); + classpath.addAll(runnerClasspath); - File base = environment.testRunnerClassesDir(); + File base = environment.runnerClassesDir(); new Mkdir().mkdirs(base); new Javac() .bootClasspath(sdkJar) @@ -126,148 +124,133 @@ abstract class Mode { .sourcepath(Vogar.HOME_JAVA) .destination(base) .extra(javacArgs) - .compile(testRunnerJava); - postCompileTestRunner(); + .compile(runnerJava); + postCompileRunner(); } /** - * Hook method called after TestRunner compilation. + * Hook method called after runner compilation. */ - abstract protected void postCompileTestRunner(); + abstract protected void postCompileRunner(); /** - * Compiles classes for the given test and makes them ready for execution. - * If the test could not be compiled successfully, it will be updated with - * the appropriate test result. + * Compiles classes for the given action and makes them ready for execution. + * + * @return null if the compilation succeeded, or an outcome describing the + * failure otherwise. */ - public void buildAndInstall(TestRun testRun) { - logger.fine("build " + testRun.getQualifiedName()); + public Outcome buildAndInstall(Action action) { + logger.fine("build " + action.getName()); - boolean testCompiled; try { - testCompiled = compileTest(testRun); - if (!testCompiled) { - testRun.setResult(Result.UNSUPPORTED, Collections.<String>emptyList()); - return; - } + compile(action); } catch (CommandFailedException e) { - testRun.setResult(Result.COMPILE_FAILED, e.getOutputLines()); - return; + return new Outcome(action.getName(), action.getName(), + Result.COMPILE_FAILED, e.getOutputLines()); } catch (IOException e) { - testRun.setResult(Result.ERROR, e); - return; + return new Outcome(action.getName(), Result.ERROR, e); } - testRun.setTestCompiled(testCompiled); - environment.prepareUserDir(testRun); + environment.prepareUserDir(action); + return null; } /** - * Compiles the classes for the described test. + * Compiles the classes for the described action. * - * @return the path to the compiled classes (directory or jar), or {@code - * null} if the test could not be compiled. * @throws CommandFailedException if javac fails */ - private boolean compileTest(TestRun testRun) throws IOException { - if (!JAVA_TEST_PATTERN.matcher(testRun.getTestJava().toString()).find()) { - return false; + private void compile(Action action) throws IOException { + if (!JAVA_SOURCE_PATTERN.matcher(action.getJavaFile().toString()).find()) { + throw new CommandFailedException(Collections.<String>emptyList(), + Collections.singletonList("Cannot compile: " + action.getJavaFile())); } - String qualifiedName = testRun.getQualifiedName(); - File testClassesDir = environment.testClassesDir(testRun); - new Mkdir().mkdirs(testClassesDir); + String qualifiedName = action.getName(); + File classesDir = environment.classesDir(action); + new Mkdir().mkdirs(classesDir); FileOutputStream propertiesOut = new FileOutputStream( - new File(testClassesDir, TestProperties.FILE)); + new File(classesDir, TestProperties.FILE)); Properties properties = new Properties(); - fillInProperties(properties, testRun); + fillInProperties(properties, action); properties.store(propertiesOut, "generated by " + Mode.class.getName()); propertiesOut.close(); Classpath classpath = new Classpath(); - classpath.addAll(testClasspath); - classpath.addAll(testRun.getRunnerClasspath()); + classpath.addAll(this.classpath); + classpath.addAll(action.getRunnerClasspath()); Set<File> sourceFiles = new HashSet<File>(); - sourceFiles.add(testRun.getTestJava()); + sourceFiles.add(action.getJavaFile()); sourceFiles.addAll(dalvikAnnotationSourceFiles()); - // compile the test case + // compile the action case new Javac() .bootClasspath(sdkJar) .classpath(classpath) - .sourcepath(testRun.getTestDirectory()) - .destination(testClassesDir) + .sourcepath(action.getJavaDirectory()) + .destination(classesDir) .extra(javacArgs) .compile(sourceFiles); - postCompileTest(testRun); - return true; + postCompile(action); } /** - * Hook method called after test compilation. - * - * @param testRun The test being compiled + * Hook method called after action compilation. */ - abstract protected void postCompileTest(TestRun testRun); + abstract protected void postCompile(Action action); /** * Fill in properties for running in this mode */ - protected void fillInProperties(Properties properties, TestRun testRun) { - properties.setProperty(TestProperties.TEST_CLASS, testRun.getTestClass()); - properties.setProperty(TestProperties.QUALIFIED_NAME, testRun.getQualifiedName()); - properties.setProperty(TestProperties.RUNNER_CLASS, testRun.getRunnerClass().getName()); + protected void fillInProperties(Properties properties, Action action) { + properties.setProperty(TestProperties.TEST_CLASS, action.getTargetClass()); + properties.setProperty(TestProperties.QUALIFIED_NAME, action.getName()); + properties.setProperty(TestProperties.RUNNER_CLASS, action.getRunnerClass().getName()); } /** - * Runs the test, and updates its test result. + * Runs the action, and returns its outcomes. */ - void runTest(TestRun testRun) { - if (!testRun.isRunnable()) { - throw new IllegalArgumentException(); - } - + Set<Outcome> run(Action action) { List<String> output; try { - output = runTestCommand(testRun); + output = executeAction(action); } catch (TimeoutException e) { - testRun.setResult(Result.EXEC_TIMEOUT, - Collections.singletonList("Exceeded timeout! (" + timeoutSeconds + "s)")); - return; + return Collections.singleton(new Outcome(action.getName(), + Result.EXEC_TIMEOUT, "Exceeded timeout! (" + timeoutSeconds + "s)")); } catch (Exception e) { - testRun.setResult(Result.ERROR, e); - return; + return Collections.singleton(new Outcome(action.getName(), Result.ERROR, e)); } // we only look at the output of the last command if (output.isEmpty()) { - testRun.setResult(Result.ERROR, - Collections.singletonList("No output returned!")); - return; + return Collections.singleton(new Outcome(action.getName(), + Result.ERROR, "No output returned!")); } Result result = TestProperties.RESULT_SUCCESS.equals(output.get(output.size() - 1)) ? Result.SUCCESS : Result.EXEC_FAILED; - testRun.setResult(result, output.subList(0, output.size() - 1)); + return Collections.singleton(new Outcome(action.getName(), + action.getName(), result, output.subList(0, output.size() - 1))); } /** - * Run the actual test to gather output + * Run the actual action to gather output */ - protected abstract List<String> runTestCommand(TestRun testRun) + protected abstract List<String> executeAction(Action action) throws TimeoutException; /** * Deletes files and releases any resources required for the execution of - * the given test. + * the given action. */ - void cleanup(TestRun testRun) { - environment.cleanup(testRun); + void cleanup(Action action) { + environment.cleanup(action); } /** - * Cleans up after all test runs have completed. + * Cleans up after all actions have completed. */ void shutdown() { environment.shutdown(); diff --git a/tools/runner/java/vogar/NamingPatternCodeFinder.java b/tools/runner/java/vogar/NamingPatternCodeFinder.java index a5524d1..1dd6166 100644 --- a/tools/runner/java/vogar/NamingPatternCodeFinder.java +++ b/tools/runner/java/vogar/NamingPatternCodeFinder.java @@ -34,9 +34,9 @@ abstract class NamingPatternCodeFinder implements CodeFinder { private final String TYPE_DECLARATION_PATTERN = "(?m)\\b(?:public|private)\\s+(?:final\\s+)?(?:interface|class|enum)\\b"; - public Set<TestRun> findTests(File testDirectory) { - Set<TestRun> result = new LinkedHashSet<TestRun>(); - findTestsRecursive(result, testDirectory); + public Set<Action> findActions(File searchDirectory) { + Set<Action> result = new LinkedHashSet<Action>(); + findActionsRecursive(result, searchDirectory); return result; } @@ -48,12 +48,10 @@ abstract class NamingPatternCodeFinder implements CodeFinder { && file.getName().endsWith(".java")); } - protected abstract String testName(File file); - - private void findTestsRecursive(Set<TestRun> sink, File file) { + private void findActionsRecursive(Set<Action> sink, File file) { if (file.isDirectory()) { for (File child : file.listFiles()) { - findTestsRecursive(sink, child); + findActionsRecursive(sink, child); } return; } @@ -63,11 +61,9 @@ abstract class NamingPatternCodeFinder implements CodeFinder { } String className = fileToClass(file); - File testDirectory = file.getParentFile(); - String testName = testName(file); - String testDescription = null; - sink.add(new TestRun(testDirectory, file, className, className, - testName, className, testDescription, + File directory = file.getParentFile(); + String description = null; + sink.add(new Action(className, className, directory, file, description, getRunnerClass(), getRunnerJava(), getRunnerClasspath())); } diff --git a/tools/runner/java/vogar/Outcome.java b/tools/runner/java/vogar/Outcome.java new file mode 100644 index 0000000..253a3cc --- /dev/null +++ b/tools/runner/java/vogar/Outcome.java @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package vogar; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * An outcome of an action. Some actions may have multiple outcomes. For + * example, JUnit tests have one outcome for each test method. + */ +final class Outcome { + + private final String outcomeName; + private final String actionName; + private final Result result; + private final List<String> outputLines; + + public Outcome(String outcomeName, String actionName, Result result, + List<String> outputLines) { + this.outcomeName = outcomeName; + this.actionName = actionName; + this.result = result; + this.outputLines = outputLines; + } + + public Outcome(String actionName, Result result, String outputLine) { + this.outcomeName = actionName; + this.actionName = actionName; + this.result = result; + this.outputLines = Collections.singletonList(outputLine); + } + + public Outcome(String actionName, Result result, Throwable throwable) { + this.outcomeName = actionName; + this.actionName = actionName; + this.result = result; + this.outputLines = throwableToLines(throwable); + } + + public String getName() { + return outcomeName; + } + + public String getActionName() { + return actionName; + } + + public Result getResult() { + return result; + } + + public List<String> getOutputLines() { + return outputLines; + } + + private static List<String> throwableToLines(Throwable t) { + StringWriter writer = new StringWriter(); + PrintWriter out = new PrintWriter(writer); + t.printStackTrace(out); + return Arrays.asList(writer.toString().split("\\n")); + } + + /** + * Returns the action's suite name, such as java.lang.Integer or + * java.lang.IntegerTest. + */ + public String getSuiteName() { + int lastDot = outcomeName.lastIndexOf('.'); + return lastDot == -1 ? "defaultpackage" : outcomeName.substring(0, lastDot); + } + + /** + * Returns the specific action name, such as BitTwiddle or testBitTwiddle. + */ + public String getTestName() { + int lastDot = outcomeName.lastIndexOf('.'); + return lastDot == -1 ? outcomeName : outcomeName.substring(lastDot + 1); + } +} diff --git a/tools/runner/java/vogar/TestRun.java b/tools/runner/java/vogar/TestRun.java deleted file mode 100644 index 1f6f00a..0000000 --- a/tools/runner/java/vogar/TestRun.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright (C) 2009 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package vogar; - -import vogar.target.Runner; - -import java.io.File; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.Arrays; -import java.util.List; - -/** - * A test run and its outcome. This class tracks the complete lifecycle of a - * single test run: - * <ol> - * <li>the test source code (test directory, java file, test class) - * <li>the test identity (suite name, test name, qualified name) - * <li>the code to execute (test classes, user dir) - * <li>the result of execution (expected result, result, output lines) - * </ol> - */ -public final class TestRun { - - private final File testDirectory; - private final File testJava; - private final String testClass; - private final Class<? extends Runner> runnerClass; - private final File runnerJava; - private final Classpath runnerClasspath; - - private final String suiteName; - private final String testName; - private final String qualifiedName; - private final String description; - - private boolean testCompiled; - private File userDir = new File(System.getProperty("user.dir")); - - private ExpectedResult expectedResult = ExpectedResult.SUCCESS; - private Result result; - private List<String> outputLines; - - public TestRun(File testDirectory, File testJava, String testClass, - String suiteName, String testName, String qualifiedName, - String description, Class<? extends Runner> runnerClass, - File runnerJava, Classpath runnerClasspath) { - this.qualifiedName = qualifiedName; - this.suiteName = suiteName; - this.testName = testName; - this.testDirectory = testDirectory; - this.testJava = testJava; - this.description = description; - this.testClass = testClass; - this.runnerClass = runnerClass; - this.runnerJava = runnerJava; - this.runnerClasspath = runnerClasspath; - } - - /** - * Returns the local directory containing this test's java file. - */ - public File getTestDirectory() { - return testDirectory; - } - - public File getTestJava() { - return testJava; - } - - /** - * Returns the executable test's classname, such as java.lang.IntegerTest - * or BitTwiddle. - */ - public String getTestClass() { - return testClass; - } - - /** - * Returns the test suite name, such as java.lang.Integer or - * java.lang.IntegerTest. - */ - public String getSuiteName() { - return suiteName; - } - - /** - * Returns the specific test name, such as BitTwiddle or testBitTwiddle. - */ - public String getTestName() { - return testName; - } - - /** - * Returns a unique identifier for this test. - */ - public String getQualifiedName() { - return qualifiedName; - } - - /** - * Returns an English description of this test, or null if no such - * description is known. - */ - public String getDescription() { - return description; - } - - public void setExpectedResult(ExpectedResult expectedResult) { - this.expectedResult = expectedResult; - } - - /** - * Set when the test is successfully compiled. - */ - public void setTestCompiled(boolean testCompiled) { - this.testCompiled = testCompiled; - } - - public boolean getTestCompiled() { - return testCompiled; - } - - /** - * Initializes the directory from which local files can be read by the test. - */ - public void setUserDir(File base) { - this.userDir = base; - } - - public File getUserDir() { - return userDir; - } - - /** - * Returns true if this test is ready for execution. Such tests have their - * classpath prepared and have not yet been assigned a result. - */ - public boolean isRunnable() { - return testCompiled && result == null; - } - - public void setResult(Result result, Throwable e) { - setResult(result, throwableToLines(e)); - } - - public void setResult(Result result, List<String> outputLines) { - if (this.result != null) { - throw new IllegalStateException("result already set"); - } - - this.result = result; - this.outputLines = outputLines; - } - - private static List<String> throwableToLines(Throwable t) { - StringWriter writer = new StringWriter(); - PrintWriter out = new PrintWriter(writer); - t.printStackTrace(out); - return Arrays.asList(writer.toString().split("\\n")); - } - - public Result getResult() { - return result; - } - - public List<String> getOutputLines() { - return outputLines; - } - - public Class<? extends Runner> getRunnerClass() { - return runnerClass; - } - - public File getRunnerJava() { - return runnerJava; - } - - public Classpath getRunnerClasspath() { - return runnerClasspath; - } - - /** - * Returns true if the outcome of this run matches what was expected. - */ - public boolean isExpectedResult() { - return result == expectedResult.getResult() && matchesExpectedPattern(); - } - - /** - * Returns true if the test's output matches the expected output. - */ - private boolean matchesExpectedPattern() { - return expectedResult.getPattern() - .matcher(Strings.join(outputLines, "\n")) - .matches(); - } - - /** - * Returns the failure message for this failed test run. This message is - * intended to help to diagnose why the test result didn't match what was - * expected. - */ - public String getFailureMessage() { - StringBuilder builder = new StringBuilder(); - - if (expectedResult.getResult() != Result.SUCCESS - && expectedResult.getResult() != result) { - builder.append("Expected result: ") - .append(expectedResult.getResult()) - .append("\n"); - } - - if (!matchesExpectedPattern()) { - builder.append("Expected output to match \"") - .append(expectedResult.getPattern().pattern()) - .append("\"\n"); - } - - for (String output : outputLines) { - builder.append(output).append("\n"); - } - - return builder.toString(); - } - - @Override public String toString() { - return qualifiedName; - } -} diff --git a/tools/runner/java/vogar/Vm.java b/tools/runner/java/vogar/Vm.java index 337e580..fe94fc2 100644 --- a/tools/runner/java/vogar/Vm.java +++ b/tools/runner/java/vogar/Vm.java @@ -34,8 +34,6 @@ import java.util.logging.Logger; */ public abstract class Vm extends Mode { - private static final Logger logger = Logger.getLogger(Vm.class.getName()); - protected final List<String> additionalVmArgs; Vm(Environment environment, long timeoutSeconds, File sdkJar, List<String> javacArgs, @@ -47,11 +45,11 @@ public abstract class Vm extends Mode { /** * Returns a VM for test execution. */ - @Override protected List<String> runTestCommand(TestRun testRun) + @Override protected List<String> executeAction(Action action) throws TimeoutException { - Command command = newVmCommandBuilder(testRun.getUserDir()) - .classpath(getRuntimeSupportClasspath(testRun)) - .userDir(testRun.getUserDir()) + Command command = newVmCommandBuilder(action.getUserDir()) + .classpath(getRuntimeSupportClasspath(action)) + .userDir(action.getUserDir()) .debugPort(environment.debugPort) .vmArgs(additionalVmArgs) .mainClass(TestRunner.class.getName()) @@ -61,15 +59,15 @@ public abstract class Vm extends Mode { } /** - * Returns a VM for test execution. + * Returns a VM for action execution. */ protected abstract VmCommandBuilder newVmCommandBuilder(File workingDirectory); /** * Returns the classpath containing JUnit and the dalvik annotations - * required for test execution. + * required for action execution. */ - protected abstract Classpath getRuntimeSupportClasspath(TestRun testRun); + protected abstract Classpath getRuntimeSupportClasspath(Action action); /** * Builds a virtual machine command. diff --git a/tools/runner/java/vogar/Vogar.java b/tools/runner/java/vogar/Vogar.java index f7de555..bfed45e 100644 --- a/tools/runner/java/vogar/Vogar.java +++ b/tools/runner/java/vogar/Vogar.java @@ -44,7 +44,7 @@ public final class Vogar { private static class Options { - private final List<File> testFiles = new ArrayList<File>(); + private final List<File> actionFiles = new ArrayList<File>(); @Option(names = { "--expectations" }) private Set<File> expectationFiles = new LinkedHashSet<File>(); @@ -102,20 +102,20 @@ public final class Vogar { private File sdkJar = new File("/home/dalvik-prebuild/android-sdk-linux/platforms/android-2.0/android.jar"); private void printUsage() { - System.out.println("Usage: Vogar [options]... <tests>..."); + System.out.println("Usage: Vogar [options]... <actions>..."); System.out.println(); - System.out.println(" <tests>: a .java file containing a jtreg test, JUnit test,"); - System.out.println(" Caliper benchmark, or a directory of such tests."); + System.out.println(" <actions>: .java files containing a jtreg tests, JUnit tests,"); + System.out.println(" Caliper benchmarks, or a directory of such tests."); System.out.println(); System.out.println("GENERAL OPTIONS"); System.out.println(); System.out.println(" --expectations <file>: include the specified file when looking for"); - System.out.println(" test expectations. The file should include qualified test names"); + System.out.println(" action expectations. The file should include qualified action names"); System.out.println(" and the corresponding expected output."); System.out.println(" Default is: " + expectationFiles); System.out.println(); System.out.println(" --mode <device|host|activity>: specify which environment to run the"); - System.out.println(" tests in. Options are on the device VM, on the host VM, and on"); + System.out.println(" actions in. Options are on the device VM, on the host VM, and on"); System.out.println(" device within an android.app.Activity."); System.out.println(" Default is: " + mode); System.out.println(); @@ -130,18 +130,18 @@ public final class Vogar { System.out.println(" --clean: synonym for --clean-before and --clean-after (default)."); System.out.println(" Disable with --no-clean if you want no files removed."); System.out.println(); - System.out.println(" --tee <file>: emit test output to file during execution."); + System.out.println(" --tee <file>: emit output to file during execution."); System.out.println(" Specify '-' for stdout."); System.out.println(); System.out.println(" --timeout-seconds <seconds>: maximum execution time of each"); - System.out.println(" test before the runner aborts it. Specifying zero seconds"); + System.out.println(" action before the runner aborts it. Specifying zero seconds"); System.out.println(" or using --debug will disable the execution timeout"); System.out.println(" Default is: " + timeoutSeconds); System.out.println(); System.out.println(" --xml-reports-directory <path>: directory to emit JUnit-style"); System.out.println(" XML test results."); System.out.println(); - System.out.println(" --ident: amount to indent test result output. Can be set to ''"); + System.out.println(" --ident: amount to indent action result output. Can be set to ''"); System.out.println(" (aka empty string) to simplify output parsing."); System.out.println(" Default is: '" + indent + "'"); System.out.println(); @@ -164,7 +164,7 @@ public final class Vogar { System.out.println(); System.out.println("HOST VM OPTIONS"); System.out.println(); - System.out.println(" --java-home <java_home>: execute the tests on the local workstation"); + System.out.println(" --java-home <java_home>: execute the actions on the local workstation"); System.out.println(" using the specified java home directory. This does not impact"); System.out.println(" which javac gets used. When unset, java is used from the PATH."); System.out.println(); @@ -182,9 +182,9 @@ public final class Vogar { } private boolean parseArgs(String[] args) { - final List<String> testFilenames; + final List<String> actionFilenames; try { - testFilenames = new OptionParser(this).parse(args); + actionFilenames = new OptionParser(this).parse(args); } catch (RuntimeException e) { System.out.println(e.getMessage()); return false; @@ -242,8 +242,8 @@ public final class Vogar { return false; } - if (testFilenames.isEmpty()) { - System.out.println("No tests provided."); + if (actionFilenames.isEmpty()) { + System.out.println("No actions provided."); return false; } @@ -261,8 +261,8 @@ public final class Vogar { timeoutSeconds = 0; } - for (String testFilename : testFilenames) { - testFiles.add(new File(testFilename)); + for (String actionFilename : actionFilenames) { + actionFiles.add(new File(actionFilename)); } if (teeName != null) { @@ -352,21 +352,28 @@ public final class Vogar { new JUnitFinder(), new CaliperFinder(), new MainFinder()); - Driver driver = new Driver( - localTemp, - mode, - options.expectationFiles, - options.xmlReportsDirectory, - options.indent, - codeFinders); + + ExpectationStore expectationStore; try { - driver.loadExpectations(); + expectationStore = ExpectationStore.parse(options.expectationFiles); } catch (IOException e) { System.out.println("Problem loading expectations: " + e); return; } - driver.buildAndRunAllTests(options.testFiles); + XmlReportPrinter xmlReportPrinter = options.xmlReportsDirectory != null + ? new XmlReportPrinter(options.xmlReportsDirectory, expectationStore) + : null; + + Driver driver = new Driver( + localTemp, + mode, + expectationStore, + options.indent, + codeFinders, + xmlReportPrinter); + + driver.buildAndRunAllActions(options.actionFiles); mode.shutdown(); } diff --git a/tools/runner/java/vogar/XmlReportPrinter.java b/tools/runner/java/vogar/XmlReportPrinter.java index 0632279..2cd8c66 100644 --- a/tools/runner/java/vogar/XmlReportPrinter.java +++ b/tools/runner/java/vogar/XmlReportPrinter.java @@ -52,7 +52,6 @@ public class XmlReportPrinter { private static final String ATTR_FAILURES = "failures"; private static final String ATTR_TESTS = "tests"; private static final String ATTR_TYPE = "type"; - private static final String ATTR_MESSAGE = "message"; private static final String PROPERTIES = "properties"; private static final String ATTR_CLASSNAME = "classname"; private static final String TIMESTAMP = "timestamp"; @@ -61,10 +60,18 @@ public class XmlReportPrinter { /** the XML namespace */ private static final String ns = null; + private final File directory; + private final ExpectationStore expectationStore; + + public XmlReportPrinter(File directory, ExpectationStore expectationStore) { + this.directory = directory; + this.expectationStore = expectationStore; + } + /** * Populates the directory with the report data from the completed tests. */ - public int generateReports(File directory, Collection<TestRun> results) { + public int generateReports(Collection<Outcome> results) { Map<String, Suite> suites = testsToSuites(results); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); @@ -100,24 +107,25 @@ public class XmlReportPrinter { return suites.size(); } - private Map<String, Suite> testsToSuites(Collection<TestRun> testRuns) { + private Map<String, Suite> testsToSuites(Collection<Outcome> outcomes) { Map<String, Suite> result = new LinkedHashMap<String, Suite>(); - for (TestRun testRun : testRuns) { - if (testRun.getResult() == Result.UNSUPPORTED) { + for (Outcome outcome : outcomes) { + if (outcome.getResult() == Result.UNSUPPORTED) { continue; } - String suiteName = testRun.getSuiteName(); + String suiteName = outcome.getSuiteName(); Suite suite = result.get(suiteName); if (suite == null) { suite = new Suite(suiteName); result.put(suiteName, suite); } - suite.tests.add(testRun); + suite.outcomes.add(outcome); - if (!testRun.isExpectedResult()) { - if (testRun.getResult() == Result.EXEC_FAILED) { + Expectation expectation = expectationStore.get(outcome.getName()); + if (!expectation.matches(outcome)) { + if (outcome.getResult() == Result.EXEC_FAILED) { suite.failuresCount++; } else { suite.errorsCount++; @@ -127,9 +135,9 @@ public class XmlReportPrinter { return result; } - static class Suite { + class Suite { private final String name; - private final List<TestRun> tests = new ArrayList<TestRun>(); + private final List<Outcome> outcomes = new ArrayList<Outcome>(); private int failuresCount; private int errorsCount; @@ -140,7 +148,7 @@ public class XmlReportPrinter { void print(KXmlSerializer serializer, String timestamp) throws IOException { serializer.startTag(ns, TESTSUITE); serializer.attribute(ns, ATTR_NAME, name); - serializer.attribute(ns, ATTR_TESTS, Integer.toString(tests.size())); + serializer.attribute(ns, ATTR_TESTS, Integer.toString(outcomes.size())); serializer.attribute(ns, ATTR_FAILURES, Integer.toString(failuresCount)); serializer.attribute(ns, ATTR_ERRORS, Integer.toString(errorsCount)); serializer.attribute(ns, ATTR_TIME, "0"); @@ -149,28 +157,25 @@ public class XmlReportPrinter { serializer.startTag(ns, PROPERTIES); serializer.endTag(ns, PROPERTIES); - for (TestRun testRun : tests) { - print(serializer, testRun); + for (Outcome outcome : outcomes) { + print(serializer, outcome); } serializer.endTag(ns, TESTSUITE); } - void print(KXmlSerializer serializer, TestRun testRun) throws IOException { + void print(KXmlSerializer serializer, Outcome outcome) throws IOException { serializer.startTag(ns, TESTCASE); - serializer.attribute(ns, ATTR_NAME, testRun.getTestName()); - serializer.attribute(ns, ATTR_CLASSNAME, testRun.getSuiteName()); + serializer.attribute(ns, ATTR_NAME, outcome.getTestName()); + serializer.attribute(ns, ATTR_CLASSNAME, outcome.getSuiteName()); serializer.attribute(ns, ATTR_TIME, "0"); - if (!testRun.isExpectedResult()) { - String result = testRun.getResult() == Result.EXEC_FAILED ? FAILURE : ERROR; + Expectation expectation = expectationStore.get(outcome.getName()); + if (!expectation.matches(outcome)) { + String result = outcome.getResult() == Result.EXEC_FAILED ? FAILURE : ERROR; serializer.startTag(ns, result); - String title = testRun.getDescription(); - if (title != null && title.length() > 0) { - serializer.attribute(ns, ATTR_MESSAGE, title); - } - serializer.attribute(ns, ATTR_TYPE, testRun.getResult().toString()); - String text = sanitize(Strings.join(testRun.getOutputLines(), "\n")); + serializer.attribute(ns, ATTR_TYPE, outcome.getResult().toString()); + String text = sanitize(Strings.join(outcome.getOutputLines(), "\n")); serializer.text(text); serializer.endTag(ns, result); } |