From fa063dab862bf76595739580f87c7bf803c42e43 Mon Sep 17 00:00:00 2001 From: Brett Chabot Date: Wed, 29 Sep 2010 16:55:00 -0700 Subject: Fix ddmlib test run failure and metric reporting. This submit fixes two bugs: 1) Test run failures were not getting reported correctly in some scenarios, and in others testRunFailed was being called before testRunStarted was called. With this submit, the order of ITestRunListener callbacks is documented and strictly adhered to. 2) test metrics were getting reported for regularly occuring instrumentation output such as 'stream' and 'id'. Also cleaned up the associated unit tests and changed them to use easymock. Change-Id: I4cee3abebdf1c607ac1dc51a240c92ca9611ca31 --- .../ddmlib/testrunner/ITestRunListener.java | 76 ++++-- .../testrunner/InstrumentationResultParser.java | 44 ++- .../ddmlib/testrunner/RemoteAndroidTestRunner.java | 28 +- ddms/libs/ddmlib/tests/.classpath | 1 + ddms/libs/ddmlib/tests/Android.mk | 4 +- .../InstrumentationResultParserTest.java | 302 +++++++++------------ .../testrunner/RemoteAndroidTestRunnerTest.java | 290 ++++---------------- .../adt/internal/launch/messages.properties | 2 +- .../META-INF/MANIFEST.MF | 3 +- .../com.android.ide.eclipse.tests/build.properties | 3 +- 10 files changed, 301 insertions(+), 452 deletions(-) diff --git a/ddms/libs/ddmlib/src/com/android/ddmlib/testrunner/ITestRunListener.java b/ddms/libs/ddmlib/src/com/android/ddmlib/testrunner/ITestRunListener.java index b235597..a8b117d 100644 --- a/ddms/libs/ddmlib/src/com/android/ddmlib/testrunner/ITestRunListener.java +++ b/ddms/libs/ddmlib/src/com/android/ddmlib/testrunner/ITestRunListener.java @@ -19,8 +19,20 @@ package com.android.ddmlib.testrunner; import java.util.Map; /** - * Receives event notifications during instrumentation test runs. + * Receives event notifications during instrumentation test runs. + *

* Patterned after {@link junit.runner.TestRunListener}. + *

+ * The sequence of calls will be: + *

*/ public interface ITestRunListener { @@ -34,7 +46,7 @@ public interface ITestRunListener { FAILURE } - /** + /** * Reports the start of a test run. * * @param runName the test run name @@ -43,49 +55,55 @@ public interface ITestRunListener { public void testRunStarted(String runName, int testCount); /** - * Reports end of test run. - * - * @param elapsedTime device reported elapsed time, in milliseconds - * @param runMetrics key-value pairs reported at the end of a test run - */ - public void testRunEnded(long elapsedTime, Map runMetrics); - - /** - * Reports test run stopped before completion. - * - * @param elapsedTime device reported elapsed time, in milliseconds + * Reports the start of an individual test case. + * + * @param test identifies the test */ - public void testRunStopped(long elapsedTime); + public void testStarted(TestIdentifier test); /** - * Reports the start of an individual test case. - * + * Reports the failure of a individual test case. + *

+ * Will be called between testStarted and testEnded. + * + * @param status failure type * @param test identifies the test + * @param trace stack trace of failure */ - public void testStarted(TestIdentifier test); + public void testFailed(TestFailure status, TestIdentifier test, String trace); /** * Reports the execution end of an individual test case. + *

* If {@link #testFailed} was not invoked, this test passed. Also returns any key/value * metrics which may have been emitted during the test case's execution. - * + * * @param test identifies the test * @param testMetrics a {@link Map} of the metrics emitted */ public void testEnded(TestIdentifier test, Map testMetrics); /** - * Reports the failure of a individual test case. - * Will be called between testStarted and testEnded. - * - * @param status failure type - * @param test identifies the test - * @param trace stack trace of failure - */ - public void testFailed(TestFailure status, TestIdentifier test, String trace); - - /** - * Reports test run failed to execute due to a fatal error. + * Reports test run failed to complete due to a fatal error. + * + * @param errorMessage {@link String} describing reason for run failure. */ public void testRunFailed(String errorMessage); + + /** + * Reports test run stopped before completion due to a user request. + *

+ * TODO: currently unused, consider removing + * + * @param elapsedTime device reported elapsed time, in milliseconds + */ + public void testRunStopped(long elapsedTime); + + /** + * Reports end of test run. + * + * @param elapsedTime device reported elapsed time, in milliseconds + * @param runMetrics key-value pairs reported at the end of a test run + */ + public void testRunEnded(long elapsedTime, Map runMetrics); } diff --git a/ddms/libs/ddmlib/src/com/android/ddmlib/testrunner/InstrumentationResultParser.java b/ddms/libs/ddmlib/src/com/android/ddmlib/testrunner/InstrumentationResultParser.java index 61236d8..4819fbc 100644 --- a/ddms/libs/ddmlib/src/com/android/ddmlib/testrunner/InstrumentationResultParser.java +++ b/ddms/libs/ddmlib/src/com/android/ddmlib/testrunner/InstrumentationResultParser.java @@ -23,7 +23,9 @@ import com.android.ddmlib.MultiLineReceiver; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -74,6 +76,21 @@ public class InstrumentationResultParser extends MultiLineReceiver { private static final String SHORTMSG = "shortMsg"; } + /** The set of expected status keys. Used to filter which keys should be stored as metrics */ + private static final Set KNOWN_KEYS = new HashSet(); + static { + KNOWN_KEYS.add(StatusKeys.TEST); + KNOWN_KEYS.add(StatusKeys.CLASS); + KNOWN_KEYS.add(StatusKeys.STACK); + KNOWN_KEYS.add(StatusKeys.NUMTESTS); + KNOWN_KEYS.add(StatusKeys.ERROR); + KNOWN_KEYS.add(StatusKeys.SHORTMSG); + // unused, but regularly occurring status keys. + KNOWN_KEYS.add("stream"); + KNOWN_KEYS.add("id"); + KNOWN_KEYS.add("current"); + } + /** Test result status codes. */ private static class StatusCodes { private static final int FAILURE = -2; @@ -183,6 +200,12 @@ public class InstrumentationResultParser extends MultiLineReceiver { /** Error message supplied when no parseable test results are received from test run. */ static final String NO_TEST_RESULTS_MSG = "No test results"; + /** Error message supplied when a test start bundle is parsed, but not the test end bundle. */ + static final String INCOMPLETE_TEST_ERR_MSG_PREFIX = "Incomplete"; + + /** Error message supplied when the test run is incomplete. */ + static final String INCOMPLETE_RUN_ERR_MSG_PREFIX = "Test run incomplete"; + /** * Creates the InstrumentationResultParser. * @@ -284,8 +307,9 @@ public class InstrumentationResultParser extends MultiLineReceiver { if (mCurrentKey != null && mCurrentValue != null) { String statusValue = mCurrentValue.toString(); if (mInInstrumentationResultKey) { - mInstrumentationResultBundle.put(mCurrentKey, statusValue); - if (mCurrentKey.equals(StatusKeys.SHORTMSG)) { + if (!KNOWN_KEYS.contains(mCurrentKey)) { + mInstrumentationResultBundle.put(mCurrentKey, statusValue); + } else if (mCurrentKey.equals(StatusKeys.SHORTMSG)) { // test run must have failed handleTestRunFailed(statusValue); } @@ -308,7 +332,7 @@ public class InstrumentationResultParser extends MultiLineReceiver { handleTestRunFailed(statusValue); } else if (mCurrentKey.equals(StatusKeys.STACK)) { testInfo.mStackTrace = statusValue; - } else { + } else if (!KNOWN_KEYS.contains(mCurrentKey)) { // Not one of the recognized key/value pairs, so dump it in mTestMetrics mTestMetrics.put(mCurrentKey, statusValue); } @@ -506,7 +530,7 @@ public class InstrumentationResultParser extends MultiLineReceiver { /** * Process a instrumentation run failure */ - private void handleTestRunFailed(String errorMsg) { + void handleTestRunFailed(String errorMsg) { errorMsg = (errorMsg == null ? "Unknown error" : errorMsg); Log.i(LOG_TAG, String.format("test run failed %s", errorMsg)); if (mLastTestResult != null && @@ -519,13 +543,19 @@ public class InstrumentationResultParser extends MultiLineReceiver { mLastTestResult.mTestName); for (ITestRunListener listener : mTestListeners) { listener.testFailed(ITestRunListener.TestFailure.ERROR, testId, - String.format("Incomplete: %s", errorMsg)); + String.format("%s: %s", INCOMPLETE_TEST_ERR_MSG_PREFIX, errorMsg)); listener.testEnded(testId, getAndResetTestMetrics()); } } for (ITestRunListener listener : mTestListeners) { + if (!mTestStartReported) { + // test run wasn't started - must have crashed before it started + listener.testRunStarted(mTestRunName, 0); + } listener.testRunFailed(errorMsg); + listener.testRunEnded(mTestTime, mInstrumentationResultBundle); } + mTestStartReported = true; mTestRunFailReported = true; } @@ -540,8 +570,8 @@ public class InstrumentationResultParser extends MultiLineReceiver { handleTestRunFailed(NO_TEST_RESULTS_MSG); } else if (!mTestRunFailReported && mNumTestsExpected > mNumTestsRun) { final String message = - String.format("Test run incomplete. Expected %d tests, received %d", - mNumTestsExpected, mNumTestsRun); + String.format("%s. Expected %d tests, received %d", + INCOMPLETE_RUN_ERR_MSG_PREFIX, mNumTestsExpected, mNumTestsRun); handleTestRunFailed(message); } else { for (ITestRunListener listener : mTestListeners) { diff --git a/ddms/libs/ddmlib/src/com/android/ddmlib/testrunner/RemoteAndroidTestRunner.java b/ddms/libs/ddmlib/src/com/android/ddmlib/testrunner/RemoteAndroidTestRunner.java index bc7b012..4c62041 100644 --- a/ddms/libs/ddmlib/src/com/android/ddmlib/testrunner/RemoteAndroidTestRunner.java +++ b/ddms/libs/ddmlib/src/com/android/ddmlib/testrunner/RemoteAndroidTestRunner.java @@ -233,7 +233,33 @@ public class RemoteAndroidTestRunner implements IRemoteAndroidTestRunner { // TODO: allow run name to be configurable mParser = new InstrumentationResultParser(mPackageName, listeners); - mRemoteDevice.executeShellCommand(runCaseCommandStr, mParser, mMaxTimeToOutputResponse); + try { + mRemoteDevice.executeShellCommand(runCaseCommandStr, mParser, mMaxTimeToOutputResponse); + } catch (IOException e) { + Log.w(LOG_TAG, String.format("IOException %s when running tests %s on %s", + e.toString(), getPackageName(), mRemoteDevice.getSerialNumber())); + // rely on parser to communicate results to listeners + mParser.handleTestRunFailed(e.toString()); + throw e; + } catch (ShellCommandUnresponsiveException e) { + Log.w(LOG_TAG, String.format( + "ShellCommandUnresponsiveException %s when running tests %s on %s", + e.toString(), getPackageName(), mRemoteDevice.getSerialNumber())); + mParser.handleTestRunFailed(e.toString()); + throw e; + } catch (TimeoutException e) { + Log.w(LOG_TAG, String.format( + "TimeoutException when running tests %s on %s", getPackageName(), + mRemoteDevice.getSerialNumber())); + mParser.handleTestRunFailed(e.toString()); + throw e; + } catch (AdbCommandRejectedException e) { + Log.w(LOG_TAG, String.format( + "AdbCommandRejectedException %s when running tests %s on %s", + e.toString(), getPackageName(), mRemoteDevice.getSerialNumber())); + mParser.handleTestRunFailed(e.toString()); + throw e; + } } /** diff --git a/ddms/libs/ddmlib/tests/.classpath b/ddms/libs/ddmlib/tests/.classpath index 2a5d2da..f2e9d17 100644 --- a/ddms/libs/ddmlib/tests/.classpath +++ b/ddms/libs/ddmlib/tests/.classpath @@ -4,5 +4,6 @@ + diff --git a/ddms/libs/ddmlib/tests/Android.mk b/ddms/libs/ddmlib/tests/Android.mk index afec3d8..238fd60 100644 --- a/ddms/libs/ddmlib/tests/Android.mk +++ b/ddms/libs/ddmlib/tests/Android.mk @@ -19,9 +19,9 @@ include $(CLEAR_VARS) # Only compile source java files in this lib. LOCAL_SRC_FILES := $(call all-java-files-under, src) -LOCAL_MODULE := ddmlibTests +LOCAL_MODULE := ddmlib-tests -LOCAL_JAVA_LIBRARIES := ddmlib junit +LOCAL_JAVA_LIBRARIES := ddmlib junit easymock include $(BUILD_HOST_JAVA_LIBRARY) diff --git a/ddms/libs/ddmlib/tests/src/com/android/ddmlib/testrunner/InstrumentationResultParserTest.java b/ddms/libs/ddmlib/tests/src/com/android/ddmlib/testrunner/InstrumentationResultParserTest.java index 946e614..032b879 100644 --- a/ddms/libs/ddmlib/tests/src/com/android/ddmlib/testrunner/InstrumentationResultParserTest.java +++ b/ddms/libs/ddmlib/tests/src/com/android/ddmlib/testrunner/InstrumentationResultParserTest.java @@ -18,24 +18,29 @@ package com.android.ddmlib.testrunner; import com.android.ddmlib.testrunner.ITestRunListener.TestFailure; +import org.easymock.Capture; +import org.easymock.EasyMock; + import junit.framework.TestCase; +import java.util.Collections; import java.util.Map; - /** - * Tests InstrumentationResultParser. + * Unit tests for {@link @InstrumentationResultParser}. */ +@SuppressWarnings("unchecked") public class InstrumentationResultParserTest extends TestCase { private InstrumentationResultParser mParser; - private VerifyingTestResult mTestResult; + private ITestRunListener mMockListener; // static dummy test names to use for validation private static final String RUN_NAME = "foo"; private static final String CLASS_NAME = "com.test.FooTest"; private static final String TEST_NAME = "testFoo"; private static final String STACK_TRACE = "java.lang.AssertionFailedException"; + private static final TestIdentifier TEST_ID = new TestIdentifier(CLASS_NAME, TEST_NAME); /** * @param name - test name @@ -50,96 +55,109 @@ public class InstrumentationResultParserTest extends TestCase { @Override protected void setUp() throws Exception { super.setUp(); - mTestResult = new VerifyingTestResult(); - mParser = new InstrumentationResultParser(RUN_NAME, mTestResult); + // use a strict mock to verify order of method calls + mMockListener = EasyMock.createStrictMock(ITestRunListener.class); + mParser = new InstrumentationResultParser(RUN_NAME, mMockListener); } /** - * Tests that the test run started and test start events is sent on first - * bundle received. + * Tests parsing empty output. */ - public void testTestStarted() { - StringBuilder output = buildCommonResult(); - addStartCode(output); + public void testParse_empty() { + mMockListener.testRunStarted(RUN_NAME, 0); + mMockListener.testRunFailed(InstrumentationResultParser.NO_TEST_RESULTS_MSG); + mMockListener.testRunEnded(0, Collections.EMPTY_MAP); - injectTestString(output.toString()); - assertCommonAttributes(); + injectAndVerifyTestString(""); } /** - * Tests basic parsing of a single successful test execution. + * Tests parsing output for a successful test run with no tests. */ - public void testTestSuccess() { - StringBuilder output = createSuccessTest(); + public void testParse_noTests() { + StringBuilder output = new StringBuilder(); + addLine(output, "INSTRUMENTATION_RESULT: stream="); + addLine(output, "Test results for InstrumentationTestRunner="); + addLine(output, "Time: 0.001"); + addLine(output, "OK (0 tests)"); + addLine(output, "INSTRUMENTATION_CODE: -1"); + + mMockListener.testRunStarted(RUN_NAME, 0); + mMockListener.testRunEnded(1, Collections.EMPTY_MAP); - injectTestString(output.toString()); - assertCommonAttributes(); - assertEquals(1, mTestResult.mNumTestsRun); - assertEquals(null, mTestResult.mTestStatus); + injectAndVerifyTestString(output.toString()); } /** - * Tests basic parsing of a successful test execution with metrics. + * Tests parsing output for a single successful test execution. */ - public void testTestSuccessMetrics() { - StringBuilder output = buildCommonResult(); + public void testParse_singleTest() { + StringBuilder output = createSuccessTest(); - addStatusKey(output, "randomKey", "randomValue"); - assertNotNull(mTestResult.mTestMetrics); - assertEquals("randomValue", mTestResult.mTestMetrics.get("randomKey")); + mMockListener.testRunStarted(RUN_NAME, 1); + mMockListener.testStarted(TEST_ID); + mMockListener.testEnded(TEST_ID, Collections.EMPTY_MAP); + mMockListener.testRunEnded(0, Collections.EMPTY_MAP); + + injectAndVerifyTestString(output.toString()); } /** - * Create instrumentation output for a successful single test case execution. + * Tests parsing output for a successful test execution with metrics. */ - private StringBuilder createSuccessTest() { + public void testParse_testMetrics() { StringBuilder output = buildCommonResult(); + + addStatusKey(output, "randomKey", "randomValue"); addSuccessCode(output); - return output; + + final Capture> captureMetrics = new Capture>(); + mMockListener.testRunStarted(RUN_NAME, 1); + mMockListener.testStarted(TEST_ID); + mMockListener.testEnded(EasyMock.eq(TEST_ID), EasyMock.capture(captureMetrics)); + mMockListener.testRunEnded(0, Collections.EMPTY_MAP); + + injectAndVerifyTestString(output.toString()); + + assertEquals("randomValue", captureMetrics.getValue().get("randomKey")); } /** - * Test basic parsing of failed test case. + * Test parsing output for a test failure. */ - public void testTestFailed() { + public void testParse_testFailed() { StringBuilder output = buildCommonResult(); - addStartCode(output); - addCommonStatus(output); addStackTrace(output); addFailureCode(output); - injectTestString(output.toString()); - assertCommonAttributes(); - - assertEquals(1, mTestResult.mNumTestsRun); - assertEquals(ITestRunListener.TestFailure.FAILURE, mTestResult.mTestStatus); - assertEquals(STACK_TRACE, mTestResult.mTrace); - } + mMockListener.testRunStarted(RUN_NAME, 1); + mMockListener.testStarted(TEST_ID); + mMockListener.testFailed(TestFailure.FAILURE, TEST_ID, STACK_TRACE); + mMockListener.testEnded(TEST_ID, Collections.EMPTY_MAP); + mMockListener.testRunEnded(0, Collections.EMPTY_MAP); - /** - * Test basic parsing and conversion of time from output. - */ - public void testTimeParsing() { - StringBuilder output = createSuccessTest(); - output.append("Time: 4.9"); - injectTestString(output.toString()); - assertEquals(4900, mTestResult.mTestTime); + injectAndVerifyTestString(output.toString()); } /** * Test parsing and conversion of time output that contains extra chars. */ - public void testTimeParsing_bracket() { + public void testParse_timeBracket() { StringBuilder output = createSuccessTest(); output.append("Time: 0.001)"); - injectTestString(output.toString()); - assertEquals(1, mTestResult.mTestTime); + + mMockListener.testRunStarted(RUN_NAME, 1); + mMockListener.testStarted(TEST_ID); + mMockListener.testEnded(TEST_ID, Collections.EMPTY_MAP); + mMockListener.testRunEnded(1, Collections.EMPTY_MAP); + + injectAndVerifyTestString(output.toString()); } /** - * Test basic parsing of a test run failure. + * Test parsing output for a test run failure. */ - public void testRunFailed() { + public void testParse_runFailed() { StringBuilder output = new StringBuilder(); final String errorMessage = "Unable to find instrumentation info"; addStatusKey(output, "Error", errorMessage); @@ -147,16 +165,20 @@ public class InstrumentationResultParserTest extends TestCase { output.append("INSTRUMENTATION_FAILED: com.dummy/android.test.InstrumentationTestRunner"); addLineBreak(output); - injectTestString(output.toString()); + mMockListener.testRunStarted(RUN_NAME, 0); + mMockListener.testRunFailed(errorMessage); + mMockListener.testRunEnded(0, Collections.EMPTY_MAP); - assertEquals(errorMessage, mTestResult.mRunFailedMessage); + injectAndVerifyTestString(output.toString()); } /** - * Test parsing of a test run failure, where an instrumentation component failed to load + * Test parsing output for a test run failure, where an instrumentation component failed to + * load. + *

* Parsing input takes the from of INSTRUMENTATION_RESULT: fff */ - public void testRunFailedResult() { + public void testParse_failedResult() { StringBuilder output = new StringBuilder(); final String errorMessage = "Unable to instantiate instrumentation"; output.append("INSTRUMENTATION_RESULT: shortMsg="); @@ -165,32 +187,40 @@ public class InstrumentationResultParserTest extends TestCase { output.append("INSTRUMENTATION_CODE: 0"); addLineBreak(output); - injectTestString(output.toString()); + mMockListener.testRunStarted(RUN_NAME, 0); + mMockListener.testRunFailed(errorMessage); + mMockListener.testRunEnded(0, Collections.EMPTY_MAP); - assertEquals(errorMessage, mTestResult.mRunFailedMessage); + injectAndVerifyTestString(output.toString()); } /** - * Test parsing of a test run that did not complete. This can occur if device spontaneously - * reboots, or if test method could not be found + * Test parsing output for a test run that did not complete. + *

+ * This can occur if device spontaneously reboots, or if test method could not be found. */ - public void testRunIncomplete() { + public void testParse_incomplete() { StringBuilder output = new StringBuilder(); // add a start test sequence, but without an end test sequence addCommonStatus(output); addStartCode(output); - injectTestString(output.toString()); + mMockListener.testRunStarted(RUN_NAME, 1); + mMockListener.testStarted(TEST_ID); + mMockListener.testFailed(EasyMock.eq(TestFailure.ERROR), EasyMock.eq(TEST_ID), + EasyMock.startsWith(InstrumentationResultParser.INCOMPLETE_TEST_ERR_MSG_PREFIX)); + mMockListener.testEnded(TEST_ID, Collections.EMPTY_MAP); + mMockListener.testRunFailed(EasyMock.startsWith( + InstrumentationResultParser.INCOMPLETE_RUN_ERR_MSG_PREFIX)); + mMockListener.testRunEnded(0, Collections.EMPTY_MAP); - assertTrue(mTestResult.mRunFailedMessage.startsWith("Test run incomplete.")); - // ensure test is marked as failed - assertEquals(TestFailure.ERROR, mTestResult.mTestStatus); + injectAndVerifyTestString(output.toString()); } /** - * Test parsing of a test run that did not start due to incorrect syntax supplied to am. + * Test parsing output for a test run that did not start due to incorrect syntax supplied to am. */ - public void testRunAmFailed() { + public void testParse_amFailed() { StringBuilder output = new StringBuilder(); addLine(output, "usage: am [subcommand] [options]"); addLine(output, "start an Activity: am start [-D] [-W] "); @@ -199,49 +229,36 @@ public class InstrumentationResultParserTest extends TestCase { addLine(output, "start a Service: am startservice "); addLine(output, "Error: Bad component name: wfsdafddfasasdf"); - injectTestString(output.toString()); + mMockListener.testRunStarted(RUN_NAME, 0); + mMockListener.testRunFailed(InstrumentationResultParser.NO_TEST_RESULTS_MSG); + mMockListener.testRunEnded(0, Collections.EMPTY_MAP); - assertEquals(InstrumentationResultParser.NO_TEST_RESULTS_MSG, - mTestResult.mRunFailedMessage); + injectAndVerifyTestString(output.toString()); } /** - * Test parsing of a test run that has no tests. + * Test parsing output for a test run that produces INSTRUMENTATION_RESULT output. *

- * Expect run to be reported as success. - */ - public void testRunNoResults() { - StringBuilder output = new StringBuilder(); - addLine(output, "INSTRUMENTATION_RESULT: stream="); - addLine(output, "Test results for InstrumentationTestRunner="); - addLine(output, "Time: 0.001"); - addLine(output, "OK (0 tests)"); - addLine(output, "INSTRUMENTATION_CODE: -1"); - - injectTestString(output.toString()); - - assertEquals(0, mTestResult.mTestCount); - assertNull(mTestResult.mRunFailedMessage); - assertEquals(1, mTestResult.mTestTime); - assertFalse(mTestResult.mStopped); - } - - /** - * Test parsing of a test run that produces INSTRUMENTATION_RESULT output. This mimics launch - * performance test output. + * This mimics launch performance test output. */ - public void testRunWithInstrumentationResults() { + public void testParse_instrumentationResults() { StringBuilder output = new StringBuilder(); addResultKey(output, "other_pss", "2390"); addResultKey(output, "java_allocated", "2539"); addResultKey(output, "foo", "bar"); + addResultKey(output, "stream", "should not be captured"); addLine(output, "INSTRUMENTATION_CODE: -1"); - injectTestString(output.toString()); + Capture> captureMetrics = new Capture>(); + mMockListener.testRunStarted(RUN_NAME, 0); + mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.capture(captureMetrics)); + + injectAndVerifyTestString(output.toString()); - assertEquals("2390", mTestResult.mResultBundle.get("other_pss")); - assertEquals("2539", mTestResult.mResultBundle.get("java_allocated")); - assertEquals("bar", mTestResult.mResultBundle.get("foo")); + assertEquals("2390", captureMetrics.getValue().get("other_pss")); + assertEquals("2539", captureMetrics.getValue().get("java_allocated")); + assertEquals("bar", captureMetrics.getValue().get("foo")); + assertEquals(3, captureMetrics.getValue().size()); } /** @@ -258,6 +275,15 @@ public class InstrumentationResultParserTest extends TestCase { } /** + * Create instrumentation output for a successful single test case execution. + */ + private StringBuilder createSuccessTest() { + StringBuilder output = buildCommonResult(); + addSuccessCode(output); + return output; + } + + /** * Adds common status results to the provided output. */ private void addCommonStatus(StringBuilder output) { @@ -334,87 +360,15 @@ public class InstrumentationResultParserTest extends TestCase { } /** - * inject a test string into the result parser. + * Inject a test string into the result parser, and verify the mock listener. * - * @param result + * @param result the string to inject into parser under test. */ - private void injectTestString(String result) { + private void injectAndVerifyTestString(String result) { + EasyMock.replay(mMockListener); byte[] data = result.getBytes(); mParser.addOutput(data, 0, data.length); mParser.flush(); - } - - private void assertCommonAttributes() { - assertEquals(RUN_NAME, mTestResult.mTestRunName); - assertEquals(CLASS_NAME, mTestResult.mSuiteName); - assertEquals(1, mTestResult.mTestCount); - assertEquals(TEST_NAME, mTestResult.mTestName); - } - - /** - * A specialized test listener that stores a single test events. - */ - private class VerifyingTestResult implements ITestRunListener { - - String mTestRunName; - String mSuiteName; - int mTestCount; - int mNumTestsRun; - String mTestName; - long mTestTime; - TestFailure mTestStatus; - String mTrace; - boolean mStopped; - /** stores the error message provided to testRunFailed */ - String mRunFailedMessage; - Map mResultBundle = null; - Map mTestMetrics = null; - - VerifyingTestResult() { - mNumTestsRun = 0; - mTestStatus = null; - mStopped = false; - mRunFailedMessage = null; - mResultBundle = null; - } - - public void testEnded(TestIdentifier test, Map testMetrics) { - mNumTestsRun++; - mTestMetrics = testMetrics; - assertEquals("Unexpected class name", mSuiteName, test.getClassName()); - assertEquals("Unexpected test ended", mTestName, test.getTestName()); - - } - - public void testFailed(TestFailure status, TestIdentifier test, String trace) { - mTestStatus = status; - mTrace = trace; - assertEquals("Unexpected class name", mSuiteName, test.getClassName()); - assertEquals("Unexpected test ended", mTestName, test.getTestName()); - } - - public void testRunEnded(long elapsedTime, Map resultBundle) { - mTestTime = elapsedTime; - mResultBundle = resultBundle; - } - - public void testRunStarted(String runName, int testCount) { - mTestRunName = runName; - mTestCount = testCount; - } - - public void testRunStopped(long elapsedTime) { - mTestTime = elapsedTime; - mStopped = true; - } - - public void testStarted(TestIdentifier test) { - mSuiteName = test.getClassName(); - mTestName = test.getTestName(); - } - - public void testRunFailed(String errorMessage) { - mRunFailedMessage = errorMessage; - } + EasyMock.verify(mMockListener); } } diff --git a/ddms/libs/ddmlib/tests/src/com/android/ddmlib/testrunner/RemoteAndroidTestRunnerTest.java b/ddms/libs/ddmlib/tests/src/com/android/ddmlib/testrunner/RemoteAndroidTestRunnerTest.java index 4ee2f4d..8bde492 100644 --- a/ddms/libs/ddmlib/tests/src/com/android/ddmlib/testrunner/RemoteAndroidTestRunnerTest.java +++ b/ddms/libs/ddmlib/tests/src/com/android/ddmlib/testrunner/RemoteAndroidTestRunnerTest.java @@ -16,30 +16,24 @@ package com.android.ddmlib.testrunner; -import com.android.ddmlib.AdbCommandRejectedException; -import com.android.ddmlib.Client; -import com.android.ddmlib.FileListingService; import com.android.ddmlib.IDevice; import com.android.ddmlib.IShellOutputReceiver; -import com.android.ddmlib.InstallException; -import com.android.ddmlib.RawImage; -import com.android.ddmlib.ShellCommandUnresponsiveException; -import com.android.ddmlib.SyncService; -import com.android.ddmlib.TimeoutException; -import com.android.ddmlib.log.LogReceiver; + +import org.easymock.EasyMock; import java.io.IOException; -import java.util.Map; +import java.util.Collections; import junit.framework.TestCase; /** - * Tests RemoteAndroidTestRunner. + * Unit tests for {@link RemoteAndroidTestRunner}. */ public class RemoteAndroidTestRunnerTest extends TestCase { private RemoteAndroidTestRunner mRunner; - private MockDevice mMockDevice; + private IDevice mMockDevice; + private ITestRunListener mMockListener; private static final String TEST_PACKAGE = "com.test"; private static final String TEST_RUNNER = "com.test.InstrumentationTestRunner"; @@ -49,272 +43,96 @@ public class RemoteAndroidTestRunnerTest extends TestCase { */ @Override protected void setUp() throws Exception { - mMockDevice = new MockDevice(); + mMockDevice = EasyMock.createMock(IDevice.class); + EasyMock.expect(mMockDevice.getSerialNumber()).andStubReturn("serial"); + mMockListener = EasyMock.createNiceMock(ITestRunListener.class); mRunner = new RemoteAndroidTestRunner(TEST_PACKAGE, TEST_RUNNER, mMockDevice); } /** * Test the basic case building of the instrumentation runner command with no arguments. - * @throws ShellCommandUnresponsiveException - * @throws AdbCommandRejectedException - * @throws TimeoutException */ - public void testRun() throws IOException, TimeoutException, AdbCommandRejectedException, - ShellCommandUnresponsiveException { - mRunner.run(new EmptyListener()); - assertStringsEquals(String.format("am instrument -w -r %s/%s", TEST_PACKAGE, TEST_RUNNER), - mMockDevice.getLastShellCommand()); + public void testRun() throws Exception { + String expectedCmd = EasyMock.eq(String.format("am instrument -w -r %s/%s", TEST_PACKAGE, + TEST_RUNNER)); + runAndVerify(expectedCmd); } /** * Test the building of the instrumentation runner command with log set. - * @throws ShellCommandUnresponsiveException - * @throws AdbCommandRejectedException - * @throws TimeoutException */ - public void testRunWithLog() throws IOException, TimeoutException, AdbCommandRejectedException, - ShellCommandUnresponsiveException { + public void testRun_withLog() throws Exception { mRunner.setLogOnly(true); - mRunner.run(new EmptyListener()); - assertStringsEquals(String.format("am instrument -w -r -e log true %s/%s", TEST_PACKAGE, - TEST_RUNNER), mMockDevice.getLastShellCommand()); + String expectedCmd = EasyMock.contains("-e log true"); + runAndVerify(expectedCmd); } /** * Test the building of the instrumentation runner command with method set. - * @throws ShellCommandUnresponsiveException - * @throws AdbCommandRejectedException - * @throws TimeoutException */ - public void testRunWithMethod() throws IOException, TimeoutException, - AdbCommandRejectedException, ShellCommandUnresponsiveException { + public void testRun_withMethod() throws Exception { final String className = "FooTest"; final String testName = "fooTest"; mRunner.setMethodName(className, testName); - mRunner.run(new EmptyListener()); - assertStringsEquals(String.format("am instrument -w -r -e class %s#%s %s/%s", className, - testName, TEST_PACKAGE, TEST_RUNNER), mMockDevice.getLastShellCommand()); + String expectedCmd = EasyMock.contains(String.format("-e class %s#%s", className, + testName)); + runAndVerify(expectedCmd); } /** * Test the building of the instrumentation runner command with test package set. - * @throws ShellCommandUnresponsiveException - * @throws AdbCommandRejectedException - * @throws TimeoutException */ - public void testRunWithPackage() throws IOException, TimeoutException, - AdbCommandRejectedException, ShellCommandUnresponsiveException { + public void testRun_withPackage() throws Exception { final String packageName = "foo.test"; mRunner.setTestPackageName(packageName); - mRunner.run(new EmptyListener()); - assertStringsEquals(String.format("am instrument -w -r -e package %s %s/%s", packageName, - TEST_PACKAGE, TEST_RUNNER), mMockDevice.getLastShellCommand()); + String expectedCmd = EasyMock.contains(String.format("-e package %s", packageName)); + runAndVerify(expectedCmd); } /** * Test the building of the instrumentation runner command with extra argument added. - * @throws ShellCommandUnresponsiveException - * @throws AdbCommandRejectedException - * @throws TimeoutException */ - public void testRunWithAddInstrumentationArg() throws IOException, TimeoutException, - AdbCommandRejectedException, ShellCommandUnresponsiveException { + public void testRun_withAddInstrumentationArg() throws Exception { final String extraArgName = "blah"; final String extraArgValue = "blahValue"; mRunner.addInstrumentationArg(extraArgName, extraArgValue); - mRunner.run(new EmptyListener()); - assertStringsEquals(String.format("am instrument -w -r -e %s %s %s/%s", extraArgName, - extraArgValue, TEST_PACKAGE, TEST_RUNNER), mMockDevice.getLastShellCommand()); - } - - /** - * Assert two strings are equal ignoring whitespace. - */ - private void assertStringsEquals(String str1, String str2) { - String strippedStr1 = str1.replaceAll(" ", ""); - String strippedStr2 = str2.replaceAll(" ", ""); - assertEquals(strippedStr1, strippedStr2); + String expectedCmd = EasyMock.contains(String.format("-e %s %s", extraArgName, + extraArgValue)); + runAndVerify(expectedCmd); } /** - * A dummy device that does nothing except store the provided executed shell command for - * later retrieval. + * Test run when the device throws a IOException */ - private static class MockDevice implements IDevice { - - private String mLastShellCommand; - - /** - * Stores the provided command for later retrieval from getLastShellCommand. - */ - public void executeShellCommand(String command, - IShellOutputReceiver receiver) throws IOException { - mLastShellCommand = command; - } - - /** - * Stores the provided command for later retrieval from getLastShellCommand. - */ - public void executeShellCommand(String command, - IShellOutputReceiver receiver, int timeout) throws IOException { - mLastShellCommand = command; - } - - /** - * Get the last command provided to executeShellCommand. - */ - public String getLastShellCommand() { - return mLastShellCommand; - } - - public void createForward(int localPort, int remotePort) { - throw new UnsupportedOperationException(); - } - - public Client getClient(String applicationName) { - throw new UnsupportedOperationException(); - } - - public String getClientName(int pid) { - throw new UnsupportedOperationException(); - } - - public Client[] getClients() { - throw new UnsupportedOperationException(); - } - - public FileListingService getFileListingService() { - throw new UnsupportedOperationException(); - } - - public Map getProperties() { - throw new UnsupportedOperationException(); - } - - public String getProperty(String name) { - throw new UnsupportedOperationException(); - } - - public int getPropertyCount() { - throw new UnsupportedOperationException(); - } - - public String getMountPoint(String name) { - throw new UnsupportedOperationException(); - } - - public RawImage getScreenshot() throws IOException { - throw new UnsupportedOperationException(); - } - - public String getSerialNumber() { - return "fakeserial"; - } - - public DeviceState getState() { - throw new UnsupportedOperationException(); - } - - public SyncService getSyncService() { - throw new UnsupportedOperationException(); - } - - public boolean hasClients() { - throw new UnsupportedOperationException(); - } - - public boolean isBootLoader() { - throw new UnsupportedOperationException(); - } - - public boolean isEmulator() { - throw new UnsupportedOperationException(); - } - - public boolean isOffline() { - throw new UnsupportedOperationException(); - } - - public boolean isOnline() { - throw new UnsupportedOperationException(); - } - - public void removeForward(int localPort, int remotePort) { - throw new UnsupportedOperationException(); - } - - public void runEventLogService(LogReceiver receiver) throws IOException { - throw new UnsupportedOperationException(); - } - - public void runLogService(String logname, LogReceiver receiver) throws IOException { - throw new UnsupportedOperationException(); - } - - public String getAvdName() { - return ""; - } - - public String installPackage(String packageFilePath, boolean reinstall) - throws InstallException { - throw new UnsupportedOperationException(); - } - - public String uninstallPackage(String packageName) throws InstallException { - throw new UnsupportedOperationException(); - } - - public String installRemotePackage(String remoteFilePath, boolean reinstall) - throws InstallException { - throw new UnsupportedOperationException(); - } - - public void removeRemotePackage(String remoteFilePath) - throws InstallException { - throw new UnsupportedOperationException(); - } - - public String syncPackageToDevice(String localFilePath) - throws IOException { - throw new UnsupportedOperationException(); - } - - public void reboot(String into) throws IOException { - throw new UnsupportedOperationException(); - } + @SuppressWarnings("unchecked") + public void testRun_ioException() throws Exception { + mMockDevice.executeShellCommand((String)EasyMock.anyObject(), (IShellOutputReceiver) + EasyMock.anyObject(), EasyMock.eq(0)); + EasyMock.expectLastCall().andThrow(new IOException()); + // verify that the listeners run started, run failure, and run ended methods are called + mMockListener.testRunStarted(TEST_PACKAGE, 0); + mMockListener.testRunFailed((String)EasyMock.anyObject()); + mMockListener.testRunEnded(EasyMock.anyLong(), EasyMock.eq(Collections.EMPTY_MAP)); + + EasyMock.replay(mMockDevice, mMockListener); + try { + mRunner.run(mMockListener); + fail("IOException not thrown"); + } catch (IOException e) { + // expected + } + EasyMock.verify(mMockDevice, mMockListener); } /** - * An empty implementation of ITestRunListener. + * Calls {@link RemoteAndroidTestRunner#run(ITestRunListener...)} and verifies the given + * expectedCmd pattern was received by the mock device. */ - private static class EmptyListener implements ITestRunListener { - - public void testEnded(TestIdentifier test, Map testMetrics) { - // ignore - } - - public void testFailed(TestFailure status, TestIdentifier test, String trace) { - // ignore - } - - public void testRunEnded(long elapsedTime, Map resultBundle) { - // ignore - } - - public void testRunFailed(String errorMessage) { - // ignore - } - - public void testRunStarted(String runName, int testCount) { - // ignore - } - - public void testRunStopped(long elapsedTime) { - // ignore - } - - public void testStarted(TestIdentifier test) { - // ignore - } + private void runAndVerify(String expectedCmd) throws Exception { + mMockDevice.executeShellCommand(expectedCmd, (IShellOutputReceiver) + EasyMock.anyObject(), EasyMock.eq(0)); + EasyMock.replay(mMockDevice); + mRunner.run(mMockListener); + EasyMock.verify(mMockDevice); } } diff --git a/eclipse/plugins/com.android.ide.eclipse.adt/src/com/android/ide/eclipse/adt/internal/launch/messages.properties b/eclipse/plugins/com.android.ide.eclipse.adt/src/com/android/ide/eclipse/adt/internal/launch/messages.properties index 2670316..730e673 100644 --- a/eclipse/plugins/com.android.ide.eclipse.adt/src/com/android/ide/eclipse/adt/internal/launch/messages.properties +++ b/eclipse/plugins/com.android.ide.eclipse.adt/src/com/android/ide/eclipse/adt/internal/launch/messages.properties @@ -33,7 +33,7 @@ AndroidJUnitTab_NoRunnerError=Instrumentation runner not specified AndroidJUnitTab_TestContainerText=Run all tests in the selected project, or package InstrValidator_NoTestLibMsg_s=The application does not declare uses-library %1$s InstrValidator_WrongRunnerTypeMsg_s=The instrumentation runner must be of type %1$s -RemoteAdtTestRunner_RunCompleteMsg=Test run complete +RemoteAdtTestRunner_RunCompleteMsg=Test run finished RemoteAdtTestRunner_RunFailedMsg_s=Test run failed: %1$s RemoteAdtTestRunner_RunTimeoutException=Connection with device timed out. RemoteAdtTestRunner_RunIOException_s=Lost connection with device: %s diff --git a/eclipse/plugins/com.android.ide.eclipse.tests/META-INF/MANIFEST.MF b/eclipse/plugins/com.android.ide.eclipse.tests/META-INF/MANIFEST.MF index 9888108..549611f 100644 --- a/eclipse/plugins/com.android.ide.eclipse.tests/META-INF/MANIFEST.MF +++ b/eclipse/plugins/com.android.ide.eclipse.tests/META-INF/MANIFEST.MF @@ -10,4 +10,5 @@ Bundle-RequiredExecutionEnvironment: J2SE-1.5 Bundle-ClassPath: kxml2-2.3.0.jar, ., layoutlib.jar, - groovy-all-1.7.0.jar + groovy-all-1.7.0.jar, + easymock.jar diff --git a/eclipse/plugins/com.android.ide.eclipse.tests/build.properties b/eclipse/plugins/com.android.ide.eclipse.tests/build.properties index 7be75ed..7c9915c 100644 --- a/eclipse/plugins/com.android.ide.eclipse.tests/build.properties +++ b/eclipse/plugins/com.android.ide.eclipse.tests/build.properties @@ -11,5 +11,6 @@ bin.includes = META-INF/,\ unittests/com/android/sdklib/testdata/,\ unittests/com/android/layoutlib/testdata/,\ unittests/com/android/ide/eclipse/testdata/,\ - groovy-all-1.7.0.jar + groovy-all-1.7.0.jar,\ + easymock.jar -- cgit v1.1