Create a catch-all testcase to handle asynchronous crashes and ANRs

Bug: 5913065
Change-Id: I391aff6919a9586159ec0898279e7254eed990f8

2 files changed, 132 insertions, 56 deletions

diff --git a/tests/SmokeTest/tests/src/com/android/smoketest/ProcessErrorsTest.java b/tests/SmokeTest/tests/src/com/android/smoketest/ProcessErrorsTest.java
index b3a2600..03c2923 100644
--- a/tests/SmokeTest/tests/src/com/android/smoketest/ProcessErrorsTest.java
+++ b/tests/SmokeTest/tests/src/com/android/smoketest/ProcessErrorsTest.java
@@ -28,17 +28,18 @@ import android.util.Log;
 
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashSet;
+import java.util.Collections;
 import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
 
 /**
- * This smoke test is designed to quickly sniff for any error conditions
- * encountered after initial startup.
+ * This smoke test is designed to check for crashes and ANRs in an attempt to quickly determine if
+ * all minimal functionality in the build is working properly.
  */
 public class ProcessErrorsTest extends AndroidTestCase {
-    
+
     private static final String TAG = "ProcessErrorsTest";
 
     private final Intent mHomeIntent;
@@ -46,15 +47,28 @@ public class ProcessErrorsTest extends AndroidTestCase {
     protected ActivityManager mActivityManager;
     protected PackageManager mPackageManager;
 
+    /**
+     * Used to buffer asynchronously-caused crashes and ANRs so that we can have a big fail-party
+     * in the catch-all testCase.
+     */
+    private static final Collection<ProcessError> mAsyncErrors =
+            Collections.synchronizedSet(new LinkedHashSet<ProcessError>());
+
     public ProcessErrorsTest() {
         mHomeIntent = new Intent(Intent.ACTION_MAIN);
         mHomeIntent.addCategory(Intent.CATEGORY_HOME);
         mHomeIntent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
     }
 
+    /**
+     * {@inheritDoc}
+     */
     @Override
     public void setUp() throws Exception {
         super.setUp();
+        // First, make sure we have a Context
+        assertNotNull("getContext() returned null!", getContext());
+
         mActivityManager = (ActivityManager)
                 getContext().getSystemService(Context.ACTIVITY_SERVICE);
         mPackageManager = getContext().getPackageManager();
@@ -75,43 +89,48 @@ public class ProcessErrorsTest extends AndroidTestCase {
         assertNull(reportMsg, reportMsg);
     }
 
-    private String checkForProcessErrors() throws Exception {
-        List<ProcessErrorStateInfo> errList;
-        errList = mActivityManager.getProcessesInErrorState();
+    /**
+     * A test that runs all Launcher-launchable activities and verifies that no ANRs or crashes
+     * happened while doing so.
+     */
+    public void testRunAllActivities() throws Exception {
+        final Set<ProcessError> errSet = new LinkedHashSet<ProcessError>();
 
-        // note: this contains information about each process that is currently in an error
-        // condition.  if the list is empty (null) then "we're good".
+        for (ResolveInfo app : getLauncherActivities(mPackageManager)) {
+            final Collection<ProcessError> errProcs = runOneActivity(app);
+            if (errProcs != null) {
+                errSet.addAll(errProcs);
+            }
+        }
 
-        // if the list is non-empty, then it's useful to report the contents of the list
-        final String reportMsg = reportListContents(errList);
-        return reportMsg;
+        if (!errSet.isEmpty()) {
+            fail(String.format("Got %d errors:\n%s", errSet.size(),
+                    reportWrappedListContents(errSet)));
+        }
     }
 
     /**
-     * A helper function to query the provided {@link PackageManager} for a list of Activities that
-     * can be launched from Launcher.
+     * This test checks for asynchronously-caused errors (crashes or ANRs) and fails in case any
+     * were found.  This prevents us from needing to fail unrelated testcases when, for instance
+     * a background thread causes a crash or ANR.
+     * <p />
+     * Because this behavior depends on the contents of static member {@link mAsyncErrors}, we clear
+     * that state here as a side-effect so that if two successive runs happen in the same process,
+     * the asynchronous errors in the second test run won't include errors produced during the first
+     * run.
      */
-    static List<ResolveInfo> getLauncherActivities(PackageManager pm) {
-        final Intent launchable = new Intent(Intent.ACTION_MAIN);
-        launchable.addCategory(Intent.CATEGORY_LAUNCHER);
-        final List<ResolveInfo> activities = pm.queryIntentActivities(launchable, 0);
-        return activities;
+    public void testZZReportAsyncErrors() throws Exception {
+        try {
+            if (!mAsyncErrors.isEmpty()) {
+                fail(String.format("Got %d asynchronous errors:\n%s", mAsyncErrors.size(),
+                        reportWrappedListContents(mAsyncErrors)));
+            }
+        } finally {
+            // Reset state just in case we should get another set of runs in the same process
+            mAsyncErrors.clear();
+        }
     }
 
-    /**
-     * A helper function to create an {@link Intent} to run, given a {@link ResolveInfo} specifying
-     * an activity to be launched.
-     */
-    static Intent intentForActivity(ResolveInfo app) {
-        // build an Intent to launch the specified app
-        final ComponentName component = new ComponentName(app.activityInfo.packageName,
-                app.activityInfo.name);
-        final Intent intent = new Intent(Intent.ACTION_MAIN);
-        intent.setComponent(component);
-        intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
-        intent.addFlags(Intent.FLAG_ACTIVITY_RESET_TASK_IF_NEEDED);
-        return intent;
-    }
 
     /**
      * A method to run the specified Activity and return a {@link Collection} of the Activities that
@@ -129,8 +148,7 @@ public class ProcessErrorsTest extends AndroidTestCase {
 
         // We check for any Crash or ANR dialogs that are already up, and we ignore them.  This is
         // so that we don't report crashes that were caused by prior apps (which those particular
-        // tests should have caught and reported already).  Otherwise, test failures would cascade
-        // from the initial broken app to many/all of the tests following that app's launch.
+        // tests should have caught and reported already).
         final Collection<ProcessError> preErrProcs =
                 ProcessError.fromCollection(mActivityManager.getProcessesInErrorState());
 
@@ -155,8 +173,24 @@ public class ProcessErrorsTest extends AndroidTestCase {
         // possible to occur.
         final Collection<ProcessError> errProcs =
                 ProcessError.fromCollection(mActivityManager.getProcessesInErrorState());
-        // Take the difference between the error processes we see now, and the ones that were
-        // present when we started
+
+        // Distinguish the asynchronous crashes/ANRs from the synchronous ones by checking the
+        // crash package name against the package name for {@code app}
+        if (errProcs != null) {
+            Iterator<ProcessError> errIter = errProcs.iterator();
+            while (errIter.hasNext()) {
+                ProcessError err = errIter.next();
+                if (!packageMatches(app, err)) {
+                    // async!  Drop into mAsyncErrors and don't report now
+                    mAsyncErrors.add(err);
+                    errIter.remove();
+                }
+            }
+        }
+        // Take the difference between the remaining current error processes and the ones that were
+        // present when we started.  The result is guaranteed to be:
+        // 1) Errors that are pertinent to this app's package
+        // 2) Errors that are pertinent to this particular app invocation
         if (errProcs != null && preErrProcs != null) {
             errProcs.removeAll(preErrProcs);
         }
@@ -164,27 +198,63 @@ public class ProcessErrorsTest extends AndroidTestCase {
         return errProcs;
     }
 
+    private String checkForProcessErrors() throws Exception {
+        List<ProcessErrorStateInfo> errList;
+        errList = mActivityManager.getProcessesInErrorState();
+
+        // note: this contains information about each process that is currently in an error
+        // condition.  if the list is empty (null) then "we're good".
+
+        // if the list is non-empty, then it's useful to report the contents of the list
+        final String reportMsg = reportListContents(errList);
+        return reportMsg;
+    }
+
     /**
-     * A test that runs all Launcher-launchable activities and verifies that no ANRs or crashes
-     * happened while doing so.
+     * A helper function that checks whether the specified error could have been caused by the
+     * specified app.
+     *
+     * @param app The app to check against
+     * @param err The error that we're considering
      */
-    public void testRunAllActivities() throws Exception {
-        final Set<ProcessError> errSet = new HashSet<ProcessError>();
+    private static boolean packageMatches(ResolveInfo app, ProcessError err) {
+        final String appPkg = app.activityInfo.packageName;
+        final String errPkg = err.info.processName;
+        Log.d(TAG, String.format("packageMatches(%s, %s)", appPkg, errPkg));
+        return appPkg.equals(errPkg);
+    }
 
-        for (ResolveInfo app : getLauncherActivities(mPackageManager)) {
-            final Collection<ProcessError> errProcs = runOneActivity(app);
-            if (errProcs != null) {
-                errSet.addAll(errProcs);
-            }
-        }
+    /**
+     * A helper function to query the provided {@link PackageManager} for a list of Activities that
+     * can be launched from Launcher.
+     */
+    static List<ResolveInfo> getLauncherActivities(PackageManager pm) {
+        final Intent launchable = new Intent(Intent.ACTION_MAIN);
+        launchable.addCategory(Intent.CATEGORY_LAUNCHER);
+        final List<ResolveInfo> activities = pm.queryIntentActivities(launchable, 0);
+        return activities;
+    }
 
-        if (!errSet.isEmpty()) {
-            fail(String.format("Got %d errors:\n%s", errSet.size(),
-                    reportWrappedListContents(errSet)));
-        }
+    /**
+     * A helper function to create an {@link Intent} to run, given a {@link ResolveInfo} specifying
+     * an activity to be launched.
+     */
+    static Intent intentForActivity(ResolveInfo app) {
+        final ComponentName component = new ComponentName(app.activityInfo.packageName,
+                app.activityInfo.name);
+        final Intent intent = new Intent(Intent.ACTION_MAIN);
+        intent.setComponent(component);
+        intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK);
+        intent.addFlags(Intent.FLAG_ACTIVITY_RESET_TASK_IF_NEEDED);
+        return intent;
     }
 
-    String reportWrappedListContents(Collection<ProcessError> errList) {
+    /**
+     * Report error reports for {@link ProcessErrorStateInfo} instances that are wrapped inside of
+     * {@link ProcessError} instances.  Just unwraps and calls
+     * {@see reportListContents(Collection<ProcessErrorStateInfo>)}.
+     */
+    static String reportWrappedListContents(Collection<ProcessError> errList) {
         List<ProcessErrorStateInfo> newList = new ArrayList<ProcessErrorStateInfo>(errList.size());
         for (ProcessError err : errList) {
             newList.add(err.info);
@@ -198,7 +268,7 @@ public class ProcessErrorsTest extends AndroidTestCase {
      * @param errList The error report containing one or more error records.
      * @return Returns a string containing all of the errors.
      */
-    private String reportListContents(Collection<ProcessErrorStateInfo> errList) {
+    private static String reportListContents(Collection<ProcessErrorStateInfo> errList) {
         if (errList == null) return null;
 
         StringBuilder builder = new StringBuilder();
diff --git a/tests/SmokeTest/tests/src/com/android/smoketest/SmokeTestRunner.java b/tests/SmokeTest/tests/src/com/android/smoketest/SmokeTestRunner.java
index 51331fe..db549a1 100644
--- a/tests/SmokeTest/tests/src/com/android/smoketest/SmokeTestRunner.java
+++ b/tests/SmokeTest/tests/src/com/android/smoketest/SmokeTestRunner.java
@@ -47,9 +47,6 @@ public class SmokeTestRunner extends InstrumentationTestRunner {
         final PackageManager pm = getTargetContext().getPackageManager();
         final List<ResolveInfo> apps = ProcessErrorsTest.getLauncherActivities(pm);
 
-        // FIXME: figure out some way to control the reported class names for these anonymous
-        // FIXME: class instances.
-
         final TestCase setupTest = new ProcessErrorsTest() {
             @Override
             public void runTest() throws Exception {
@@ -88,6 +85,15 @@ public class SmokeTestRunner extends InstrumentationTestRunner {
             suite.addTest(appTest);
         }
 
+        final TestCase asyncErrorTest = new ProcessErrorsTest() {
+            @Override
+            public void runTest() throws Exception {
+                testZZReportAsyncErrors();
+            }
+        };
+        asyncErrorTest.setName("testAsynchronousErrors");
+        suite.addTest(asyncErrorTest);
+
         return suite;
     }
 }