Update DumpRenderTree2 to handle failing tests where no meaningful diff is available

Currently, DumpRenderTree2 asigns the following result codes
{PASS, FAIL_RESULT_DIFFERS, FAIL_NO_EXPECTED_RESULT, FAIL_TIMED_OUT,
FAIL_CRASHED}

This is not strictly correct, as a test may fail for any of three
reasons - crashing, timing out, or the the result not being as
expected. Therefore, it's possible for a test to fail even if the
result is as expected.

This patch updates AbstractResult to handle each of the three reasons
for failure separately. We then test all three to determine if the
test has passed.

This allows us to correctly report whether or not the result differs
from expected for a failing test.

Change-Id: I7adcfe72c4dd0bd3de2e1b868d9807be6eb5bddf

5 files changed, 161 insertions, 88 deletions

diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java
index 7cbb397..6048338a 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java
@@ -46,12 +46,14 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
         public abstract AbstractResult createResult(Bundle bundle);
     }
 
+    /**
+     * A code representing the result of comparing actual and expected results.
+     */
     public enum ResultCode {
-        PASS("Passed"),
-        FAIL_RESULT_DIFFERS("Result differs"),
-        FAIL_NO_EXPECTED_RESULT("No expected result"),
-        FAIL_TIMED_OUT("Timed out"),
-        FAIL_CRASHED("Crashed");
+        RESULTS_MATCH("Results match"),
+        RESULTS_DIFFER("Results differ"),
+        NO_EXPECTED_RESULT("No expected result"),
+        NO_ACTUAL_RESULT("No actual result");
 
         private String mTitle;
 
@@ -123,14 +125,48 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
     public abstract String getActualTextResult();
 
     /**
-     * Returns the code of this result.
+     * Returns the status code representing the result of comparing actual and expected results.
      *
      * @return
-     *      the code of this result
+     *      the status code from comparing actual and expected results
      */
     public abstract ResultCode getResultCode();
 
     /**
+     * Returns whether this test crashed.
+     *
+     * @return
+     *      whether this test crashed
+     */
+    public abstract boolean didCrash();
+
+    /**
+     * Returns whether this test timed out.
+     *
+     * @return
+     *      whether this test timed out
+     */
+    public abstract boolean didTimeOut();
+
+    /**
+     * Sets that this test timed out.
+     */
+    public abstract void setDidTimeOut();
+
+    /**
+     * Returns whether the test passed.
+     *
+     * @return
+     *      whether the test passed
+     */
+    public boolean didPass() {
+        // Tests that crash can't have timed out or have an actual result.
+        assert !(didCrash() && didTimeOut());
+        assert !(didCrash() && getResultCode() != ResultCode.NO_ACTUAL_RESULT);
+        return !didCrash() && !didTimeOut() && getResultCode() == ResultCode.RESULTS_MATCH;
+    }
+
+    /**
      * Return the type of the result data.
      *
      * @return
@@ -150,4 +186,4 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
     public abstract String getDiffAsHtml();
 
     public abstract Bundle getBundle();
-}
\ No newline at end of file
+}
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java
index 31da776..4831168 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java
@@ -61,7 +61,23 @@ public class CrashedDummyResult extends AbstractResult {
 
     @Override
     public ResultCode getResultCode() {
-        return ResultCode.FAIL_CRASHED;
+        return ResultCode.NO_ACTUAL_RESULT;
+    }
+
+    @Override
+    public boolean didCrash() {
+        return true;
+    }
+
+    @Override
+    public boolean didTimeOut() {
+        return false;
+    }
+
+    @Override
+    public void setDidTimeOut() {
+        /** This method is not applicable for this type of result */
+        assert false;
     }
 
     @Override
@@ -106,4 +122,4 @@ public class CrashedDummyResult extends AbstractResult {
     public void setExpectedTextResultPath(String relativePath) {
         /** TODO */
     }
-}
\ No newline at end of file
+}
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java
index efea192..b9fc274 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java
@@ -426,6 +426,7 @@ public class LayoutTestsExecutor extends Activity {
         assert mCurrentState.isRunningState() : "mCurrentState = " + mCurrentState.name();
 
         Log.i(LOG_TAG, "onTestFinished(): " + mCurrentTestRelativePath);
+        mResultHandler.removeMessages(MSG_TEST_TIMED_OUT);
         obtainActualResultsFromWebView();
     }
 
@@ -441,6 +442,9 @@ public class LayoutTestsExecutor extends Activity {
 
         mCurrentState = CurrentState.OBTAINING_RESULT;
 
+        if (mCurrentTestTimedOut) {
+            mCurrentResult.setDidTimeOut();
+        }
         mCurrentResult.obtainActualResults(mCurrentWebView,
                 mResultHandler.obtainMessage(MSG_ACTUAL_RESULT_OBTAINED));
     }
@@ -452,7 +456,6 @@ public class LayoutTestsExecutor extends Activity {
         Log.i(LOG_TAG, "onActualResultsObtained(): " + mCurrentTestRelativePath);
         mCurrentState = CurrentState.IDLE;
 
-        mResultHandler.removeMessages(MSG_TEST_TIMED_OUT);
         reportResultToService();
         mCurrentTestIndex++;
         updateProgressBar();
@@ -470,9 +473,6 @@ public class LayoutTestsExecutor extends Activity {
 
             Bundle bundle = mCurrentResult.getBundle();
             bundle.putInt("testIndex", mCurrentTestIndex);
-            if (mCurrentTestTimedOut) {
-                bundle.putString("resultCode", AbstractResult.ResultCode.FAIL_TIMED_OUT.name());
-            }
             if (!mTestsList.isEmpty()) {
                 bundle.putString("nextTest", mTestsList.get(0));
             }
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
index e31e747..b1dea6e 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
@@ -75,9 +75,12 @@ public class Summarizer {
             "       width: 20px;}" +
             "h3 span.sqr {" +
             "       text-decoration: none;" +
-            "       color: #8ee100;" +
             "       float: left;" +
             "       width: 20px;}" +
+            "h3 span.sqr_pass {" +
+            "       color: #8ee100;}" +
+            "h3 span.sqr_fail {" +
+            "       color: #c30000;}" +
             "span.source {" +
             "       display: block;" +
             "       font-size: 10px;" +
@@ -145,16 +148,16 @@ public class Summarizer {
             "       text-transform: uppercase;" +
             "       padding: 3px;" +
             "       -webkit-border-radius: 4px;}" +
-            "span." + AbstractResult.ResultCode.FAIL_RESULT_DIFFERS.name() + "{" +
+            "span." + AbstractResult.ResultCode.RESULTS_DIFFER.name() + "{" +
             "       background-color: #ccc;" +
             "       color: black;}" +
-            "span." + AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT.name() + "{" +
+            "span." + AbstractResult.ResultCode.NO_EXPECTED_RESULT.name() + "{" +
             "       background-color: #a700e4;" +
             "       color: #fff;}" +
-            "span." + AbstractResult.ResultCode.FAIL_TIMED_OUT.name() + "{" +
+            "span.timed_out {" +
             "       background-color: #f3cb00;" +
             "       color: black;}" +
-            "span." + AbstractResult.ResultCode.FAIL_CRASHED.name() + "{" +
+            "span.crashed {" +
             "       background-color: #c30000;" +
             "       color: #fff;}" +
             "span.noLtc {" +
@@ -208,11 +211,11 @@ public class Summarizer {
     public void appendTest(AbstractResult result) {
         String relativePath = result.getRelativePath();
 
-        if (result.getResultCode() == AbstractResult.ResultCode.FAIL_CRASHED) {
+        if (result.didCrash()) {
             mCrashedTestsCount++;
         }
 
-        if (result.getResultCode() == AbstractResult.ResultCode.PASS) {
+        if (result.didPass()) {
             if (mFileFilter.isFail(relativePath)) {
                 mUnexpectedPasses.add(result);
             } else {
@@ -276,13 +279,10 @@ public class Summarizer {
 
         createTopSummaryTable(webKitRevision, html);
 
-        createResultsListWithDiff(html, "Unexpected failures", mUnexpectedFailures);
-
-        createResultsListNoDiff(html, "Unexpected passes", mUnexpectedPasses);
-
-        createResultsListWithDiff(html, "Expected failures", mExpectedFailures);
-
-        createResultsListNoDiff(html, "Expected passes", mExpectedPasses);
+        createResultsList(html, "Unexpected failures", mUnexpectedFailures);
+        createResultsList(html, "Unexpected passes", mUnexpectedPasses);
+        createResultsList(html, "Expected failures", mExpectedFailures);
+        createResultsList(html, "Expected passes", mExpectedPasses);
 
         html.append("</body></html>");
 
@@ -357,8 +357,8 @@ public class Summarizer {
         html.append("</tr>");
     }
 
-    private void createResultsListWithDiff(StringBuilder html, String title,
-            List<AbstractResult> resultsList) {
+    private void createResultsList(
+            StringBuilder html, String title, List<AbstractResult> resultsList) {
         String relativePath;
         String id = "";
         AbstractResult.ResultCode resultCode;
@@ -368,7 +368,6 @@ public class Summarizer {
         for (AbstractResult result : resultsList) {
             relativePath = result.getRelativePath();
             resultCode = result.getResultCode();
-            assert resultCode != AbstractResult.ResultCode.PASS : "resultCode=" + resultCode;
 
             html.append("<h3>");
 
@@ -378,61 +377,73 @@ public class Summarizer {
              * to cause any problems in this case
              */
             id = relativePath.replace(File.separator, ":");
-            html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
-            html.append("return false;\">");
-            html.append("<span class=\"tri\" id=\"tri." + id + "\">&#x25b6; </span>");
-            html.append("<span class=\"path\">" + relativePath + "</span>");
-            html.append("</a>");
-
-            html.append(" <span class=\"listItem " + resultCode.name() + "\">");
-            html.append(resultCode.toString());
-            html.append("</span>");
-
-            /** Detect missing LTC function */
-            String additionalTextOutputString = result.getAdditionalTextOutputString();
-            if (additionalTextOutputString != null &&
-                    additionalTextOutputString.contains("com.android.dumprendertree") &&
-                    additionalTextOutputString.contains("has no method")) {
-                if (additionalTextOutputString.contains("LayoutTestController")) {
-                    html.append(" <span class=\"listItem noLtc\">LTC function missing</span>");
-                }
-                if (additionalTextOutputString.contains("EventSender")) {
-                    html.append(" <span class=\"listItem noEventSender\">");
-                    html.append("ES function missing</span>");
-                }
+
+            /** Write the test name */
+            if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) {
+                html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
+                html.append("return false;\">");
+                html.append("<span class=\"tri\" id=\"tri." + id + "\">&#x25b6; </span>");
+                html.append("<span class=\"path\">" + relativePath + "</span>");
+                html.append("</a>");
+            } else {
+                html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() + "\"");
+                html.append(" target=\"_blank\">");
+                html.append("<span class=\"sqr sqr_" + (result.didPass() ? "pass" : "fail"));
+                html.append("\">&#x25a0; </span>");
+                html.append("<span class=\"path\">" + result.getRelativePath() + "</span>");
+                html.append("</a>");
+            }
+
+            if (!result.didPass()) {
+                appendTags(html, result);
             }
 
             html.append("</h3>");
             appendExpectedResultsSources(result, html);
 
-            html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">");
-            html.append(result.getDiffAsHtml());
-            html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
-            html.append("return false;\">Hide</a>");
-            html.append(" | ");
-            html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\"");
-            html.append(" target=\"_blank\">Show source</a>");
-            html.append("</div>");
+            if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) {
+                html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">");
+                html.append(result.getDiffAsHtml());
+                html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
+                html.append("return false;\">Hide</a>");
+                html.append(" | ");
+                html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\"");
+                html.append(" target=\"_blank\">Show source</a>");
+                html.append("</div>");
+            }
 
             html.append("<div class=\"space\"></div>");
         }
     }
 
-    private void createResultsListNoDiff(StringBuilder html, String title,
-            List<AbstractResult> resultsList) {
-        Collections.sort(resultsList);
-        html.append("<h2>" + title + " [" + resultsList.size() + "]</h2>");
-        for (AbstractResult result : resultsList) {
-            html.append("<h3>");
-            html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() +
-                    "\"");
-            html.append(" target=\"_blank\">");
-            html.append("<span class=\"sqr\">&#x25a0; </span>");
-            html.append("<span class=\"path\">" + result.getRelativePath() + "</span>");
-            html.append("</a>");
-            html.append("</h3>");
-            appendExpectedResultsSources(result, html);
-            html.append("<div class=\"space\"></div>");
+    private void appendTags(StringBuilder html, AbstractResult result) {
+        /** Tag tests which crash, time out or where results don't match */
+        if (result.didCrash()) {
+            html.append(" <span class=\"listItem crashed\">Crashed</span>");
+        } else {
+            if (result.didTimeOut()) {
+                html.append(" <span class=\"listItem timed_out\">Timed out</span>");
+            }
+            AbstractResult.ResultCode resultCode = result.getResultCode();
+            if (resultCode != AbstractResult.ResultCode.RESULTS_MATCH) {
+                html.append(" <span class=\"listItem " + resultCode.name() + "\">");
+                html.append(resultCode.toString());
+                html.append("</span>");
+            }
+        }
+
+        /** Detect missing LTC function */
+        String additionalTextOutputString = result.getAdditionalTextOutputString();
+        if (additionalTextOutputString != null &&
+                additionalTextOutputString.contains("com.android.dumprendertree") &&
+                additionalTextOutputString.contains("has no method")) {
+            if (additionalTextOutputString.contains("LayoutTestController")) {
+                html.append(" <span class=\"listItem noLtc\">LTC function missing</span>");
+            }
+            if (additionalTextOutputString.contains("EventSender")) {
+                html.append(" <span class=\"listItem noEventSender\">");
+                html.append("ES function missing</span>");
+            }
         }
     }
 
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java
index 9664efe..21e5430 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java
@@ -39,6 +39,7 @@ public class TextResult extends AbstractResult {
     private String mExpectedResultPath;
     private String mActualResult;
     private String mRelativePath;
+    private boolean mDidTimeOut;
     private ResultCode mResultCode;
     private Message mResultObtainedMsg;
 
@@ -74,10 +75,7 @@ public class TextResult extends AbstractResult {
         mActualResult = bundle.getString("actualTextualResult");
         setAdditionalTextOutputString(bundle.getString("additionalTextOutputString"));
         mRelativePath = bundle.getString("relativePath");
-        String resultCode = bundle.getString("resultCode");
-        if (resultCode != null) {
-            mResultCode = ResultCode.valueOf(resultCode);
-        }
+        mDidTimeOut = bundle.getBoolean("didTimeOut");
     }
 
     @Override
@@ -87,17 +85,31 @@ public class TextResult extends AbstractResult {
         }
 
         if (mExpectedResult == null) {
-            mResultCode = AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT;
+            mResultCode = AbstractResult.ResultCode.NO_EXPECTED_RESULT;
         } else if (!mExpectedResult.equals(mActualResult)) {
-            mResultCode = AbstractResult.ResultCode.FAIL_RESULT_DIFFERS;
+            mResultCode = AbstractResult.ResultCode.RESULTS_DIFFER;
         } else {
-            mResultCode = AbstractResult.ResultCode.PASS;
+            mResultCode = AbstractResult.ResultCode.RESULTS_MATCH;
         }
-
         return mResultCode;
     }
 
     @Override
+    public boolean didCrash() {
+        return false;
+    }
+
+    @Override
+    public boolean didTimeOut() {
+        return mDidTimeOut;
+    }
+
+    @Override
+    public void setDidTimeOut() {
+        mDidTimeOut = true;
+    }
+
+    @Override
     public byte[] getActualImageResult() {
         return null;
     }
@@ -239,9 +251,7 @@ public class TextResult extends AbstractResult {
         bundle.putString("actualTextualResult", getActualTextResult());
         bundle.putString("additionalTextOutputString", getAdditionalTextOutputString());
         bundle.putString("relativePath", mRelativePath);
-        if (mResultCode != null) {
-            bundle.putString("resultCode", mResultCode.name());
-        }
+        bundle.putBoolean("didTimeOut", mDidTimeOut);
         bundle.putString("type", getType().name());
         return bundle;
     }
@@ -250,4 +260,4 @@ public class TextResult extends AbstractResult {
     public String getRelativePath() {
         return mRelativePath;
     }
-}
\ No newline at end of file
+}