Merge "Update DumpRenderTree2 to handle failing tests where no meaningful diff is available"

5 files changed, 161 insertions, 88 deletions

diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java
index 7cbb397..6048338a 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java
@@ -46,12 +46,14 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
         public abstract AbstractResult createResult(Bundle bundle);
     }
 
+    /**
+     * A code representing the result of comparing actual and expected results.
+     */
     public enum ResultCode {
-        PASS("Passed"),
-        FAIL_RESULT_DIFFERS("Result differs"),
-        FAIL_NO_EXPECTED_RESULT("No expected result"),
-        FAIL_TIMED_OUT("Timed out"),
-        FAIL_CRASHED("Crashed");
+        RESULTS_MATCH("Results match"),
+        RESULTS_DIFFER("Results differ"),
+        NO_EXPECTED_RESULT("No expected result"),
+        NO_ACTUAL_RESULT("No actual result");
 
         private String mTitle;
 
@@ -123,14 +125,48 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
     public abstract String getActualTextResult();
 
     /**
-     * Returns the code of this result.
+     * Returns the status code representing the result of comparing actual and expected results.
      *
      * @return
-     *      the code of this result
+     *      the status code from comparing actual and expected results
      */
     public abstract ResultCode getResultCode();
 
     /**
+     * Returns whether this test crashed.
+     *
+     * @return
+     *      whether this test crashed
+     */
+    public abstract boolean didCrash();
+
+    /**
+     * Returns whether this test timed out.
+     *
+     * @return
+     *      whether this test timed out
+     */
+    public abstract boolean didTimeOut();
+
+    /**
+     * Sets that this test timed out.
+     */
+    public abstract void setDidTimeOut();
+
+    /**
+     * Returns whether the test passed.
+     *
+     * @return
+     *      whether the test passed
+     */
+    public boolean didPass() {
+        // Tests that crash can't have timed out or have an actual result.
+        assert !(didCrash() && didTimeOut());
+        assert !(didCrash() && getResultCode() != ResultCode.NO_ACTUAL_RESULT);
+        return !didCrash() && !didTimeOut() && getResultCode() == ResultCode.RESULTS_MATCH;
+    }
+
+    /**
      * Return the type of the result data.
      *
      * @return
@@ -150,4 +186,4 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
     public abstract String getDiffAsHtml();
 
     public abstract Bundle getBundle();
-}
\ No newline at end of file
+}
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java
index 31da776..4831168 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java
@@ -61,7 +61,23 @@ public class CrashedDummyResult extends AbstractResult {
 
     @Override
     public ResultCode getResultCode() {
-        return ResultCode.FAIL_CRASHED;
+        return ResultCode.NO_ACTUAL_RESULT;
+    }
+
+    @Override
+    public boolean didCrash() {
+        return true;
+    }
+
+    @Override
+    public boolean didTimeOut() {
+        return false;
+    }
+
+    @Override
+    public void setDidTimeOut() {
+        /** This method is not applicable for this type of result */
+        assert false;
     }
 
     @Override
@@ -106,4 +122,4 @@ public class CrashedDummyResult extends AbstractResult {
     public void setExpectedTextResultPath(String relativePath) {
         /** TODO */
     }
-}
\ No newline at end of file
+}
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java
index efea192..b9fc274 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java
@@ -426,6 +426,7 @@ public class LayoutTestsExecutor extends Activity {
         assert mCurrentState.isRunningState() : "mCurrentState = " + mCurrentState.name();
 
         Log.i(LOG_TAG, "onTestFinished(): " + mCurrentTestRelativePath);
+        mResultHandler.removeMessages(MSG_TEST_TIMED_OUT);
         obtainActualResultsFromWebView();
     }
 
@@ -441,6 +442,9 @@ public class LayoutTestsExecutor extends Activity {
 
         mCurrentState = CurrentState.OBTAINING_RESULT;
 
+        if (mCurrentTestTimedOut) {
+            mCurrentResult.setDidTimeOut();
+        }
         mCurrentResult.obtainActualResults(mCurrentWebView,
                 mResultHandler.obtainMessage(MSG_ACTUAL_RESULT_OBTAINED));
     }
@@ -452,7 +456,6 @@ public class LayoutTestsExecutor extends Activity {
         Log.i(LOG_TAG, "onActualResultsObtained(): " + mCurrentTestRelativePath);
         mCurrentState = CurrentState.IDLE;
 
-        mResultHandler.removeMessages(MSG_TEST_TIMED_OUT);
         reportResultToService();
         mCurrentTestIndex++;
         updateProgressBar();
@@ -470,9 +473,6 @@ public class LayoutTestsExecutor extends Activity {
 
             Bundle bundle = mCurrentResult.getBundle();
             bundle.putInt("testIndex", mCurrentTestIndex);
-            if (mCurrentTestTimedOut) {
-                bundle.putString("resultCode", AbstractResult.ResultCode.FAIL_TIMED_OUT.name());
-            }
             if (!mTestsList.isEmpty()) {
                 bundle.putString("nextTest", mTestsList.get(0));
             }
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
index e31e747..b1dea6e 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
@@ -75,9 +75,12 @@ public class Summarizer {
             "       width: 20px;}" +
             "h3 span.sqr {" +
             "       text-decoration: none;" +
-            "       color: #8ee100;" +
             "       float: left;" +
             "       width: 20px;}" +
+            "h3 span.sqr_pass {" +
+            "       color: #8ee100;}" +
+            "h3 span.sqr_fail {" +
+            "       color: #c30000;}" +
             "span.source {" +
             "       display: block;" +
             "       font-size: 10px;" +
@@ -145,16 +148,16 @@ public class Summarizer {
             "       text-transform: uppercase;" +
             "       padding: 3px;" +
             "       -webkit-border-radius: 4px;}" +
-            "span." + AbstractResult.ResultCode.FAIL_RESULT_DIFFERS.name() + "{" +
+            "span." + AbstractResult.ResultCode.RESULTS_DIFFER.name() + "{" +
             "       background-color: #ccc;" +
             "       color: black;}" +
-            "span." + AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT.name() + "{" +
+            "span." + AbstractResult.ResultCode.NO_EXPECTED_RESULT.name() + "{" +
             "       background-color: #a700e4;" +
             "       color: #fff;}" +
-            "span." + AbstractResult.ResultCode.FAIL_TIMED_OUT.name() + "{" +
+            "span.timed_out {" +
             "       background-color: #f3cb00;" +
             "       color: black;}" +
-            "span." + AbstractResult.ResultCode.FAIL_CRASHED.name() + "{" +
+            "span.crashed {" +
             "       background-color: #c30000;" +
             "       color: #fff;}" +
             "span.noLtc {" +
@@ -208,11 +211,11 @@ public class Summarizer {
     public void appendTest(AbstractResult result) {
         String relativePath = result.getRelativePath();
 
-        if (result.getResultCode() == AbstractResult.ResultCode.FAIL_CRASHED) {
+        if (result.didCrash()) {
             mCrashedTestsCount++;
         }
 
-        if (result.getResultCode() == AbstractResult.ResultCode.PASS) {
+        if (result.didPass()) {
             if (mFileFilter.isFail(relativePath)) {
                 mUnexpectedPasses.add(result);
             } else {
@@ -276,13 +279,10 @@ public class Summarizer {
 
         createTopSummaryTable(webKitRevision, html);
 
-        createResultsListWithDiff(html, "Unexpected failures", mUnexpectedFailures);
-
-        createResultsListNoDiff(html, "Unexpected passes", mUnexpectedPasses);
-
-        createResultsListWithDiff(html, "Expected failures", mExpectedFailures);
-
-        createResultsListNoDiff(html, "Expected passes", mExpectedPasses);
+        createResultsList(html, "Unexpected failures", mUnexpectedFailures);
+        createResultsList(html, "Unexpected passes", mUnexpectedPasses);
+        createResultsList(html, "Expected failures", mExpectedFailures);
+        createResultsList(html, "Expected passes", mExpectedPasses);
 
         html.append("</body></html>");
 
@@ -357,8 +357,8 @@ public class Summarizer {
         html.append("</tr>");
     }
 
-    private void createResultsListWithDiff(StringBuilder html, String title,
-            List<AbstractResult> resultsList) {
+    private void createResultsList(
+            StringBuilder html, String title, List<AbstractResult> resultsList) {
         String relativePath;
         String id = "";
         AbstractResult.ResultCode resultCode;
@@ -368,7 +368,6 @@ public class Summarizer {
         for (AbstractResult result : resultsList) {
             relativePath = result.getRelativePath();
             resultCode = result.getResultCode();
-            assert resultCode != AbstractResult.ResultCode.PASS : "resultCode=" + resultCode;
 
             html.append("<h3>");
 
@@ -378,61 +377,73 @@ public class Summarizer {
              * to cause any problems in this case
              */
             id = relativePath.replace(File.separator, ":");
-            html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
-            html.append("return false;\">");
-            html.append("<span class=\"tri\" id=\"tri." + id + "\">&#x25b6; </span>");
-            html.append("<span class=\"path\">" + relativePath + "</span>");
-            html.append("</a>");
-
-            html.append(" <span class=\"listItem " + resultCode.name() + "\">");
-            html.append(resultCode.toString());
-            html.append("</span>");
-
-            /** Detect missing LTC function */
-            String additionalTextOutputString = result.getAdditionalTextOutputString();
-            if (additionalTextOutputString != null &&
-                    additionalTextOutputString.contains("com.android.dumprendertree") &&
-                    additionalTextOutputString.contains("has no method")) {
-                if (additionalTextOutputString.contains("LayoutTestController")) {
-                    html.append(" <span class=\"listItem noLtc\">LTC function missing</span>");
-                }
-                if (additionalTextOutputString.contains("EventSender")) {
-                    html.append(" <span class=\"listItem noEventSender\">");
-                    html.append("ES function missing</span>");
-                }
+
+            /** Write the test name */
+            if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) {
+                html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
+                html.append("return false;\">");
+                html.append("<span class=\"tri\" id=\"tri." + id + "\">&#x25b6; </span>");
+                html.append("<span class=\"path\">" + relativePath + "</span>");
+                html.append("</a>");
+            } else {
+                html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() + "\"");
+                html.append(" target=\"_blank\">");
+                html.append("<span class=\"sqr sqr_" + (result.didPass() ? "pass" : "fail"));
+                html.append("\">&#x25a0; </span>");
+                html.append("<span class=\"path\">" + result.getRelativePath() + "</span>");
+                html.append("</a>");
+            }
+
+            if (!result.didPass()) {
+                appendTags(html, result);
             }
 
             html.append("</h3>");
             appendExpectedResultsSources(result, html);
 
-            html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">");
-            html.append(result.getDiffAsHtml());
-            html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
-            html.append("return false;\">Hide</a>");
-            html.append(" | ");
-            html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\"");
-            html.append(" target=\"_blank\">Show source</a>");
-            html.append("</div>");
+            if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) {
+                html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">");
+                html.append(result.getDiffAsHtml());
+                html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
+                html.append("return false;\">Hide</a>");
+                html.append(" | ");
+                html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\"");
+                html.append(" target=\"_blank\">Show source</a>");
+                html.append("</div>");
+            }
 
             html.append("<div class=\"space\"></div>");
         }
     }
 
-    private void createResultsListNoDiff(StringBuilder html, String title,
-            List<AbstractResult> resultsList) {
-        Collections.sort(resultsList);
-        html.append("<h2>" + title + " [" + resultsList.size() + "]</h2>");
-        for (AbstractResult result : resultsList) {
-            html.append("<h3>");
-            html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() +
-                    "\"");
-            html.append(" target=\"_blank\">");
-            html.append("<span class=\"sqr\">&#x25a0; </span>");
-            html.append("<span class=\"path\">" + result.getRelativePath() + "</span>");
-            html.append("</a>");
-            html.append("</h3>");
-            appendExpectedResultsSources(result, html);
-            html.append("<div class=\"space\"></div>");
+    private void appendTags(StringBuilder html, AbstractResult result) {
+        /** Tag tests which crash, time out or where results don't match */
+        if (result.didCrash()) {
+            html.append(" <span class=\"listItem crashed\">Crashed</span>");
+        } else {
+            if (result.didTimeOut()) {
+                html.append(" <span class=\"listItem timed_out\">Timed out</span>");
+            }
+            AbstractResult.ResultCode resultCode = result.getResultCode();
+            if (resultCode != AbstractResult.ResultCode.RESULTS_MATCH) {
+                html.append(" <span class=\"listItem " + resultCode.name() + "\">");
+                html.append(resultCode.toString());
+                html.append("</span>");
+            }
+        }
+
+        /** Detect missing LTC function */
+        String additionalTextOutputString = result.getAdditionalTextOutputString();
+        if (additionalTextOutputString != null &&
+                additionalTextOutputString.contains("com.android.dumprendertree") &&
+                additionalTextOutputString.contains("has no method")) {
+            if (additionalTextOutputString.contains("LayoutTestController")) {
+                html.append(" <span class=\"listItem noLtc\">LTC function missing</span>");
+            }
+            if (additionalTextOutputString.contains("EventSender")) {
+                html.append(" <span class=\"listItem noEventSender\">");
+                html.append("ES function missing</span>");
+            }
         }
     }
 
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java
index 9664efe..21e5430 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java
@@ -39,6 +39,7 @@ public class TextResult extends AbstractResult {
     private String mExpectedResultPath;
     private String mActualResult;
     private String mRelativePath;
+    private boolean mDidTimeOut;
     private ResultCode mResultCode;
     private Message mResultObtainedMsg;
 
@@ -74,10 +75,7 @@ public class TextResult extends AbstractResult {
         mActualResult = bundle.getString("actualTextualResult");
         setAdditionalTextOutputString(bundle.getString("additionalTextOutputString"));
         mRelativePath = bundle.getString("relativePath");
-        String resultCode = bundle.getString("resultCode");
-        if (resultCode != null) {
-            mResultCode = ResultCode.valueOf(resultCode);
-        }
+        mDidTimeOut = bundle.getBoolean("didTimeOut");
     }
 
     @Override
@@ -87,17 +85,31 @@ public class TextResult extends AbstractResult {
         }
 
         if (mExpectedResult == null) {
-            mResultCode = AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT;
+            mResultCode = AbstractResult.ResultCode.NO_EXPECTED_RESULT;
         } else if (!mExpectedResult.equals(mActualResult)) {
-            mResultCode = AbstractResult.ResultCode.FAIL_RESULT_DIFFERS;
+            mResultCode = AbstractResult.ResultCode.RESULTS_DIFFER;
         } else {
-            mResultCode = AbstractResult.ResultCode.PASS;
+            mResultCode = AbstractResult.ResultCode.RESULTS_MATCH;
         }
-
         return mResultCode;
     }
 
     @Override
+    public boolean didCrash() {
+        return false;
+    }
+
+    @Override
+    public boolean didTimeOut() {
+        return mDidTimeOut;
+    }
+
+    @Override
+    public void setDidTimeOut() {
+        mDidTimeOut = true;
+    }
+
+    @Override
     public byte[] getActualImageResult() {
         return null;
     }
@@ -239,9 +251,7 @@ public class TextResult extends AbstractResult {
         bundle.putString("actualTextualResult", getActualTextResult());
         bundle.putString("additionalTextOutputString", getAdditionalTextOutputString());
         bundle.putString("relativePath", mRelativePath);
-        if (mResultCode != null) {
-            bundle.putString("resultCode", mResultCode.name());
-        }
+        bundle.putBoolean("didTimeOut", mDidTimeOut);
         bundle.putString("type", getType().name());
         return bundle;
     }
@@ -250,4 +260,4 @@ public class TextResult extends AbstractResult {
     public String getRelativePath() {
         return mRelativePath;
     }
-}
\ No newline at end of file
+}