diff options
5 files changed, 161 insertions, 88 deletions
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java index 7cbb397..6048338a 100644 --- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java +++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java @@ -46,12 +46,14 @@ public abstract class AbstractResult implements Comparable<AbstractResult> { public abstract AbstractResult createResult(Bundle bundle); } + /** + * A code representing the result of comparing actual and expected results. + */ public enum ResultCode { - PASS("Passed"), - FAIL_RESULT_DIFFERS("Result differs"), - FAIL_NO_EXPECTED_RESULT("No expected result"), - FAIL_TIMED_OUT("Timed out"), - FAIL_CRASHED("Crashed"); + RESULTS_MATCH("Results match"), + RESULTS_DIFFER("Results differ"), + NO_EXPECTED_RESULT("No expected result"), + NO_ACTUAL_RESULT("No actual result"); private String mTitle; @@ -123,14 +125,48 @@ public abstract class AbstractResult implements Comparable<AbstractResult> { public abstract String getActualTextResult(); /** - * Returns the code of this result. + * Returns the status code representing the result of comparing actual and expected results. * * @return - * the code of this result + * the status code from comparing actual and expected results */ public abstract ResultCode getResultCode(); /** + * Returns whether this test crashed. + * + * @return + * whether this test crashed + */ + public abstract boolean didCrash(); + + /** + * Returns whether this test timed out. + * + * @return + * whether this test timed out + */ + public abstract boolean didTimeOut(); + + /** + * Sets that this test timed out. + */ + public abstract void setDidTimeOut(); + + /** + * Returns whether the test passed. + * + * @return + * whether the test passed + */ + public boolean didPass() { + // Tests that crash can't have timed out or have an actual result. + assert !(didCrash() && didTimeOut()); + assert !(didCrash() && getResultCode() != ResultCode.NO_ACTUAL_RESULT); + return !didCrash() && !didTimeOut() && getResultCode() == ResultCode.RESULTS_MATCH; + } + + /** * Return the type of the result data. * * @return @@ -150,4 +186,4 @@ public abstract class AbstractResult implements Comparable<AbstractResult> { public abstract String getDiffAsHtml(); public abstract Bundle getBundle(); -}
\ No newline at end of file +} diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java index 31da776..4831168 100644 --- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java +++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java @@ -61,7 +61,23 @@ public class CrashedDummyResult extends AbstractResult { @Override public ResultCode getResultCode() { - return ResultCode.FAIL_CRASHED; + return ResultCode.NO_ACTUAL_RESULT; + } + + @Override + public boolean didCrash() { + return true; + } + + @Override + public boolean didTimeOut() { + return false; + } + + @Override + public void setDidTimeOut() { + /** This method is not applicable for this type of result */ + assert false; } @Override @@ -106,4 +122,4 @@ public class CrashedDummyResult extends AbstractResult { public void setExpectedTextResultPath(String relativePath) { /** TODO */ } -}
\ No newline at end of file +} diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java index efea192..b9fc274 100644 --- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java +++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java @@ -426,6 +426,7 @@ public class LayoutTestsExecutor extends Activity { assert mCurrentState.isRunningState() : "mCurrentState = " + mCurrentState.name(); Log.i(LOG_TAG, "onTestFinished(): " + mCurrentTestRelativePath); + mResultHandler.removeMessages(MSG_TEST_TIMED_OUT); obtainActualResultsFromWebView(); } @@ -441,6 +442,9 @@ public class LayoutTestsExecutor extends Activity { mCurrentState = CurrentState.OBTAINING_RESULT; + if (mCurrentTestTimedOut) { + mCurrentResult.setDidTimeOut(); + } mCurrentResult.obtainActualResults(mCurrentWebView, mResultHandler.obtainMessage(MSG_ACTUAL_RESULT_OBTAINED)); } @@ -452,7 +456,6 @@ public class LayoutTestsExecutor extends Activity { Log.i(LOG_TAG, "onActualResultsObtained(): " + mCurrentTestRelativePath); mCurrentState = CurrentState.IDLE; - mResultHandler.removeMessages(MSG_TEST_TIMED_OUT); reportResultToService(); mCurrentTestIndex++; updateProgressBar(); @@ -470,9 +473,6 @@ public class LayoutTestsExecutor extends Activity { Bundle bundle = mCurrentResult.getBundle(); bundle.putInt("testIndex", mCurrentTestIndex); - if (mCurrentTestTimedOut) { - bundle.putString("resultCode", AbstractResult.ResultCode.FAIL_TIMED_OUT.name()); - } if (!mTestsList.isEmpty()) { bundle.putString("nextTest", mTestsList.get(0)); } diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java index e31e747..b1dea6e 100644 --- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java +++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java @@ -75,9 +75,12 @@ public class Summarizer { " width: 20px;}" + "h3 span.sqr {" + " text-decoration: none;" + - " color: #8ee100;" + " float: left;" + " width: 20px;}" + + "h3 span.sqr_pass {" + + " color: #8ee100;}" + + "h3 span.sqr_fail {" + + " color: #c30000;}" + "span.source {" + " display: block;" + " font-size: 10px;" + @@ -145,16 +148,16 @@ public class Summarizer { " text-transform: uppercase;" + " padding: 3px;" + " -webkit-border-radius: 4px;}" + - "span." + AbstractResult.ResultCode.FAIL_RESULT_DIFFERS.name() + "{" + + "span." + AbstractResult.ResultCode.RESULTS_DIFFER.name() + "{" + " background-color: #ccc;" + " color: black;}" + - "span." + AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT.name() + "{" + + "span." + AbstractResult.ResultCode.NO_EXPECTED_RESULT.name() + "{" + " background-color: #a700e4;" + " color: #fff;}" + - "span." + AbstractResult.ResultCode.FAIL_TIMED_OUT.name() + "{" + + "span.timed_out {" + " background-color: #f3cb00;" + " color: black;}" + - "span." + AbstractResult.ResultCode.FAIL_CRASHED.name() + "{" + + "span.crashed {" + " background-color: #c30000;" + " color: #fff;}" + "span.noLtc {" + @@ -208,11 +211,11 @@ public class Summarizer { public void appendTest(AbstractResult result) { String relativePath = result.getRelativePath(); - if (result.getResultCode() == AbstractResult.ResultCode.FAIL_CRASHED) { + if (result.didCrash()) { mCrashedTestsCount++; } - if (result.getResultCode() == AbstractResult.ResultCode.PASS) { + if (result.didPass()) { if (mFileFilter.isFail(relativePath)) { mUnexpectedPasses.add(result); } else { @@ -276,13 +279,10 @@ public class Summarizer { createTopSummaryTable(webKitRevision, html); - createResultsListWithDiff(html, "Unexpected failures", mUnexpectedFailures); - - createResultsListNoDiff(html, "Unexpected passes", mUnexpectedPasses); - - createResultsListWithDiff(html, "Expected failures", mExpectedFailures); - - createResultsListNoDiff(html, "Expected passes", mExpectedPasses); + createResultsList(html, "Unexpected failures", mUnexpectedFailures); + createResultsList(html, "Unexpected passes", mUnexpectedPasses); + createResultsList(html, "Expected failures", mExpectedFailures); + createResultsList(html, "Expected passes", mExpectedPasses); html.append("</body></html>"); @@ -357,8 +357,8 @@ public class Summarizer { html.append("</tr>"); } - private void createResultsListWithDiff(StringBuilder html, String title, - List<AbstractResult> resultsList) { + private void createResultsList( + StringBuilder html, String title, List<AbstractResult> resultsList) { String relativePath; String id = ""; AbstractResult.ResultCode resultCode; @@ -368,7 +368,6 @@ public class Summarizer { for (AbstractResult result : resultsList) { relativePath = result.getRelativePath(); resultCode = result.getResultCode(); - assert resultCode != AbstractResult.ResultCode.PASS : "resultCode=" + resultCode; html.append("<h3>"); @@ -378,61 +377,73 @@ public class Summarizer { * to cause any problems in this case */ id = relativePath.replace(File.separator, ":"); - html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');"); - html.append("return false;\">"); - html.append("<span class=\"tri\" id=\"tri." + id + "\">▶ </span>"); - html.append("<span class=\"path\">" + relativePath + "</span>"); - html.append("</a>"); - - html.append(" <span class=\"listItem " + resultCode.name() + "\">"); - html.append(resultCode.toString()); - html.append("</span>"); - - /** Detect missing LTC function */ - String additionalTextOutputString = result.getAdditionalTextOutputString(); - if (additionalTextOutputString != null && - additionalTextOutputString.contains("com.android.dumprendertree") && - additionalTextOutputString.contains("has no method")) { - if (additionalTextOutputString.contains("LayoutTestController")) { - html.append(" <span class=\"listItem noLtc\">LTC function missing</span>"); - } - if (additionalTextOutputString.contains("EventSender")) { - html.append(" <span class=\"listItem noEventSender\">"); - html.append("ES function missing</span>"); - } + + /** Write the test name */ + if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) { + html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');"); + html.append("return false;\">"); + html.append("<span class=\"tri\" id=\"tri." + id + "\">▶ </span>"); + html.append("<span class=\"path\">" + relativePath + "</span>"); + html.append("</a>"); + } else { + html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() + "\""); + html.append(" target=\"_blank\">"); + html.append("<span class=\"sqr sqr_" + (result.didPass() ? "pass" : "fail")); + html.append("\">■ </span>"); + html.append("<span class=\"path\">" + result.getRelativePath() + "</span>"); + html.append("</a>"); + } + + if (!result.didPass()) { + appendTags(html, result); } html.append("</h3>"); appendExpectedResultsSources(result, html); - html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">"); - html.append(result.getDiffAsHtml()); - html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');"); - html.append("return false;\">Hide</a>"); - html.append(" | "); - html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\""); - html.append(" target=\"_blank\">Show source</a>"); - html.append("</div>"); + if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) { + html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">"); + html.append(result.getDiffAsHtml()); + html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');"); + html.append("return false;\">Hide</a>"); + html.append(" | "); + html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\""); + html.append(" target=\"_blank\">Show source</a>"); + html.append("</div>"); + } html.append("<div class=\"space\"></div>"); } } - private void createResultsListNoDiff(StringBuilder html, String title, - List<AbstractResult> resultsList) { - Collections.sort(resultsList); - html.append("<h2>" + title + " [" + resultsList.size() + "]</h2>"); - for (AbstractResult result : resultsList) { - html.append("<h3>"); - html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() + - "\""); - html.append(" target=\"_blank\">"); - html.append("<span class=\"sqr\">■ </span>"); - html.append("<span class=\"path\">" + result.getRelativePath() + "</span>"); - html.append("</a>"); - html.append("</h3>"); - appendExpectedResultsSources(result, html); - html.append("<div class=\"space\"></div>"); + private void appendTags(StringBuilder html, AbstractResult result) { + /** Tag tests which crash, time out or where results don't match */ + if (result.didCrash()) { + html.append(" <span class=\"listItem crashed\">Crashed</span>"); + } else { + if (result.didTimeOut()) { + html.append(" <span class=\"listItem timed_out\">Timed out</span>"); + } + AbstractResult.ResultCode resultCode = result.getResultCode(); + if (resultCode != AbstractResult.ResultCode.RESULTS_MATCH) { + html.append(" <span class=\"listItem " + resultCode.name() + "\">"); + html.append(resultCode.toString()); + html.append("</span>"); + } + } + + /** Detect missing LTC function */ + String additionalTextOutputString = result.getAdditionalTextOutputString(); + if (additionalTextOutputString != null && + additionalTextOutputString.contains("com.android.dumprendertree") && + additionalTextOutputString.contains("has no method")) { + if (additionalTextOutputString.contains("LayoutTestController")) { + html.append(" <span class=\"listItem noLtc\">LTC function missing</span>"); + } + if (additionalTextOutputString.contains("EventSender")) { + html.append(" <span class=\"listItem noEventSender\">"); + html.append("ES function missing</span>"); + } } } diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java index 9664efe..21e5430 100644 --- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java +++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java @@ -39,6 +39,7 @@ public class TextResult extends AbstractResult { private String mExpectedResultPath; private String mActualResult; private String mRelativePath; + private boolean mDidTimeOut; private ResultCode mResultCode; private Message mResultObtainedMsg; @@ -74,10 +75,7 @@ public class TextResult extends AbstractResult { mActualResult = bundle.getString("actualTextualResult"); setAdditionalTextOutputString(bundle.getString("additionalTextOutputString")); mRelativePath = bundle.getString("relativePath"); - String resultCode = bundle.getString("resultCode"); - if (resultCode != null) { - mResultCode = ResultCode.valueOf(resultCode); - } + mDidTimeOut = bundle.getBoolean("didTimeOut"); } @Override @@ -87,17 +85,31 @@ public class TextResult extends AbstractResult { } if (mExpectedResult == null) { - mResultCode = AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT; + mResultCode = AbstractResult.ResultCode.NO_EXPECTED_RESULT; } else if (!mExpectedResult.equals(mActualResult)) { - mResultCode = AbstractResult.ResultCode.FAIL_RESULT_DIFFERS; + mResultCode = AbstractResult.ResultCode.RESULTS_DIFFER; } else { - mResultCode = AbstractResult.ResultCode.PASS; + mResultCode = AbstractResult.ResultCode.RESULTS_MATCH; } - return mResultCode; } @Override + public boolean didCrash() { + return false; + } + + @Override + public boolean didTimeOut() { + return mDidTimeOut; + } + + @Override + public void setDidTimeOut() { + mDidTimeOut = true; + } + + @Override public byte[] getActualImageResult() { return null; } @@ -239,9 +251,7 @@ public class TextResult extends AbstractResult { bundle.putString("actualTextualResult", getActualTextResult()); bundle.putString("additionalTextOutputString", getAdditionalTextOutputString()); bundle.putString("relativePath", mRelativePath); - if (mResultCode != null) { - bundle.putString("resultCode", mResultCode.name()); - } + bundle.putBoolean("didTimeOut", mDidTimeOut); bundle.putString("type", getType().name()); return bundle; } @@ -250,4 +260,4 @@ public class TextResult extends AbstractResult { public String getRelativePath() { return mRelativePath; } -}
\ No newline at end of file +} |