summaryrefslogtreecommitdiffstats
path: root/tests/DumpRenderTree2
diff options
context:
space:
mode:
authorSteve Block <steveblock@google.com>2010-09-17 06:40:24 -0700
committerAndroid (Google) Code Review <android-gerrit@google.com>2010-09-17 06:40:24 -0700
commitb6a2364ebad11596858892fa9806346e1c011ad6 (patch)
treee38cd9896a043687737f8e1ba7b166f6b6352d4d /tests/DumpRenderTree2
parent44ce3242a984bc3959e43fc832966d7f0ed61159 (diff)
parent8a6def02473ee4fbffcd1b34173daf751d316202 (diff)
downloadframeworks_base-b6a2364ebad11596858892fa9806346e1c011ad6.zip
frameworks_base-b6a2364ebad11596858892fa9806346e1c011ad6.tar.gz
frameworks_base-b6a2364ebad11596858892fa9806346e1c011ad6.tar.bz2
Merge "Update DumpRenderTree2 to handle failing tests where no meaningful diff is available"
Diffstat (limited to 'tests/DumpRenderTree2')
-rw-r--r--tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java52
-rw-r--r--tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java20
-rw-r--r--tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java8
-rw-r--r--tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java135
-rw-r--r--tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java34
5 files changed, 161 insertions, 88 deletions
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java
index 7cbb397..6048338a 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java
@@ -46,12 +46,14 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
public abstract AbstractResult createResult(Bundle bundle);
}
+ /**
+ * A code representing the result of comparing actual and expected results.
+ */
public enum ResultCode {
- PASS("Passed"),
- FAIL_RESULT_DIFFERS("Result differs"),
- FAIL_NO_EXPECTED_RESULT("No expected result"),
- FAIL_TIMED_OUT("Timed out"),
- FAIL_CRASHED("Crashed");
+ RESULTS_MATCH("Results match"),
+ RESULTS_DIFFER("Results differ"),
+ NO_EXPECTED_RESULT("No expected result"),
+ NO_ACTUAL_RESULT("No actual result");
private String mTitle;
@@ -123,14 +125,48 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
public abstract String getActualTextResult();
/**
- * Returns the code of this result.
+ * Returns the status code representing the result of comparing actual and expected results.
*
* @return
- * the code of this result
+ * the status code from comparing actual and expected results
*/
public abstract ResultCode getResultCode();
/**
+ * Returns whether this test crashed.
+ *
+ * @return
+ * whether this test crashed
+ */
+ public abstract boolean didCrash();
+
+ /**
+ * Returns whether this test timed out.
+ *
+ * @return
+ * whether this test timed out
+ */
+ public abstract boolean didTimeOut();
+
+ /**
+ * Sets that this test timed out.
+ */
+ public abstract void setDidTimeOut();
+
+ /**
+ * Returns whether the test passed.
+ *
+ * @return
+ * whether the test passed
+ */
+ public boolean didPass() {
+ // Tests that crash can't have timed out or have an actual result.
+ assert !(didCrash() && didTimeOut());
+ assert !(didCrash() && getResultCode() != ResultCode.NO_ACTUAL_RESULT);
+ return !didCrash() && !didTimeOut() && getResultCode() == ResultCode.RESULTS_MATCH;
+ }
+
+ /**
* Return the type of the result data.
*
* @return
@@ -150,4 +186,4 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
public abstract String getDiffAsHtml();
public abstract Bundle getBundle();
-} \ No newline at end of file
+}
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java
index 31da776..4831168 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/CrashedDummyResult.java
@@ -61,7 +61,23 @@ public class CrashedDummyResult extends AbstractResult {
@Override
public ResultCode getResultCode() {
- return ResultCode.FAIL_CRASHED;
+ return ResultCode.NO_ACTUAL_RESULT;
+ }
+
+ @Override
+ public boolean didCrash() {
+ return true;
+ }
+
+ @Override
+ public boolean didTimeOut() {
+ return false;
+ }
+
+ @Override
+ public void setDidTimeOut() {
+ /** This method is not applicable for this type of result */
+ assert false;
}
@Override
@@ -106,4 +122,4 @@ public class CrashedDummyResult extends AbstractResult {
public void setExpectedTextResultPath(String relativePath) {
/** TODO */
}
-} \ No newline at end of file
+}
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java
index efea192..b9fc274 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/LayoutTestsExecutor.java
@@ -426,6 +426,7 @@ public class LayoutTestsExecutor extends Activity {
assert mCurrentState.isRunningState() : "mCurrentState = " + mCurrentState.name();
Log.i(LOG_TAG, "onTestFinished(): " + mCurrentTestRelativePath);
+ mResultHandler.removeMessages(MSG_TEST_TIMED_OUT);
obtainActualResultsFromWebView();
}
@@ -441,6 +442,9 @@ public class LayoutTestsExecutor extends Activity {
mCurrentState = CurrentState.OBTAINING_RESULT;
+ if (mCurrentTestTimedOut) {
+ mCurrentResult.setDidTimeOut();
+ }
mCurrentResult.obtainActualResults(mCurrentWebView,
mResultHandler.obtainMessage(MSG_ACTUAL_RESULT_OBTAINED));
}
@@ -452,7 +456,6 @@ public class LayoutTestsExecutor extends Activity {
Log.i(LOG_TAG, "onActualResultsObtained(): " + mCurrentTestRelativePath);
mCurrentState = CurrentState.IDLE;
- mResultHandler.removeMessages(MSG_TEST_TIMED_OUT);
reportResultToService();
mCurrentTestIndex++;
updateProgressBar();
@@ -470,9 +473,6 @@ public class LayoutTestsExecutor extends Activity {
Bundle bundle = mCurrentResult.getBundle();
bundle.putInt("testIndex", mCurrentTestIndex);
- if (mCurrentTestTimedOut) {
- bundle.putString("resultCode", AbstractResult.ResultCode.FAIL_TIMED_OUT.name());
- }
if (!mTestsList.isEmpty()) {
bundle.putString("nextTest", mTestsList.get(0));
}
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
index e31e747..b1dea6e 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
@@ -75,9 +75,12 @@ public class Summarizer {
" width: 20px;}" +
"h3 span.sqr {" +
" text-decoration: none;" +
- " color: #8ee100;" +
" float: left;" +
" width: 20px;}" +
+ "h3 span.sqr_pass {" +
+ " color: #8ee100;}" +
+ "h3 span.sqr_fail {" +
+ " color: #c30000;}" +
"span.source {" +
" display: block;" +
" font-size: 10px;" +
@@ -145,16 +148,16 @@ public class Summarizer {
" text-transform: uppercase;" +
" padding: 3px;" +
" -webkit-border-radius: 4px;}" +
- "span." + AbstractResult.ResultCode.FAIL_RESULT_DIFFERS.name() + "{" +
+ "span." + AbstractResult.ResultCode.RESULTS_DIFFER.name() + "{" +
" background-color: #ccc;" +
" color: black;}" +
- "span." + AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT.name() + "{" +
+ "span." + AbstractResult.ResultCode.NO_EXPECTED_RESULT.name() + "{" +
" background-color: #a700e4;" +
" color: #fff;}" +
- "span." + AbstractResult.ResultCode.FAIL_TIMED_OUT.name() + "{" +
+ "span.timed_out {" +
" background-color: #f3cb00;" +
" color: black;}" +
- "span." + AbstractResult.ResultCode.FAIL_CRASHED.name() + "{" +
+ "span.crashed {" +
" background-color: #c30000;" +
" color: #fff;}" +
"span.noLtc {" +
@@ -208,11 +211,11 @@ public class Summarizer {
public void appendTest(AbstractResult result) {
String relativePath = result.getRelativePath();
- if (result.getResultCode() == AbstractResult.ResultCode.FAIL_CRASHED) {
+ if (result.didCrash()) {
mCrashedTestsCount++;
}
- if (result.getResultCode() == AbstractResult.ResultCode.PASS) {
+ if (result.didPass()) {
if (mFileFilter.isFail(relativePath)) {
mUnexpectedPasses.add(result);
} else {
@@ -276,13 +279,10 @@ public class Summarizer {
createTopSummaryTable(webKitRevision, html);
- createResultsListWithDiff(html, "Unexpected failures", mUnexpectedFailures);
-
- createResultsListNoDiff(html, "Unexpected passes", mUnexpectedPasses);
-
- createResultsListWithDiff(html, "Expected failures", mExpectedFailures);
-
- createResultsListNoDiff(html, "Expected passes", mExpectedPasses);
+ createResultsList(html, "Unexpected failures", mUnexpectedFailures);
+ createResultsList(html, "Unexpected passes", mUnexpectedPasses);
+ createResultsList(html, "Expected failures", mExpectedFailures);
+ createResultsList(html, "Expected passes", mExpectedPasses);
html.append("</body></html>");
@@ -357,8 +357,8 @@ public class Summarizer {
html.append("</tr>");
}
- private void createResultsListWithDiff(StringBuilder html, String title,
- List<AbstractResult> resultsList) {
+ private void createResultsList(
+ StringBuilder html, String title, List<AbstractResult> resultsList) {
String relativePath;
String id = "";
AbstractResult.ResultCode resultCode;
@@ -368,7 +368,6 @@ public class Summarizer {
for (AbstractResult result : resultsList) {
relativePath = result.getRelativePath();
resultCode = result.getResultCode();
- assert resultCode != AbstractResult.ResultCode.PASS : "resultCode=" + resultCode;
html.append("<h3>");
@@ -378,61 +377,73 @@ public class Summarizer {
* to cause any problems in this case
*/
id = relativePath.replace(File.separator, ":");
- html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
- html.append("return false;\">");
- html.append("<span class=\"tri\" id=\"tri." + id + "\">&#x25b6; </span>");
- html.append("<span class=\"path\">" + relativePath + "</span>");
- html.append("</a>");
-
- html.append(" <span class=\"listItem " + resultCode.name() + "\">");
- html.append(resultCode.toString());
- html.append("</span>");
-
- /** Detect missing LTC function */
- String additionalTextOutputString = result.getAdditionalTextOutputString();
- if (additionalTextOutputString != null &&
- additionalTextOutputString.contains("com.android.dumprendertree") &&
- additionalTextOutputString.contains("has no method")) {
- if (additionalTextOutputString.contains("LayoutTestController")) {
- html.append(" <span class=\"listItem noLtc\">LTC function missing</span>");
- }
- if (additionalTextOutputString.contains("EventSender")) {
- html.append(" <span class=\"listItem noEventSender\">");
- html.append("ES function missing</span>");
- }
+
+ /** Write the test name */
+ if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) {
+ html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
+ html.append("return false;\">");
+ html.append("<span class=\"tri\" id=\"tri." + id + "\">&#x25b6; </span>");
+ html.append("<span class=\"path\">" + relativePath + "</span>");
+ html.append("</a>");
+ } else {
+ html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() + "\"");
+ html.append(" target=\"_blank\">");
+ html.append("<span class=\"sqr sqr_" + (result.didPass() ? "pass" : "fail"));
+ html.append("\">&#x25a0; </span>");
+ html.append("<span class=\"path\">" + result.getRelativePath() + "</span>");
+ html.append("</a>");
+ }
+
+ if (!result.didPass()) {
+ appendTags(html, result);
}
html.append("</h3>");
appendExpectedResultsSources(result, html);
- html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">");
- html.append(result.getDiffAsHtml());
- html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
- html.append("return false;\">Hide</a>");
- html.append(" | ");
- html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\"");
- html.append(" target=\"_blank\">Show source</a>");
- html.append("</div>");
+ if (resultCode == AbstractResult.ResultCode.RESULTS_DIFFER) {
+ html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">");
+ html.append(result.getDiffAsHtml());
+ html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
+ html.append("return false;\">Hide</a>");
+ html.append(" | ");
+ html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\"");
+ html.append(" target=\"_blank\">Show source</a>");
+ html.append("</div>");
+ }
html.append("<div class=\"space\"></div>");
}
}
- private void createResultsListNoDiff(StringBuilder html, String title,
- List<AbstractResult> resultsList) {
- Collections.sort(resultsList);
- html.append("<h2>" + title + " [" + resultsList.size() + "]</h2>");
- for (AbstractResult result : resultsList) {
- html.append("<h3>");
- html.append("<a href=\"" + getViewSourceUrl(result.getRelativePath()).toString() +
- "\"");
- html.append(" target=\"_blank\">");
- html.append("<span class=\"sqr\">&#x25a0; </span>");
- html.append("<span class=\"path\">" + result.getRelativePath() + "</span>");
- html.append("</a>");
- html.append("</h3>");
- appendExpectedResultsSources(result, html);
- html.append("<div class=\"space\"></div>");
+ private void appendTags(StringBuilder html, AbstractResult result) {
+ /** Tag tests which crash, time out or where results don't match */
+ if (result.didCrash()) {
+ html.append(" <span class=\"listItem crashed\">Crashed</span>");
+ } else {
+ if (result.didTimeOut()) {
+ html.append(" <span class=\"listItem timed_out\">Timed out</span>");
+ }
+ AbstractResult.ResultCode resultCode = result.getResultCode();
+ if (resultCode != AbstractResult.ResultCode.RESULTS_MATCH) {
+ html.append(" <span class=\"listItem " + resultCode.name() + "\">");
+ html.append(resultCode.toString());
+ html.append("</span>");
+ }
+ }
+
+ /** Detect missing LTC function */
+ String additionalTextOutputString = result.getAdditionalTextOutputString();
+ if (additionalTextOutputString != null &&
+ additionalTextOutputString.contains("com.android.dumprendertree") &&
+ additionalTextOutputString.contains("has no method")) {
+ if (additionalTextOutputString.contains("LayoutTestController")) {
+ html.append(" <span class=\"listItem noLtc\">LTC function missing</span>");
+ }
+ if (additionalTextOutputString.contains("EventSender")) {
+ html.append(" <span class=\"listItem noEventSender\">");
+ html.append("ES function missing</span>");
+ }
}
}
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java
index 9664efe..21e5430 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java
@@ -39,6 +39,7 @@ public class TextResult extends AbstractResult {
private String mExpectedResultPath;
private String mActualResult;
private String mRelativePath;
+ private boolean mDidTimeOut;
private ResultCode mResultCode;
private Message mResultObtainedMsg;
@@ -74,10 +75,7 @@ public class TextResult extends AbstractResult {
mActualResult = bundle.getString("actualTextualResult");
setAdditionalTextOutputString(bundle.getString("additionalTextOutputString"));
mRelativePath = bundle.getString("relativePath");
- String resultCode = bundle.getString("resultCode");
- if (resultCode != null) {
- mResultCode = ResultCode.valueOf(resultCode);
- }
+ mDidTimeOut = bundle.getBoolean("didTimeOut");
}
@Override
@@ -87,17 +85,31 @@ public class TextResult extends AbstractResult {
}
if (mExpectedResult == null) {
- mResultCode = AbstractResult.ResultCode.FAIL_NO_EXPECTED_RESULT;
+ mResultCode = AbstractResult.ResultCode.NO_EXPECTED_RESULT;
} else if (!mExpectedResult.equals(mActualResult)) {
- mResultCode = AbstractResult.ResultCode.FAIL_RESULT_DIFFERS;
+ mResultCode = AbstractResult.ResultCode.RESULTS_DIFFER;
} else {
- mResultCode = AbstractResult.ResultCode.PASS;
+ mResultCode = AbstractResult.ResultCode.RESULTS_MATCH;
}
-
return mResultCode;
}
@Override
+ public boolean didCrash() {
+ return false;
+ }
+
+ @Override
+ public boolean didTimeOut() {
+ return mDidTimeOut;
+ }
+
+ @Override
+ public void setDidTimeOut() {
+ mDidTimeOut = true;
+ }
+
+ @Override
public byte[] getActualImageResult() {
return null;
}
@@ -239,9 +251,7 @@ public class TextResult extends AbstractResult {
bundle.putString("actualTextualResult", getActualTextResult());
bundle.putString("additionalTextOutputString", getAdditionalTextOutputString());
bundle.putString("relativePath", mRelativePath);
- if (mResultCode != null) {
- bundle.putString("resultCode", mResultCode.name());
- }
+ bundle.putBoolean("didTimeOut", mDidTimeOut);
bundle.putString("type", getType().name());
return bundle;
}
@@ -250,4 +260,4 @@ public class TextResult extends AbstractResult {
public String getRelativePath() {
return mRelativePath;
}
-} \ No newline at end of file
+}