Merge "Updated Summarizer to report tests under all 4 possible categories now."

2 files changed, 65 insertions, 65 deletions

diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/FileFilter.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/FileFilter.java
index 4f2fc77..9427719 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/FileFilter.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/FileFilter.java
@@ -49,11 +49,11 @@ public class FileFilter {
     private static final String SSL_PATH = "ssl/";
 
     private static final String TOKEN_SKIP = "SKIP";
-    private static final String TOKEN_IGNORE_RESULT = "IGNORE_RESULT";
+    private static final String TOKEN_FAIL = "FAIL";
     private static final String TOKEN_SLOW = "SLOW";
 
     private final Set<String> mSkipList = new HashSet<String>();
-    private final Set<String> mIgnoreResultList = new HashSet<String>();
+    private final Set<String> mFailList = new HashSet<String>();
     private final Set<String> mSlowList = new HashSet<String>();
 
     private final String mRootDirPath;
@@ -91,7 +91,6 @@ public class FileFilter {
                 String[] parts;
                 String path;
                 Set<String> tokens;
-                Boolean skipped;
                 while (true) {
                     line = bufferedReader.readLine();
                     if (line == null) {
@@ -122,21 +121,16 @@ public class FileFilter {
                     tokens = new HashSet<String>(Arrays.asList(parts[1].split("\\s", 0)));
 
                     /** Chose the right collections to add to */
-                    skipped = false;
                     if (tokens.contains(TOKEN_SKIP)) {
                         mSkipList.add(path);
-                        skipped = true;
-                    }
 
-                    /** If test is on skip list we ignore any further options */
-                    if (skipped) {
+                        /** If test is on skip list we ignore any further options */
                         continue;
                     }
 
-                    if (tokens.contains(TOKEN_IGNORE_RESULT)) {
-                        mIgnoreResultList.add(path);
+                    if (tokens.contains(TOKEN_FAIL)) {
+                        mFailList.add(path);
                     }
-
                     if (tokens.contains(TOKEN_SLOW)) {
                         mSlowList.add(path);
                     }
@@ -177,18 +171,18 @@ public class FileFilter {
     }
 
     /**
-     * Checks if test result is supposed to be ignored.
+     * Checks if test result is supposed to be "failed".
      *
      * <p>
      * Path given should relative within LayoutTests folder, e.g. fast/dom/foo.html
      *
      * @param testPath
      *            - a relative path within LayoutTests folder
-     * @return if the test result is supposed to be ignored
+     * @return if the test result is supposed to be "failed"
      */
-    public boolean isIgnoreRes(String testPath) {
+    public boolean isFail(String testPath) {
         for (String prefix : getPrefixes(testPath)) {
-            if (mIgnoreResultList.contains(prefix)) {
+            if (mFailList.contains(prefix)) {
                 return true;
             }
         }
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
index 7bbbc05..94ba35f 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
@@ -183,9 +183,10 @@ public class Summarizer {
     private static final String TXT_SUMMARY_RELATIVE_PATH = "summary.txt";
 
     private int mCrashedTestsCount = 0;
-    private List<AbstractResult> mFailedNotIgnoredTests = new ArrayList<AbstractResult>();
-    private List<AbstractResult> mIgnoredTests = new ArrayList<AbstractResult>();
-    private List<String> mPassedNotIgnoredTests = new ArrayList<String>();
+    private List<AbstractResult> mUnexpectedFailures = new ArrayList<AbstractResult>();
+    private List<AbstractResult> mExpectedFailures = new ArrayList<AbstractResult>();
+    private List<String> mExpectedPasses = new ArrayList<String>();
+    private List<String> mUnexpectedPasses = new ArrayList<String>();
 
     private FileFilter mFileFilter;
     private String mResultsRootDirPath;
@@ -206,12 +207,18 @@ public class Summarizer {
             mCrashedTestsCount++;
         }
 
-        if (mFileFilter.isIgnoreRes(relativePath)) {
-            mIgnoredTests.add(result);
-        } else if (result.getResultCode() == AbstractResult.ResultCode.PASS) {
-            mPassedNotIgnoredTests.add(relativePath);
+        if (result.getResultCode() == AbstractResult.ResultCode.PASS) {
+            if (mFileFilter.isFail(relativePath)) {
+                mUnexpectedPasses.add(relativePath);
+            } else {
+                mExpectedPasses.add(relativePath);
+            }
         } else {
-            mFailedNotIgnoredTests.add(result);
+            if (mFileFilter.isFail(relativePath)) {
+                mExpectedFailures.add(result);
+            } else {
+                mUnexpectedFailures.add(result);
+            }
         }
     }
 
@@ -226,9 +233,9 @@ public class Summarizer {
 
     public void reset() {
         mCrashedTestsCount = 0;
-        mFailedNotIgnoredTests.clear();
-        mIgnoredTests.clear();
-        mPassedNotIgnoredTests.clear();
+        mUnexpectedFailures.clear();
+        mExpectedFailures.clear();
+        mExpectedPasses.clear();
         mDate = new Date();
     }
 
@@ -246,9 +253,10 @@ public class Summarizer {
             txt.append("CRASHED (total among all tests): " + mCrashedTestsCount + "\n");
             txt.append("-------------");
         }
-        txt.append("FAILED:  " + mFailedNotIgnoredTests.size() + "\n");
-        txt.append("IGNORED: " + mIgnoredTests.size() + "\n");
-        txt.append("PASSED:  " + mPassedNotIgnoredTests.size() + "\n");
+        txt.append("UNEXPECTED FAILURES: " + mUnexpectedFailures.size() + "\n");
+        txt.append("UNEXPECTED PASSES:   " + mUnexpectedPasses.size() + "\n");
+        txt.append("EXPECTED FAILURES:   " + mExpectedFailures.size() + "\n");
+        txt.append("EXPECTED PASSES:     " + mExpectedPasses.size() + "\n");
 
         FsUtils.writeDataToStorage(new File(mResultsRootDirPath, TXT_SUMMARY_RELATIVE_PATH),
                 txt.toString().getBytes(), false);
@@ -264,11 +272,13 @@ public class Summarizer {
 
         createTopSummaryTable(html);
 
-        createResultsListWithDiff(html, "Failed", mFailedNotIgnoredTests);
+        createResultsListWithDiff(html, "Unexpected failures", mUnexpectedFailures);
+
+        createResultsListNoDiff(html, "Unexpected passes", mUnexpectedPasses);
 
-        createResultsListWithDiff(html, "Ignored", mIgnoredTests);
+        createResultsListWithDiff(html, "Expected failures", mExpectedFailures);
 
-        createResultsListNoDiff(html, "Passed", mPassedNotIgnoredTests);
+        createResultsListNoDiff(html, "Expected passes", mExpectedPasses);
 
         html.append("</body></html>");
 
@@ -277,9 +287,10 @@ public class Summarizer {
     }
 
     private int getTotalTestCount() {
-        return mFailedNotIgnoredTests.size() +
-                mPassedNotIgnoredTests.size() +
-                mIgnoredTests.size();
+        return mUnexpectedFailures.size() +
+                mUnexpectedPasses.size() +
+                mExpectedPasses.size() +
+                mExpectedFailures.size();
     }
 
     private String getWebKitVersionFromUserAgentString() {
@@ -305,9 +316,10 @@ public class Summarizer {
         html.append("<table class=\"summary\">");
         createSummaryTableRow(html, "TOTAL", getTotalTestCount());
         createSummaryTableRow(html, "CRASHED", mCrashedTestsCount);
-        createSummaryTableRow(html, "FAILED", mFailedNotIgnoredTests.size());
-        createSummaryTableRow(html, "IGNORED", mIgnoredTests.size());
-        createSummaryTableRow(html, "PASSED", mPassedNotIgnoredTests.size());
+        createSummaryTableRow(html, "UNEXPECTED FAILURES", mUnexpectedFailures.size());
+        createSummaryTableRow(html, "UNEXPECTED PASSES", mUnexpectedPasses.size());
+        createSummaryTableRow(html, "EXPECTED FAILURES", mExpectedFailures.size());
+        createSummaryTableRow(html, "EXPECTED PASSES", mExpectedPasses.size());
         html.append("</table>");
     }
 
@@ -329,25 +341,21 @@ public class Summarizer {
         for (AbstractResult result : resultsList) {
             relativePath = result.getRelativePath();
             resultCode = result.getResultCode();
+            assert resultCode != AbstractResult.ResultCode.PASS : "resultCode=" + resultCode;
 
             html.append("<h3>");
 
-            if (resultCode == AbstractResult.ResultCode.PASS) {
-                html.append("<span class=\"sqr\">&#x25a0; </span>");
-                html.append("<span class=\"path\">" + relativePath + "</span>");
-            } else {
-                /**
-                 * Technically, two different paths could end up being the same, because
-                 * ':' is a valid  character in a path. However, it is probably not going
-                 * to cause any problems in this case
-                 */
-                id = relativePath.replace(File.separator, ":");
-                html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
-                html.append("return false;\">");
-                html.append("<span class=\"tri\" id=\"tri." + id + "\">&#x25b6; </span>");
-                html.append("<span class=\"path\">" + relativePath + "</span>");
-                html.append("</a>");
-            }
+            /**
+             * Technically, two different paths could end up being the same, because
+             * ':' is a valid  character in a path. However, it is probably not going
+             * to cause any problems in this case
+             */
+            id = relativePath.replace(File.separator, ":");
+            html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
+            html.append("return false;\">");
+            html.append("<span class=\"tri\" id=\"tri." + id + "\">&#x25b6; </span>");
+            html.append("<span class=\"path\">" + relativePath + "</span>");
+            html.append("</a>");
 
             html.append(" <span class=\"listItem " + resultCode.name() + "\">");
             html.append(resultCode.toString());
@@ -369,16 +377,14 @@ public class Summarizer {
 
             html.append("</h3>");
 
-            if (resultCode != AbstractResult.ResultCode.PASS) {
-                html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">");
-                html.append(result.getDiffAsHtml());
-                html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
-                html.append("return false;\">Hide</a>");
-                html.append(" | ");
-                html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\"");
-                html.append(" target=\"_blank\">Show source</a>");
-                html.append("</div>");
-            }
+            html.append("<div class=\"diff\" style=\"display: none;\" id=\"" + id + "\">");
+            html.append(result.getDiffAsHtml());
+            html.append("<a href=\"#\" onClick=\"toggleDisplay('" + id + "');");
+            html.append("return false;\">Hide</a>");
+            html.append(" | ");
+            html.append("<a href=\"" + getViewSourceUrl(relativePath).toString() + "\"");
+            html.append(" target=\"_blank\">Show source</a>");
+            html.append("</div>");
 
             html.append("<div class=\"space\"></div>");
         }
@@ -387,7 +393,7 @@ public class Summarizer {
     private void createResultsListNoDiff(StringBuilder html, String title,
             List<String> resultsList) {
         Collections.sort(resultsList);
-        html.append("<h2>Passed [" + resultsList.size() + "]</h2>");
+        html.append("<h2>" + title + "[" + resultsList.size() + "]</h2>");
         for (String result : resultsList) {
             html.append("<h3>");
             html.append("<a href=\"" + getViewSourceUrl(result).toString() + "\"");