Updated Summarizer to use SQLite DB.

It avoids OOM issues by dumping the memory to the DB every
few hundred tests and then reading them in chunks when
generating summaries, etc.

Change-Id: I5f63d960d456f17e9f2189230a7ae0ca5607ce51

5 files changed, 297 insertions, 30 deletions

diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java
index 6048338a..96c1e5e 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/AbstractResult.java
@@ -18,8 +18,16 @@ package com.android.dumprendertree2;
 
 import android.os.Bundle;
 import android.os.Message;
+import android.util.Log;
 import android.webkit.WebView;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+
 /**
  * A class that represent a result of the test. It is responsible for returning the result's
  * raw data and generating its own diff in HTML format.
@@ -49,7 +57,7 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
     /**
      * A code representing the result of comparing actual and expected results.
      */
-    public enum ResultCode {
+    public enum ResultCode implements Serializable {
         RESULTS_MATCH("Results match"),
         RESULTS_DIFFER("Results differ"),
         NO_EXPECTED_RESULT("No expected result"),
@@ -81,6 +89,58 @@ public abstract class AbstractResult implements Comparable<AbstractResult> {
         return mAdditionalTextOutputString;
     }
 
+    public byte[] getBytes() {
+        ByteArrayOutputStream baos = null;
+        ObjectOutputStream oos = null;
+        try {
+            try {
+                baos = new ByteArrayOutputStream();
+                oos = new ObjectOutputStream(baos);
+                oos.writeObject(this);
+            } finally {
+                if (baos != null) {
+                    baos.close();
+                }
+                if (oos != null) {
+                    oos.close();
+                }
+            }
+        } catch (IOException e) {
+            Log.e(LOG_TAG, "Unable to serialize result: " + getRelativePath(), e);
+        }
+
+        return baos == null ? null : baos.toByteArray();
+    }
+
+    public static AbstractResult create(byte[] bytes) {
+        ByteArrayInputStream bais = null;
+        ObjectInputStream ois = null;
+        AbstractResult result = null;
+        try {
+            try {
+                bais = new ByteArrayInputStream(bytes);
+                ois = new ObjectInputStream(bais);
+                result = (AbstractResult)ois.readObject();
+            } finally {
+                if (bais != null) {
+                    bais.close();
+                }
+                if (ois != null) {
+                    ois.close();
+                }
+            }
+        } catch (IOException e) {
+            Log.e(LOG_TAG, "Unable to deserialize result!", e);
+        } catch (ClassNotFoundException e) {
+            Log.e(LOG_TAG, "Unable to deserialize result!", e);
+        }
+        return result;
+    }
+
+    public void clearResults() {
+        mAdditionalTextOutputString = null;
+    }
+
     /**
      * Makes the result object obtain the results of the test from the webview
      * and store them in the format that suits itself bests. This method is asynchronous.
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/ManagerService.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/ManagerService.java
index d4bb0d38..17e19d0 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/ManagerService.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/ManagerService.java
@@ -147,7 +147,7 @@ public class ManagerService extends Service {
         super.onCreate();
 
         mFileFilter = new FileFilter();
-        mSummarizer = new Summarizer(mFileFilter, RESULTS_ROOT_DIR_PATH);
+        mSummarizer = new Summarizer(mFileFilter, RESULTS_ROOT_DIR_PATH, getApplicationContext());
     }
 
     @Override
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
index ffb3f8f..25c5ad5 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/Summarizer.java
@@ -16,9 +16,11 @@
 
 package com.android.dumprendertree2;
 
+import android.content.Context;
 import android.content.res.AssetManager;
 import android.content.res.Configuration;
 import android.content.res.Resources;
+import android.database.Cursor;
 import android.os.Build;
 import android.os.Message;
 import android.util.DisplayMetrics;
@@ -32,7 +34,6 @@ import java.net.URI;
 import java.net.URL;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.Date;
 import java.util.List;
 import java.util.regex.Matcher;
@@ -189,6 +190,7 @@ public class Summarizer {
     private static final String TXT_SUMMARY_RELATIVE_PATH = "summary.txt";
 
     private static final int RESULTS_PER_DUMP = 500;
+    private static final int RESULTS_PER_DB_ACCESS = 50;
 
     private int mCrashedTestsCount = 0;
     private List<AbstractResult> mUnexpectedFailures = new ArrayList<AbstractResult>();
@@ -196,16 +198,31 @@ public class Summarizer {
     private List<AbstractResult> mExpectedPasses = new ArrayList<AbstractResult>();
     private List<AbstractResult> mUnexpectedPasses = new ArrayList<AbstractResult>();
 
+    private Cursor mUnexpectedFailuresCursor;
+    private Cursor mExpectedFailuresCursor;
+    private Cursor mUnexpectedPassesCursor;
+    private Cursor mExpectedPassesCursor;
+
     private FileFilter mFileFilter;
     private String mResultsRootDirPath;
     private String mTestsRelativePath;
     private Date mDate;
 
     private int mResultsSinceLastHtmlDump = 0;
+    private int mResultsSinceLastDbAccess = 0;
+
+    private SummarizerDBHelper mDbHelper;
 
-    public Summarizer(FileFilter fileFilter, String resultsRootDirPath) {
+    public Summarizer(FileFilter fileFilter, String resultsRootDirPath, Context context) {
         mFileFilter = fileFilter;
         mResultsRootDirPath = resultsRootDirPath;
+
+        /**
+         * We don't run the database I/O in a separate thread to avoid consumer/producer problem
+         * and to simplify code.
+         */
+        mDbHelper = new SummarizerDBHelper(context);
+        mDbHelper.open();
     }
 
     public static URI getDetailsUri() {
@@ -221,6 +238,7 @@ public class Summarizer {
         }
 
         if (result.didPass()) {
+            result.clearResults();
             if (mFileFilter.isFail(relativePath)) {
                 mUnexpectedPasses.add(result);
             } else {
@@ -233,6 +251,32 @@ public class Summarizer {
                 mUnexpectedFailures.add(result);
             }
         }
+
+        if (++mResultsSinceLastDbAccess == RESULTS_PER_DB_ACCESS) {
+            persistLists();
+            clearLists();
+        }
+    }
+
+    private void clearLists() {
+        mUnexpectedFailures.clear();
+        mExpectedFailures.clear();
+        mUnexpectedPasses.clear();
+        mExpectedPasses.clear();
+    }
+
+    private void persistLists() {
+        persistListToTable(mUnexpectedFailures, SummarizerDBHelper.UNEXPECTED_FAILURES_TABLE);
+        persistListToTable(mExpectedFailures, SummarizerDBHelper.EXPECTED_FAILURES_TABLE);
+        persistListToTable(mUnexpectedPasses, SummarizerDBHelper.UNEXPECTED_PASSES_TABLE);
+        persistListToTable(mExpectedPasses, SummarizerDBHelper.EXPECTED_PASSES_TABLE);
+        mResultsSinceLastDbAccess = 0;
+    }
+
+    private void persistListToTable(List<AbstractResult> results, String table) {
+        for (AbstractResult abstractResult : results) {
+            mDbHelper.insertAbstractResult(abstractResult, table);
+        }
     }
 
     public void setTestsRelativePath(String testsRelativePath) {
@@ -240,17 +284,35 @@ public class Summarizer {
     }
 
     public void summarize(Message onFinishMessage) {
+        persistLists();
+        clearLists();
+
+        mUnexpectedFailuresCursor =
+            mDbHelper.getAbstractResults(SummarizerDBHelper.UNEXPECTED_FAILURES_TABLE);
+        mUnexpectedPassesCursor =
+            mDbHelper.getAbstractResults(SummarizerDBHelper.UNEXPECTED_PASSES_TABLE);
+        mExpectedFailuresCursor =
+            mDbHelper.getAbstractResults(SummarizerDBHelper.EXPECTED_FAILURES_TABLE);
+        mExpectedPassesCursor =
+            mDbHelper.getAbstractResults(SummarizerDBHelper.EXPECTED_PASSES_TABLE);
+
         String webKitRevision = getWebKitRevision();
         createHtmlDetails(webKitRevision);
         createTxtSummary(webKitRevision);
+
+        clearLists();
+        mUnexpectedFailuresCursor.close();
+        mUnexpectedPassesCursor.close();
+        mExpectedFailuresCursor.close();
+        mExpectedPassesCursor.close();
+
         onFinishMessage.sendToTarget();
     }
 
     public void reset() {
         mCrashedTestsCount = 0;
-        mUnexpectedFailures.clear();
-        mExpectedFailures.clear();
-        mExpectedPasses.clear();
+        clearLists();
+        mDbHelper.reset();
         mDate = new Date();
     }
 
@@ -273,10 +335,10 @@ public class Summarizer {
 
         txt.append("TOTAL:                     " + getTotalTestCount() + "\n");
         txt.append("CRASHED (among all tests): " + mCrashedTestsCount + "\n");
-        txt.append("UNEXPECTED FAILURES:       " + mUnexpectedFailures.size() + "\n");
-        txt.append("UNEXPECTED PASSES:         " + mUnexpectedPasses.size() + "\n");
-        txt.append("EXPECTED FAILURES:         " + mExpectedFailures.size() + "\n");
-        txt.append("EXPECTED PASSES:           " + mExpectedPasses.size() + "\n");
+        txt.append("UNEXPECTED FAILURES:       " + mUnexpectedFailuresCursor.getCount() + "\n");
+        txt.append("UNEXPECTED PASSES:         " + mUnexpectedPassesCursor.getCount() + "\n");
+        txt.append("EXPECTED FAILURES:         " + mExpectedFailuresCursor.getCount() + "\n");
+        txt.append("EXPECTED PASSES:           " + mExpectedPassesCursor.getCount() + "\n");
 
         FsUtils.writeDataToStorage(new File(mResultsRootDirPath, TXT_SUMMARY_RELATIVE_PATH),
                 txt.toString().getBytes(), false);
@@ -293,20 +355,20 @@ public class Summarizer {
         createTopSummaryTable(webKitRevision, html);
         dumpHtmlToFile(html, false);
 
-        createResultsList(html, "Unexpected failures", mUnexpectedFailures);
-        createResultsList(html, "Unexpected passes", mUnexpectedPasses);
-        createResultsList(html, "Expected failures", mExpectedFailures);
-        createResultsList(html, "Expected passes", mExpectedPasses);
+        createResultsList(html, "Unexpected failures", mUnexpectedFailuresCursor);
+        createResultsList(html, "Unexpected passes", mUnexpectedPassesCursor);
+        createResultsList(html, "Expected failures", mExpectedFailuresCursor);
+        createResultsList(html, "Expected passes", mExpectedPassesCursor);
 
         html.append("</body></html>");
         dumpHtmlToFile(html, true);
     }
 
     private int getTotalTestCount() {
-        return mUnexpectedFailures.size() +
-                mUnexpectedPasses.size() +
-                mExpectedPasses.size() +
-                mExpectedFailures.size();
+        return mUnexpectedFailuresCursor.getCount() +
+                mUnexpectedPassesCursor.getCount() +
+                mExpectedPassesCursor.getCount() +
+                mExpectedFailuresCursor.getCount();
     }
 
     private String getWebKitVersionFromUserAgentString() {
@@ -355,10 +417,10 @@ public class Summarizer {
         html.append("<table class=\"summary\">");
         createSummaryTableRow(html, "TOTAL", getTotalTestCount());
         createSummaryTableRow(html, "CRASHED (among all tests)", mCrashedTestsCount);
-        createSummaryTableRow(html, "UNEXPECTED FAILURES", mUnexpectedFailures.size());
-        createSummaryTableRow(html, "UNEXPECTED PASSES", mUnexpectedPasses.size());
-        createSummaryTableRow(html, "EXPECTED FAILURES", mExpectedFailures.size());
-        createSummaryTableRow(html, "EXPECTED PASSES", mExpectedPasses.size());
+        createSummaryTableRow(html, "UNEXPECTED FAILURES", mUnexpectedFailuresCursor.getCount());
+        createSummaryTableRow(html, "UNEXPECTED PASSES", mUnexpectedPassesCursor.getCount());
+        createSummaryTableRow(html, "EXPECTED FAILURES", mExpectedFailuresCursor.getCount());
+        createSummaryTableRow(html, "EXPECTED PASSES", mExpectedPassesCursor.getCount());
         html.append("</table>");
     }
 
@@ -370,14 +432,21 @@ public class Summarizer {
     }
 
     private void createResultsList(
-            StringBuilder html, String title, List<AbstractResult> resultsList) {
+            StringBuilder html, String title, Cursor cursor) {
         String relativePath;
         String id = "";
         AbstractResult.ResultCode resultCode;
 
-        Collections.sort(resultsList);
-        html.append("<h2>" + title + " [" + resultsList.size() + "]</h2>");
-        for (AbstractResult result : resultsList) {
+        html.append("<h2>" + title + " [" + cursor.getCount() + "]</h2>");
+
+        if (!cursor.moveToFirst()) {
+            return;
+        }
+
+        AbstractResult result;
+        do {
+            result = SummarizerDBHelper.getAbstractResult(cursor);
+
             relativePath = result.getRelativePath();
             resultCode = result.getResultCode();
 
@@ -429,7 +498,9 @@ public class Summarizer {
             if (++mResultsSinceLastHtmlDump == RESULTS_PER_DUMP) {
                 dumpHtmlToFile(html, true);
             }
-        }
+
+            cursor.moveToNext();
+        } while (!cursor.isAfterLast());
     }
 
     private void appendTags(StringBuilder html, AbstractResult result) {
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/SummarizerDBHelper.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/SummarizerDBHelper.java
new file mode 100644
index 0000000..23e13ec
--- /dev/null
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/SummarizerDBHelper.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.dumprendertree2;
+
+import android.content.ContentValues;
+import android.content.Context;
+import android.database.Cursor;
+import android.database.SQLException;
+import android.database.sqlite.SQLiteDatabase;
+import android.database.sqlite.SQLiteOpenHelper;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * A basic class that wraps database accesses inside itself and provides functionality to
+ * store and retrieve AbstractResults.
+ */
+public class SummarizerDBHelper {
+    private static final String KEY_ID = "id";
+    private static final String KEY_PATH = "path";
+    private static final String KEY_BYTES = "bytes";
+
+    private static final String DATABASE_NAME = "SummarizerDB";
+    private static final int DATABASE_VERSION = 1;
+
+    static final String EXPECTED_FAILURES_TABLE = "expectedFailures";
+    static final String UNEXPECTED_FAILURES_TABLE = "unexpectedFailures";
+    static final String EXPECTED_PASSES_TABLE = "expextedPasses";
+    static final String UNEXPECTED_PASSES_TABLE = "unexpextedPasses";
+    private static final Set<String> TABLES_NAMES = new HashSet<String>();
+    {
+        TABLES_NAMES.add(EXPECTED_FAILURES_TABLE);
+        TABLES_NAMES.add(EXPECTED_PASSES_TABLE);
+        TABLES_NAMES.add(UNEXPECTED_FAILURES_TABLE);
+        TABLES_NAMES.add(UNEXPECTED_PASSES_TABLE);
+    }
+
+    private static final void createTables(SQLiteDatabase db) {
+        String cmd;
+        for (String tableName : TABLES_NAMES) {
+            cmd = "create table " + tableName + " ("
+                    + KEY_ID + " integer primary key autoincrement, "
+                    + KEY_PATH + " text not null, "
+                    + KEY_BYTES + " blob not null);";
+            db.execSQL(cmd);
+        }
+    }
+
+    private static final void dropTables(SQLiteDatabase db) {
+        for (String tableName : TABLES_NAMES) {
+            db.execSQL("DROP TABLE IF EXISTS " + tableName);
+        }
+    }
+
+    private static class DatabaseHelper extends SQLiteOpenHelper {
+        DatabaseHelper(Context context) {
+            super(context, DATABASE_NAME, null, DATABASE_VERSION);
+        }
+
+        @Override
+        public void onCreate(SQLiteDatabase db) {
+            dropTables(db);
+            createTables(db);
+        }
+
+        @Override
+        public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
+            /** NOOP for now, because we will never upgrade the db */
+        }
+
+        public void reset(SQLiteDatabase db) {
+            dropTables(db);
+            createTables(db);
+        }
+    }
+
+    private DatabaseHelper mDbHelper;
+    private SQLiteDatabase mDb;
+
+    private final Context mContext;
+
+    public SummarizerDBHelper(Context ctx) {
+        mContext = ctx;
+        mDbHelper = new DatabaseHelper(mContext);
+    }
+
+    public void reset() {
+        mDbHelper.reset(this.mDb);
+    }
+
+    public void open() throws SQLException {
+        mDb = mDbHelper.getWritableDatabase();
+    }
+
+    public void close() {
+        mDbHelper.close();
+    }
+
+    public void insertAbstractResult(AbstractResult result, String table) {
+        ContentValues cv = new ContentValues();
+        cv.put(KEY_PATH, result.getRelativePath());
+        cv.put(KEY_BYTES, result.getBytes());
+        mDb.insert(table, null, cv);
+    }
+
+    public Cursor getAbstractResults(String table) throws SQLException {
+        return mDb.query(false, table, new String[] {KEY_BYTES}, null, null, null, null,
+                KEY_PATH + " ASC", null);
+    }
+
+    public static AbstractResult getAbstractResult(Cursor cursor) {
+        return AbstractResult.create(cursor.getBlob(cursor.getColumnIndex(KEY_BYTES)));
+    }
+}
\ No newline at end of file
diff --git a/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java b/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java
index 21e5430..f835b6a 100644
--- a/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java
+++ b/tests/DumpRenderTree2/src/com/android/dumprendertree2/TextResult.java
@@ -41,11 +41,11 @@ public class TextResult extends AbstractResult {
     private String mRelativePath;
     private boolean mDidTimeOut;
     private ResultCode mResultCode;
-    private Message mResultObtainedMsg;
+    transient private Message mResultObtainedMsg;
 
     private boolean mDumpChildFramesAsText;
 
-    private Handler mHandler = new Handler() {
+    transient private Handler mHandler = new Handler() {
         @Override
         public void handleMessage(Message msg) {
             if (msg.what == MSG_DOCUMENT_AS_TEXT) {
@@ -79,6 +79,13 @@ public class TextResult extends AbstractResult {
     }
 
     @Override
+    public void clearResults() {
+        super.clearResults();
+        mExpectedResult = null;
+        mActualResult = null;
+    }
+
+    @Override
     public ResultCode getResultCode() {
         if (mResultCode != null) {
             return mResultCode;