Add input/output JSON data for baseline comparison

CanvasCompare will output a JSON file with test results, and can take
such files as input for baseline comparison. The new logcat output
breaks down results into per-test and per-modifier improvement and
regressions.

Change-Id: I4da0251db0637841173ac95e9f431a7ff52c8b61

3 files changed, 213 insertions, 38 deletions

diff --git a/tests/CanvasCompare/AndroidManifest.xml b/tests/CanvasCompare/AndroidManifest.xml
index 1cb8c37..b55e290 100644
--- a/tests/CanvasCompare/AndroidManifest.xml
+++ b/tests/CanvasCompare/AndroidManifest.xml
@@ -15,8 +15,9 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
     package="com.android.test.hwuicompare" >
 
-    <!-- for perfhud -->
     <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE"/>
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
 
     <application
         android:label="@string/app_name"
diff --git a/tests/CanvasCompare/src/com/android/test/hwuicompare/AutomaticActivity.java b/tests/CanvasCompare/src/com/android/test/hwuicompare/AutomaticActivity.java
index e0ff1dc..1ed4723 100644
--- a/tests/CanvasCompare/src/com/android/test/hwuicompare/AutomaticActivity.java
+++ b/tests/CanvasCompare/src/com/android/test/hwuicompare/AutomaticActivity.java
@@ -16,11 +16,20 @@
 
 package com.android.test.hwuicompare;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.TreeSet;
 
-import com.android.test.hwuicompare.R;
+import org.json.JSONException;
+import org.json.JSONObject;
 
 import android.os.Bundle;
+import android.os.Environment;
 import android.os.Trace;
 import android.util.Log;
 import android.widget.ImageView;
@@ -31,20 +40,31 @@ public class AutomaticActivity extends CompareActivity {
     private static final float ERROR_DISPLAY_THRESHOLD = 0.01f;
     protected static final boolean DRAW_BITMAPS = false;
 
-    private ImageView mSoftwareImageView = null;
-    private ImageView mHardwareImageView = null;
+    /**
+     * Threshold of error change required to consider a test regressed/improved
+     */
+    private static final float ERROR_CHANGE_THRESHOLD = 0.001f;
 
-    private static final float[] ERROR_CUTOFFS = {0, 0.005f, 0.01f, 0.02f, 0.05f, 0.1f, 0.25f, 0.5f, 1f, 2f};
-    private float[] mErrorRates = new float[ERROR_CUTOFFS.length];
+    private static final float[] ERROR_CUTOFFS = {
+            0, 0.005f, 0.01f, 0.02f, 0.05f, 0.1f, 0.25f, 0.5f, 1f, 2f
+    };
+
+    private final float[] mErrorRates = new float[ERROR_CUTOFFS.length];
     private float mTotalTests = 0;
     private float mTotalError = 0;
+    private int mTestsRegressed = 0;
+    private int mTestsImproved = 0;
 
-    public abstract static class TestCallback {
+    private ImageView mSoftwareImageView = null;
+    private ImageView mHardwareImageView = null;
+
+
+    public abstract static class FinalCallback {
         abstract void report(String name, float value);
-        void complete() {}
+        void complete() {};
     }
 
-    private ArrayList<TestCallback> mTestCallbacks = new ArrayList<TestCallback>();
+    private final ArrayList<FinalCallback> mFinalCallbacks = new ArrayList<FinalCallback>();
 
     Runnable mRunnable = new Runnable() {
         @Override
@@ -64,32 +84,11 @@ public class AutomaticActivity extends CompareActivity {
             float error = mErrorCalculator.calcErrorRS(mSoftwareBitmap, mHardwareBitmap);
             Trace.traceEnd(Trace.TRACE_TAG_ALWAYS);
 
-            if (error > ERROR_DISPLAY_THRESHOLD) {
-                String modname = "";
-                for (String s : DisplayModifier.getLastAppliedModifications()) {
-                    modname = modname.concat(s + ".");
-                }
-                Log.d(LOG_TAG, String.format("error for %s was %2.9f", modname, error));
-            }
-            for (int i = 0; i < ERROR_CUTOFFS.length; i++) {
-                if (error <= ERROR_CUTOFFS[i]) break;
-                mErrorRates[i]++;
-            }
-            mTotalError += error;
-            mTotalTests++;
+            final String[] modifierNames = DisplayModifier.getLastAppliedModifications();
+            handleError(modifierNames, error);
 
             if (DisplayModifier.step()) {
-                for (TestCallback c : mTestCallbacks) {
-                    c.report("averageError", (mTotalError / mTotalTests));
-                    for (int i = 1; i < ERROR_CUTOFFS.length; i++) {
-                        c.report(String.format("error over %1.3f", ERROR_CUTOFFS[i]),
-                                mErrorRates[i]/mTotalTests);
-                    }
-                    c.complete();
-                }
-
-                Toast.makeText(getApplicationContext(), "done!", Toast.LENGTH_SHORT).show();
-                finish();
+                finishTest();
             } else {
                 mHardwareView.invalidate();
                 if (DRAW_BITMAPS) {
@@ -116,11 +115,186 @@ public class AutomaticActivity extends CompareActivity {
         mHardwareImageView = (ImageView) findViewById(R.id.hardware_image_view);
 
         onCreateCommon(mRunnable);
-        mTestCallbacks.add(new TestCallback() {
+        beginTest();
+    }
+
+    private static class TestResult {
+        TestResult(String label, float error) {
+            mLabel = label;
+            mTotalError = error;
+            mCount = 1;
+        }
+        public void addInto(float error) {
+            mTotalError += error;
+            mCount++;
+        }
+        public float getAverage() {
+            return mTotalError / mCount;
+        }
+        final String mLabel;
+        float mTotalError;
+        int mCount;
+    }
+
+    JSONObject mOutputJson = null;
+    JSONObject mInputJson = null;
+    final HashMap<String, TestResult> mModifierResults = new HashMap<String, TestResult>();
+    final HashMap<String, TestResult> mIndividualResults = new HashMap<String, TestResult>();
+    final HashMap<String, TestResult> mModifierDiffResults = new HashMap<String, TestResult>();
+    final HashMap<String, TestResult> mIndividualDiffResults = new HashMap<String, TestResult>();
+    private void beginTest() {
+        mFinalCallbacks.add(new FinalCallback() {
+            @Override
             void report(String name, float value) {
                 Log.d(LOG_TAG, name + " " + value);
             };
         });
+
+        File inputFile = new File(Environment.getExternalStorageDirectory(),
+                "CanvasCompareInput.json");
+        if (inputFile.exists() && inputFile.canRead() && inputFile.length() > 0) {
+            try {
+                FileInputStream inputStream = new FileInputStream(inputFile);
+                Log.d(LOG_TAG, "Parsing input file...");
+                StringBuffer content = new StringBuffer((int)inputFile.length());
+                byte[] buffer = new byte[1024];
+                while (inputStream.read(buffer) != -1) {
+                    content.append(new String(buffer));
+                }
+                mInputJson = new JSONObject(content.toString());
+                inputStream.close();
+                Log.d(LOG_TAG, "Parsed input file with " + mInputJson.length() + " entries");
+            } catch (JSONException e) {
+                Log.e(LOG_TAG, "error parsing input json", e);
+            } catch (IOException e) {
+                Log.e(LOG_TAG, "error reading input json from sd", e);
+            }
+        }
+
+        mOutputJson = new JSONObject();
+    }
+
+    private static void logTestResultHash(String label, HashMap<String, TestResult> map) {
+        Log.d(LOG_TAG, "---------------");
+        Log.d(LOG_TAG, label + ":");
+        Log.d(LOG_TAG, "---------------");
+        TreeSet<TestResult> set = new TreeSet<TestResult>(new Comparator<TestResult>() {
+            @Override
+            public int compare(TestResult lhs, TestResult rhs) {
+                if (lhs == rhs) return 0; // don't need to worry about complex equality
+
+                int cmp = Float.compare(lhs.getAverage(), rhs.getAverage());
+                if (cmp != 0) {
+                    return cmp;
+                }
+                return lhs.mLabel.compareTo(rhs.mLabel);
+            }
+        });
+
+        for (TestResult t : map.values()) {
+            set.add(t);
+        }
+
+        for (TestResult t : set.descendingSet()) {
+            if (Math.abs(t.getAverage()) > ERROR_DISPLAY_THRESHOLD) {
+                Log.d(LOG_TAG, String.format("%2.4f : %s", t.getAverage(), t.mLabel));
+            }
+        }
+        Log.d(LOG_TAG, "");
+    }
+
+    private void finishTest() {
+        for (FinalCallback c : mFinalCallbacks) {
+            c.report("averageError", (mTotalError / mTotalTests));
+            for (int i = 1; i < ERROR_CUTOFFS.length; i++) {
+                c.report(String.format("tests with error over %1.3f", ERROR_CUTOFFS[i]),
+                        mErrorRates[i]);
+            }
+            if (mInputJson != null) {
+                c.report("tests regressed", mTestsRegressed);
+                c.report("tests improved", mTestsImproved);
+            }
+            c.complete();
+        }
+
+        try {
+            if (mOutputJson != null) {
+                String outputString = mOutputJson.toString(4);
+                File outputFile = new File(Environment.getExternalStorageDirectory(),
+                        "CanvasCompareOutput.json");
+                FileOutputStream outputStream = new FileOutputStream(outputFile);
+                outputStream.write(outputString.getBytes());
+                outputStream.close();
+                Log.d(LOG_TAG, "Saved output file with " + mOutputJson.length() + " entries");
+            }
+        } catch (JSONException e) {
+            Log.e(LOG_TAG, "error during JSON stringify", e);
+        } catch (IOException e) {
+            Log.e(LOG_TAG, "error storing JSON output on sd", e);
+        }
+
+        logTestResultHash("Modifier change vs previous", mModifierDiffResults);
+        logTestResultHash("Invidual test change vs previous", mIndividualDiffResults);
+        logTestResultHash("Modifier average test results", mModifierResults);
+        logTestResultHash("Individual test results", mIndividualResults);
+
+        Toast.makeText(getApplicationContext(), "done!", Toast.LENGTH_SHORT).show();
+        finish();
+    }
+
+    /**
+     * Inserts the error value into all TestResult objects, associated with each of its modifiers
+     */
+    private static void addForAllModifiers(String fullName, float error, String[] modifierNames,
+            HashMap<String, TestResult> modifierResults) {
+        for (String modifierName : modifierNames) {
+            TestResult r = modifierResults.get(fullName);
+            if (r == null) {
+                modifierResults.put(modifierName, new TestResult(modifierName, error));
+            } else {
+                r.addInto(error);
+            }
+        }
+    }
+
+    private void handleError(final String[] modifierNames, final float error) {
+        String fullName = "";
+        for (String s : modifierNames) {
+            fullName = fullName.concat("." + s);
+        }
+        fullName = fullName.substring(1);
+
+        float deltaError = 0;
+        if (mInputJson != null) {
+            try {
+                deltaError = error - (float)mInputJson.getDouble(fullName);
+            } catch (JSONException e) {
+                Log.w(LOG_TAG, "Warning: unable to read from input json", e);
+            }
+            if (deltaError > ERROR_CHANGE_THRESHOLD) mTestsRegressed++;
+            if (deltaError < -ERROR_CHANGE_THRESHOLD) mTestsImproved++;
+            mIndividualDiffResults.put(fullName, new TestResult(fullName, deltaError));
+            addForAllModifiers(fullName, deltaError, modifierNames, mModifierDiffResults);
+        }
+
+        mIndividualResults.put(fullName, new TestResult(fullName, error));
+        addForAllModifiers(fullName, error, modifierNames, mModifierResults);
+
+        try {
+            if (mOutputJson != null) {
+                mOutputJson.put(fullName, error);
+            }
+        } catch (JSONException e) {
+            Log.e(LOG_TAG, "exception during JSON recording", e);
+            mOutputJson = null;
+        }
+
+        for (int i = 0; i < ERROR_CUTOFFS.length; i++) {
+            if (error <= ERROR_CUTOFFS[i]) break;
+            mErrorRates[i]++;
+        }
+        mTotalError += error;
+        mTotalTests++;
     }
 
     @Override
@@ -130,7 +304,7 @@ public class AutomaticActivity extends CompareActivity {
     }
 
     // FOR TESTING
-    public void setCallback(TestCallback c) {
-        mTestCallbacks.add(c);
+    public void setFinalCallback(FinalCallback c) {
+        mFinalCallbacks.add(c);
     }
 }
diff --git a/tests/CanvasCompare/src/com/android/test/hwuicompare/Test.java b/tests/CanvasCompare/src/com/android/test/hwuicompare/Test.java
index 6ea8237..1ff153c 100644
--- a/tests/CanvasCompare/src/com/android/test/hwuicompare/Test.java
+++ b/tests/CanvasCompare/src/com/android/test/hwuicompare/Test.java
@@ -1,6 +1,6 @@
 package com.android.test.hwuicompare;
 
-import com.android.test.hwuicompare.AutomaticActivity.TestCallback;
+import com.android.test.hwuicompare.AutomaticActivity.FinalCallback;
 
 import android.os.Bundle;
 import android.test.ActivityInstrumentationTestCase2;
@@ -18,7 +18,7 @@ public class Test extends ActivityInstrumentationTestCase2<AutomaticActivity> {
         super.setUp();
         mBundle = new Bundle();
         mActivity = getActivity();
-        mActivity.setCallback(new TestCallback() {
+        mActivity.setFinalCallback(new FinalCallback() {
 
             @Override
             void report(String key, float value) {