From 70d8be7616a450e9b7ef09c8bda5a8b25936a29a Mon Sep 17 00:00:00 2001
From: Dianne Hackborn <hackbod@google.com>
Date: Tue, 23 Jun 2015 19:33:02 +0000
Subject: Fix issue #22013372: Assist should take translationX and friends...

...into account when calculating the position information

Actually what we need here is the full transformation matrix, if it
is available.  And that means actually computing the location of
views on the screen requires doing this all through transformations,
so the AssistVisualizer has been changed to do this (while still
also keeping the old mechanism for comparison to verify that things
are working correctly).

Also added new properties for elevation and alpha.

And optimized the parcelling of AssistStructure to not write things
that aren't needed; this reduces the parcelled size by about half.

Change-Id: I50b0dd2e6599c74701a5d188617a3eff64b07d03
---
 .../res/layout/voice_interaction_session.xml       |  41 +++++---
 tests/VoiceInteraction/res/values/strings.xml      |   2 +
 .../test/voiceinteraction/AssistVisualizer.java    | 103 ++++++++++++++++++---
 .../voiceinteraction/MainInteractionSession.java   |  29 +++++-
 4 files changed, 143 insertions(+), 32 deletions(-)

(limited to 'tests')

diff --git a/tests/VoiceInteraction/res/layout/voice_interaction_session.xml b/tests/VoiceInteraction/res/layout/voice_interaction_session.xml
index d44afb0..610f30b 100644
--- a/tests/VoiceInteraction/res/layout/voice_interaction_session.xml
+++ b/tests/VoiceInteraction/res/layout/voice_interaction_session.xml
@@ -18,6 +18,11 @@
     android:layout_width="match_parent"
     android:layout_height="match_parent">
 
+    <ImageView android:id="@+id/full_screenshot"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        android:visibility="invisible"/>
+
     <com.android.test.voiceinteraction.AssistVisualizer android:id="@+id/assist_visualizer"
             android:layout_width="match_parent"
             android:layout_height="match_parent" />
@@ -30,19 +35,29 @@
             android:layout_width="match_parent"
             android:layout_height="wrap_content"
             android:layout_gravity="top"
-            android:orientation="vertical"
+            android:orientation="horizontal"
             android:background="#ffffffff"
             android:elevation="8dp"
             >
-            <Button android:id="@+id/start"
+            <ImageView android:id="@+id/screenshot"
+                android:layout_width="wrap_content"
+                android:layout_height="46dp"
+                android:adjustViewBounds="true" />
+            <View android:layout_width="0dp"
+                android:layout_height="0dp"
+                android:layout_weight="1" />
+            <Button android:id="@+id/do_tree"
                 android:layout_width="wrap_content"
                 android:layout_height="wrap_content"
-                android:layout_gravity="top|right"
-                android:text="@string/start"
-                />
-            <ImageView android:id="@+id/screenshot"
+                android:text="@string/tree" />
+            <Button android:id="@+id/do_text"
                 android:layout_width="wrap_content"
-                android:layout_height="wrap_content"/>
+                android:layout_height="wrap_content"
+                android:text="@string/text" />
+            <Button android:id="@+id/start"
+                android:layout_width="wrap_content"
+                android:layout_height="wrap_content"
+                android:text="@string/start" />
         </LinearLayout>
 
         <LinearLayout android:id="@+id/bottom_content"
@@ -58,26 +73,22 @@
                 android:layout_width="match_parent"
                 android:layout_height="wrap_content"
                 android:layout_marginBottom="16dp"
-                android:textAppearance="?android:attr/textAppearanceMedium"
-                />
+                android:textAppearance="?android:attr/textAppearanceMedium" />
 
             <LinearLayout android:layout_width="match_parent" android:layout_height="wrap_content"
                     android:orientation="horizontal">
                 <Button android:id="@+id/confirm"
                     android:layout_width="wrap_content"
                     android:layout_height="wrap_content"
-                    android:text="@string/confirm"
-                    />
+                    android:text="@string/confirm" />
                 <Button android:id="@+id/complete"
                     android:layout_width="wrap_content"
                     android:layout_height="wrap_content"
-                    android:text="@string/complete"
-                    />
+                    android:text="@string/complete" />
                 <Button android:id="@+id/abort"
                     android:layout_width="wrap_content"
                     android:layout_height="wrap_content"
-                    android:text="@string/abort"
-                    />
+                    android:text="@string/abort" />
             </LinearLayout>
 
         </LinearLayout>
diff --git a/tests/VoiceInteraction/res/values/strings.xml b/tests/VoiceInteraction/res/values/strings.xml
index 6289929..4cf4104 100644
--- a/tests/VoiceInteraction/res/values/strings.xml
+++ b/tests/VoiceInteraction/res/values/strings.xml
@@ -17,6 +17,8 @@
 <resources>
 
     <string name="start">Start</string>
+    <string name="tree">Tree</string>
+    <string name="text">Text</string>
     <string name="asyncStructure">(Async structure goes here)</string>
     <string name="confirm">Confirm</string>
     <string name="abort">Abort</string>
diff --git a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AssistVisualizer.java b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AssistVisualizer.java
index 439ace8..339755f 100644
--- a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AssistVisualizer.java
+++ b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AssistVisualizer.java
@@ -20,6 +20,7 @@ import android.annotation.Nullable;
 import android.app.assist.AssistStructure;
 import android.content.Context;
 import android.graphics.Canvas;
+import android.graphics.Matrix;
 import android.graphics.Paint;
 import android.graphics.Rect;
 import android.util.AttributeSet;
@@ -31,10 +32,32 @@ import java.util.ArrayList;
 public class AssistVisualizer extends View {
     static final String TAG = "AssistVisualizer";
 
+    static class TextEntry {
+        final Rect bounds;
+        final int parentLeft, parentTop;
+        final Matrix matrix;
+        final String className;
+        final CharSequence text;
+
+        TextEntry(AssistStructure.ViewNode node, int parentLeft, int parentTop, Matrix matrix) {
+            int left = parentLeft+node.getLeft();
+            int top = parentTop+node.getTop();
+            bounds = new Rect(left, top, left+node.getWidth(), top+node.getHeight());
+            this.parentLeft = parentLeft;
+            this.parentTop = parentTop;
+            this.matrix = new Matrix(matrix);
+            this.className = node.getClassName();
+            this.text = node.getText() != null ? node.getText() : node.getContentDescription();
+        }
+    }
+
     AssistStructure mAssistStructure;
     final Paint mFramePaint = new Paint();
-    final ArrayList<Rect> mTextRects = new ArrayList<>();
+    final Paint mFrameNoTransformPaint = new Paint();
+    final ArrayList<Matrix> mMatrixStack = new ArrayList<>();
+    final ArrayList<TextEntry> mTextRects = new ArrayList<>();
     final int[] mTmpLocation = new int[2];
+    final float[] mTmpMatrixPoint = new float[2];
 
     public AssistVisualizer(Context context, @Nullable AttributeSet attrs) {
         super(context, attrs);
@@ -42,17 +65,26 @@ public class AssistVisualizer extends View {
         mFramePaint.setColor(0xffff0000);
         mFramePaint.setStyle(Paint.Style.STROKE);
         mFramePaint.setStrokeWidth(0);
+        float density = getResources().getDisplayMetrics().density;
+        mFramePaint.setShadowLayer(density, density, density, 0xff000000);
+        mFrameNoTransformPaint.setColor(0xff0000ff);
+        mFrameNoTransformPaint.setStyle(Paint.Style.STROKE);
+        mFrameNoTransformPaint.setStrokeWidth(0);
+        mFrameNoTransformPaint.setShadowLayer(density, density, density, 0xff000000);
     }
 
     public void setAssistStructure(AssistStructure as) {
         mAssistStructure = as;
-        mAssistStructure.dump();
         mTextRects.clear();
         final int N = as.getWindowNodeCount();
         if (N > 0) {
             for (int i=0; i<N; i++) {
                 AssistStructure.WindowNode windowNode = as.getWindowNodeAt(i);
-                buildTextRects(windowNode.getRootViewNode(), windowNode.getLeft(),
+                mMatrixStack.clear();
+                Matrix matrix = new Matrix();
+                matrix.setTranslate(windowNode.getLeft(), windowNode.getTop());
+                mMatrixStack.add(matrix);
+                buildTextRects(windowNode.getRootViewNode(), 0, windowNode.getLeft(),
                         windowNode.getTop());
             }
         }
@@ -60,31 +92,62 @@ public class AssistVisualizer extends View {
         invalidate();
     }
 
+    public void logTree() {
+        if (mAssistStructure != null) {
+            mAssistStructure.dump();
+        }
+    }
+
+    public void logText() {
+        final int N = mTextRects.size();
+        for (int i=0; i<N; i++) {
+            TextEntry te = mTextRects.get(i);
+            Log.d(TAG, "View " + te.className + " " + te.bounds.toShortString()
+                    + " in " + te.parentLeft + "," + te.parentTop
+                    + " matrix=" + te.matrix.toShortString() + ": "
+                    + te.text);
+        }
+    }
+
     public void clearAssistData() {
         mAssistStructure = null;
         mTextRects.clear();
     }
 
-    void buildTextRects(AssistStructure.ViewNode root, int parentLeft, int parentTop) {
+    void buildTextRects(AssistStructure.ViewNode root, int matrixStackIndex,
+            int parentLeft, int parentTop) {
         if (root.getVisibility() != View.VISIBLE) {
             return;
         }
-        int left = parentLeft+root.getLeft();
-        int top = parentTop+root.getTop();
+        Matrix parentMatrix = mMatrixStack.get(matrixStackIndex);
+        matrixStackIndex++;
+        Matrix matrix;
+        if (mMatrixStack.size() > matrixStackIndex) {
+            matrix = mMatrixStack.get(matrixStackIndex);
+            matrix.set(parentMatrix);
+        } else {
+            matrix = new Matrix(parentMatrix);
+            mMatrixStack.add(matrix);
+        }
+        matrix.preTranslate(root.getLeft(), root.getTop());
+        int left = parentLeft + root.getLeft();
+        int top = parentTop + root.getTop();
+        Matrix transform = root.getTransformation();
+        if (transform != null) {
+            matrix.preConcat(transform);
+        }
         if (root.getText() != null || root.getContentDescription() != null) {
-            Rect r = new Rect(left, top, left+root.getWidth(), top+root.getHeight());
-            Log.d(TAG, "View " + root.getClassName() + " " + left + "," + top + " tr "
-                    + r.toShortString() + ": "
-                    + (root.getText() != null ? root.getText() : root.getContentDescription()));
-            mTextRects.add(r);
+            TextEntry te = new TextEntry(root, parentLeft, parentTop, matrix);
+            mTextRects.add(te);
         }
         final int N = root.getChildCount();
         if (N > 0) {
             left -= root.getScrollX();
             top -= root.getScrollY();
+            matrix.preTranslate(-root.getScrollX(), -root.getScrollY());
             for (int i=0; i<N; i++) {
                 AssistStructure.ViewNode child = root.getChildAt(i);
-                buildTextRects(child, left, top);
+                buildTextRects(child, matrixStackIndex, left, top);
             }
         }
     }
@@ -96,9 +159,19 @@ public class AssistVisualizer extends View {
         final int N = mTextRects.size();
         Log.d(TAG, "Drawing text rects in " + this + ": found " + mTextRects.size());
         for (int i=0; i<N; i++) {
-            Rect r = mTextRects.get(i);
-            canvas.drawRect(r.left-mTmpLocation[0], r.top-mTmpLocation[1],
-                    r.right-mTmpLocation[0], r.bottom-mTmpLocation[1], mFramePaint);
+            TextEntry te = mTextRects.get(i);
+            canvas.drawRect(te.bounds.left - mTmpLocation[0], te.bounds.top - mTmpLocation[1],
+                    te.bounds.right - mTmpLocation[0], te.bounds.bottom - mTmpLocation[1],
+                    mFrameNoTransformPaint);
+        }
+        for (int i=0; i<N; i++) {
+            TextEntry te = mTextRects.get(i);
+            canvas.save();
+            canvas.translate(-mTmpLocation[0], -mTmpLocation[1]);
+            canvas.concat(te.matrix);
+            canvas.drawRect(0, 0, te.bounds.right - te.bounds.left, te.bounds.bottom - te.bounds.top,
+                    mFramePaint);
+            canvas.restore();
         }
     }
 }
diff --git a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java
index 90a781c..97c1e85 100644
--- a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java
+++ b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java
@@ -42,8 +42,11 @@ public class MainInteractionSession extends VoiceInteractionSession
     View mTopContent;
     View mBottomContent;
     TextView mText;
+    Button mTreeButton;
+    Button mTextButton;
     Button mStartButton;
     ImageView mScreenshot;
+    ImageView mFullScreenshot;
     Button mConfirmButton;
     Button mCompleteButton;
     Button mAbortButton;
@@ -110,9 +113,15 @@ public class MainInteractionSession extends VoiceInteractionSession
         mTopContent = mContentView.findViewById(R.id.top_content);
         mBottomContent = mContentView.findViewById(R.id.bottom_content);
         mText = (TextView)mContentView.findViewById(R.id.text);
+        mTreeButton = (Button)mContentView.findViewById(R.id.do_tree);
+        mTreeButton.setOnClickListener(this);
+        mTextButton = (Button)mContentView.findViewById(R.id.do_text);
+        mTextButton.setOnClickListener(this);
         mStartButton = (Button)mContentView.findViewById(R.id.start);
         mStartButton.setOnClickListener(this);
         mScreenshot = (ImageView)mContentView.findViewById(R.id.screenshot);
+        mScreenshot.setOnClickListener(this);
+        mFullScreenshot = (ImageView)mContentView.findViewById(R.id.full_screenshot);
         mConfirmButton = (Button)mContentView.findViewById(R.id.confirm);
         mConfirmButton.setOnClickListener(this);
         mCompleteButton = (Button)mContentView.findViewById(R.id.complete);
@@ -156,8 +165,10 @@ public class MainInteractionSession extends VoiceInteractionSession
             mScreenshot.setAdjustViewBounds(true);
             mScreenshot.setMaxWidth(screenshot.getWidth()/3);
             mScreenshot.setMaxHeight(screenshot.getHeight()/3);
+            mFullScreenshot.setImageBitmap(screenshot);
         } else {
             mScreenshot.setImageDrawable(null);
+            mFullScreenshot.setImageDrawable(null);
         }
     }
 
@@ -183,7 +194,15 @@ public class MainInteractionSession extends VoiceInteractionSession
     }
 
     public void onClick(View v) {
-        if (v == mStartButton) {
+        if (v == mTreeButton) {
+            if (mAssistVisualizer != null) {
+                mAssistVisualizer.logTree();
+            }
+        } else if (v == mTextButton) {
+            if (mAssistVisualizer != null) {
+                mAssistVisualizer.logText();
+            }
+        } else if (v == mStartButton) {
             mState = STATE_LAUNCHING;
             updateState();
             startVoiceActivity(mStartIntent);
@@ -219,9 +238,15 @@ public class MainInteractionSession extends VoiceInteractionSession
         } else if (v == mAbortButton) {
             mPendingRequest.sendAbortVoiceResult(null);
             mPendingRequest = null;
-        } else if (v== mCompleteButton) {
+        } else if (v == mCompleteButton) {
             mPendingRequest.sendCompleteVoiceResult(null);
             mPendingRequest = null;
+        } else if (v == mScreenshot) {
+            if (mFullScreenshot.getVisibility() != View.VISIBLE) {
+                mFullScreenshot.setVisibility(View.VISIBLE);
+            } else {
+                mFullScreenshot.setVisibility(View.INVISIBLE);
+            }
         }
         updateState();
     }
-- 
cgit v1.1