take the state transform into account with h/w composer hal

if the state transform didn't preserve rectangles, we
would still try to use h/w composer hal using the bounds
of the transformed rect, which isn't correct.

now we correctly fall back to composition.

Change-Id: Iff78f4339ece415d4987e95a5717b04934d370ab

2 files changed, 21 insertions, 5 deletions

diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp
index 35e29a6..f3b6c4d 100644
--- a/services/surfaceflinger/Layer.cpp
+++ b/services/surfaceflinger/Layer.cpp
@@ -188,22 +188,37 @@ void Layer::setGeometry(hwc_layer_t* hwcl)
         return;
     }
 
+    /*
+     * Transformations are applied in this order:
+     * 1) buffer orientation/flip/mirror
+     * 2) state transformation (window manager)
+     * 3) layer orientation (screen orientation)
+     * (NOTE: the matrices are multiplied in reverse order)
+     */
+
+    const Transform bufferOrientation(mCurrentTransform);
+    const Transform& stateTransform(s.transform);
+    const Transform layerOrientation(mOrientation);
+
+    const Transform tr(layerOrientation * stateTransform * bufferOrientation);
+
+    // this gives us only the "orientation" component of the transform
+    const uint32_t finalTransform = tr.getOrientation();
+
     // we can only handle simple transformation
-    if (mOrientation & Transform::ROT_INVALID) {
+    if (finalTransform & Transform::ROT_INVALID) {
         hwcl->flags = HWC_SKIP_LAYER;
         return;
     }
 
-    // FIXME: shouldn't we take the state's transform into account here?
-
-    Transform tr(Transform(mOrientation) * Transform(mCurrentTransform));
-    hwcl->transform = tr.getOrientation();
+    hwcl->transform = finalTransform;
 
     if (!isOpaque()) {
         hwcl->blending = mPremultipliedAlpha ?
                 HWC_BLENDING_PREMULT : HWC_BLENDING_COVERAGE;
     }
 
+    // scaling is already applied in mTransformedBounds
     hwcl->displayFrame.left   = mTransformedBounds.left;
     hwcl->displayFrame.top    = mTransformedBounds.top;
     hwcl->displayFrame.right  = mTransformedBounds.right;
diff --git a/services/surfaceflinger/Transform.cpp b/services/surfaceflinger/Transform.cpp
index 0467a14..4cedcbf 100644
--- a/services/surfaceflinger/Transform.cpp
+++ b/services/surfaceflinger/Transform.cpp
@@ -308,6 +308,7 @@ uint32_t Transform::type() const
                 scale = true;
             }
         } else {
+            // there is a skew component and/or a non 90 degrees rotation
             flags = ROT_INVALID;
         }