stagefright: fix cropping handling for SoftAVC and SoftMEPEG4.

Bug: 17326758
Change-Id: I9b0c281d92cf1803e275b1768e9edab9404ea577

5 files changed, 31 insertions, 18 deletions

diff --git a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp
index 5b2ab84..d98fa80 100644
--- a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp
+++ b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp
@@ -295,19 +295,23 @@ bool SoftMPEG4::handlePortSettingsChange() {
     ALOGV("disp_width = %d, disp_height = %d, buf_width = %d, buf_height = %d",
             disp_width, disp_height, buf_width, buf_height);
 
-    bool cropChanged = false;
-    if (mCropWidth != disp_width || mCropHeight != disp_height) {
-        mCropLeft = 0;
-        mCropTop = 0;
-        mCropWidth = disp_width;
-        mCropHeight = disp_height;
-        cropChanged = true;
+    CropSettingsMode cropSettingsMode = kCropUnSet;
+    if (disp_width != buf_width || disp_height != buf_height) {
+        cropSettingsMode = kCropSet;
+
+        if (mCropWidth != disp_width || mCropHeight != disp_height) {
+            mCropLeft = 0;
+            mCropTop = 0;
+            mCropWidth = disp_width;
+            mCropHeight = disp_height;
+            cropSettingsMode = kCropChanged;
+        }
     }
 
     bool portWillReset = false;
     const bool fakeStride = true;
     SoftVideoDecoderOMXComponent::handlePortSettingsChange(
-            &portWillReset, buf_width, buf_height, cropChanged, fakeStride);
+            &portWillReset, buf_width, buf_height, cropSettingsMode, fakeStride);
     if (portWillReset) {
         if (mMode == MODE_H263) {
             PVCleanUpVideoDecoder(mHandle);
diff --git a/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp b/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp
index cf3c3e3..168208f 100644
--- a/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp
+++ b/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp
@@ -160,10 +160,11 @@ void SoftAVC::onQueueFilled(OMX_U32 /* portIndex */) {
                     H264SwDecInfo decoderInfo;
                     CHECK(H264SwDecGetInfo(mHandle, &decoderInfo) == H264SWDEC_OK);
 
-                    bool cropChanged = handleCropChange(decoderInfo);
+                    SoftVideoDecoderOMXComponent::CropSettingsMode cropSettingsMode =
+                        handleCropParams(decoderInfo);
                     handlePortSettingsChange(
                             &portWillReset, decoderInfo.picWidth, decoderInfo.picHeight,
-                            cropChanged);
+                            cropSettingsMode);
                 }
             } else {
                 if (portWillReset) {
@@ -209,9 +210,10 @@ void SoftAVC::onQueueFilled(OMX_U32 /* portIndex */) {
     }
 }
 
-bool SoftAVC::handleCropChange(const H264SwDecInfo& decInfo) {
+SoftVideoDecoderOMXComponent::CropSettingsMode SoftAVC::handleCropParams(
+        const H264SwDecInfo& decInfo) {
     if (!decInfo.croppingFlag) {
-        return false;
+        return kCropUnSet;
     }
 
     const CropParams& crop = decInfo.cropParams;
@@ -219,14 +221,14 @@ bool SoftAVC::handleCropChange(const H264SwDecInfo& decInfo) {
         mCropTop == crop.cropTopOffset &&
         mCropWidth == crop.cropOutWidth &&
         mCropHeight == crop.cropOutHeight) {
-        return false;
+        return kCropSet;
     }
 
     mCropLeft = crop.cropLeftOffset;
     mCropTop = crop.cropTopOffset;
     mCropWidth = crop.cropOutWidth;
     mCropHeight = crop.cropOutHeight;
-    return true;
+    return kCropChanged;
 }
 
 void SoftAVC::saveFirstOutputBuffer(int32_t picId, uint8_t *data) {
diff --git a/media/libstagefright/codecs/on2/h264dec/SoftAVC.h b/media/libstagefright/codecs/on2/h264dec/SoftAVC.h
index 253a406..069107d 100644
--- a/media/libstagefright/codecs/on2/h264dec/SoftAVC.h
+++ b/media/libstagefright/codecs/on2/h264dec/SoftAVC.h
@@ -73,7 +73,7 @@ private:
     void drainAllOutputBuffers(bool eos);
     void drainOneOutputBuffer(int32_t picId, uint8_t *data);
     void saveFirstOutputBuffer(int32_t pidId, uint8_t *data);
-    bool handleCropChange(const H264SwDecInfo& decInfo);
+    CropSettingsMode handleCropParams(const H264SwDecInfo& decInfo);
 
     DISALLOW_EVIL_CONSTRUCTORS(SoftAVC);
 };
diff --git a/media/libstagefright/include/SoftVideoDecoderOMXComponent.h b/media/libstagefright/include/SoftVideoDecoderOMXComponent.h
index 37b1fe1..9e97ebd 100644
--- a/media/libstagefright/include/SoftVideoDecoderOMXComponent.h
+++ b/media/libstagefright/include/SoftVideoDecoderOMXComponent.h
@@ -68,9 +68,14 @@ protected:
     uint32_t outputBufferWidth();
     uint32_t outputBufferHeight();
 
+    enum CropSettingsMode {
+        kCropUnSet = 0,
+        kCropSet,
+        kCropChanged,
+    };
     void handlePortSettingsChange(
             bool *portWillReset, uint32_t width, uint32_t height,
-            bool cropChanged = false, bool fakeStride = false);
+            CropSettingsMode cropSettingsMode = kCropUnSet, bool fakeStride = false);
 
     void copyYV12FrameToOutputBuffer(
             uint8_t *dst, const uint8_t *srcY, const uint8_t *srcU, const uint8_t *srcV,
diff --git a/media/libstagefright/omx/SoftVideoDecoderOMXComponent.cpp b/media/libstagefright/omx/SoftVideoDecoderOMXComponent.cpp
index 5853469..a7f7a07 100644
--- a/media/libstagefright/omx/SoftVideoDecoderOMXComponent.cpp
+++ b/media/libstagefright/omx/SoftVideoDecoderOMXComponent.cpp
@@ -160,15 +160,17 @@ uint32_t SoftVideoDecoderOMXComponent::outputBufferHeight() {
 }
 
 void SoftVideoDecoderOMXComponent::handlePortSettingsChange(
-        bool *portWillReset, uint32_t width, uint32_t height, bool cropChanged, bool fakeStride) {
+        bool *portWillReset, uint32_t width, uint32_t height,
+        CropSettingsMode cropSettingsMode, bool fakeStride) {
     *portWillReset = false;
     bool sizeChanged = (width != mWidth || height != mHeight);
+    bool updateCrop = (cropSettingsMode == kCropUnSet);
+    bool cropChanged = (cropSettingsMode == kCropChanged);
 
     if (sizeChanged || cropChanged) {
         mWidth = width;
         mHeight = height;
 
-        bool updateCrop = !cropChanged;
         if ((sizeChanged && !mIsAdaptive)
             || width > mAdaptiveMaxWidth
             || height > mAdaptiveMaxHeight) {