libstagefright: set actual stride info for SW encoder input port

This fixes encoding flexible YUV content using SW encoders.
Also skip setting input color format if it is flexible YUV, as it
has already been translated to a color format supported by the codec.

Bug: 18124320
Change-Id: I423782936986f4d6cf65ea9ef89ae77a92e30140

4 files changed, 17 insertions, 1 deletions

diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index 2f2f9cf..0e9d734 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -2221,7 +2221,11 @@ status_t ACodec::setupVideoEncoder(const char *mime, const sp<AMessage> &msg) {
 
     video_def->xFramerate = (OMX_U32)(frameRate * 65536.0f);
     video_def->eCompressionFormat = OMX_VIDEO_CodingUnused;
-    video_def->eColorFormat = colorFormat;
+    // this is redundant as it was already set up in setVideoPortFormatType
+    // FIXME for now skip this only for flexible YUV formats
+    if (colorFormat != OMX_COLOR_FormatYUV420Flexible) {
+        video_def->eColorFormat = colorFormat;
+    }
 
     err = mOMX->setParameter(
             mNode, OMX_IndexParamPortDefinition, &def, sizeof(def));
diff --git a/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp b/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp
index ed3dca0..bb55871 100644
--- a/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp
+++ b/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp
@@ -575,9 +575,13 @@ OMX_ERRORTYPE SoftAVCEncoder::internalSetParameter(
                     &editPortInfo(0)->mDef;
                 portDef->format.video.nFrameWidth = mVideoWidth;
                 portDef->format.video.nFrameHeight = mVideoHeight;
+                portDef->format.video.nStride = portDef->format.video.nFrameWidth;
+                portDef->format.video.nSliceHeight = portDef->format.video.nFrameHeight;
                 portDef->format.video.xFramerate = def->format.video.xFramerate;
                 portDef->format.video.eColorFormat =
                     (OMX_COLOR_FORMATTYPE) mVideoColorFormat;
+                portDef->nBufferSize =
+                    (portDef->format.video.nStride * portDef->format.video.nSliceHeight * 3) / 2;
                 portDef = &editPortInfo(1)->mDef;
                 portDef->format.video.nFrameWidth = mVideoWidth;
                 portDef->format.video.nFrameHeight = mVideoHeight;
diff --git a/media/libstagefright/codecs/m4v_h263/enc/SoftMPEG4Encoder.cpp b/media/libstagefright/codecs/m4v_h263/enc/SoftMPEG4Encoder.cpp
index c87d19c..400f320 100644
--- a/media/libstagefright/codecs/m4v_h263/enc/SoftMPEG4Encoder.cpp
+++ b/media/libstagefright/codecs/m4v_h263/enc/SoftMPEG4Encoder.cpp
@@ -462,9 +462,13 @@ OMX_ERRORTYPE SoftMPEG4Encoder::internalSetParameter(
                     &editPortInfo(0)->mDef;
                 portDef->format.video.nFrameWidth = mVideoWidth;
                 portDef->format.video.nFrameHeight = mVideoHeight;
+                portDef->format.video.nStride = portDef->format.video.nFrameWidth;
+                portDef->format.video.nSliceHeight = portDef->format.video.nFrameHeight;
                 portDef->format.video.xFramerate = def->format.video.xFramerate;
                 portDef->format.video.eColorFormat =
                     (OMX_COLOR_FORMATTYPE) mVideoColorFormat;
+                portDef->nBufferSize =
+                    (portDef->format.video.nStride * portDef->format.video.nSliceHeight * 3) / 2;
                 portDef = &editPortInfo(1)->mDef;
                 portDef->format.video.nFrameWidth = mVideoWidth;
                 portDef->format.video.nFrameHeight = mVideoHeight;
diff --git a/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.cpp b/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.cpp
index eb621d5..0285feb 100644
--- a/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.cpp
+++ b/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.cpp
@@ -805,8 +805,12 @@ OMX_ERRORTYPE SoftVPXEncoder::internalSetPortParams(
         OMX_PARAM_PORTDEFINITIONTYPE *def = &editPortInfo(kInputPortIndex)->mDef;
         def->format.video.nFrameWidth = mWidth;
         def->format.video.nFrameHeight = mHeight;
+        def->format.video.nStride = def->format.video.nFrameWidth;
+        def->format.video.nSliceHeight = def->format.video.nFrameHeight;
         def->format.video.xFramerate = mFramerate;
         def->format.video.eColorFormat = mColorFormat;
+        def->nBufferSize =
+            (def->format.video.nStride * def->format.video.nSliceHeight * 3) / 2;
         def = &editPortInfo(kOutputPortIndex)->mDef;
         def->format.video.nFrameWidth = mWidth;
         def->format.video.nFrameHeight = mHeight;