codecs: check OMX buffer size before use in (avc|hevc|mpeg2)dec

Bug: 27833616
Change-Id: Ic4045a3f56f53b08d0b1264b2a91b8f43e91b738
(cherry picked from commit 87fdee0bc9e3ac4d2a88ef0a8e150cfdf08c161d)

6 files changed, 62 insertions, 33 deletions

diff --git a/media/libstagefright/codecs/avcdec/SoftAVCDec.cpp b/media/libstagefright/codecs/avcdec/SoftAVCDec.cpp
index afbe230..2130ccf 100644
--- a/media/libstagefright/codecs/avcdec/SoftAVCDec.cpp
+++ b/media/libstagefright/codecs/avcdec/SoftAVCDec.cpp
@@ -381,7 +381,7 @@ void SoftAVC::onReset() {
     resetPlugin();
 }
 
-void SoftAVC::setDecodeArgs(
+bool SoftAVC::setDecodeArgs(
         ivd_video_decode_ip_t *ps_dec_ip,
         ivd_video_decode_op_t *ps_dec_op,
         OMX_BUFFERHEADERTYPE *inHeader,
@@ -389,7 +389,6 @@ void SoftAVC::setDecodeArgs(
         size_t timeStampIx) {
     size_t sizeY = outputBufferWidth() * outputBufferHeight();
     size_t sizeUV;
-    uint8_t *pBuf;
 
     ps_dec_ip->u4_size = sizeof(ivd_video_decode_ip_t);
     ps_dec_op->u4_size = sizeof(ivd_video_decode_op_t);
@@ -409,22 +408,28 @@ void SoftAVC::setDecodeArgs(
         ps_dec_ip->u4_num_Bytes = 0;
     }
 
+    sizeUV = sizeY / 4;
+    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[0] = sizeY;
+    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[1] = sizeUV;
+    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[2] = sizeUV;
+
+    uint8_t *pBuf;
     if (outHeader) {
+        if (outHeader->nAllocLen < sizeY + (sizeUV * 2)) {
+            android_errorWriteLog(0x534e4554, "27569635");
+            return false;
+        }
         pBuf = outHeader->pBuffer;
     } else {
+        // mFlushOutBuffer always has the right size.
         pBuf = mFlushOutBuffer;
     }
 
-    sizeUV = sizeY / 4;
-    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[0] = sizeY;
-    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[1] = sizeUV;
-    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[2] = sizeUV;
-
     ps_dec_ip->s_out_buffer.pu1_bufs[0] = pBuf;
     ps_dec_ip->s_out_buffer.pu1_bufs[1] = pBuf + sizeY;
     ps_dec_ip->s_out_buffer.pu1_bufs[2] = pBuf + sizeY + sizeUV;
     ps_dec_ip->s_out_buffer.u4_num_bufs = 3;
-    return;
+    return true;
 }
 void SoftAVC::onPortFlushCompleted(OMX_U32 portIndex) {
     /* Once the output buffers are flushed, ignore any buffers that are held in decoder */
@@ -573,7 +578,12 @@ void SoftAVC::onQueueFilled(OMX_U32 portIndex) {
             WORD32 timeDelay, timeTaken;
             size_t sizeY, sizeUV;
 
-            setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx);
+            if (!setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx)) {
+                ALOGE("Decoder arg setup failed");
+                notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+                mSignalledError = true;
+                return;
+            }
             // If input dump is enabled, then write to file
             DUMP_TO_FILE(mInFile, s_dec_ip.pv_stream_buffer, s_dec_ip.u4_num_Bytes);
 
diff --git a/media/libstagefright/codecs/avcdec/SoftAVCDec.h b/media/libstagefright/codecs/avcdec/SoftAVCDec.h
index 9dcabb4..c710c76 100644
--- a/media/libstagefright/codecs/avcdec/SoftAVCDec.h
+++ b/media/libstagefright/codecs/avcdec/SoftAVCDec.h
@@ -109,7 +109,7 @@ private:
     status_t resetPlugin();
 
 
-    void setDecodeArgs(
+    bool setDecodeArgs(
             ivd_video_decode_ip_t *ps_dec_ip,
             ivd_video_decode_op_t *ps_dec_op,
             OMX_BUFFERHEADERTYPE *inHeader,
diff --git a/media/libstagefright/codecs/hevcdec/SoftHEVC.cpp b/media/libstagefright/codecs/hevcdec/SoftHEVC.cpp
index e601125..a70755c 100644
--- a/media/libstagefright/codecs/hevcdec/SoftHEVC.cpp
+++ b/media/libstagefright/codecs/hevcdec/SoftHEVC.cpp
@@ -343,14 +343,13 @@ void SoftHEVC::onReset() {
     resetPlugin();
 }
 
-void SoftHEVC::setDecodeArgs(ivd_video_decode_ip_t *ps_dec_ip,
+bool SoftHEVC::setDecodeArgs(ivd_video_decode_ip_t *ps_dec_ip,
         ivd_video_decode_op_t *ps_dec_op,
         OMX_BUFFERHEADERTYPE *inHeader,
         OMX_BUFFERHEADERTYPE *outHeader,
         size_t timeStampIx) {
     size_t sizeY = outputBufferWidth() * outputBufferHeight();
     size_t sizeUV;
-    uint8_t *pBuf;
 
     ps_dec_ip->u4_size = sizeof(ivd_video_decode_ip_t);
     ps_dec_op->u4_size = sizeof(ivd_video_decode_op_t);
@@ -370,22 +369,28 @@ void SoftHEVC::setDecodeArgs(ivd_video_decode_ip_t *ps_dec_ip,
         ps_dec_ip->u4_num_Bytes = 0;
     }
 
+    sizeUV = sizeY / 4;
+    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[0] = sizeY;
+    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[1] = sizeUV;
+    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[2] = sizeUV;
+
+    uint8_t *pBuf;
     if (outHeader) {
+        if (outHeader->nAllocLen < sizeY + (sizeUV * 2)) {
+            android_errorWriteLog(0x534e4554, "27569635");
+            return false;
+        }
         pBuf = outHeader->pBuffer;
     } else {
+        // mFlushOutBuffer always has the right size.
         pBuf = mFlushOutBuffer;
     }
 
-    sizeUV = sizeY / 4;
-    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[0] = sizeY;
-    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[1] = sizeUV;
-    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[2] = sizeUV;
-
     ps_dec_ip->s_out_buffer.pu1_bufs[0] = pBuf;
     ps_dec_ip->s_out_buffer.pu1_bufs[1] = pBuf + sizeY;
     ps_dec_ip->s_out_buffer.pu1_bufs[2] = pBuf + sizeY + sizeUV;
     ps_dec_ip->s_out_buffer.u4_num_bufs = 3;
-    return;
+    return true;
 }
 void SoftHEVC::onPortFlushCompleted(OMX_U32 portIndex) {
     /* Once the output buffers are flushed, ignore any buffers that are held in decoder */
@@ -520,7 +525,12 @@ void SoftHEVC::onQueueFilled(OMX_U32 portIndex) {
             WORD32 timeDelay, timeTaken;
             size_t sizeY, sizeUV;
 
-            setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx);
+            if (!setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx)) {
+                ALOGE("Decoder arg setup failed");
+                notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+                mSignalledError = true;
+                return;
+            }
 
             GETTIME(&mTimeStart, NULL);
             /* Compute time elapsed between end of previous decode()
diff --git a/media/libstagefright/codecs/hevcdec/SoftHEVC.h b/media/libstagefright/codecs/hevcdec/SoftHEVC.h
index 21bb99e..943edfd 100644
--- a/media/libstagefright/codecs/hevcdec/SoftHEVC.h
+++ b/media/libstagefright/codecs/hevcdec/SoftHEVC.h
@@ -106,7 +106,7 @@ private:
     status_t resetDecoder();
     status_t resetPlugin();
 
-    void setDecodeArgs(ivd_video_decode_ip_t *ps_dec_ip,
+    bool setDecodeArgs(ivd_video_decode_ip_t *ps_dec_ip,
         ivd_video_decode_op_t *ps_dec_op,
         OMX_BUFFERHEADERTYPE *inHeader,
         OMX_BUFFERHEADERTYPE *outHeader,
diff --git a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp
index 4307c4e..e134d38 100644
--- a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp
+++ b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp
@@ -466,7 +466,7 @@ OMX_ERRORTYPE SoftMPEG2::internalSetParameter(OMX_INDEXTYPE index, const OMX_PTR
     return ret;
 }
 
-void SoftMPEG2::setDecodeArgs(
+bool SoftMPEG2::setDecodeArgs(
         ivd_video_decode_ip_t *ps_dec_ip,
         ivd_video_decode_op_t *ps_dec_op,
         OMX_BUFFERHEADERTYPE *inHeader,
@@ -474,7 +474,6 @@ void SoftMPEG2::setDecodeArgs(
         size_t timeStampIx) {
     size_t sizeY = outputBufferWidth() * outputBufferHeight();
     size_t sizeUV;
-    uint8_t *pBuf;
 
     ps_dec_ip->u4_size = sizeof(ivd_video_decode_ip_t);
     ps_dec_op->u4_size = sizeof(ivd_video_decode_op_t);
@@ -494,22 +493,28 @@ void SoftMPEG2::setDecodeArgs(
         ps_dec_ip->u4_num_Bytes = 0;
     }
 
+    sizeUV = sizeY / 4;
+    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[0] = sizeY;
+    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[1] = sizeUV;
+    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[2] = sizeUV;
+
+    uint8_t *pBuf;
     if (outHeader) {
+        if (outHeader->nAllocLen < sizeY + (sizeUV * 2)) {
+            android_errorWriteLog(0x534e4554, "27569635");
+            return false;
+        }
         pBuf = outHeader->pBuffer;
     } else {
+        // mFlushOutBuffer always has the right size.
         pBuf = mFlushOutBuffer;
     }
 
-    sizeUV = sizeY / 4;
-    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[0] = sizeY;
-    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[1] = sizeUV;
-    ps_dec_ip->s_out_buffer.u4_min_out_buf_size[2] = sizeUV;
-
     ps_dec_ip->s_out_buffer.pu1_bufs[0] = pBuf;
     ps_dec_ip->s_out_buffer.pu1_bufs[1] = pBuf + sizeY;
     ps_dec_ip->s_out_buffer.pu1_bufs[2] = pBuf + sizeY + sizeUV;
     ps_dec_ip->s_out_buffer.u4_num_bufs = 3;
-    return;
+    return true;
 }
 void SoftMPEG2::onPortFlushCompleted(OMX_U32 portIndex) {
     /* Once the output buffers are flushed, ignore any buffers that are held in decoder */
@@ -622,7 +627,11 @@ void SoftMPEG2::onQueueFilled(OMX_U32 portIndex) {
             WORD32 timeDelay, timeTaken;
             size_t sizeY, sizeUV;
 
-            setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx);
+            if (!setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx)) {
+                ALOGE("Decoder arg setup failed");
+                notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+                return;
+            }
             // If input dump is enabled, then write to file
             DUMP_TO_FILE(mInFile, s_dec_ip.pv_stream_buffer, s_dec_ip.u4_num_Bytes);
 
@@ -665,9 +674,9 @@ void SoftMPEG2::onQueueFilled(OMX_U32 portIndex) {
 
                 CHECK_EQ(reInitDecoder(), (status_t)OK);
 
-                setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx);
-
-                ivdec_api_function(mCodecCtx, (void *)&s_dec_ip, (void *)&s_dec_op);
+                if (setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx)) {
+                    ivdec_api_function(mCodecCtx, (void *)&s_dec_ip, (void *)&s_dec_op);
+                }
                 return;
             }
 
diff --git a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h
index a625e08..f48b70b 100644
--- a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h
+++ b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h
@@ -117,7 +117,7 @@ private:
     status_t resetPlugin();
     status_t reInitDecoder();
 
-    void setDecodeArgs(
+    bool setDecodeArgs(
             ivd_video_decode_ip_t *ps_dec_ip,
             ivd_video_decode_op_t *ps_dec_op,
             OMX_BUFFERHEADERTYPE *inHeader,