SoftVPX: fix nFilledLen overflow

Bug: 29421675
Change-Id: I25d4cf54a5df22c2130c37e95c7c7f75063111f3

2 files changed, 20 insertions, 3 deletions

diff --git a/media/libstagefright/codecs/on2/dec/SoftVPX.cpp b/media/libstagefright/codecs/on2/dec/SoftVPX.cpp
index 02e85a1..58a2660 100644
--- a/media/libstagefright/codecs/on2/dec/SoftVPX.cpp
+++ b/media/libstagefright/codecs/on2/dec/SoftVPX.cpp
@@ -149,7 +149,7 @@ bool SoftVPX::outputBuffers(bool flushDecoder, bool display, bool eos, bool *por
         outHeader->nFlags = 0;
         outHeader->nFilledLen = (outputBufferWidth() * outputBufferHeight() * 3) / 2;
         outHeader->nTimeStamp = *(OMX_TICKS *)mImg->user_priv;
-        if (outHeader->nAllocLen >= outHeader->nFilledLen) {
+        if (outputBufferSafe(outHeader)) {
             uint8_t *dst = outHeader->pBuffer;
             const uint8_t *srcY = (const uint8_t *)mImg->planes[VPX_PLANE_Y];
             const uint8_t *srcU = (const uint8_t *)mImg->planes[VPX_PLANE_U];
@@ -159,8 +159,6 @@ bool SoftVPX::outputBuffers(bool flushDecoder, bool display, bool eos, bool *por
             size_t srcVStride = mImg->stride[VPX_PLANE_V];
             copyYV12FrameToOutputBuffer(dst, srcY, srcU, srcV, srcYStride, srcUStride, srcVStride);
         } else {
-            ALOGE("b/27597103, buffer too small");
-            android_errorWriteLog(0x534e4554, "27597103");
             outHeader->nFilledLen = 0;
         }
 
@@ -190,6 +188,24 @@ bool SoftVPX::outputBuffers(bool flushDecoder, bool display, bool eos, bool *por
     return true;
 }
 
+bool SoftVPX::outputBufferSafe(OMX_BUFFERHEADERTYPE *outHeader) {
+    uint32_t width = outputBufferWidth();
+    uint32_t height = outputBufferHeight();
+    uint64_t nFilledLen = width;
+    nFilledLen *= height;
+    if (nFilledLen > UINT32_MAX / 3) {
+        ALOGE("b/29421675, nFilledLen overflow %llu w %u h %u", nFilledLen, width, height);
+        android_errorWriteLog(0x534e4554, "29421675");
+        return false;
+    } else if (outHeader->nAllocLen < outHeader->nFilledLen) {
+        ALOGE("b/27597103, buffer too small");
+        android_errorWriteLog(0x534e4554, "27597103");
+        return false;
+    }
+
+    return true;
+}
+
 void SoftVPX::onQueueFilled(OMX_U32 /* portIndex */) {
     if (mOutputPortSettingsChange != NONE || mEOSStatus == OUTPUT_FRAMES_FLUSHED) {
         return;
diff --git a/media/libstagefright/codecs/on2/dec/SoftVPX.h b/media/libstagefright/codecs/on2/dec/SoftVPX.h
index 8ccbae2..84cf79c 100644
--- a/media/libstagefright/codecs/on2/dec/SoftVPX.h
+++ b/media/libstagefright/codecs/on2/dec/SoftVPX.h
@@ -66,6 +66,7 @@ private:
     status_t initDecoder();
     status_t destroyDecoder();
     bool outputBuffers(bool flushDecoder, bool display, bool eos, bool *portWillReset);
+    bool outputBufferSafe(OMX_BUFFERHEADERTYPE *outHeader);
 
     DISALLOW_EVIL_CONSTRUCTORS(SoftVPX);
 };