From ee44d7cdbdea9a8a67e967b3bc05f0cd409ae2b1 Mon Sep 17 00:00:00 2001 From: Robert Shih Date: Fri, 24 Jun 2016 12:37:45 -0700 Subject: SoftVPX: fix nFilledLen overflow Bug: 29421675 Change-Id: I25d4cf54a5df22c2130c37e95c7c7f75063111f3 --- media/libstagefright/codecs/on2/dec/SoftVPX.cpp | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) (limited to 'media/libstagefright/codecs/on2/dec/SoftVPX.cpp') diff --git a/media/libstagefright/codecs/on2/dec/SoftVPX.cpp b/media/libstagefright/codecs/on2/dec/SoftVPX.cpp index 02e85a1..58a2660 100644 --- a/media/libstagefright/codecs/on2/dec/SoftVPX.cpp +++ b/media/libstagefright/codecs/on2/dec/SoftVPX.cpp @@ -149,7 +149,7 @@ bool SoftVPX::outputBuffers(bool flushDecoder, bool display, bool eos, bool *por outHeader->nFlags = 0; outHeader->nFilledLen = (outputBufferWidth() * outputBufferHeight() * 3) / 2; outHeader->nTimeStamp = *(OMX_TICKS *)mImg->user_priv; - if (outHeader->nAllocLen >= outHeader->nFilledLen) { + if (outputBufferSafe(outHeader)) { uint8_t *dst = outHeader->pBuffer; const uint8_t *srcY = (const uint8_t *)mImg->planes[VPX_PLANE_Y]; const uint8_t *srcU = (const uint8_t *)mImg->planes[VPX_PLANE_U]; @@ -159,8 +159,6 @@ bool SoftVPX::outputBuffers(bool flushDecoder, bool display, bool eos, bool *por size_t srcVStride = mImg->stride[VPX_PLANE_V]; copyYV12FrameToOutputBuffer(dst, srcY, srcU, srcV, srcYStride, srcUStride, srcVStride); } else { - ALOGE("b/27597103, buffer too small"); - android_errorWriteLog(0x534e4554, "27597103"); outHeader->nFilledLen = 0; } @@ -190,6 +188,24 @@ bool SoftVPX::outputBuffers(bool flushDecoder, bool display, bool eos, bool *por return true; } +bool SoftVPX::outputBufferSafe(OMX_BUFFERHEADERTYPE *outHeader) { + uint32_t width = outputBufferWidth(); + uint32_t height = outputBufferHeight(); + uint64_t nFilledLen = width; + nFilledLen *= height; + if (nFilledLen > UINT32_MAX / 3) { + ALOGE("b/29421675, nFilledLen overflow %llu w %u h %u", nFilledLen, width, height); + android_errorWriteLog(0x534e4554, "29421675"); + return false; + } else if (outHeader->nAllocLen < outHeader->nFilledLen) { + ALOGE("b/27597103, buffer too small"); + android_errorWriteLog(0x534e4554, "27597103"); + return false; + } + + return true; +} + void SoftVPX::onQueueFilled(OMX_U32 /* portIndex */) { if (mOutputPortSettingsChange != NONE || mEOSStatus == OUTPUT_FRAMES_FLUSHED) { return; -- cgit v1.1 From 50643aadeb8dfe53da7848a5d5e995d5486678e1 Mon Sep 17 00:00:00 2001 From: Lajos Molnar Date: Wed, 13 Jul 2016 20:27:32 -0700 Subject: fix build Change-Id: I9bb8c659d3fc97a8e748451d82d0f3448faa242b --- media/libstagefright/codecs/on2/dec/SoftVPX.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'media/libstagefright/codecs/on2/dec/SoftVPX.cpp') diff --git a/media/libstagefright/codecs/on2/dec/SoftVPX.cpp b/media/libstagefright/codecs/on2/dec/SoftVPX.cpp index 58a2660..912fac2 100644 --- a/media/libstagefright/codecs/on2/dec/SoftVPX.cpp +++ b/media/libstagefright/codecs/on2/dec/SoftVPX.cpp @@ -194,7 +194,8 @@ bool SoftVPX::outputBufferSafe(OMX_BUFFERHEADERTYPE *outHeader) { uint64_t nFilledLen = width; nFilledLen *= height; if (nFilledLen > UINT32_MAX / 3) { - ALOGE("b/29421675, nFilledLen overflow %llu w %u h %u", nFilledLen, width, height); + ALOGE("b/29421675, nFilledLen overflow %llu w %u h %u", + (unsigned long long)nFilledLen, width, height); android_errorWriteLog(0x534e4554, "29421675"); return false; } else if (outHeader->nAllocLen < outHeader->nFilledLen) { -- cgit v1.1