From b569311fe413fef4f35114f00485340cc6e322ee Mon Sep 17 00:00:00 2001 From: Pawin Vongmasa Date: Mon, 18 Jul 2016 20:12:02 -0700 Subject: SoftMPEG4: Check the buffer size before writing the reference frame. Also prevent overflow in SoftMPEG4 and division by zero in SoftMPEG4Encoder. Bug: 30033990 Change-Id: I7701f5fc54c2670587d122330e5dc851f64ed3c2 (cherry picked from commit 695123195034402ca76169b195069c28c30342d3) --- .../codecs/m4v_h263/dec/SoftMPEG4.cpp | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) (limited to 'media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp') diff --git a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp index bb59ae4..e1cfc06 100644 --- a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp +++ b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp @@ -210,8 +210,17 @@ void SoftMPEG4::onQueueFilled(OMX_U32 /* portIndex */) { PortInfo *port = editPortInfo(1); OMX_BUFFERHEADERTYPE *outHeader = port->mBuffers.editItemAt(1).mHeader; + OMX_U32 yFrameSize = sizeof(uint8) * mHandle->size; + if ((outHeader->nAllocLen < yFrameSize) || + (outHeader->nAllocLen - yFrameSize < yFrameSize / 2)) { + ALOGE("Too small output buffer for reference frame: %zu bytes", + outHeader->nAllocLen); + android_errorWriteLog(0x534e4554, "30033990"); + notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); + mSignalledError = true; + return; + } PVSetReferenceYUV(mHandle, outHeader->pBuffer); - mFramesConfigured = true; } @@ -229,7 +238,16 @@ void SoftMPEG4::onQueueFilled(OMX_U32 /* portIndex */) { int32_t bufferSize = inHeader->nFilledLen; int32_t tmp = bufferSize; - OMX_U32 frameSize = (mWidth * mHeight * 3) / 2; + OMX_U32 frameSize; + OMX_U64 yFrameSize = (OMX_U64)mWidth * (OMX_U64)mHeight; + if (yFrameSize > ((OMX_U64)UINT32_MAX / 3) * 2) { + ALOGE("Frame size too large"); + notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); + mSignalledError = true; + return; + } + frameSize = (OMX_U32)(yFrameSize + (yFrameSize / 2)); + if (outHeader->nAllocLen < frameSize) { android_errorWriteLog(0x534e4554, "27833616"); ALOGE("Insufficient output buffer size"); -- cgit v1.1 From 089104004f5aeb2634764c56b948ef83318a9f95 Mon Sep 17 00:00:00 2001 From: Pawin Vongmasa Date: Fri, 19 Aug 2016 02:07:33 -0700 Subject: Fix build breakage caused by commit 940829f69b52d6038db66a9c727534636ecc456d. Change-Id: I4776db4a26fb3c31bb994d48788373fe569c812a (cherry picked from commit baa9146401e28c5acf54dea21ddd197f0d3a8fcd) --- media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp') diff --git a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp index e1cfc06..1dd631a 100644 --- a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp +++ b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp @@ -213,8 +213,8 @@ void SoftMPEG4::onQueueFilled(OMX_U32 /* portIndex */) { OMX_U32 yFrameSize = sizeof(uint8) * mHandle->size; if ((outHeader->nAllocLen < yFrameSize) || (outHeader->nAllocLen - yFrameSize < yFrameSize / 2)) { - ALOGE("Too small output buffer for reference frame: %zu bytes", - outHeader->nAllocLen); + ALOGE("Too small output buffer for reference frame: %lu bytes", + (unsigned long)outHeader->nAllocLen); android_errorWriteLog(0x534e4554, "30033990"); notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); mSignalledError = true; -- cgit v1.1