diff options
Diffstat (limited to 'media/libstagefright/codecs/on2')
16 files changed, 222 insertions, 88 deletions
diff --git a/media/libstagefright/codecs/on2/dec/SoftVPX.cpp b/media/libstagefright/codecs/on2/dec/SoftVPX.cpp index e161fb8..444a7fc 100644 --- a/media/libstagefright/codecs/on2/dec/SoftVPX.cpp +++ b/media/libstagefright/codecs/on2/dec/SoftVPX.cpp @@ -102,7 +102,6 @@ status_t SoftVPX::destroyDecoder() { } bool SoftVPX::outputBuffers(bool flushDecoder, bool display, bool eos, bool *portWillReset) { - List<BufferInfo *> &inQueue = getPortQueue(0); List<BufferInfo *> &outQueue = getPortQueue(1); BufferInfo *outInfo = NULL; OMX_BUFFERHEADERTYPE *outHeader = NULL; @@ -149,15 +148,18 @@ bool SoftVPX::outputBuffers(bool flushDecoder, bool display, bool eos, bool *por outHeader->nFlags = 0; outHeader->nFilledLen = (outputBufferWidth() * outputBufferHeight() * 3) / 2; outHeader->nTimeStamp = *(OMX_TICKS *)mImg->user_priv; - - uint8_t *dst = outHeader->pBuffer; - const uint8_t *srcY = (const uint8_t *)mImg->planes[VPX_PLANE_Y]; - const uint8_t *srcU = (const uint8_t *)mImg->planes[VPX_PLANE_U]; - const uint8_t *srcV = (const uint8_t *)mImg->planes[VPX_PLANE_V]; - size_t srcYStride = mImg->stride[VPX_PLANE_Y]; - size_t srcUStride = mImg->stride[VPX_PLANE_U]; - size_t srcVStride = mImg->stride[VPX_PLANE_V]; - copyYV12FrameToOutputBuffer(dst, srcY, srcU, srcV, srcYStride, srcUStride, srcVStride); + if (outputBufferSafe(outHeader)) { + uint8_t *dst = outHeader->pBuffer; + const uint8_t *srcY = (const uint8_t *)mImg->planes[VPX_PLANE_Y]; + const uint8_t *srcU = (const uint8_t *)mImg->planes[VPX_PLANE_U]; + const uint8_t *srcV = (const uint8_t *)mImg->planes[VPX_PLANE_V]; + size_t srcYStride = mImg->stride[VPX_PLANE_Y]; + size_t srcUStride = mImg->stride[VPX_PLANE_U]; + size_t srcVStride = mImg->stride[VPX_PLANE_V]; + copyYV12FrameToOutputBuffer(dst, srcY, srcU, srcV, srcYStride, srcUStride, srcVStride); + } else { + outHeader->nFilledLen = 0; + } mImg = NULL; outInfo->mOwnedByUs = false; @@ -185,6 +187,25 @@ bool SoftVPX::outputBuffers(bool flushDecoder, bool display, bool eos, bool *por return true; } +bool SoftVPX::outputBufferSafe(OMX_BUFFERHEADERTYPE *outHeader) { + uint32_t width = outputBufferWidth(); + uint32_t height = outputBufferHeight(); + uint64_t nFilledLen = width; + nFilledLen *= height; + if (nFilledLen > UINT32_MAX / 3) { + ALOGE("b/29421675, nFilledLen overflow %llu w %u h %u", + (unsigned long long)nFilledLen, width, height); + android_errorWriteLog(0x534e4554, "29421675"); + return false; + } else if (outHeader->nAllocLen < outHeader->nFilledLen) { + ALOGE("b/27597103, buffer too small"); + android_errorWriteLog(0x534e4554, "27597103"); + return false; + } + + return true; +} + void SoftVPX::onQueueFilled(OMX_U32 /* portIndex */) { if (mOutputPortSettingsChange != NONE || mEOSStatus == OUTPUT_FRAMES_FLUSHED) { return; @@ -193,7 +214,6 @@ void SoftVPX::onQueueFilled(OMX_U32 /* portIndex */) { List<BufferInfo *> &inQueue = getPortQueue(0); List<BufferInfo *> &outQueue = getPortQueue(1); bool EOSseen = false; - vpx_codec_err_t err; bool portWillReset = false; while ((mEOSStatus == INPUT_EOS_SEEN || !inQueue.empty()) @@ -217,8 +237,6 @@ void SoftVPX::onQueueFilled(OMX_U32 /* portIndex */) { OMX_BUFFERHEADERTYPE *inHeader = inInfo->mHeader; mTimeStamps[mTimeStampIdx] = inHeader->nTimeStamp; - BufferInfo *outInfo = *outQueue.begin(); - OMX_BUFFERHEADERTYPE *outHeader = outInfo->mHeader; if (inHeader->nFlags & OMX_BUFFERFLAG_EOS) { mEOSStatus = INPUT_EOS_SEEN; EOSseen = true; diff --git a/media/libstagefright/codecs/on2/dec/SoftVPX.h b/media/libstagefright/codecs/on2/dec/SoftVPX.h index 8ccbae2..84cf79c 100644 --- a/media/libstagefright/codecs/on2/dec/SoftVPX.h +++ b/media/libstagefright/codecs/on2/dec/SoftVPX.h @@ -66,6 +66,7 @@ private: status_t initDecoder(); status_t destroyDecoder(); bool outputBuffers(bool flushDecoder, bool display, bool eos, bool *portWillReset); + bool outputBufferSafe(OMX_BUFFERHEADERTYPE *outHeader); DISALLOW_EVIL_CONSTRUCTORS(SoftVPX); }; diff --git a/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.cpp b/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.cpp index 410f9d0..5c950c7 100644 --- a/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.cpp +++ b/media/libstagefright/codecs/on2/enc/SoftVPXEncoder.cpp @@ -368,20 +368,24 @@ OMX_ERRORTYPE SoftVPXEncoder::internalGetParameter(OMX_INDEXTYPE index, OMX_VIDEO_PARAM_BITRATETYPE *bitrate = (OMX_VIDEO_PARAM_BITRATETYPE *)param; - if (bitrate->nPortIndex != kOutputPortIndex) { - return OMX_ErrorUnsupportedIndex; - } + if (!isValidOMXParam(bitrate)) { + return OMX_ErrorBadParameter; + } - bitrate->nTargetBitrate = mBitrate; + if (bitrate->nPortIndex != kOutputPortIndex) { + return OMX_ErrorUnsupportedIndex; + } - if (mBitrateControlMode == VPX_VBR) { - bitrate->eControlRate = OMX_Video_ControlRateVariable; - } else if (mBitrateControlMode == VPX_CBR) { - bitrate->eControlRate = OMX_Video_ControlRateConstant; - } else { - return OMX_ErrorUnsupportedSetting; - } - return OMX_ErrorNone; + bitrate->nTargetBitrate = mBitrate; + + if (mBitrateControlMode == VPX_VBR) { + bitrate->eControlRate = OMX_Video_ControlRateVariable; + } else if (mBitrateControlMode == VPX_CBR) { + bitrate->eControlRate = OMX_Video_ControlRateConstant; + } else { + return OMX_ErrorUnsupportedSetting; + } + return OMX_ErrorNone; } // VP8 specific parameters that use extension headers @@ -389,33 +393,41 @@ OMX_ERRORTYPE SoftVPXEncoder::internalGetParameter(OMX_INDEXTYPE index, OMX_VIDEO_PARAM_VP8TYPE *vp8Params = (OMX_VIDEO_PARAM_VP8TYPE *)param; - if (vp8Params->nPortIndex != kOutputPortIndex) { - return OMX_ErrorUnsupportedIndex; - } + if (!isValidOMXParam(vp8Params)) { + return OMX_ErrorBadParameter; + } + + if (vp8Params->nPortIndex != kOutputPortIndex) { + return OMX_ErrorUnsupportedIndex; + } - vp8Params->eProfile = OMX_VIDEO_VP8ProfileMain; - vp8Params->eLevel = mLevel; - vp8Params->nDCTPartitions = mDCTPartitions; - vp8Params->bErrorResilientMode = mErrorResilience; - return OMX_ErrorNone; + vp8Params->eProfile = OMX_VIDEO_VP8ProfileMain; + vp8Params->eLevel = mLevel; + vp8Params->nDCTPartitions = mDCTPartitions; + vp8Params->bErrorResilientMode = mErrorResilience; + return OMX_ErrorNone; } case OMX_IndexParamVideoAndroidVp8Encoder: { OMX_VIDEO_PARAM_ANDROID_VP8ENCODERTYPE *vp8AndroidParams = (OMX_VIDEO_PARAM_ANDROID_VP8ENCODERTYPE *)param; - if (vp8AndroidParams->nPortIndex != kOutputPortIndex) { - return OMX_ErrorUnsupportedIndex; - } + if (!isValidOMXParam(vp8AndroidParams)) { + return OMX_ErrorBadParameter; + } - vp8AndroidParams->nKeyFrameInterval = mKeyFrameInterval; - vp8AndroidParams->eTemporalPattern = mTemporalPatternType; - vp8AndroidParams->nTemporalLayerCount = mTemporalLayers; - vp8AndroidParams->nMinQuantizer = mMinQuantizer; - vp8AndroidParams->nMaxQuantizer = mMaxQuantizer; - memcpy(vp8AndroidParams->nTemporalLayerBitrateRatio, - mTemporalLayerBitrateRatio, sizeof(mTemporalLayerBitrateRatio)); - return OMX_ErrorNone; + if (vp8AndroidParams->nPortIndex != kOutputPortIndex) { + return OMX_ErrorUnsupportedIndex; + } + + vp8AndroidParams->nKeyFrameInterval = mKeyFrameInterval; + vp8AndroidParams->eTemporalPattern = mTemporalPatternType; + vp8AndroidParams->nTemporalLayerCount = mTemporalLayers; + vp8AndroidParams->nMinQuantizer = mMinQuantizer; + vp8AndroidParams->nMaxQuantizer = mMaxQuantizer; + memcpy(vp8AndroidParams->nTemporalLayerBitrateRatio, + mTemporalLayerBitrateRatio, sizeof(mTemporalLayerBitrateRatio)); + return OMX_ErrorNone; } default: @@ -430,17 +442,38 @@ OMX_ERRORTYPE SoftVPXEncoder::internalSetParameter(OMX_INDEXTYPE index, const int32_t indexFull = index; switch (indexFull) { - case OMX_IndexParamVideoBitrate: - return internalSetBitrateParams( - (const OMX_VIDEO_PARAM_BITRATETYPE *)param); + case OMX_IndexParamVideoBitrate: { + const OMX_VIDEO_PARAM_BITRATETYPE *bitRate = + (const OMX_VIDEO_PARAM_BITRATETYPE*) param; + + if (!isValidOMXParam(bitRate)) { + return OMX_ErrorBadParameter; + } - case OMX_IndexParamVideoVp8: - return internalSetVp8Params( - (const OMX_VIDEO_PARAM_VP8TYPE *)param); + return internalSetBitrateParams(bitRate); + } - case OMX_IndexParamVideoAndroidVp8Encoder: - return internalSetAndroidVp8Params( - (const OMX_VIDEO_PARAM_ANDROID_VP8ENCODERTYPE *)param); + case OMX_IndexParamVideoVp8: { + const OMX_VIDEO_PARAM_VP8TYPE *vp8Params = + (const OMX_VIDEO_PARAM_VP8TYPE*) param; + + if (!isValidOMXParam(vp8Params)) { + return OMX_ErrorBadParameter; + } + + return internalSetVp8Params(vp8Params); + } + + case OMX_IndexParamVideoAndroidVp8Encoder: { + const OMX_VIDEO_PARAM_ANDROID_VP8ENCODERTYPE *vp8AndroidParams = + (const OMX_VIDEO_PARAM_ANDROID_VP8ENCODERTYPE*) param; + + if (!isValidOMXParam(vp8AndroidParams)) { + return OMX_ErrorBadParameter; + } + + return internalSetAndroidVp8Params(vp8AndroidParams); + } default: return SoftVideoEncoderOMXComponent::internalSetParameter(index, param); @@ -455,6 +488,10 @@ OMX_ERRORTYPE SoftVPXEncoder::setConfig( OMX_CONFIG_INTRAREFRESHVOPTYPE *params = (OMX_CONFIG_INTRAREFRESHVOPTYPE *)_params; + if (!isValidOMXParam(params)) { + return OMX_ErrorBadParameter; + } + if (params->nPortIndex != kOutputPortIndex) { return OMX_ErrorBadPortIndex; } @@ -468,6 +505,10 @@ OMX_ERRORTYPE SoftVPXEncoder::setConfig( OMX_VIDEO_CONFIG_BITRATETYPE *params = (OMX_VIDEO_CONFIG_BITRATETYPE *)_params; + if (!isValidOMXParam(params)) { + return OMX_ErrorBadParameter; + } + if (params->nPortIndex != kOutputPortIndex) { return OMX_ErrorBadPortIndex; } @@ -688,9 +729,10 @@ void SoftVPXEncoder::onQueueFilled(OMX_U32 /* portIndex */) { const uint8_t *source = inputBufferHeader->pBuffer + inputBufferHeader->nOffset; + size_t frameSize = mWidth * mHeight * 3 / 2; if (mInputDataIsMeta) { source = extractGraphicBuffer( - mConversionBuffer, mWidth * mHeight * 3 / 2, + mConversionBuffer, frameSize, source, inputBufferHeader->nFilledLen, mWidth, mHeight); if (source == NULL) { @@ -698,11 +740,21 @@ void SoftVPXEncoder::onQueueFilled(OMX_U32 /* portIndex */) { notify(OMX_EventError, OMX_ErrorUndefined, 0, 0); return; } - } else if (mColorFormat == OMX_COLOR_FormatYUV420SemiPlanar) { - ConvertYUV420SemiPlanarToYUV420Planar( - source, mConversionBuffer, mWidth, mHeight); + } else { + if (inputBufferHeader->nFilledLen < frameSize) { + android_errorWriteLog(0x534e4554, "27569635"); + notify(OMX_EventError, OMX_ErrorUndefined, 0, 0); + return; + } else if (inputBufferHeader->nFilledLen > frameSize) { + ALOGW("Input buffer contains too many pixels"); + } - source = mConversionBuffer; + if (mColorFormat == OMX_COLOR_FormatYUV420SemiPlanar) { + ConvertYUV420SemiPlanarToYUV420Planar( + source, mConversionBuffer, mWidth, mHeight); + + source = mConversionBuffer; + } } vpx_image_t raw_frame; vpx_img_wrap(&raw_frame, VPX_IMG_FMT_I420, mWidth, mHeight, @@ -764,9 +816,14 @@ void SoftVPXEncoder::onQueueFilled(OMX_U32 /* portIndex */) { outputBufferHeader->nTimeStamp = encoded_packet->data.frame.pts; outputBufferHeader->nFlags = 0; if (encoded_packet->data.frame.flags & VPX_FRAME_IS_KEY) - outputBufferHeader->nFlags |= OMX_BUFFERFLAG_SYNCFRAME; + outputBufferHeader->nFlags |= OMX_BUFFERFLAG_SYNCFRAME; outputBufferHeader->nOffset = 0; outputBufferHeader->nFilledLen = encoded_packet->data.frame.sz; + if (outputBufferHeader->nFilledLen > outputBufferHeader->nAllocLen) { + android_errorWriteLog(0x534e4554, "27569635"); + notify(OMX_EventError, OMX_ErrorUndefined, 0, 0); + return; + } memcpy(outputBufferHeader->pBuffer, encoded_packet->data.frame.buf, encoded_packet->data.frame.sz); diff --git a/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp b/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp index 6b8b395..2f61d12 100644 --- a/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp +++ b/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp @@ -202,7 +202,12 @@ void SoftAVC::onQueueFilled(OMX_U32 /* portIndex */) { } if (mFirstPicture && !outQueue.empty()) { - drainOneOutputBuffer(mFirstPictureId, mFirstPicture); + if (!drainOneOutputBuffer(mFirstPictureId, mFirstPicture)) { + ALOGE("Drain failed"); + notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); + mSignalledError = true; + return; + } delete[] mFirstPicture; mFirstPicture = NULL; mFirstPictureId = -1; @@ -242,15 +247,20 @@ void SoftAVC::saveFirstOutputBuffer(int32_t picId, uint8_t *data) { memcpy(mFirstPicture, data, pictureSize); } -void SoftAVC::drainOneOutputBuffer(int32_t picId, uint8_t* data) { +bool SoftAVC::drainOneOutputBuffer(int32_t picId, uint8_t* data) { List<BufferInfo *> &outQueue = getPortQueue(kOutputPortIndex); BufferInfo *outInfo = *outQueue.begin(); - outQueue.erase(outQueue.begin()); OMX_BUFFERHEADERTYPE *outHeader = outInfo->mHeader; + OMX_U32 frameSize = mWidth * mHeight * 3 / 2; + if (outHeader->nAllocLen - outHeader->nOffset < frameSize) { + android_errorWriteLog(0x534e4554, "27833616"); + return false; + } + outQueue.erase(outQueue.begin()); OMX_BUFFERHEADERTYPE *header = mPicToHeaderMap.valueFor(picId); outHeader->nTimeStamp = header->nTimeStamp; outHeader->nFlags = header->nFlags; - outHeader->nFilledLen = mWidth * mHeight * 3 / 2; + outHeader->nFilledLen = frameSize; uint8_t *dst = outHeader->pBuffer + outHeader->nOffset; const uint8_t *srcY = data; @@ -265,6 +275,7 @@ void SoftAVC::drainOneOutputBuffer(int32_t picId, uint8_t* data) { delete header; outInfo->mOwnedByUs = false; notifyFillBufferDone(outHeader); + return true; } void SoftAVC::drainAllOutputBuffers(bool eos) { @@ -277,7 +288,12 @@ void SoftAVC::drainAllOutputBuffers(bool eos) { mHandle, &decodedPicture, eos /* flush */)) { int32_t picId = decodedPicture.picId; uint8_t *data = (uint8_t *) decodedPicture.pOutputPicture; - drainOneOutputBuffer(picId, data); + if (!drainOneOutputBuffer(picId, data)) { + ALOGE("Drain failed"); + notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); + mSignalledError = true; + return; + } } } diff --git a/media/libstagefright/codecs/on2/h264dec/SoftAVC.h b/media/libstagefright/codecs/on2/h264dec/SoftAVC.h index 069107d..b8c1807 100644 --- a/media/libstagefright/codecs/on2/h264dec/SoftAVC.h +++ b/media/libstagefright/codecs/on2/h264dec/SoftAVC.h @@ -71,7 +71,7 @@ private: status_t initDecoder(); void drainAllOutputBuffers(bool eos); - void drainOneOutputBuffer(int32_t picId, uint8_t *data); + bool drainOneOutputBuffer(int32_t picId, uint8_t *data); void saveFirstOutputBuffer(int32_t pidId, uint8_t *data); CropSettingsMode handleCropParams(const H264SwDecInfo& decInfo); diff --git a/media/libstagefright/codecs/on2/h264dec/inc/H264SwDecApi.h b/media/libstagefright/codecs/on2/h264dec/inc/H264SwDecApi.h index fe112bc..9814e73 100644 --- a/media/libstagefright/codecs/on2/h264dec/inc/H264SwDecApi.h +++ b/media/libstagefright/codecs/on2/h264dec/inc/H264SwDecApi.h @@ -161,7 +161,7 @@ extern "C" void H264SwDecTrace(char *); /* function prototype for memory allocation */ - void* H264SwDecMalloc(u32 size); + void* H264SwDecMalloc(u32 size, u32 num); /* function prototype for memory free */ void H264SwDecFree(void *ptr); diff --git a/media/libstagefright/codecs/on2/h264dec/omxdl/arm_neon/api/armCOMM.h b/media/libstagefright/codecs/on2/h264dec/omxdl/arm_neon/api/armCOMM.h index 91e38b8..1992885 100644 --- a/media/libstagefright/codecs/on2/h264dec/omxdl/arm_neon/api/armCOMM.h +++ b/media/libstagefright/codecs/on2/h264dec/omxdl/arm_neon/api/armCOMM.h @@ -86,7 +86,7 @@ typedef OMX_S16 ARM_BLOCK8x8[64]; /* Alignment operation */ -#define armAlignToBytes(Ptr,N) (Ptr + ( ((N-(int)Ptr)&(N-1)) / sizeof(*Ptr) )) +#define armAlignToBytes(Ptr,N) (Ptr + ( ((N-(intptr_t)Ptr)&(N-1)) / sizeof(*Ptr) )) #define armAlignTo2Bytes(Ptr) armAlignToBytes(Ptr,2) #define armAlignTo4Bytes(Ptr) armAlignToBytes(Ptr,4) #define armAlignTo8Bytes(Ptr) armAlignToBytes(Ptr,8) @@ -98,8 +98,8 @@ typedef OMX_S16 ARM_BLOCK8x8[64]; #define armRetDataErrIf(condition, code) if(condition) { return (code); } #ifndef ALIGNMENT_DOESNT_MATTER -#define armIsByteAligned(Ptr,N) ((((int)(Ptr)) % N)==0) -#define armNotByteAligned(Ptr,N) ((((int)(Ptr)) % N)!=0) +#define armIsByteAligned(Ptr,N) ((((intptr_t)(Ptr)) % N)==0) +#define armNotByteAligned(Ptr,N) ((((intptr_t)(Ptr)) % N)!=0) #else #define armIsByteAligned(Ptr,N) (1) #define armNotByteAligned(Ptr,N) (0) diff --git a/media/libstagefright/codecs/on2/h264dec/omxdl/reference/api/armCOMM.h b/media/libstagefright/codecs/on2/h264dec/omxdl/reference/api/armCOMM.h index fbb97e2..7304863 100644 --- a/media/libstagefright/codecs/on2/h264dec/omxdl/reference/api/armCOMM.h +++ b/media/libstagefright/codecs/on2/h264dec/omxdl/reference/api/armCOMM.h @@ -86,7 +86,7 @@ typedef OMX_S16 ARM_BLOCK8x8[64]; /* Alignment operation */ -#define armAlignToBytes(Ptr,N) (Ptr + ( ((N-(int)Ptr)&(N-1)) / sizeof(*Ptr) )) +#define armAlignToBytes(Ptr,N) (Ptr + ( ((N-(intptr_t)Ptr)&(N-1)) / sizeof(*Ptr) )) #define armAlignTo2Bytes(Ptr) armAlignToBytes(Ptr,2) #define armAlignTo4Bytes(Ptr) armAlignToBytes(Ptr,4) #define armAlignTo8Bytes(Ptr) armAlignToBytes(Ptr,8) @@ -98,8 +98,8 @@ typedef OMX_S16 ARM_BLOCK8x8[64]; #define armRetDataErrIf(condition, code) if(condition) { return (code); } #ifndef ALIGNMENT_DOESNT_MATTER -#define armIsByteAligned(Ptr,N) ((((int)(Ptr)) % N)==0) -#define armNotByteAligned(Ptr,N) ((((int)(Ptr)) % N)!=0) +#define armIsByteAligned(Ptr,N) ((((intptr_t)(Ptr)) % N)==0) +#define armNotByteAligned(Ptr,N) ((((intptr_t)(Ptr)) % N)!=0) #else #define armIsByteAligned(Ptr,N) (1) #define armNotByteAligned(Ptr,N) (0) diff --git a/media/libstagefright/codecs/on2/h264dec/source/DecTestBench.c b/media/libstagefright/codecs/on2/h264dec/source/DecTestBench.c index dcf2ef6..55c0065 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/DecTestBench.c +++ b/media/libstagefright/codecs/on2/h264dec/source/DecTestBench.c @@ -700,18 +700,21 @@ void H264SwDecTrace(char *string) library function malloc for allocation of memory. ------------------------------------------------------------------------------*/ -void* H264SwDecMalloc(u32 size) +void* H264SwDecMalloc(u32 size, u32 num) { + if (size > UINT32_MAX / num) { + return NULL; + } #if defined(CHECK_MEMORY_USAGE) /* Note that if the decoder has to free and reallocate some of the buffers * the total value will be invalid */ static u32 numBytes = 0; - numBytes += size; + numBytes += size * num; DEBUG(("Allocated %d bytes, total %d\n", size, numBytes)); #endif - return malloc(size); + return malloc(size * num); } /*------------------------------------------------------------------------------ diff --git a/media/libstagefright/codecs/on2/h264dec/source/EvaluationTestBench.c b/media/libstagefright/codecs/on2/h264dec/source/EvaluationTestBench.c index aadc75f..e756a1f 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/EvaluationTestBench.c +++ b/media/libstagefright/codecs/on2/h264dec/source/EvaluationTestBench.c @@ -85,7 +85,7 @@ int main(int argc, char **argv) rewind(finput); /* allocate memory for stream buffer, exit if unsuccessful */ - byteStrm = byteStrmStart = (u8 *)H264SwDecMalloc(sizeof(u8)*strmLen); + byteStrm = byteStrmStart = (u8 *)H264SwDecMalloc(sizeof(u8), strmLen); if (byteStrm == NULL) { printf("UNABLE TO ALLOCATE MEMORY\n"); @@ -298,9 +298,12 @@ void H264SwDecTrace(char *string) library function malloc for allocation of memory. ------------------------------------------------------------------------------*/ -void* H264SwDecMalloc(u32 size) +void* H264SwDecMalloc(u32 size, u32 num) { - return malloc(size); + if (size > UINT32_MAX / num) { + return NULL; + } + return malloc(size * num); } /*------------------------------------------------------------------------------ diff --git a/media/libstagefright/codecs/on2/h264dec/source/H264SwDecApi.c b/media/libstagefright/codecs/on2/h264dec/source/H264SwDecApi.c index a073dcb..f820dfd 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/H264SwDecApi.c +++ b/media/libstagefright/codecs/on2/h264dec/source/H264SwDecApi.c @@ -35,6 +35,8 @@ /*------------------------------------------------------------------------------ 1. Include headers ------------------------------------------------------------------------------*/ +#include <log/log.h> + #include <stdlib.h> #include <string.h> #include "basetype.h" @@ -79,8 +81,13 @@ void H264SwDecTrace(char *string) { UNUSED(string); } -void* H264SwDecMalloc(u32 size) { - return malloc(size); +void* H264SwDecMalloc(u32 size, u32 num) { + if (size > UINT32_MAX / num) { + ALOGE("can't allocate %u * %u bytes", size, num); + android_errorWriteLog(0x534e4554, "27855419"); + return NULL; + } + return malloc(size * num); } void H264SwDecFree(void *ptr) { @@ -144,7 +151,7 @@ H264SwDecRet H264SwDecInit(H264SwDecInst *decInst, u32 noOutputReordering) return(H264SWDEC_PARAM_ERR); } - pDecCont = (decContainer_t *)H264SwDecMalloc(sizeof(decContainer_t)); + pDecCont = (decContainer_t *)H264SwDecMalloc(sizeof(decContainer_t), 1); if (pDecCont == NULL) { diff --git a/media/libstagefright/codecs/on2/h264dec/source/TestBenchMultipleInstance.c b/media/libstagefright/codecs/on2/h264dec/source/TestBenchMultipleInstance.c index 42170d3..9a386bb 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/TestBenchMultipleInstance.c +++ b/media/libstagefright/codecs/on2/h264dec/source/TestBenchMultipleInstance.c @@ -413,9 +413,12 @@ void H264SwDecTrace(char *string) Function name: H264SwDecmalloc ------------------------------------------------------------------------------*/ -void* H264SwDecMalloc(u32 size) +void* H264SwDecMalloc(u32 size, u32 num) { - return malloc(size); + if (size > UINT32_MAX / num) { + return NULL; + } + return malloc(size * num); } /*------------------------------------------------------------------------------ diff --git a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_decoder.c b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_decoder.c index a816871..0ac480f 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_decoder.c +++ b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_decoder.c @@ -101,7 +101,7 @@ u32 h264bsdInit(storage_t *pStorage, u32 noOutputReordering) * specific NEON optimized "memset" for clearing the structure */ size = (sizeof(macroblockLayer_t) + 63) & ~0x3F; - pStorage->mbLayer = (macroblockLayer_t*)H264SwDecMalloc(size); + pStorage->mbLayer = (macroblockLayer_t*)H264SwDecMalloc(size, 1); if (!pStorage->mbLayer) return HANTRO_NOK; diff --git a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_dpb.c b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_dpb.c index 9517d0a..799bd16 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_dpb.c +++ b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_dpb.c @@ -60,6 +60,7 @@ #include "h264bsd_util.h" #include "basetype.h" +#include <log/log.h> /*------------------------------------------------------------------------------ 2. External compiler flags -------------------------------------------------------------------------------- @@ -998,6 +999,13 @@ u32 h264bsdInitDpb( ASSERT(maxFrameNum); ASSERT(dpbSize); + // see comment in loop below about size calculation + if (picSizeInMbs > (UINT32_MAX - 32 - 15) / 384) { + ALOGE("b/28533562"); + android_errorWriteLog(0x534e4554, "28533562"); + return(MEMORY_ALLOCATION_ERROR); + } + dpb->maxLongTermFrameIdx = NO_LONG_TERM_FRAME_INDICES; dpb->maxRefFrames = MAX(maxRefFrames, 1); if (noReordering) diff --git a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_storage.c b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_storage.c index 3234754..ff7a42a 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_storage.c +++ b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_storage.c @@ -58,6 +58,10 @@ 3. Module defines ------------------------------------------------------------------------------*/ +#ifndef UINT32_MAX +#define UINT32_MAX (4294967295U) +#endif + /*------------------------------------------------------------------------------ 4. Local function prototypes ------------------------------------------------------------------------------*/ @@ -326,9 +330,23 @@ u32 h264bsdActivateParamSets(storage_t *pStorage, u32 ppsId, u32 isIdr) pStorage->activePps = pStorage->pps[ppsId]; pStorage->activeSpsId = pStorage->activePps->seqParameterSetId; pStorage->activeSps = pStorage->sps[pStorage->activeSpsId]; - pStorage->picSizeInMbs = - pStorage->activeSps->picWidthInMbs * - pStorage->activeSps->picHeightInMbs; + + /* report error before multiplication to prevent integer overflow */ + if (pStorage->activeSps->picWidthInMbs == 0) + { + pStorage->picSizeInMbs = 0; + } + else if (pStorage->activeSps->picHeightInMbs > + UINT32_MAX / pStorage->activeSps->picWidthInMbs) + { + return(MEMORY_ALLOCATION_ERROR); + } + else + { + pStorage->picSizeInMbs = + pStorage->activeSps->picWidthInMbs * + pStorage->activeSps->picHeightInMbs; + } pStorage->currImage->width = pStorage->activeSps->picWidthInMbs; pStorage->currImage->height = pStorage->activeSps->picHeightInMbs; diff --git a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_util.h b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_util.h index 216ad04..9f0eb7d 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_util.h +++ b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_util.h @@ -141,7 +141,7 @@ /* macro to allocate memory */ #define ALLOCATE(ptr, count, type) \ { \ - (ptr) = H264SwDecMalloc((count) * sizeof(type)); \ + (ptr) = H264SwDecMalloc(sizeof(type), (count)); \ } /* macro to free allocated memory */ |