From 2b6f22dc64d456471a1dc6df09d515771d1427c8 Mon Sep 17 00:00:00 2001 From: Marco Nelissen Date: Fri, 8 Apr 2016 10:04:48 -0700 Subject: h264dec: check for overflows when calculating allocation size. Bug: 27855419 Change-Id: Idabedca52913ec31ea5cb6a6109ab94e3fb2badd --- .../libstagefright/codecs/on2/h264dec/source/DecTestBench.c | 9 ++++++--- .../codecs/on2/h264dec/source/EvaluationTestBench.c | 9 ++++++--- .../libstagefright/codecs/on2/h264dec/source/H264SwDecApi.c | 13 ++++++++++--- .../codecs/on2/h264dec/source/TestBenchMultipleInstance.c | 7 +++++-- .../codecs/on2/h264dec/source/h264bsd_decoder.c | 2 +- .../libstagefright/codecs/on2/h264dec/source/h264bsd_util.h | 2 +- 6 files changed, 29 insertions(+), 13 deletions(-) (limited to 'media/libstagefright/codecs/on2/h264dec/source') diff --git a/media/libstagefright/codecs/on2/h264dec/source/DecTestBench.c b/media/libstagefright/codecs/on2/h264dec/source/DecTestBench.c index dcf2ef6..55c0065 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/DecTestBench.c +++ b/media/libstagefright/codecs/on2/h264dec/source/DecTestBench.c @@ -700,18 +700,21 @@ void H264SwDecTrace(char *string) library function malloc for allocation of memory. ------------------------------------------------------------------------------*/ -void* H264SwDecMalloc(u32 size) +void* H264SwDecMalloc(u32 size, u32 num) { + if (size > UINT32_MAX / num) { + return NULL; + } #if defined(CHECK_MEMORY_USAGE) /* Note that if the decoder has to free and reallocate some of the buffers * the total value will be invalid */ static u32 numBytes = 0; - numBytes += size; + numBytes += size * num; DEBUG(("Allocated %d bytes, total %d\n", size, numBytes)); #endif - return malloc(size); + return malloc(size * num); } /*------------------------------------------------------------------------------ diff --git a/media/libstagefright/codecs/on2/h264dec/source/EvaluationTestBench.c b/media/libstagefright/codecs/on2/h264dec/source/EvaluationTestBench.c index aadc75f..e756a1f 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/EvaluationTestBench.c +++ b/media/libstagefright/codecs/on2/h264dec/source/EvaluationTestBench.c @@ -85,7 +85,7 @@ int main(int argc, char **argv) rewind(finput); /* allocate memory for stream buffer, exit if unsuccessful */ - byteStrm = byteStrmStart = (u8 *)H264SwDecMalloc(sizeof(u8)*strmLen); + byteStrm = byteStrmStart = (u8 *)H264SwDecMalloc(sizeof(u8), strmLen); if (byteStrm == NULL) { printf("UNABLE TO ALLOCATE MEMORY\n"); @@ -298,9 +298,12 @@ void H264SwDecTrace(char *string) library function malloc for allocation of memory. ------------------------------------------------------------------------------*/ -void* H264SwDecMalloc(u32 size) +void* H264SwDecMalloc(u32 size, u32 num) { - return malloc(size); + if (size > UINT32_MAX / num) { + return NULL; + } + return malloc(size * num); } /*------------------------------------------------------------------------------ diff --git a/media/libstagefright/codecs/on2/h264dec/source/H264SwDecApi.c b/media/libstagefright/codecs/on2/h264dec/source/H264SwDecApi.c index a073dcb..f820dfd 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/H264SwDecApi.c +++ b/media/libstagefright/codecs/on2/h264dec/source/H264SwDecApi.c @@ -35,6 +35,8 @@ /*------------------------------------------------------------------------------ 1. Include headers ------------------------------------------------------------------------------*/ +#include + #include #include #include "basetype.h" @@ -79,8 +81,13 @@ void H264SwDecTrace(char *string) { UNUSED(string); } -void* H264SwDecMalloc(u32 size) { - return malloc(size); +void* H264SwDecMalloc(u32 size, u32 num) { + if (size > UINT32_MAX / num) { + ALOGE("can't allocate %u * %u bytes", size, num); + android_errorWriteLog(0x534e4554, "27855419"); + return NULL; + } + return malloc(size * num); } void H264SwDecFree(void *ptr) { @@ -144,7 +151,7 @@ H264SwDecRet H264SwDecInit(H264SwDecInst *decInst, u32 noOutputReordering) return(H264SWDEC_PARAM_ERR); } - pDecCont = (decContainer_t *)H264SwDecMalloc(sizeof(decContainer_t)); + pDecCont = (decContainer_t *)H264SwDecMalloc(sizeof(decContainer_t), 1); if (pDecCont == NULL) { diff --git a/media/libstagefright/codecs/on2/h264dec/source/TestBenchMultipleInstance.c b/media/libstagefright/codecs/on2/h264dec/source/TestBenchMultipleInstance.c index 42170d3..9a386bb 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/TestBenchMultipleInstance.c +++ b/media/libstagefright/codecs/on2/h264dec/source/TestBenchMultipleInstance.c @@ -413,9 +413,12 @@ void H264SwDecTrace(char *string) Function name: H264SwDecmalloc ------------------------------------------------------------------------------*/ -void* H264SwDecMalloc(u32 size) +void* H264SwDecMalloc(u32 size, u32 num) { - return malloc(size); + if (size > UINT32_MAX / num) { + return NULL; + } + return malloc(size * num); } /*------------------------------------------------------------------------------ diff --git a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_decoder.c b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_decoder.c index a816871..0ac480f 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_decoder.c +++ b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_decoder.c @@ -101,7 +101,7 @@ u32 h264bsdInit(storage_t *pStorage, u32 noOutputReordering) * specific NEON optimized "memset" for clearing the structure */ size = (sizeof(macroblockLayer_t) + 63) & ~0x3F; - pStorage->mbLayer = (macroblockLayer_t*)H264SwDecMalloc(size); + pStorage->mbLayer = (macroblockLayer_t*)H264SwDecMalloc(size, 1); if (!pStorage->mbLayer) return HANTRO_NOK; diff --git a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_util.h b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_util.h index 216ad04..9f0eb7d 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_util.h +++ b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_util.h @@ -141,7 +141,7 @@ /* macro to allocate memory */ #define ALLOCATE(ptr, count, type) \ { \ - (ptr) = H264SwDecMalloc((count) * sizeof(type)); \ + (ptr) = H264SwDecMalloc(sizeof(type), (count)); \ } /* macro to free allocated memory */ -- cgit v1.1 From 60547808ca4e9cfac50028c00c58a6ceb2319301 Mon Sep 17 00:00:00 2001 From: Pawin Vongmasa Date: Wed, 11 May 2016 16:08:21 -0700 Subject: h264bsdActivateParamSets: Prevent multiplication overflow. Report MEMORY_ALLOCATION_ERROR if pStorage->picSizeInMbs would exceed UINT32_MAX bytes. Bug: 28532266 Change-Id: Ia6f11efb18818afcdb5fa2a38a14f2a2d8c8447a --- .../codecs/on2/h264dec/source/h264bsd_storage.c | 24 +++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) (limited to 'media/libstagefright/codecs/on2/h264dec/source') diff --git a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_storage.c b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_storage.c index 3234754..ff7a42a 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_storage.c +++ b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_storage.c @@ -58,6 +58,10 @@ 3. Module defines ------------------------------------------------------------------------------*/ +#ifndef UINT32_MAX +#define UINT32_MAX (4294967295U) +#endif + /*------------------------------------------------------------------------------ 4. Local function prototypes ------------------------------------------------------------------------------*/ @@ -326,9 +330,23 @@ u32 h264bsdActivateParamSets(storage_t *pStorage, u32 ppsId, u32 isIdr) pStorage->activePps = pStorage->pps[ppsId]; pStorage->activeSpsId = pStorage->activePps->seqParameterSetId; pStorage->activeSps = pStorage->sps[pStorage->activeSpsId]; - pStorage->picSizeInMbs = - pStorage->activeSps->picWidthInMbs * - pStorage->activeSps->picHeightInMbs; + + /* report error before multiplication to prevent integer overflow */ + if (pStorage->activeSps->picWidthInMbs == 0) + { + pStorage->picSizeInMbs = 0; + } + else if (pStorage->activeSps->picHeightInMbs > + UINT32_MAX / pStorage->activeSps->picWidthInMbs) + { + return(MEMORY_ALLOCATION_ERROR); + } + else + { + pStorage->picSizeInMbs = + pStorage->activeSps->picWidthInMbs * + pStorage->activeSps->picHeightInMbs; + } pStorage->currImage->width = pStorage->activeSps->picWidthInMbs; pStorage->currImage->height = pStorage->activeSps->picHeightInMbs; -- cgit v1.1 From 590d1729883f700ab905cdc9ad850f3ddd7e1f56 Mon Sep 17 00:00:00 2001 From: Marco Nelissen Date: Tue, 7 Jun 2016 15:48:07 -0700 Subject: Fix potential overflow Bug: 28533562 Change-Id: I798ab24caa4c81f3ba564cad7c9ee019284fb702 --- media/libstagefright/codecs/on2/h264dec/source/h264bsd_dpb.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'media/libstagefright/codecs/on2/h264dec/source') diff --git a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_dpb.c b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_dpb.c index 9517d0a..799bd16 100644 --- a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_dpb.c +++ b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_dpb.c @@ -60,6 +60,7 @@ #include "h264bsd_util.h" #include "basetype.h" +#include /*------------------------------------------------------------------------------ 2. External compiler flags -------------------------------------------------------------------------------- @@ -998,6 +999,13 @@ u32 h264bsdInitDpb( ASSERT(maxFrameNum); ASSERT(dpbSize); + // see comment in loop below about size calculation + if (picSizeInMbs > (UINT32_MAX - 32 - 15) / 384) { + ALOGE("b/28533562"); + android_errorWriteLog(0x534e4554, "28533562"); + return(MEMORY_ALLOCATION_ERROR); + } + dpb->maxLongTermFrameIdx = NO_LONG_TERM_FRAME_INDICES; dpb->maxRefFrames = MAX(maxRefFrames, 1); if (noReordering) -- cgit v1.1