h264dec: check for overflows when calculating allocation size.

Bug: 27855419
Change-Id: Idabedca52913ec31ea5cb6a6109ab94e3fb2badd

7 files changed, 30 insertions, 14 deletions

diff --git a/media/libstagefright/codecs/on2/h264dec/inc/H264SwDecApi.h b/media/libstagefright/codecs/on2/h264dec/inc/H264SwDecApi.h
index fe112bc..9814e73 100644
--- a/media/libstagefright/codecs/on2/h264dec/inc/H264SwDecApi.h
+++ b/media/libstagefright/codecs/on2/h264dec/inc/H264SwDecApi.h
@@ -161,7 +161,7 @@ extern "C"
     void H264SwDecTrace(char *);
 
     /* function prototype for memory allocation */
-    void* H264SwDecMalloc(u32 size);
+    void* H264SwDecMalloc(u32 size, u32 num);
 
     /* function prototype for memory free */
     void H264SwDecFree(void *ptr);
diff --git a/media/libstagefright/codecs/on2/h264dec/source/DecTestBench.c b/media/libstagefright/codecs/on2/h264dec/source/DecTestBench.c
index dcf2ef6..55c0065 100644
--- a/media/libstagefright/codecs/on2/h264dec/source/DecTestBench.c
+++ b/media/libstagefright/codecs/on2/h264dec/source/DecTestBench.c
@@ -700,18 +700,21 @@ void H264SwDecTrace(char *string)
         library function malloc for allocation of memory.
 
 ------------------------------------------------------------------------------*/
-void* H264SwDecMalloc(u32 size)
+void* H264SwDecMalloc(u32 size, u32 num)
 {
+    if (size > UINT32_MAX / num) {
+        return NULL;
+    }
 
 #if defined(CHECK_MEMORY_USAGE)
     /* Note that if the decoder has to free and reallocate some of the buffers
      * the total value will be invalid */
     static u32 numBytes = 0;
-    numBytes += size;
+    numBytes += size * num;
     DEBUG(("Allocated %d bytes, total %d\n", size, numBytes));
 #endif
 
-    return malloc(size);
+    return malloc(size * num);
 }
 
 /*------------------------------------------------------------------------------
diff --git a/media/libstagefright/codecs/on2/h264dec/source/EvaluationTestBench.c b/media/libstagefright/codecs/on2/h264dec/source/EvaluationTestBench.c
index aadc75f..e756a1f 100644
--- a/media/libstagefright/codecs/on2/h264dec/source/EvaluationTestBench.c
+++ b/media/libstagefright/codecs/on2/h264dec/source/EvaluationTestBench.c
@@ -85,7 +85,7 @@ int main(int argc, char **argv)
     rewind(finput);
 
     /* allocate memory for stream buffer, exit if unsuccessful */
-    byteStrm = byteStrmStart = (u8 *)H264SwDecMalloc(sizeof(u8)*strmLen);
+    byteStrm = byteStrmStart = (u8 *)H264SwDecMalloc(sizeof(u8), strmLen);
     if (byteStrm == NULL)
     {
         printf("UNABLE TO ALLOCATE MEMORY\n");
@@ -298,9 +298,12 @@ void H264SwDecTrace(char *string)
         library function malloc for allocation of memory.
 
 ------------------------------------------------------------------------------*/
-void* H264SwDecMalloc(u32 size)
+void* H264SwDecMalloc(u32 size, u32 num)
 {
-    return malloc(size);
+    if (size > UINT32_MAX / num) {
+        return NULL;
+    }
+    return malloc(size * num);
 }
 
 /*------------------------------------------------------------------------------
diff --git a/media/libstagefright/codecs/on2/h264dec/source/H264SwDecApi.c b/media/libstagefright/codecs/on2/h264dec/source/H264SwDecApi.c
index a073dcb..f820dfd 100644
--- a/media/libstagefright/codecs/on2/h264dec/source/H264SwDecApi.c
+++ b/media/libstagefright/codecs/on2/h264dec/source/H264SwDecApi.c
@@ -35,6 +35,8 @@
 /*------------------------------------------------------------------------------
     1. Include headers
 ------------------------------------------------------------------------------*/
+#include <log/log.h>
+
 #include <stdlib.h>
 #include <string.h>
 #include "basetype.h"
@@ -79,8 +81,13 @@ void H264SwDecTrace(char *string) {
     UNUSED(string);
 }
 
-void* H264SwDecMalloc(u32 size) {
-    return malloc(size);
+void* H264SwDecMalloc(u32 size, u32 num) {
+    if (size > UINT32_MAX / num) {
+        ALOGE("can't allocate %u * %u bytes", size, num);
+        android_errorWriteLog(0x534e4554, "27855419");
+        return NULL;
+    }
+    return malloc(size * num);
 }
 
 void H264SwDecFree(void *ptr) {
@@ -144,7 +151,7 @@ H264SwDecRet H264SwDecInit(H264SwDecInst *decInst, u32 noOutputReordering)
         return(H264SWDEC_PARAM_ERR);
     }
 
-    pDecCont = (decContainer_t *)H264SwDecMalloc(sizeof(decContainer_t));
+    pDecCont = (decContainer_t *)H264SwDecMalloc(sizeof(decContainer_t), 1);
 
     if (pDecCont == NULL)
     {
diff --git a/media/libstagefright/codecs/on2/h264dec/source/TestBenchMultipleInstance.c b/media/libstagefright/codecs/on2/h264dec/source/TestBenchMultipleInstance.c
index 42170d3..9a386bb 100644
--- a/media/libstagefright/codecs/on2/h264dec/source/TestBenchMultipleInstance.c
+++ b/media/libstagefright/codecs/on2/h264dec/source/TestBenchMultipleInstance.c
@@ -413,9 +413,12 @@ void H264SwDecTrace(char *string)
     Function name:  H264SwDecmalloc
 
 ------------------------------------------------------------------------------*/
-void* H264SwDecMalloc(u32 size)
+void* H264SwDecMalloc(u32 size, u32 num)
 {
-    return malloc(size);
+    if (size > UINT32_MAX / num) {
+        return NULL;
+    }
+    return malloc(size * num);
 }
 
 /*------------------------------------------------------------------------------
diff --git a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_decoder.c b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_decoder.c
index a816871..0ac480f 100644
--- a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_decoder.c
+++ b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_decoder.c
@@ -101,7 +101,7 @@ u32 h264bsdInit(storage_t *pStorage, u32 noOutputReordering)
      * specific NEON optimized "memset" for clearing the structure */
     size = (sizeof(macroblockLayer_t) + 63) & ~0x3F;
 
-    pStorage->mbLayer = (macroblockLayer_t*)H264SwDecMalloc(size);
+    pStorage->mbLayer = (macroblockLayer_t*)H264SwDecMalloc(size, 1);
     if (!pStorage->mbLayer)
         return HANTRO_NOK;
 
diff --git a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_util.h b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_util.h
index 216ad04..9f0eb7d 100644
--- a/media/libstagefright/codecs/on2/h264dec/source/h264bsd_util.h
+++ b/media/libstagefright/codecs/on2/h264dec/source/h264bsd_util.h
@@ -141,7 +141,7 @@
 /* macro to allocate memory */
 #define ALLOCATE(ptr, count, type) \
 { \
-    (ptr) = H264SwDecMalloc((count) * sizeof(type)); \
+    (ptr) = H264SwDecMalloc(sizeof(type), (count)); \
 }
 
 /* macro to free allocated memory */