SoftAVCEnc: check requested memory size before allocation.

Bug: 20674674
Change-Id: I569e7a9b33fe64779a40e55539929c3dc4303c19

1 files changed, 8 insertions, 0 deletions

diff --git a/media/libstagefright/codecs/avcenc/SoftAVCEnc.cpp b/media/libstagefright/codecs/avcenc/SoftAVCEnc.cpp
index 449d195..a00f324 100755
--- a/media/libstagefright/codecs/avcenc/SoftAVCEnc.cpp
+++ b/media/libstagefright/codecs/avcenc/SoftAVCEnc.cpp
@@ -634,6 +634,10 @@ OMX_ERRORTYPE SoftAVC::initEncoder() {
         }
 
         if (mConversionBuffer == NULL) {
+            if (((uint64_t)mStride * mHeight) > (((uint64_t)INT32_MAX / 3) * 2)) {
+                ALOGE("Buffer size is too big.");
+                return OMX_ErrorUndefined;
+            }
             mConversionBuffer = (uint8_t *)malloc(mStride * mHeight * 3 / 2);
             if (mConversionBuffer == NULL) {
                 ALOGE("Allocating conversion buffer failed.");
@@ -679,6 +683,10 @@ OMX_ERRORTYPE SoftAVC::initEncoder() {
     }
 
     /* Allocate array to hold memory records */
+    if (mNumMemRecords > SIZE_MAX / sizeof(iv_mem_rec_t)) {
+        ALOGE("requested memory size is too big.");
+        return OMX_ErrorUndefined;
+    }
     mMemRecords = (iv_mem_rec_t *)malloc(mNumMemRecords * sizeof(iv_mem_rec_t));
     if (NULL == mMemRecords) {
         ALOGE("Unable to allocate memory for hold memory records: Size %zu",