Merge commit 'b46eb8d9' into HEAD

SoftAVCEnc: check requested memory size before allocation.

Bug: 20674674
(cherry picked from commit f6fe4340219a8e674f3250fe32d4697ec8184b24)

Change-Id: Ib52a26de912fd5a9a08dd9948885fb4a9b32e9e6

1 files changed, 8 insertions, 0 deletions

diff --git a/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp b/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp
index ed3dca0..523e785 100644
--- a/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp
+++ b/media/libstagefright/codecs/avc/enc/SoftAVCEncoder.cpp
@@ -234,6 +234,10 @@ OMX_ERRORTYPE SoftAVCEncoder::initEncParams() {
             || mStoreMetaDataInBuffers) {
         // Color conversion is needed.
         free(mInputFrameData);
+        if (((uint64_t)mVideoWidth * mVideoHeight) > ((uint64_t)INT32_MAX / 3)) {
+            ALOGE("Buffer size is too big.");
+            return OMX_ErrorUndefined;
+        }
         mInputFrameData =
             (uint8_t *) malloc((mVideoWidth * mVideoHeight * 3 ) >> 1);
         CHECK(mInputFrameData != NULL);
@@ -255,6 +259,10 @@ OMX_ERRORTYPE SoftAVCEncoder::initEncParams() {
     int32_t nMacroBlocks = ((((mVideoWidth + 15) >> 4) << 4) *
             (((mVideoHeight + 15) >> 4) << 4)) >> 8;
     CHECK(mSliceGroup == NULL);
+    if (nMacroBlocks > SIZE_MAX / sizeof(uint32_t)) {
+        ALOGE("requested memory size is too big.");
+        return OMX_ErrorUndefined;
+    }
     mSliceGroup = (uint32_t *) malloc(sizeof(uint32_t) * nMacroBlocks);
     CHECK(mSliceGroup != NULL);
     for (int ii = 0, idx = 0; ii < nMacroBlocks; ++ii) {