From 918eeaa29d99d257282fafec931b4bda0e3bae12 Mon Sep 17 00:00:00 2001 From: Wonsik Kim Date: Sun, 20 Mar 2016 10:44:44 +0900 Subject: codecs: check OMX buffer size before use in (avc|hevc|mpeg2)dec Bug: 27833616 Change-Id: Ic4045a3f56f53b08d0b1264b2a91b8f43e91b738 (cherry picked from commit 87fdee0bc9e3ac4d2a88ef0a8e150cfdf08c161d) --- media/libstagefright/codecs/avcdec/SoftAVCDec.cpp | 28 ++++++++++++------ media/libstagefright/codecs/avcdec/SoftAVCDec.h | 2 +- media/libstagefright/codecs/hevcdec/SoftHEVC.cpp | 28 ++++++++++++------ media/libstagefright/codecs/hevcdec/SoftHEVC.h | 2 +- media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp | 33 ++++++++++++++-------- media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h | 2 +- 6 files changed, 62 insertions(+), 33 deletions(-) (limited to 'media/libstagefright') diff --git a/media/libstagefright/codecs/avcdec/SoftAVCDec.cpp b/media/libstagefright/codecs/avcdec/SoftAVCDec.cpp index afbe230..2130ccf 100644 --- a/media/libstagefright/codecs/avcdec/SoftAVCDec.cpp +++ b/media/libstagefright/codecs/avcdec/SoftAVCDec.cpp @@ -381,7 +381,7 @@ void SoftAVC::onReset() { resetPlugin(); } -void SoftAVC::setDecodeArgs( +bool SoftAVC::setDecodeArgs( ivd_video_decode_ip_t *ps_dec_ip, ivd_video_decode_op_t *ps_dec_op, OMX_BUFFERHEADERTYPE *inHeader, @@ -389,7 +389,6 @@ void SoftAVC::setDecodeArgs( size_t timeStampIx) { size_t sizeY = outputBufferWidth() * outputBufferHeight(); size_t sizeUV; - uint8_t *pBuf; ps_dec_ip->u4_size = sizeof(ivd_video_decode_ip_t); ps_dec_op->u4_size = sizeof(ivd_video_decode_op_t); @@ -409,22 +408,28 @@ void SoftAVC::setDecodeArgs( ps_dec_ip->u4_num_Bytes = 0; } + sizeUV = sizeY / 4; + ps_dec_ip->s_out_buffer.u4_min_out_buf_size[0] = sizeY; + ps_dec_ip->s_out_buffer.u4_min_out_buf_size[1] = sizeUV; + ps_dec_ip->s_out_buffer.u4_min_out_buf_size[2] = sizeUV; + + uint8_t *pBuf; if (outHeader) { + if (outHeader->nAllocLen < sizeY + (sizeUV * 2)) { + android_errorWriteLog(0x534e4554, "27569635"); + return false; + } pBuf = outHeader->pBuffer; } else { + // mFlushOutBuffer always has the right size. pBuf = mFlushOutBuffer; } - sizeUV = sizeY / 4; - ps_dec_ip->s_out_buffer.u4_min_out_buf_size[0] = sizeY; - ps_dec_ip->s_out_buffer.u4_min_out_buf_size[1] = sizeUV; - ps_dec_ip->s_out_buffer.u4_min_out_buf_size[2] = sizeUV; - ps_dec_ip->s_out_buffer.pu1_bufs[0] = pBuf; ps_dec_ip->s_out_buffer.pu1_bufs[1] = pBuf + sizeY; ps_dec_ip->s_out_buffer.pu1_bufs[2] = pBuf + sizeY + sizeUV; ps_dec_ip->s_out_buffer.u4_num_bufs = 3; - return; + return true; } void SoftAVC::onPortFlushCompleted(OMX_U32 portIndex) { /* Once the output buffers are flushed, ignore any buffers that are held in decoder */ @@ -573,7 +578,12 @@ void SoftAVC::onQueueFilled(OMX_U32 portIndex) { WORD32 timeDelay, timeTaken; size_t sizeY, sizeUV; - setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx); + if (!setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx)) { + ALOGE("Decoder arg setup failed"); + notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); + mSignalledError = true; + return; + } // If input dump is enabled, then write to file DUMP_TO_FILE(mInFile, s_dec_ip.pv_stream_buffer, s_dec_ip.u4_num_Bytes); diff --git a/media/libstagefright/codecs/avcdec/SoftAVCDec.h b/media/libstagefright/codecs/avcdec/SoftAVCDec.h index 9dcabb4..c710c76 100644 --- a/media/libstagefright/codecs/avcdec/SoftAVCDec.h +++ b/media/libstagefright/codecs/avcdec/SoftAVCDec.h @@ -109,7 +109,7 @@ private: status_t resetPlugin(); - void setDecodeArgs( + bool setDecodeArgs( ivd_video_decode_ip_t *ps_dec_ip, ivd_video_decode_op_t *ps_dec_op, OMX_BUFFERHEADERTYPE *inHeader, diff --git a/media/libstagefright/codecs/hevcdec/SoftHEVC.cpp b/media/libstagefright/codecs/hevcdec/SoftHEVC.cpp index e601125..a70755c 100644 --- a/media/libstagefright/codecs/hevcdec/SoftHEVC.cpp +++ b/media/libstagefright/codecs/hevcdec/SoftHEVC.cpp @@ -343,14 +343,13 @@ void SoftHEVC::onReset() { resetPlugin(); } -void SoftHEVC::setDecodeArgs(ivd_video_decode_ip_t *ps_dec_ip, +bool SoftHEVC::setDecodeArgs(ivd_video_decode_ip_t *ps_dec_ip, ivd_video_decode_op_t *ps_dec_op, OMX_BUFFERHEADERTYPE *inHeader, OMX_BUFFERHEADERTYPE *outHeader, size_t timeStampIx) { size_t sizeY = outputBufferWidth() * outputBufferHeight(); size_t sizeUV; - uint8_t *pBuf; ps_dec_ip->u4_size = sizeof(ivd_video_decode_ip_t); ps_dec_op->u4_size = sizeof(ivd_video_decode_op_t); @@ -370,22 +369,28 @@ void SoftHEVC::setDecodeArgs(ivd_video_decode_ip_t *ps_dec_ip, ps_dec_ip->u4_num_Bytes = 0; } + sizeUV = sizeY / 4; + ps_dec_ip->s_out_buffer.u4_min_out_buf_size[0] = sizeY; + ps_dec_ip->s_out_buffer.u4_min_out_buf_size[1] = sizeUV; + ps_dec_ip->s_out_buffer.u4_min_out_buf_size[2] = sizeUV; + + uint8_t *pBuf; if (outHeader) { + if (outHeader->nAllocLen < sizeY + (sizeUV * 2)) { + android_errorWriteLog(0x534e4554, "27569635"); + return false; + } pBuf = outHeader->pBuffer; } else { + // mFlushOutBuffer always has the right size. pBuf = mFlushOutBuffer; } - sizeUV = sizeY / 4; - ps_dec_ip->s_out_buffer.u4_min_out_buf_size[0] = sizeY; - ps_dec_ip->s_out_buffer.u4_min_out_buf_size[1] = sizeUV; - ps_dec_ip->s_out_buffer.u4_min_out_buf_size[2] = sizeUV; - ps_dec_ip->s_out_buffer.pu1_bufs[0] = pBuf; ps_dec_ip->s_out_buffer.pu1_bufs[1] = pBuf + sizeY; ps_dec_ip->s_out_buffer.pu1_bufs[2] = pBuf + sizeY + sizeUV; ps_dec_ip->s_out_buffer.u4_num_bufs = 3; - return; + return true; } void SoftHEVC::onPortFlushCompleted(OMX_U32 portIndex) { /* Once the output buffers are flushed, ignore any buffers that are held in decoder */ @@ -520,7 +525,12 @@ void SoftHEVC::onQueueFilled(OMX_U32 portIndex) { WORD32 timeDelay, timeTaken; size_t sizeY, sizeUV; - setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx); + if (!setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx)) { + ALOGE("Decoder arg setup failed"); + notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); + mSignalledError = true; + return; + } GETTIME(&mTimeStart, NULL); /* Compute time elapsed between end of previous decode() diff --git a/media/libstagefright/codecs/hevcdec/SoftHEVC.h b/media/libstagefright/codecs/hevcdec/SoftHEVC.h index 21bb99e..943edfd 100644 --- a/media/libstagefright/codecs/hevcdec/SoftHEVC.h +++ b/media/libstagefright/codecs/hevcdec/SoftHEVC.h @@ -106,7 +106,7 @@ private: status_t resetDecoder(); status_t resetPlugin(); - void setDecodeArgs(ivd_video_decode_ip_t *ps_dec_ip, + bool setDecodeArgs(ivd_video_decode_ip_t *ps_dec_ip, ivd_video_decode_op_t *ps_dec_op, OMX_BUFFERHEADERTYPE *inHeader, OMX_BUFFERHEADERTYPE *outHeader, diff --git a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp index 4307c4e..e134d38 100644 --- a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp +++ b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.cpp @@ -466,7 +466,7 @@ OMX_ERRORTYPE SoftMPEG2::internalSetParameter(OMX_INDEXTYPE index, const OMX_PTR return ret; } -void SoftMPEG2::setDecodeArgs( +bool SoftMPEG2::setDecodeArgs( ivd_video_decode_ip_t *ps_dec_ip, ivd_video_decode_op_t *ps_dec_op, OMX_BUFFERHEADERTYPE *inHeader, @@ -474,7 +474,6 @@ void SoftMPEG2::setDecodeArgs( size_t timeStampIx) { size_t sizeY = outputBufferWidth() * outputBufferHeight(); size_t sizeUV; - uint8_t *pBuf; ps_dec_ip->u4_size = sizeof(ivd_video_decode_ip_t); ps_dec_op->u4_size = sizeof(ivd_video_decode_op_t); @@ -494,22 +493,28 @@ void SoftMPEG2::setDecodeArgs( ps_dec_ip->u4_num_Bytes = 0; } + sizeUV = sizeY / 4; + ps_dec_ip->s_out_buffer.u4_min_out_buf_size[0] = sizeY; + ps_dec_ip->s_out_buffer.u4_min_out_buf_size[1] = sizeUV; + ps_dec_ip->s_out_buffer.u4_min_out_buf_size[2] = sizeUV; + + uint8_t *pBuf; if (outHeader) { + if (outHeader->nAllocLen < sizeY + (sizeUV * 2)) { + android_errorWriteLog(0x534e4554, "27569635"); + return false; + } pBuf = outHeader->pBuffer; } else { + // mFlushOutBuffer always has the right size. pBuf = mFlushOutBuffer; } - sizeUV = sizeY / 4; - ps_dec_ip->s_out_buffer.u4_min_out_buf_size[0] = sizeY; - ps_dec_ip->s_out_buffer.u4_min_out_buf_size[1] = sizeUV; - ps_dec_ip->s_out_buffer.u4_min_out_buf_size[2] = sizeUV; - ps_dec_ip->s_out_buffer.pu1_bufs[0] = pBuf; ps_dec_ip->s_out_buffer.pu1_bufs[1] = pBuf + sizeY; ps_dec_ip->s_out_buffer.pu1_bufs[2] = pBuf + sizeY + sizeUV; ps_dec_ip->s_out_buffer.u4_num_bufs = 3; - return; + return true; } void SoftMPEG2::onPortFlushCompleted(OMX_U32 portIndex) { /* Once the output buffers are flushed, ignore any buffers that are held in decoder */ @@ -622,7 +627,11 @@ void SoftMPEG2::onQueueFilled(OMX_U32 portIndex) { WORD32 timeDelay, timeTaken; size_t sizeY, sizeUV; - setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx); + if (!setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx)) { + ALOGE("Decoder arg setup failed"); + notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); + return; + } // If input dump is enabled, then write to file DUMP_TO_FILE(mInFile, s_dec_ip.pv_stream_buffer, s_dec_ip.u4_num_Bytes); @@ -665,9 +674,9 @@ void SoftMPEG2::onQueueFilled(OMX_U32 portIndex) { CHECK_EQ(reInitDecoder(), (status_t)OK); - setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx); - - ivdec_api_function(mCodecCtx, (void *)&s_dec_ip, (void *)&s_dec_op); + if (setDecodeArgs(&s_dec_ip, &s_dec_op, inHeader, outHeader, timeStampIx)) { + ivdec_api_function(mCodecCtx, (void *)&s_dec_ip, (void *)&s_dec_op); + } return; } diff --git a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h index a625e08..f48b70b 100644 --- a/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h +++ b/media/libstagefright/codecs/mpeg2dec/SoftMPEG2.h @@ -117,7 +117,7 @@ private: status_t resetPlugin(); status_t reInitDecoder(); - void setDecodeArgs( + bool setDecodeArgs( ivd_video_decode_ip_t *ps_dec_ip, ivd_video_decode_op_t *ps_dec_op, OMX_BUFFERHEADERTYPE *inHeader, -- cgit v1.1