From 8bd4d16aa5636e98522c07ae31236420788aa749 Mon Sep 17 00:00:00 2001 From: Chong Zhang Date: Fri, 10 Jan 2014 17:36:57 -0800 Subject: Cap pts gap between adjacent frames to specified value - In the scenario of cast mirroring, encoding could be suspended for prolonged periods. Limiting the pts gap to workaround the problem where encoder's rate control logic produces huge frames after a long period of suspension. - Repeat last frame a couple more times to get better quality on static scenes. - Fix the timestamp on repeat frames (it was not set) Bug: 11971963 Change-Id: I1d68ab3d269874bf3921aa429a985c5f63e428c7 (cherry picked from commit 94ee4b708acfa941581160b267afb79192b1d816) --- include/media/IOMX.h | 1 + include/media/stagefright/ACodec.h | 1 + media/libstagefright/ACodec.cpp | 22 ++++- media/libstagefright/omx/GraphicBufferSource.cpp | 100 ++++++++++++++++++++++- media/libstagefright/omx/GraphicBufferSource.h | 25 +++++- media/libstagefright/omx/OMXNodeInstance.cpp | 26 +++++- 6 files changed, 169 insertions(+), 6 deletions(-) diff --git a/include/media/IOMX.h b/include/media/IOMX.h index 9c8451c..6643736 100644 --- a/include/media/IOMX.h +++ b/include/media/IOMX.h @@ -142,6 +142,7 @@ public: enum InternalOptionType { INTERNAL_OPTION_SUSPEND, // data is a bool INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY, // data is an int64_t + INTERNAL_OPTION_MAX_TIMESTAMP_GAP, // data is int64_t }; virtual status_t setInternalOption( node_id node, diff --git a/include/media/stagefright/ACodec.h b/include/media/stagefright/ACodec.h index 510c482..bf3a998 100644 --- a/include/media/stagefright/ACodec.h +++ b/include/media/stagefright/ACodec.h @@ -205,6 +205,7 @@ private: int32_t mMetaDataBuffersToSubmit; int64_t mRepeatFrameDelayUs; + int64_t mMaxPtsGapUs; status_t setCyclicIntraMacroblockRefresh(const sp &msg, int32_t mode); status_t allocateBuffersOnPort(OMX_U32 portIndex); diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp index 3810ac1..9276818 100644 --- a/media/libstagefright/ACodec.cpp +++ b/media/libstagefright/ACodec.cpp @@ -371,7 +371,8 @@ ACodec::ACodec() mDequeueCounter(0), mStoreMetaDataInOutputBuffers(false), mMetaDataBuffersToSubmit(0), - mRepeatFrameDelayUs(-1ll) { + mRepeatFrameDelayUs(-1ll), + mMaxPtsGapUs(-1l) { mUninitializedState = new UninitializedState(this); mLoadedState = new LoadedState(this); mLoadedToIdleState = new LoadedToIdleState(this); @@ -1109,6 +1110,10 @@ status_t ACodec::configureCodec( &mRepeatFrameDelayUs)) { mRepeatFrameDelayUs = -1ll; } + + if (!msg->findInt64("max-pts-gap-to-encoder", &mMaxPtsGapUs)) { + mMaxPtsGapUs = -1l; + } } // Always try to enable dynamic output buffers on native surface @@ -3855,6 +3860,21 @@ void ACodec::LoadedState::onCreateInputSurface( } } + if (err == OK && mCodec->mMaxPtsGapUs > 0l) { + err = mCodec->mOMX->setInternalOption( + mCodec->mNode, + kPortIndexInput, + IOMX::INTERNAL_OPTION_MAX_TIMESTAMP_GAP, + &mCodec->mMaxPtsGapUs, + sizeof(mCodec->mMaxPtsGapUs)); + + if (err != OK) { + ALOGE("[%s] Unable to configure max timestamp gap (err %d)", + mCodec->mComponentName.c_str(), + err); + } + } + if (err == OK) { notify->setObject("input-surface", new BufferProducerWrapper(bufferProducer)); diff --git a/media/libstagefright/omx/GraphicBufferSource.cpp b/media/libstagefright/omx/GraphicBufferSource.cpp index b8970ad..f1bb4f1 100644 --- a/media/libstagefright/omx/GraphicBufferSource.cpp +++ b/media/libstagefright/omx/GraphicBufferSource.cpp @@ -42,7 +42,11 @@ GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance, mEndOfStream(false), mEndOfStreamSent(false), mRepeatAfterUs(-1ll), + mMaxTimestampGapUs(-1ll), + mPrevOriginalTimeUs(-1ll), + mPrevModifiedTimeUs(-1ll), mRepeatLastFrameGeneration(0), + mRepeatLastFrameTimestamp(-1ll), mLatestSubmittedBufferId(-1), mLatestSubmittedBufferFrameNum(0), mLatestSubmittedBufferUseCount(0), @@ -299,6 +303,32 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) { return; } +void GraphicBufferSource::codecBufferFilled(OMX_BUFFERHEADERTYPE* header) { + Mutex::Autolock autoLock(mMutex); + + if (mMaxTimestampGapUs > 0ll + && !(header->nFlags & OMX_BUFFERFLAG_CODECCONFIG)) { + ssize_t index = mOriginalTimeUs.indexOfKey(header->nTimeStamp); + if (index >= 0) { + ALOGV("OUT timestamp: %lld -> %lld", + header->nTimeStamp, mOriginalTimeUs[index]); + header->nTimeStamp = mOriginalTimeUs[index]; + mOriginalTimeUs.removeItemsAt(index); + } else { + // giving up the effort as encoder doesn't appear to preserve pts + ALOGW("giving up limiting timestamp gap (pts = %lld)", + header->nTimeStamp); + mMaxTimestampGapUs = -1ll; + } + if (mOriginalTimeUs.size() > BufferQueue::NUM_BUFFER_SLOTS) { + // something terribly wrong must have happened, giving up... + ALOGE("mOriginalTimeUs has too many entries (%d)", + mOriginalTimeUs.size()); + mMaxTimestampGapUs = -1ll; + } + } +} + void GraphicBufferSource::suspend(bool suspend) { Mutex::Autolock autoLock(mMutex); @@ -431,6 +461,7 @@ bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() { BufferQueue::BufferItem item; item.mBuf = mLatestSubmittedBufferId; item.mFrameNumber = mLatestSubmittedBufferFrameNum; + item.mTimestamp = mRepeatLastFrameTimestamp; status_t err = submitBuffer_l(item, cbi); @@ -440,6 +471,20 @@ bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() { ++mLatestSubmittedBufferUseCount; + /* repeat last frame up to kRepeatLastFrameCount times. + * in case of static scene, a single repeat might not get rid of encoder + * ghosting completely, refresh a couple more times to get better quality + */ + if (--mRepeatLastFrameCount > 0) { + mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000; + + if (mReflector != NULL) { + sp msg = new AMessage(kWhatRepeatLastFrame, mReflector->id()); + msg->setInt32("generation", ++mRepeatLastFrameGeneration); + msg->post(mRepeatAfterUs); + } + } + return true; } @@ -460,8 +505,11 @@ void GraphicBufferSource::setLatestSubmittedBuffer_l( mLatestSubmittedBufferId = item.mBuf; mLatestSubmittedBufferFrameNum = item.mFrameNumber; + mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000; + mLatestSubmittedBufferUseCount = 1; mRepeatBufferDeferred = false; + mRepeatLastFrameCount = kRepeatLastFrameCount; if (mReflector != NULL) { sp msg = new AMessage(kWhatRepeatLastFrame, mReflector->id()); @@ -497,6 +545,39 @@ status_t GraphicBufferSource::signalEndOfInputStream() { return OK; } +int64_t GraphicBufferSource::getTimestamp(const BufferQueue::BufferItem &item) { + int64_t timeUs = item.mTimestamp / 1000; + + if (mMaxTimestampGapUs > 0ll) { + /* Cap timestamp gap between adjacent frames to specified max + * + * In the scenario of cast mirroring, encoding could be suspended for + * prolonged periods. Limiting the pts gap to workaround the problem + * where encoder's rate control logic produces huge frames after a + * long period of suspension. + */ + + int64_t originalTimeUs = timeUs; + if (mPrevOriginalTimeUs >= 0ll) { + if (originalTimeUs < mPrevOriginalTimeUs) { + // Drop the frame if it's going backward in time. Bad timestamp + // could disrupt encoder's rate control completely. + ALOGV("Dropping frame that's going backward in time"); + return -1; + } + int64_t timestampGapUs = originalTimeUs - mPrevOriginalTimeUs; + timeUs = (timestampGapUs < mMaxTimestampGapUs ? + timestampGapUs : mMaxTimestampGapUs) + mPrevModifiedTimeUs; + } + mPrevOriginalTimeUs = originalTimeUs; + mPrevModifiedTimeUs = timeUs; + mOriginalTimeUs.add(timeUs, originalTimeUs); + ALOGV("IN timestamp: %lld -> %lld", originalTimeUs, timeUs); + } + + return timeUs; +} + status_t GraphicBufferSource::submitBuffer_l( const BufferQueue::BufferItem &item, int cbi) { ALOGV("submitBuffer_l cbi=%d", cbi); @@ -513,9 +594,15 @@ status_t GraphicBufferSource::submitBuffer_l( memcpy(data, &type, 4); memcpy(data + 4, &handle, sizeof(buffer_handle_t)); + int64_t timeUs = getTimestamp(item); + if (timeUs < 0ll) { + ALOGE("Dropping frame with bad timestamp"); + return UNKNOWN_ERROR; + } + status_t err = mNodeInstance->emptyDirectBuffer(header, 0, 4 + sizeof(buffer_handle_t), OMX_BUFFERFLAG_ENDOFFRAME, - item.mTimestamp / 1000); + timeUs); if (err != OK) { ALOGW("WARNING: emptyDirectBuffer failed: 0x%x", err); codecBuffer.mGraphicBuffer = NULL; @@ -658,6 +745,17 @@ status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs( return OK; } +status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) { + Mutex::Autolock autoLock(mMutex); + + if (mExecuting || maxGapUs <= 0ll) { + return INVALID_OPERATION; + } + + mMaxTimestampGapUs = maxGapUs; + + return OK; +} void GraphicBufferSource::onMessageReceived(const sp &msg) { switch (msg->what()) { case kWhatRepeatLastFrame: diff --git a/media/libstagefright/omx/GraphicBufferSource.h b/media/libstagefright/omx/GraphicBufferSource.h index 9e5eee6..3b0e454 100644 --- a/media/libstagefright/omx/GraphicBufferSource.h +++ b/media/libstagefright/omx/GraphicBufferSource.h @@ -87,6 +87,10 @@ public: // fill it with a new frame of data; otherwise, just mark it as available. void codecBufferEmptied(OMX_BUFFERHEADERTYPE* header); + // Called when omx_message::FILL_BUFFER_DONE is received. (Currently the + // buffer source will fix timestamp in the header if needed.) + void codecBufferFilled(OMX_BUFFERHEADERTYPE* header); + // This is called after the last input frame has been submitted. We // need to submit an empty buffer with the EOS flag set. If we don't // have a codec buffer ready, we just set the mEndOfStream flag. @@ -105,6 +109,15 @@ public: // state and once this behaviour is specified it cannot be reset. status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs); + // When set, the timestamp fed to the encoder will be modified such that + // the gap between two adjacent frames is capped at maxGapUs. Timestamp + // will be restored to the original when the encoded frame is returned to + // the client. + // This is to solve a problem in certain real-time streaming case, where + // encoder's rate control logic produces huge frames after a long period + // of suspension on input. + status_t setMaxTimestampGapUs(int64_t maxGapUs); + protected: // BufferQueue::ConsumerListener interface, called when a new frame of // data is available. If we're executing and a codec buffer is @@ -165,6 +178,7 @@ private: void setLatestSubmittedBuffer_l(const BufferQueue::BufferItem &item); bool repeatLatestSubmittedBuffer_l(); + int64_t getTimestamp(const BufferQueue::BufferItem &item); // Lock, covers all member variables. mutable Mutex mMutex; @@ -206,13 +220,22 @@ private: enum { kWhatRepeatLastFrame, }; - + enum { + kRepeatLastFrameCount = 10, + }; int64_t mRepeatAfterUs; + int64_t mMaxTimestampGapUs; + + KeyedVector mOriginalTimeUs; + int64_t mPrevOriginalTimeUs; + int64_t mPrevModifiedTimeUs; sp mLooper; sp > mReflector; int32_t mRepeatLastFrameGeneration; + int64_t mRepeatLastFrameTimestamp; + int32_t mRepeatLastFrameCount; int mLatestSubmittedBufferId; uint64_t mLatestSubmittedBufferFrameNum; diff --git a/media/libstagefright/omx/OMXNodeInstance.cpp b/media/libstagefright/omx/OMXNodeInstance.cpp index 5f104fc..6c5c857 100644 --- a/media/libstagefright/omx/OMXNodeInstance.cpp +++ b/media/libstagefright/omx/OMXNodeInstance.cpp @@ -849,6 +849,7 @@ status_t OMXNodeInstance::setInternalOption( switch (type) { case IOMX::INTERNAL_OPTION_SUSPEND: case IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY: + case IOMX::INTERNAL_OPTION_MAX_TIMESTAMP_GAP: { const sp &bufferSource = getGraphicBufferSource(); @@ -864,7 +865,8 @@ status_t OMXNodeInstance::setInternalOption( bool suspend = *(bool *)data; bufferSource->suspend(suspend); - } else { + } else if (type == + IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY){ if (size != sizeof(int64_t)) { return INVALID_OPERATION; } @@ -872,6 +874,14 @@ status_t OMXNodeInstance::setInternalOption( int64_t delayUs = *(int64_t *)data; return bufferSource->setRepeatPreviousFrameDelayUs(delayUs); + } else { + if (size != sizeof(int64_t)) { + return INVALID_OPERATION; + } + + int64_t maxGapUs = *(int64_t *)data; + + return bufferSource->setMaxTimestampGapUs(maxGapUs); } return OK; @@ -883,6 +893,8 @@ status_t OMXNodeInstance::setInternalOption( } void OMXNodeInstance::onMessage(const omx_message &msg) { + const sp& bufferSource(getGraphicBufferSource()); + if (msg.type == omx_message::FILL_BUFFER_DONE) { OMX_BUFFERHEADERTYPE *buffer = static_cast( @@ -892,10 +904,18 @@ void OMXNodeInstance::onMessage(const omx_message &msg) { static_cast(buffer->pAppPrivate); buffer_meta->CopyFromOMX(buffer); - } else if (msg.type == omx_message::EMPTY_BUFFER_DONE) { - const sp& bufferSource(getGraphicBufferSource()); if (bufferSource != NULL) { + // fix up the buffer info (especially timestamp) if needed + bufferSource->codecBufferFilled(buffer); + + omx_message newMsg = msg; + newMsg.u.extended_buffer_data.timestamp = buffer->nTimeStamp; + mObserver->onMessage(newMsg); + return; + } + } else if (msg.type == omx_message::EMPTY_BUFFER_DONE) { + if (bufferSource != NULL) { // This is one of the buffers used exclusively by // GraphicBufferSource. // Don't dispatch a message back to ACodec, since it doesn't -- cgit v1.1