From 90fcf68fd29f3cb695bd53a830ad984cb7d430c0 Mon Sep 17 00:00:00 2001 From: Lajos Molnar Date: Thu, 4 Jun 2015 10:29:19 -0700 Subject: stagefright: add support for output frame rendered callback - Added FRAME_RENDERED event in OMX, used by tunneled video decoders to signal rendered event timing - Track buffers sent for rendering in ACodec and in SoftwareRenderer, and determine when they have rendered - Propagate render times to MediaCodec Bug: 20503131 Change-Id: Idf0a8714d5368b237c2285dd39fa82db847c232f --- media/libstagefright/ACodec.cpp | 123 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 1 deletion(-) (limited to 'media/libstagefright/ACodec.cpp') diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp index 172e19c..71cb194 100644 --- a/media/libstagefright/ACodec.cpp +++ b/media/libstagefright/ACodec.cpp @@ -37,6 +37,7 @@ #include #include +#include #include #include #include @@ -175,6 +176,15 @@ struct CodecObserver : public BnOMXObserver { break; } + case omx_message::FRAME_RENDERED: + { + msg->setInt64( + "media_time_us", omx_msg.u.render_data.timestamp); + msg->setInt64( + "system_nano", omx_msg.u.render_data.nanoTime); + break; + } + default: ALOGE("Unrecognized message type: %d", omx_msg.type); break; @@ -238,6 +248,8 @@ private: int64_t timeUs, int fenceFd); + virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano); + void getMoreInputDataIfPossible(); DISALLOW_EVIL_CONSTRUCTORS(BaseState); @@ -354,6 +366,7 @@ protected: virtual void stateEntered(); virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2); + virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano); private: bool mActive; @@ -372,6 +385,7 @@ protected: virtual void stateEntered(); virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2); + virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano); private: DISALLOW_EVIL_CONSTRUCTORS(OutputPortSettingsChangedState); @@ -786,6 +800,7 @@ status_t ACodec::allocateBuffersOnPort(OMX_U32 portIndex) { BufferInfo info; info.mStatus = BufferInfo::OWNED_BY_US; info.mFenceFd = -1; + info.mRenderInfo = NULL; uint32_t requiresAllocateBufferBit = (portIndex == kPortIndexInput) @@ -1002,6 +1017,7 @@ status_t ACodec::allocateOutputBuffersFromNativeWindow() { info.mStatus = BufferInfo::OWNED_BY_US; info.mFenceFd = fenceFd; info.mIsReadFence = false; + info.mRenderInfo = NULL; info.mData = new ABuffer(NULL /* data */, bufferSize /* capacity */); info.mGraphicBuffer = graphicBuffer; mBuffers[kPortIndexOutput].push(info); @@ -1073,6 +1089,7 @@ status_t ACodec::allocateOutputMetadataBuffers() { BufferInfo info; info.mStatus = BufferInfo::OWNED_BY_NATIVE_WINDOW; info.mFenceFd = -1; + info.mRenderInfo = NULL; info.mGraphicBuffer = NULL; info.mDequeuedAt = mDequeueCounter; @@ -1219,6 +1236,42 @@ status_t ACodec::cancelBufferToNativeWindow(BufferInfo *info) { return err; } +void ACodec::updateRenderInfoForDequeuedBuffer( + ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info) { + + info->mRenderInfo = + mRenderTracker.updateInfoForDequeuedBuffer( + buf, fenceFd, info - &mBuffers[kPortIndexOutput][0]); + + // check for any fences already signaled + notifyOfRenderedFrames(false /* dropIncomplete */, info->mRenderInfo); +} + +void ACodec::onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) { + if (mRenderTracker.onFrameRendered(mediaTimeUs, systemNano) != OK) { + mRenderTracker.dumpRenderQueue(); + } +} + +void ACodec::notifyOfRenderedFrames(bool dropIncomplete, FrameRenderTracker::Info *until) { + sp msg = mNotify->dup(); + msg->setInt32("what", CodecBase::kWhatOutputFramesRendered); + std::list done = + mRenderTracker.checkFencesAndGetRenderedFrames(until, dropIncomplete); + + // unlink untracked frames + for (std::list::const_iterator it = done.cbegin(); + it != done.cend(); ++it) { + if (it->getIndex() >= 0) { + mBuffers[kPortIndexOutput].editItemAt(it->getIndex()).mRenderInfo = NULL; + } + } + + if (MediaCodec::CreateFramesRenderedMessage(done, msg)) { + msg->post(); + } +} + ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { ANativeWindowBuffer *buf; CHECK(mNativeWindow.get() != NULL); @@ -1242,7 +1295,7 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { BufferInfo *info = &mBuffers[kPortIndexOutput].editItemAt(i); if (info->mGraphicBuffer != NULL && - info->mGraphicBuffer->handle == buf->handle) { + info->mGraphicBuffer->handle == buf->handle) { // Since consumers can attach buffers to BufferQueues, it is possible // that a known yet stale buffer can return from a surface that we // once used. We can simply ignore this as we have already dequeued @@ -1255,9 +1308,11 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { stale = true; break; } + ALOGV("dequeued buffer %p", info->mGraphicBuffer->getNativeBuffer()); info->mStatus = BufferInfo::OWNED_BY_US; info->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow"); + updateRenderInfoForDequeuedBuffer(buf, fenceFd, info); return info; } } @@ -1301,6 +1356,8 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { oldest->mGraphicBuffer = new GraphicBuffer(buf, false); oldest->mStatus = BufferInfo::OWNED_BY_US; oldest->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow for oldest"); + mRenderTracker.untrackFrame(oldest->mRenderInfo); + oldest->mRenderInfo = NULL; mOMX->updateGraphicBufferInMeta( mNode, kPortIndexOutput, oldest->mGraphicBuffer, @@ -1324,6 +1381,7 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { oldest->mGraphicBuffer->getNativeBuffer(), oldest->mData->base()); } + updateRenderInfoForDequeuedBuffer(buf, fenceFd, oldest); return oldest; } @@ -1398,6 +1456,9 @@ status_t ACodec::freeBuffer(OMX_U32 portIndex, size_t i) { ::close(info->mFenceFd); } + mRenderTracker.untrackFrame(info->mRenderInfo); + info->mRenderInfo = NULL; + // remove buffer even if mOMX->freeBuffer fails mBuffers[portIndex].removeAt(i); @@ -4518,9 +4579,20 @@ bool ACodec::BaseState::onOMXMessageList(const sp &msg) { CHECK(msg->findObject("messages", &obj)); sp msgList = static_cast(obj.get()); + bool receivedRenderedEvents = false; for (std::list>::const_iterator it = msgList->getList().cbegin(); it != msgList->getList().cend(); ++it) { onOMXMessage(*it); + int32_t type; + CHECK((*it)->findInt32("type", &type)); + if (type == omx_message::FRAME_RENDERED) { + receivedRenderedEvents = true; + } + } + + if (receivedRenderedEvents) { + // NOTE: all buffers are rendered in this case + mCodec->notifyOfRenderedFrames(); } return true; } @@ -4587,12 +4659,29 @@ bool ACodec::BaseState::onOMXMessage(const sp &msg) { fenceFd); } + case omx_message::FRAME_RENDERED: + { + int64_t mediaTimeUs, systemNano; + + CHECK(msg->findInt64("media_time_us", &mediaTimeUs)); + CHECK(msg->findInt64("system_nano", &systemNano)); + + return onOMXFrameRendered( + mediaTimeUs, systemNano); + } + default: ALOGE("Unexpected message type: %d", type); return false; } } +bool ACodec::BaseState::onOMXFrameRendered( + int64_t mediaTimeUs __unused, nsecs_t systemNano __unused) { + // ignore outside of Executing and PortSettingsChanged states + return true; +} + bool ACodec::BaseState::onOMXEvent( OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) { if (event != OMX_EventError) { @@ -4949,6 +5038,15 @@ bool ACodec::BaseState::onOMXFillBufferDone( info->mDequeuedAt = ++mCodec->mDequeueCounter; info->mStatus = BufferInfo::OWNED_BY_US; + if (info->mRenderInfo != NULL) { + // The fence for an emptied buffer must have signaled, but there still could be queued + // or out-of-order dequeued buffers in the render queue prior to this buffer. Drop these, + // as we will soon requeue this buffer to the surface. While in theory we could still keep + // track of buffers that are requeued to the surface, it is better to add support to the + // buffer-queue to notify us of released buffers and their fences (in the future). + mCodec->notifyOfRenderedFrames(true /* dropIncomplete */); + } + // byte buffers cannot take fences, so wait for any fence now if (mCodec->mNativeWindow == NULL) { (void)mCodec->waitForFence(fenceFd, "onOMXFillBufferDone"); @@ -5085,6 +5183,14 @@ void ACodec::BaseState::onOutputBufferDrained(const sp &msg) { ATRACE_NAME("render"); // The client wants this buffer to be rendered. + // save buffers sent to the surface so we can get render time when they return + int64_t mediaTimeUs = -1; + info->mData->meta()->findInt64("timeUs", &mediaTimeUs); + if (mediaTimeUs >= 0) { + mCodec->mRenderTracker.onFrameQueued( + mediaTimeUs, info->mGraphicBuffer, new Fence(::dup(info->mFenceFd))); + } + int64_t timestampNs = 0; if (!msg->findInt64("timestampNs", ×tampNs)) { // TODO: it seems like we should use the timestamp @@ -5367,6 +5473,7 @@ bool ACodec::UninitializedState::onAllocateComponent(const sp &msg) { observer->setNotificationMessage(notify); mCodec->mComponentName = componentName; + mCodec->mRenderTracker.setComponentName(componentName); mCodec->mFlags = 0; if (componentName.endsWith(".secure")) { @@ -5969,6 +6076,7 @@ void ACodec::ExecutingState::resume() { void ACodec::ExecutingState::stateEntered() { ALOGV("[%s] Now Executing", mCodec->mComponentName.c_str()); + mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC)); mCodec->processDeferredMessages(); } @@ -6179,6 +6287,11 @@ void ACodec::onSignalEndOfInputStream() { notify->post(); } +bool ACodec::ExecutingState::onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) { + mCodec->onFrameRendered(mediaTimeUs, systemNano); + return true; +} + bool ACodec::ExecutingState::onOMXEvent( OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) { switch (event) { @@ -6266,6 +6379,12 @@ void ACodec::OutputPortSettingsChangedState::stateEntered() { mCodec->mComponentName.c_str()); } +bool ACodec::OutputPortSettingsChangedState::onOMXFrameRendered( + int64_t mediaTimeUs, nsecs_t systemNano) { + mCodec->onFrameRendered(mediaTimeUs, systemNano); + return true; +} + bool ACodec::OutputPortSettingsChangedState::onOMXEvent( OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) { switch (event) { @@ -6644,6 +6763,8 @@ void ACodec::FlushingState::changeStateIfWeOwnAllBuffers() { // the native window for rendering. Let's get those back as well. mCodec->waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs(); + mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC)); + sp notify = mCodec->mNotify->dup(); notify->setInt32("what", CodecBase::kWhatFlushCompleted); notify->post(); -- cgit v1.1