From 90fcf68fd29f3cb695bd53a830ad984cb7d430c0 Mon Sep 17 00:00:00 2001 From: Lajos Molnar Date: Thu, 4 Jun 2015 10:29:19 -0700 Subject: stagefright: add support for output frame rendered callback - Added FRAME_RENDERED event in OMX, used by tunneled video decoders to signal rendered event timing - Track buffers sent for rendering in ACodec and in SoftwareRenderer, and determine when they have rendered - Propagate render times to MediaCodec Bug: 20503131 Change-Id: Idf0a8714d5368b237c2285dd39fa82db847c232f --- media/libstagefright/ACodec.cpp | 123 +++++++++++++- media/libstagefright/Android.mk | 1 + media/libstagefright/AwesomePlayer.cpp | 6 +- media/libstagefright/FrameRenderTracker.cpp | 184 +++++++++++++++++++++ media/libstagefright/MediaCodec.cpp | 73 +++++++- .../colorconversion/SoftwareRenderer.cpp | 39 +++-- media/libstagefright/include/SoftwareRenderer.h | 9 +- media/libstagefright/omx/OMX.cpp | 28 +++- 8 files changed, 435 insertions(+), 28 deletions(-) create mode 100644 media/libstagefright/FrameRenderTracker.cpp (limited to 'media') diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp index 172e19c..71cb194 100644 --- a/media/libstagefright/ACodec.cpp +++ b/media/libstagefright/ACodec.cpp @@ -37,6 +37,7 @@ #include #include +#include #include #include #include @@ -175,6 +176,15 @@ struct CodecObserver : public BnOMXObserver { break; } + case omx_message::FRAME_RENDERED: + { + msg->setInt64( + "media_time_us", omx_msg.u.render_data.timestamp); + msg->setInt64( + "system_nano", omx_msg.u.render_data.nanoTime); + break; + } + default: ALOGE("Unrecognized message type: %d", omx_msg.type); break; @@ -238,6 +248,8 @@ private: int64_t timeUs, int fenceFd); + virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano); + void getMoreInputDataIfPossible(); DISALLOW_EVIL_CONSTRUCTORS(BaseState); @@ -354,6 +366,7 @@ protected: virtual void stateEntered(); virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2); + virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano); private: bool mActive; @@ -372,6 +385,7 @@ protected: virtual void stateEntered(); virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2); + virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano); private: DISALLOW_EVIL_CONSTRUCTORS(OutputPortSettingsChangedState); @@ -786,6 +800,7 @@ status_t ACodec::allocateBuffersOnPort(OMX_U32 portIndex) { BufferInfo info; info.mStatus = BufferInfo::OWNED_BY_US; info.mFenceFd = -1; + info.mRenderInfo = NULL; uint32_t requiresAllocateBufferBit = (portIndex == kPortIndexInput) @@ -1002,6 +1017,7 @@ status_t ACodec::allocateOutputBuffersFromNativeWindow() { info.mStatus = BufferInfo::OWNED_BY_US; info.mFenceFd = fenceFd; info.mIsReadFence = false; + info.mRenderInfo = NULL; info.mData = new ABuffer(NULL /* data */, bufferSize /* capacity */); info.mGraphicBuffer = graphicBuffer; mBuffers[kPortIndexOutput].push(info); @@ -1073,6 +1089,7 @@ status_t ACodec::allocateOutputMetadataBuffers() { BufferInfo info; info.mStatus = BufferInfo::OWNED_BY_NATIVE_WINDOW; info.mFenceFd = -1; + info.mRenderInfo = NULL; info.mGraphicBuffer = NULL; info.mDequeuedAt = mDequeueCounter; @@ -1219,6 +1236,42 @@ status_t ACodec::cancelBufferToNativeWindow(BufferInfo *info) { return err; } +void ACodec::updateRenderInfoForDequeuedBuffer( + ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info) { + + info->mRenderInfo = + mRenderTracker.updateInfoForDequeuedBuffer( + buf, fenceFd, info - &mBuffers[kPortIndexOutput][0]); + + // check for any fences already signaled + notifyOfRenderedFrames(false /* dropIncomplete */, info->mRenderInfo); +} + +void ACodec::onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) { + if (mRenderTracker.onFrameRendered(mediaTimeUs, systemNano) != OK) { + mRenderTracker.dumpRenderQueue(); + } +} + +void ACodec::notifyOfRenderedFrames(bool dropIncomplete, FrameRenderTracker::Info *until) { + sp msg = mNotify->dup(); + msg->setInt32("what", CodecBase::kWhatOutputFramesRendered); + std::list done = + mRenderTracker.checkFencesAndGetRenderedFrames(until, dropIncomplete); + + // unlink untracked frames + for (std::list::const_iterator it = done.cbegin(); + it != done.cend(); ++it) { + if (it->getIndex() >= 0) { + mBuffers[kPortIndexOutput].editItemAt(it->getIndex()).mRenderInfo = NULL; + } + } + + if (MediaCodec::CreateFramesRenderedMessage(done, msg)) { + msg->post(); + } +} + ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { ANativeWindowBuffer *buf; CHECK(mNativeWindow.get() != NULL); @@ -1242,7 +1295,7 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { BufferInfo *info = &mBuffers[kPortIndexOutput].editItemAt(i); if (info->mGraphicBuffer != NULL && - info->mGraphicBuffer->handle == buf->handle) { + info->mGraphicBuffer->handle == buf->handle) { // Since consumers can attach buffers to BufferQueues, it is possible // that a known yet stale buffer can return from a surface that we // once used. We can simply ignore this as we have already dequeued @@ -1255,9 +1308,11 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { stale = true; break; } + ALOGV("dequeued buffer %p", info->mGraphicBuffer->getNativeBuffer()); info->mStatus = BufferInfo::OWNED_BY_US; info->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow"); + updateRenderInfoForDequeuedBuffer(buf, fenceFd, info); return info; } } @@ -1301,6 +1356,8 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { oldest->mGraphicBuffer = new GraphicBuffer(buf, false); oldest->mStatus = BufferInfo::OWNED_BY_US; oldest->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow for oldest"); + mRenderTracker.untrackFrame(oldest->mRenderInfo); + oldest->mRenderInfo = NULL; mOMX->updateGraphicBufferInMeta( mNode, kPortIndexOutput, oldest->mGraphicBuffer, @@ -1324,6 +1381,7 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { oldest->mGraphicBuffer->getNativeBuffer(), oldest->mData->base()); } + updateRenderInfoForDequeuedBuffer(buf, fenceFd, oldest); return oldest; } @@ -1398,6 +1456,9 @@ status_t ACodec::freeBuffer(OMX_U32 portIndex, size_t i) { ::close(info->mFenceFd); } + mRenderTracker.untrackFrame(info->mRenderInfo); + info->mRenderInfo = NULL; + // remove buffer even if mOMX->freeBuffer fails mBuffers[portIndex].removeAt(i); @@ -4518,9 +4579,20 @@ bool ACodec::BaseState::onOMXMessageList(const sp &msg) { CHECK(msg->findObject("messages", &obj)); sp msgList = static_cast(obj.get()); + bool receivedRenderedEvents = false; for (std::list>::const_iterator it = msgList->getList().cbegin(); it != msgList->getList().cend(); ++it) { onOMXMessage(*it); + int32_t type; + CHECK((*it)->findInt32("type", &type)); + if (type == omx_message::FRAME_RENDERED) { + receivedRenderedEvents = true; + } + } + + if (receivedRenderedEvents) { + // NOTE: all buffers are rendered in this case + mCodec->notifyOfRenderedFrames(); } return true; } @@ -4587,12 +4659,29 @@ bool ACodec::BaseState::onOMXMessage(const sp &msg) { fenceFd); } + case omx_message::FRAME_RENDERED: + { + int64_t mediaTimeUs, systemNano; + + CHECK(msg->findInt64("media_time_us", &mediaTimeUs)); + CHECK(msg->findInt64("system_nano", &systemNano)); + + return onOMXFrameRendered( + mediaTimeUs, systemNano); + } + default: ALOGE("Unexpected message type: %d", type); return false; } } +bool ACodec::BaseState::onOMXFrameRendered( + int64_t mediaTimeUs __unused, nsecs_t systemNano __unused) { + // ignore outside of Executing and PortSettingsChanged states + return true; +} + bool ACodec::BaseState::onOMXEvent( OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) { if (event != OMX_EventError) { @@ -4949,6 +5038,15 @@ bool ACodec::BaseState::onOMXFillBufferDone( info->mDequeuedAt = ++mCodec->mDequeueCounter; info->mStatus = BufferInfo::OWNED_BY_US; + if (info->mRenderInfo != NULL) { + // The fence for an emptied buffer must have signaled, but there still could be queued + // or out-of-order dequeued buffers in the render queue prior to this buffer. Drop these, + // as we will soon requeue this buffer to the surface. While in theory we could still keep + // track of buffers that are requeued to the surface, it is better to add support to the + // buffer-queue to notify us of released buffers and their fences (in the future). + mCodec->notifyOfRenderedFrames(true /* dropIncomplete */); + } + // byte buffers cannot take fences, so wait for any fence now if (mCodec->mNativeWindow == NULL) { (void)mCodec->waitForFence(fenceFd, "onOMXFillBufferDone"); @@ -5085,6 +5183,14 @@ void ACodec::BaseState::onOutputBufferDrained(const sp &msg) { ATRACE_NAME("render"); // The client wants this buffer to be rendered. + // save buffers sent to the surface so we can get render time when they return + int64_t mediaTimeUs = -1; + info->mData->meta()->findInt64("timeUs", &mediaTimeUs); + if (mediaTimeUs >= 0) { + mCodec->mRenderTracker.onFrameQueued( + mediaTimeUs, info->mGraphicBuffer, new Fence(::dup(info->mFenceFd))); + } + int64_t timestampNs = 0; if (!msg->findInt64("timestampNs", ×tampNs)) { // TODO: it seems like we should use the timestamp @@ -5367,6 +5473,7 @@ bool ACodec::UninitializedState::onAllocateComponent(const sp &msg) { observer->setNotificationMessage(notify); mCodec->mComponentName = componentName; + mCodec->mRenderTracker.setComponentName(componentName); mCodec->mFlags = 0; if (componentName.endsWith(".secure")) { @@ -5969,6 +6076,7 @@ void ACodec::ExecutingState::resume() { void ACodec::ExecutingState::stateEntered() { ALOGV("[%s] Now Executing", mCodec->mComponentName.c_str()); + mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC)); mCodec->processDeferredMessages(); } @@ -6179,6 +6287,11 @@ void ACodec::onSignalEndOfInputStream() { notify->post(); } +bool ACodec::ExecutingState::onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) { + mCodec->onFrameRendered(mediaTimeUs, systemNano); + return true; +} + bool ACodec::ExecutingState::onOMXEvent( OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) { switch (event) { @@ -6266,6 +6379,12 @@ void ACodec::OutputPortSettingsChangedState::stateEntered() { mCodec->mComponentName.c_str()); } +bool ACodec::OutputPortSettingsChangedState::onOMXFrameRendered( + int64_t mediaTimeUs, nsecs_t systemNano) { + mCodec->onFrameRendered(mediaTimeUs, systemNano); + return true; +} + bool ACodec::OutputPortSettingsChangedState::onOMXEvent( OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) { switch (event) { @@ -6644,6 +6763,8 @@ void ACodec::FlushingState::changeStateIfWeOwnAllBuffers() { // the native window for rendering. Let's get those back as well. mCodec->waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs(); + mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC)); + sp notify = mCodec->mNotify->dup(); notify->setInt32("what", CodecBase::kWhatFlushCompleted); notify->post(); diff --git a/media/libstagefright/Android.mk b/media/libstagefright/Android.mk index 0dfa300..69128bd 100644 --- a/media/libstagefright/Android.mk +++ b/media/libstagefright/Android.mk @@ -23,6 +23,7 @@ LOCAL_SRC_FILES:= \ ESDS.cpp \ FileSource.cpp \ FLACExtractor.cpp \ + FrameRenderTracker.cpp \ HTTPBase.cpp \ JPEGSource.cpp \ MP3Extractor.cpp \ diff --git a/media/libstagefright/AwesomePlayer.cpp b/media/libstagefright/AwesomePlayer.cpp index df01e7c..4e6c2a6 100644 --- a/media/libstagefright/AwesomePlayer.cpp +++ b/media/libstagefright/AwesomePlayer.cpp @@ -113,11 +113,11 @@ struct AwesomeLocalRenderer : public AwesomeRenderer { CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs)); render((const uint8_t *)buffer->data() + buffer->range_offset(), - buffer->range_length(), timeUs * 1000); + buffer->range_length(), timeUs, timeUs * 1000); } - void render(const void *data, size_t size, int64_t timestampNs) { - mTarget->render(data, size, timestampNs, NULL, mFormat); + void render(const void *data, size_t size, int64_t mediaTimeUs, nsecs_t renderTimeNs) { + (void)mTarget->render(data, size, mediaTimeUs, renderTimeNs, NULL, mFormat); } protected: diff --git a/media/libstagefright/FrameRenderTracker.cpp b/media/libstagefright/FrameRenderTracker.cpp new file mode 100644 index 0000000..ebd2197 --- /dev/null +++ b/media/libstagefright/FrameRenderTracker.cpp @@ -0,0 +1,184 @@ +/* + * Copyright 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "FrameRenderTracker" + +#include +#include + +#include +#include + +namespace android { + +FrameRenderTracker::FrameRenderTracker() + : mLastRenderTimeNs(-1), + mComponentName("unknown component") { +} + +FrameRenderTracker::~FrameRenderTracker() { +} + +void FrameRenderTracker::setComponentName(const AString &componentName) { + mComponentName = componentName; +} + +void FrameRenderTracker::clear(nsecs_t lastRenderTimeNs) { + mRenderQueue.clear(); + mLastRenderTimeNs = lastRenderTimeNs; +} + +void FrameRenderTracker::onFrameQueued( + int64_t mediaTimeUs, const sp &graphicBuffer, const sp &fence) { + mRenderQueue.emplace_back(mediaTimeUs, graphicBuffer, fence); +} + +FrameRenderTracker::Info *FrameRenderTracker::updateInfoForDequeuedBuffer( + ANativeWindowBuffer *buf, int fenceFd, int index) { + if (index < 0) { + return NULL; + } + + // see if this is a buffer that was to be rendered + std::list::iterator renderInfo = mRenderQueue.end(); + for (std::list::iterator it = mRenderQueue.begin(); + it != mRenderQueue.end(); ++it) { + if (it->mGraphicBuffer->handle == buf->handle) { + renderInfo = it; + break; + } + } + if (renderInfo == mRenderQueue.end()) { + // could have been canceled after fence has signaled + return NULL; + } + + if (renderInfo->mIndex >= 0) { + // buffer has been dequeued before, so there is nothing to do + return NULL; + } + + // was this frame dropped (we could also infer this if the fence is invalid or a dup of + // the queued fence; however, there is no way to figure that out.) + if (fenceFd < 0) { + // frame is new or was dropped + mRenderQueue.erase(renderInfo); + return NULL; + } + + // store dequeue fence and buffer index + renderInfo->mFence = new Fence(::dup(fenceFd)); + renderInfo->mIndex = index; + return &*renderInfo; +} + +status_t FrameRenderTracker::onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) { + // ensure monotonic timestamps + if (mLastRenderTimeNs >= systemNano) { + ALOGW("[%s] Ignoring out of order/stale system nano %lld for media time %lld from codec.", + mComponentName.c_str(), (long long)systemNano, (long long)mediaTimeUs); + return BAD_VALUE; + } + + nsecs_t now = systemTime(SYSTEM_TIME_MONOTONIC); + if (systemNano > now) { + ALOGW("[%s] Ignoring system nano %lld in the future for media time %lld from codec.", + mComponentName.c_str(), (long long)systemNano, (long long)mediaTimeUs); + return OK; + } + + mRenderQueue.emplace_back(mediaTimeUs, systemNano); + mLastRenderTimeNs = systemNano; + return OK; +} + +std::list FrameRenderTracker::checkFencesAndGetRenderedFrames( + const FrameRenderTracker::Info *until, bool dropIncomplete) { + std::list done; + + // complete any frames queued prior to this and drop any incomplete ones if requested + for (std::list::iterator it = mRenderQueue.begin(); + it != mRenderQueue.end(); ) { + bool drop = false; // whether to drop each frame + if (it->mIndex < 0) { + // frame not yet dequeued (or already rendered on a tunneled surface) + drop = dropIncomplete; + } else if (it->mFence != NULL) { + // check if fence signaled + nsecs_t signalTime = it->mFence->getSignalTime(); + if (signalTime < 0) { // invalid fence + drop = true; + } else if (signalTime == INT64_MAX) { // unsignaled fence + drop = dropIncomplete; + } else { // signaled + // save render time + it->mFence.clear(); + it->mRenderTimeNs = signalTime; + } + } + bool foundFrame = (Info *)&*it == until; + + // Return frames with signaled fences at the start of the queue, as they are + // in submit order, and we don't have to wait for any in-between frames. + // Also return any dropped frames. + if (drop || (it->mFence == NULL && it == mRenderQueue.begin())) { + // (unrendered) dropped frames have their mRenderTimeNs still set to -1 + done.splice(done.end(), mRenderQueue, it++); + } else { + ++it; + } + if (foundFrame) { + break; + } + } + + return done; +} + +void FrameRenderTracker::untrackFrame(const FrameRenderTracker::Info *info) { + if (info != NULL) { + for (std::list::iterator it = mRenderQueue.begin(); + it != mRenderQueue.end(); ++it) { + if (&*it == info) { + mRenderQueue.erase(it); + return; + } + } + } +} + +void FrameRenderTracker::dumpRenderQueue() const { + ALOGI("[%s] Render Queue: (last render time: %lldns)", + mComponentName.c_str(), (long long)mLastRenderTimeNs); + for (std::list::const_iterator it = mRenderQueue.cbegin(); + it != mRenderQueue.cend(); ++it) { + if (it->mFence == NULL) { + ALOGI(" RENDERED: handle: %p, media time: %lldus, index: %zd, render time: %lldns", + it->mGraphicBuffer == NULL ? NULL : it->mGraphicBuffer->handle, + (long long)it->mMediaTimeUs, it->mIndex, (long long)it->mRenderTimeNs); + } else if (it->mIndex < 0) { + ALOGI(" QUEUED: handle: %p, media time: %lldus, fence: %s", + it->mGraphicBuffer->handle, (long long)it->mMediaTimeUs, + it->mFence->isValid() ? "YES" : "NO"); + } else { + ALOGI(" DEQUEUED: handle: %p, media time: %lldus, index: %zd", + it->mGraphicBuffer->handle, (long long)it->mMediaTimeUs, it->mIndex); + } + } +} + +} // namespace android diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp index f918d2d..7ee84a8 100644 --- a/media/libstagefright/MediaCodec.cpp +++ b/media/libstagefright/MediaCodec.cpp @@ -397,6 +397,12 @@ status_t MediaCodec::setCallback(const sp &callback) { return PostAndAwaitResponse(msg, &response); } +status_t MediaCodec::setOnFrameRenderedNotification(const sp ¬ify) { + sp msg = new AMessage(kWhatSetNotification, this); + msg->setMessage("on-frame-rendered", notify); + return msg->post(); +} + status_t MediaCodec::configure( const sp &format, const sp &surface, @@ -1333,6 +1339,18 @@ void MediaCodec::onMessageReceived(const sp &msg) { break; } + case CodecBase::kWhatOutputFramesRendered: + { + // ignore these in all states except running, and check that we have a + // notification set + if (mState == STARTED && mOnFrameRenderedNotification != NULL) { + sp notify = mOnFrameRenderedNotification->dup(); + notify->setMessage("data", msg); + notify->post(); + } + break; + } + case CodecBase::kWhatFillThisBuffer: { /* size_t index = */updateBuffers(kPortIndexInput, msg); @@ -1530,6 +1548,15 @@ void MediaCodec::onMessageReceived(const sp &msg) { break; } + case kWhatSetNotification: + { + sp notify; + if (msg->findMessage("on-frame-rendered", ¬ify)) { + mOnFrameRenderedNotification = notify; + } + break; + } + case kWhatSetCallback: { sp replyID; @@ -2372,6 +2399,23 @@ status_t MediaCodec::onQueueInputBuffer(const sp &msg) { return OK; } +//static +size_t MediaCodec::CreateFramesRenderedMessage( + std::list done, sp &msg) { + size_t index = 0; + + for (std::list::const_iterator it = done.cbegin(); + it != done.cend(); ++it) { + if (it->getRenderTimeNs() < 0) { + continue; // dropped frame from tracking + } + msg->setInt64(AStringPrintf("%zu-media-time-us", index).c_str(), it->getMediaTimeUs()); + msg->setInt64(AStringPrintf("%zu-system-nano", index).c_str(), it->getRenderTimeNs()); + ++index; + } + return index; +} + status_t MediaCodec::onReleaseOutputBuffer(const sp &msg) { size_t index; CHECK(msg->findSize("index", &index)); @@ -2404,26 +2448,37 @@ status_t MediaCodec::onReleaseOutputBuffer(const sp &msg) { if (render && info->mData != NULL && info->mData->size() != 0) { info->mNotify->setInt32("render", true); - int64_t timestampNs = 0; - if (msg->findInt64("timestampNs", ×tampNs)) { - info->mNotify->setInt64("timestampNs", timestampNs); + int64_t mediaTimeUs = -1; + info->mData->meta()->findInt64("timeUs", &mediaTimeUs); + + int64_t renderTimeNs = 0; + if (msg->findInt64("timestampNs", &renderTimeNs)) { + info->mNotify->setInt64("timestampNs", renderTimeNs); } else { // TODO: it seems like we should use the timestamp // in the (media)buffer as it potentially came from // an input surface, but we did not propagate it prior to // API 20. Perhaps check for target SDK version. #if 0 - if (info->mData->meta()->findInt64("timeUs", ×tampNs)) { - ALOGV("using buffer PTS of %" PRId64, timestampNs); - timestampNs *= 1000; - } + ALOGV("using buffer PTS of %" PRId64, timestampNs); + renderTimeNs = mediaTimeUs * 1000; #endif } if (mSoftRenderer != NULL) { - mSoftRenderer->render( + std::list doneFrames = mSoftRenderer->render( info->mData->data(), info->mData->size(), - timestampNs, NULL, info->mFormat); + mediaTimeUs, renderTimeNs, NULL, info->mFormat); + + // if we are running, notify rendered frames + if (!doneFrames.empty() && mState == STARTED && mOnFrameRenderedNotification != NULL) { + sp notify = mOnFrameRenderedNotification->dup(); + sp data = new AMessage; + if (CreateFramesRenderedMessage(doneFrames, data)) { + notify->setMessage("data", data); + notify->post(); + } + } } } diff --git a/media/libstagefright/colorconversion/SoftwareRenderer.cpp b/media/libstagefright/colorconversion/SoftwareRenderer.cpp index 695cfc8..d22451b 100644 --- a/media/libstagefright/colorconversion/SoftwareRenderer.cpp +++ b/media/libstagefright/colorconversion/SoftwareRenderer.cpp @@ -196,17 +196,29 @@ void SoftwareRenderer::resetFormatIfChanged(const sp &format) { mNativeWindow.get(), transform)); } -void SoftwareRenderer::render( - const void *data, size_t size, int64_t timestampNs, +void SoftwareRenderer::clearTracker() { + mRenderTracker.clear(-1 /* lastRenderTimeNs */); +} + +std::list SoftwareRenderer::render( + const void *data, size_t size, int64_t mediaTimeUs, nsecs_t renderTimeNs, void* /*platformPrivate*/, const sp& format) { resetFormatIfChanged(format); + FrameRenderTracker::Info *info = NULL; ANativeWindowBuffer *buf; - int err; - if ((err = native_window_dequeue_buffer_and_wait(mNativeWindow.get(), - &buf)) != 0) { + int fenceFd = -1; + int err = mNativeWindow->dequeueBuffer(mNativeWindow.get(), &buf, &fenceFd); + if (err == 0 && fenceFd >= 0) { + info = mRenderTracker.updateInfoForDequeuedBuffer(buf, fenceFd, 0); + sp fence = new Fence(fenceFd); + err = fence->waitForever("SoftwareRenderer::render"); + } + if (err != 0) { ALOGW("Surface::dequeueBuffer returned error %d", err); - return; + // complete (drop) dequeued frame if fence wait failed; otherwise, + // this returns an empty list as no frames should have rendered and not yet returned. + return mRenderTracker.checkFencesAndGetRenderedFrames(info, false /* dropIncomplete */); } GraphicBufferMapper &mapper = GraphicBufferMapper::get(); @@ -342,16 +354,21 @@ void SoftwareRenderer::render( skip_copying: CHECK_EQ(0, mapper.unlock(buf->handle)); - if ((err = native_window_set_buffers_timestamp(mNativeWindow.get(), - timestampNs)) != 0) { - ALOGW("Surface::set_buffers_timestamp returned error %d", err); + if (renderTimeNs >= 0) { + if ((err = native_window_set_buffers_timestamp(mNativeWindow.get(), + renderTimeNs)) != 0) { + ALOGW("Surface::set_buffers_timestamp returned error %d", err); + } } - if ((err = mNativeWindow->queueBuffer(mNativeWindow.get(), buf, - -1)) != 0) { + if ((err = mNativeWindow->queueBuffer(mNativeWindow.get(), buf, -1)) != 0) { ALOGW("Surface::queueBuffer returned error %d", err); + } else { + mRenderTracker.onFrameQueued(mediaTimeUs, (GraphicBuffer *)buf, Fence::NO_FENCE); } + buf = NULL; + return mRenderTracker.checkFencesAndGetRenderedFrames(info, info != NULL /* dropIncomplete */); } } // namespace android diff --git a/media/libstagefright/include/SoftwareRenderer.h b/media/libstagefright/include/SoftwareRenderer.h index fa3ea89..9e652d5 100644 --- a/media/libstagefright/include/SoftwareRenderer.h +++ b/media/libstagefright/include/SoftwareRenderer.h @@ -19,9 +19,12 @@ #define SOFTWARE_RENDERER_H_ #include +#include #include #include +#include + namespace android { struct AMessage; @@ -32,9 +35,10 @@ public: ~SoftwareRenderer(); - void render( - const void *data, size_t size, int64_t timestampNs, + std::list render( + const void *data, size_t size, int64_t mediaTimeUs, nsecs_t renderTimeNs, void *platformPrivate, const sp &format); + void clearTracker(); private: enum YUVMode { @@ -48,6 +52,7 @@ private: int32_t mWidth, mHeight; int32_t mCropLeft, mCropTop, mCropRight, mCropBottom; int32_t mCropWidth, mCropHeight; + FrameRenderTracker mRenderTracker; SoftwareRenderer(const SoftwareRenderer &); SoftwareRenderer &operator=(const SoftwareRenderer &); diff --git a/media/libstagefright/omx/OMX.cpp b/media/libstagefright/omx/OMX.cpp index e94adbd..cb7ab5e 100644 --- a/media/libstagefright/omx/OMX.cpp +++ b/media/libstagefright/omx/OMX.cpp @@ -34,6 +34,7 @@ #include #include +#include namespace android { @@ -455,12 +456,35 @@ OMX_ERRORTYPE OMX::OnEvent( OMX_IN OMX_EVENTTYPE eEvent, OMX_IN OMX_U32 nData1, OMX_IN OMX_U32 nData2, - OMX_IN OMX_PTR /* pEventData */) { + OMX_IN OMX_PTR pEventData) { ALOGV("OnEvent(%d, %" PRIu32", %" PRIu32 ")", eEvent, nData1, nData2); // Forward to OMXNodeInstance. findInstance(node)->onEvent(eEvent, nData1, nData2); + sp dispatcher = findDispatcher(node); + + // output rendered events are not processed as regular events until they hit the observer + if (eEvent == OMX_EventOutputRendered) { + if (pEventData == NULL) { + return OMX_ErrorBadParameter; + } + + // process data from array + OMX_VIDEO_RENDEREVENTTYPE *renderData = (OMX_VIDEO_RENDEREVENTTYPE *)pEventData; + for (size_t i = 0; i < nData1; ++i) { + omx_message msg; + msg.type = omx_message::FRAME_RENDERED; + msg.node = node; + msg.fenceFd = -1; + msg.u.render_data.timestamp = renderData[i].nMediaTimeUs; + msg.u.render_data.nanoTime = renderData[i].nSystemTimeNs; + + dispatcher->post(msg, false /* realTime */); + } + return OMX_ErrorNone; + } + omx_message msg; msg.type = omx_message::EVENT; msg.node = node; @@ -469,7 +493,7 @@ OMX_ERRORTYPE OMX::OnEvent( msg.u.event_data.data1 = nData1; msg.u.event_data.data2 = nData2; - findDispatcher(node)->post(msg); + dispatcher->post(msg, true /* realTime */); return OMX_ErrorNone; } -- cgit v1.1