From 90fcf68fd29f3cb695bd53a830ad984cb7d430c0 Mon Sep 17 00:00:00 2001 From: Lajos Molnar Date: Thu, 4 Jun 2015 10:29:19 -0700 Subject: stagefright: add support for output frame rendered callback - Added FRAME_RENDERED event in OMX, used by tunneled video decoders to signal rendered event timing - Track buffers sent for rendering in ACodec and in SoftwareRenderer, and determine when they have rendered - Propagate render times to MediaCodec Bug: 20503131 Change-Id: Idf0a8714d5368b237c2285dd39fa82db847c232f --- include/media/IOMX.h | 7 +- include/media/stagefright/ACodec.h | 20 +++ include/media/stagefright/CodecBase.h | 1 + include/media/stagefright/FrameRenderTracker.h | 142 ++++++++++++++++ include/media/stagefright/MediaCodec.h | 12 ++ media/libstagefright/ACodec.cpp | 123 +++++++++++++- media/libstagefright/Android.mk | 1 + media/libstagefright/AwesomePlayer.cpp | 6 +- media/libstagefright/FrameRenderTracker.cpp | 184 +++++++++++++++++++++ media/libstagefright/MediaCodec.cpp | 73 +++++++- .../colorconversion/SoftwareRenderer.cpp | 39 +++-- media/libstagefright/include/SoftwareRenderer.h | 9 +- media/libstagefright/omx/OMX.cpp | 28 +++- 13 files changed, 616 insertions(+), 29 deletions(-) create mode 100644 include/media/stagefright/FrameRenderTracker.h create mode 100644 media/libstagefright/FrameRenderTracker.cpp diff --git a/include/media/IOMX.h b/include/media/IOMX.h index 7023453..3d29e4a 100644 --- a/include/media/IOMX.h +++ b/include/media/IOMX.h @@ -198,7 +198,7 @@ struct omx_message { EVENT, EMPTY_BUFFER_DONE, FILL_BUFFER_DONE, - + FRAME_RENDERED, } type; IOMX::node_id node; @@ -226,6 +226,11 @@ struct omx_message { OMX_TICKS timestamp; } extended_buffer_data; + // if type == FRAME_RENDERED + struct { + OMX_TICKS timestamp; + OMX_S64 nanoTime; + } render_data; } u; }; diff --git a/include/media/stagefright/ACodec.h b/include/media/stagefright/ACodec.h index d4891a1..f9ea38e 100644 --- a/include/media/stagefright/ACodec.h +++ b/include/media/stagefright/ACodec.h @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -162,6 +163,7 @@ private: sp mData; sp mGraphicBuffer; int mFenceFd; + FrameRenderTracker::Info *mRenderInfo; // The following field and 4 methods are used for debugging only bool mIsReadFence; @@ -214,6 +216,7 @@ private: sp mOutputFormat; sp mBaseOutputFormat; + FrameRenderTracker mRenderTracker; // render information for buffers rendered by ACodec Vector mBuffers[2]; bool mPortEOS[2]; status_t mInputEOSResult; @@ -375,6 +378,23 @@ private: void deferMessage(const sp &msg); void processDeferredMessages(); + void onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano); + // called when we have dequeued a buffer |buf| from the native window to track render info. + // |fenceFd| is the dequeue fence, and |info| points to the buffer info where this buffer is + // stored. + void updateRenderInfoForDequeuedBuffer( + ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info); + + // Checks to see if any frames have rendered up until |until|, and to notify client + // (MediaCodec) of rendered frames up-until the frame pointed to by |until| or the first + // unrendered frame. These frames are removed from the render queue. + // If |dropIncomplete| is true, unrendered frames up-until |until| will be dropped from the + // queue, allowing all rendered framed up till then to be notified of. + // (This will effectively clear the render queue up-until (and including) |until|.) + // If |until| is NULL, or is not in the rendered queue, this method will check all frames. + void notifyOfRenderedFrames( + bool dropIncomplete = false, FrameRenderTracker::Info *until = NULL); + void sendFormatChange(const sp &reply); status_t getPortFormat(OMX_U32 portIndex, sp ¬ify); diff --git a/include/media/stagefright/CodecBase.h b/include/media/stagefright/CodecBase.h index 989df4f..bb36052 100644 --- a/include/media/stagefright/CodecBase.h +++ b/include/media/stagefright/CodecBase.h @@ -43,6 +43,7 @@ struct CodecBase : public AHandler { kWhatInputSurfaceAccepted = 'isfa', kWhatSignaledInputEOS = 'seos', kWhatBuffersAllocated = 'allc', + kWhatOutputFramesRendered = 'outR', }; virtual void setNotificationMessage(const sp &msg) = 0; diff --git a/include/media/stagefright/FrameRenderTracker.h b/include/media/stagefright/FrameRenderTracker.h new file mode 100644 index 0000000..3b0db5a --- /dev/null +++ b/include/media/stagefright/FrameRenderTracker.h @@ -0,0 +1,142 @@ +/* + * Copyright 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FRAME_RENDER_TRACKER_H_ + +#define FRAME_RENDER_TRACKER_H_ + +#include +#include +#include + +#include +#include + +#include + +namespace android { + +class Fence; +class GraphicBuffer; + +struct FrameRenderTracker : public RefBase { + // Tracks the render information about a frame. Frames go through several states while + // the render information is tracked: + // + // 1. queued frame: mMediaTime and mGraphicBuffer are set for the frame. mFence is the + // queue fence (read fence). mIndex is negative, and mRenderTimeNs is invalid. + // Key characteristics: mFence is not NULL and mIndex is negative. + // + // 2. dequeued frame: mFence is updated with the dequeue fence (write fence). mIndex is set. + // Key characteristics: mFence is not NULL and mIndex is non-negative. mRenderTime is still + // invalid. + // + // 3. rendered frame or frame: mFence is cleared, mRenderTimeNs is set. + // Key characteristics: mFence is NULL. + // + struct Info { + // set by client during onFrameQueued or onFrameRendered + int64_t getMediaTimeUs() const { return mMediaTimeUs; } + + // -1 if frame is not yet rendered + nsecs_t getRenderTimeNs() const { return mRenderTimeNs; } + + // set by client during updateRenderInfoForDequeuedBuffer; -1 otherwise + ssize_t getIndex() const { return mIndex; } + + // creates information for a queued frame + Info(int64_t mediaTimeUs, const sp &graphicBuffer, const sp &fence) + : mMediaTimeUs(mediaTimeUs), + mRenderTimeNs(-1), + mIndex(-1), + mGraphicBuffer(graphicBuffer), + mFence(fence) { + } + + // creates information for a frame rendered on a tunneled surface + Info(int64_t mediaTimeUs, nsecs_t renderTimeNs) + : mMediaTimeUs(mediaTimeUs), + mRenderTimeNs(renderTimeNs), + mIndex(-1), + mGraphicBuffer(NULL), + mFence(NULL) { + } + + private: + int64_t mMediaTimeUs; + nsecs_t mRenderTimeNs; + ssize_t mIndex; // to be used by client + sp mGraphicBuffer; + sp mFence; + + friend class FrameRenderTracker; + }; + + FrameRenderTracker(); + + void setComponentName(const AString &componentName); + + // clears all tracked frames, and resets last render time + void clear(nsecs_t lastRenderTimeNs); + + // called when |graphicBuffer| corresponding to |mediaTimeUs| is + // queued to the output surface using |fence|. + void onFrameQueued( + int64_t mediaTimeUs, const sp &graphicBuffer, const sp &fence); + + // Called when we have dequeued a buffer |buf| from the native window to track render info. + // |fenceFd| is the dequeue fence, and |index| is a positive buffer ID to be usable by the + // client to track this render info among the dequeued buffers. + // Returns pointer to the tracked info, or NULL if buffer is not tracked or if |index| + // is negative. + Info *updateInfoForDequeuedBuffer(ANativeWindowBuffer *buf, int fenceFd, int index); + + // called when tunneled codec signals frame rendered event + // returns BAD_VALUE if systemNano is not monotonic. Otherwise, returns OK. + status_t onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano); + + // Checks to see if any frames have rendered up until |until|. If |until| is NULL or not a + // tracked info, this method searches the entire render queue. + // Returns list of rendered frames up-until the frame pointed to by |until| or the first + // unrendered frame, as well as any dropped frames (those with invalid fence) up-until |until|. + // These frames are removed from the render queue. + // If |dropIncomplete| is true, unrendered frames up-until |until| will also be dropped from the + // queue, allowing all rendered framed up till then to be notified of. + // (This will effectively clear the render queue up-until (and including) |until|.) + std::list checkFencesAndGetRenderedFrames(const Info *until, bool dropIncomplete); + + // Stop tracking a queued frame (e.g. if the frame has been discarded). If |info| is NULL or is + // not tracked, this method is a no-op. + void untrackFrame(const Info *info); + + void dumpRenderQueue() const; + + virtual ~FrameRenderTracker(); + +private: + + // Render information for buffers. Regular surface buffers are queued in the order of + // rendering. Tunneled buffers are queued in the order of receipt. + std::list mRenderQueue; + nsecs_t mLastRenderTimeNs; + AString mComponentName; + + DISALLOW_EVIL_CONSTRUCTORS(FrameRenderTracker); +}; + +} // namespace android + +#endif // FRAME_RENDER_TRACKER_H_ diff --git a/include/media/stagefright/MediaCodec.h b/include/media/stagefright/MediaCodec.h index d62b35d..09cbe8f 100644 --- a/include/media/stagefright/MediaCodec.h +++ b/include/media/stagefright/MediaCodec.h @@ -22,6 +22,7 @@ #include #include #include +#include #include namespace android { @@ -76,6 +77,8 @@ struct MediaCodec : public AHandler { status_t setCallback(const sp &callback); + status_t setOnFrameRenderedNotification(const sp ¬ify); + status_t createInputSurface(sp* bufferProducer); status_t setInputSurface(const sp &surface); @@ -157,6 +160,12 @@ struct MediaCodec : public AHandler { status_t setParameters(const sp ¶ms); + // Create a MediaCodec notification message from a list of rendered or dropped render infos + // by adding rendered frame information to a base notification message. Returns the number + // of frames that were rendered. + static size_t CreateFramesRenderedMessage( + std::list done, sp &msg); + protected: virtual ~MediaCodec(); virtual void onMessageReceived(const sp &msg); @@ -212,6 +221,7 @@ private: kWhatGetName = 'getN', kWhatSetParameters = 'setP', kWhatSetCallback = 'setC', + kWhatSetNotification = 'setN', }; enum { @@ -275,9 +285,11 @@ private: status_t mStickyError; sp mSurface; SoftwareRenderer *mSoftRenderer; + sp mOutputFormat; sp mInputFormat; sp mCallback; + sp mOnFrameRenderedNotification; sp mDealer; sp mResourceManagerClient; diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp index 172e19c..71cb194 100644 --- a/media/libstagefright/ACodec.cpp +++ b/media/libstagefright/ACodec.cpp @@ -37,6 +37,7 @@ #include #include +#include #include #include #include @@ -175,6 +176,15 @@ struct CodecObserver : public BnOMXObserver { break; } + case omx_message::FRAME_RENDERED: + { + msg->setInt64( + "media_time_us", omx_msg.u.render_data.timestamp); + msg->setInt64( + "system_nano", omx_msg.u.render_data.nanoTime); + break; + } + default: ALOGE("Unrecognized message type: %d", omx_msg.type); break; @@ -238,6 +248,8 @@ private: int64_t timeUs, int fenceFd); + virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano); + void getMoreInputDataIfPossible(); DISALLOW_EVIL_CONSTRUCTORS(BaseState); @@ -354,6 +366,7 @@ protected: virtual void stateEntered(); virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2); + virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano); private: bool mActive; @@ -372,6 +385,7 @@ protected: virtual void stateEntered(); virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2); + virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano); private: DISALLOW_EVIL_CONSTRUCTORS(OutputPortSettingsChangedState); @@ -786,6 +800,7 @@ status_t ACodec::allocateBuffersOnPort(OMX_U32 portIndex) { BufferInfo info; info.mStatus = BufferInfo::OWNED_BY_US; info.mFenceFd = -1; + info.mRenderInfo = NULL; uint32_t requiresAllocateBufferBit = (portIndex == kPortIndexInput) @@ -1002,6 +1017,7 @@ status_t ACodec::allocateOutputBuffersFromNativeWindow() { info.mStatus = BufferInfo::OWNED_BY_US; info.mFenceFd = fenceFd; info.mIsReadFence = false; + info.mRenderInfo = NULL; info.mData = new ABuffer(NULL /* data */, bufferSize /* capacity */); info.mGraphicBuffer = graphicBuffer; mBuffers[kPortIndexOutput].push(info); @@ -1073,6 +1089,7 @@ status_t ACodec::allocateOutputMetadataBuffers() { BufferInfo info; info.mStatus = BufferInfo::OWNED_BY_NATIVE_WINDOW; info.mFenceFd = -1; + info.mRenderInfo = NULL; info.mGraphicBuffer = NULL; info.mDequeuedAt = mDequeueCounter; @@ -1219,6 +1236,42 @@ status_t ACodec::cancelBufferToNativeWindow(BufferInfo *info) { return err; } +void ACodec::updateRenderInfoForDequeuedBuffer( + ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info) { + + info->mRenderInfo = + mRenderTracker.updateInfoForDequeuedBuffer( + buf, fenceFd, info - &mBuffers[kPortIndexOutput][0]); + + // check for any fences already signaled + notifyOfRenderedFrames(false /* dropIncomplete */, info->mRenderInfo); +} + +void ACodec::onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) { + if (mRenderTracker.onFrameRendered(mediaTimeUs, systemNano) != OK) { + mRenderTracker.dumpRenderQueue(); + } +} + +void ACodec::notifyOfRenderedFrames(bool dropIncomplete, FrameRenderTracker::Info *until) { + sp msg = mNotify->dup(); + msg->setInt32("what", CodecBase::kWhatOutputFramesRendered); + std::list done = + mRenderTracker.checkFencesAndGetRenderedFrames(until, dropIncomplete); + + // unlink untracked frames + for (std::list::const_iterator it = done.cbegin(); + it != done.cend(); ++it) { + if (it->getIndex() >= 0) { + mBuffers[kPortIndexOutput].editItemAt(it->getIndex()).mRenderInfo = NULL; + } + } + + if (MediaCodec::CreateFramesRenderedMessage(done, msg)) { + msg->post(); + } +} + ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { ANativeWindowBuffer *buf; CHECK(mNativeWindow.get() != NULL); @@ -1242,7 +1295,7 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { BufferInfo *info = &mBuffers[kPortIndexOutput].editItemAt(i); if (info->mGraphicBuffer != NULL && - info->mGraphicBuffer->handle == buf->handle) { + info->mGraphicBuffer->handle == buf->handle) { // Since consumers can attach buffers to BufferQueues, it is possible // that a known yet stale buffer can return from a surface that we // once used. We can simply ignore this as we have already dequeued @@ -1255,9 +1308,11 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { stale = true; break; } + ALOGV("dequeued buffer %p", info->mGraphicBuffer->getNativeBuffer()); info->mStatus = BufferInfo::OWNED_BY_US; info->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow"); + updateRenderInfoForDequeuedBuffer(buf, fenceFd, info); return info; } } @@ -1301,6 +1356,8 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { oldest->mGraphicBuffer = new GraphicBuffer(buf, false); oldest->mStatus = BufferInfo::OWNED_BY_US; oldest->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow for oldest"); + mRenderTracker.untrackFrame(oldest->mRenderInfo); + oldest->mRenderInfo = NULL; mOMX->updateGraphicBufferInMeta( mNode, kPortIndexOutput, oldest->mGraphicBuffer, @@ -1324,6 +1381,7 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { oldest->mGraphicBuffer->getNativeBuffer(), oldest->mData->base()); } + updateRenderInfoForDequeuedBuffer(buf, fenceFd, oldest); return oldest; } @@ -1398,6 +1456,9 @@ status_t ACodec::freeBuffer(OMX_U32 portIndex, size_t i) { ::close(info->mFenceFd); } + mRenderTracker.untrackFrame(info->mRenderInfo); + info->mRenderInfo = NULL; + // remove buffer even if mOMX->freeBuffer fails mBuffers[portIndex].removeAt(i); @@ -4518,9 +4579,20 @@ bool ACodec::BaseState::onOMXMessageList(const sp &msg) { CHECK(msg->findObject("messages", &obj)); sp msgList = static_cast(obj.get()); + bool receivedRenderedEvents = false; for (std::list>::const_iterator it = msgList->getList().cbegin(); it != msgList->getList().cend(); ++it) { onOMXMessage(*it); + int32_t type; + CHECK((*it)->findInt32("type", &type)); + if (type == omx_message::FRAME_RENDERED) { + receivedRenderedEvents = true; + } + } + + if (receivedRenderedEvents) { + // NOTE: all buffers are rendered in this case + mCodec->notifyOfRenderedFrames(); } return true; } @@ -4587,12 +4659,29 @@ bool ACodec::BaseState::onOMXMessage(const sp &msg) { fenceFd); } + case omx_message::FRAME_RENDERED: + { + int64_t mediaTimeUs, systemNano; + + CHECK(msg->findInt64("media_time_us", &mediaTimeUs)); + CHECK(msg->findInt64("system_nano", &systemNano)); + + return onOMXFrameRendered( + mediaTimeUs, systemNano); + } + default: ALOGE("Unexpected message type: %d", type); return false; } } +bool ACodec::BaseState::onOMXFrameRendered( + int64_t mediaTimeUs __unused, nsecs_t systemNano __unused) { + // ignore outside of Executing and PortSettingsChanged states + return true; +} + bool ACodec::BaseState::onOMXEvent( OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) { if (event != OMX_EventError) { @@ -4949,6 +5038,15 @@ bool ACodec::BaseState::onOMXFillBufferDone( info->mDequeuedAt = ++mCodec->mDequeueCounter; info->mStatus = BufferInfo::OWNED_BY_US; + if (info->mRenderInfo != NULL) { + // The fence for an emptied buffer must have signaled, but there still could be queued + // or out-of-order dequeued buffers in the render queue prior to this buffer. Drop these, + // as we will soon requeue this buffer to the surface. While in theory we could still keep + // track of buffers that are requeued to the surface, it is better to add support to the + // buffer-queue to notify us of released buffers and their fences (in the future). + mCodec->notifyOfRenderedFrames(true /* dropIncomplete */); + } + // byte buffers cannot take fences, so wait for any fence now if (mCodec->mNativeWindow == NULL) { (void)mCodec->waitForFence(fenceFd, "onOMXFillBufferDone"); @@ -5085,6 +5183,14 @@ void ACodec::BaseState::onOutputBufferDrained(const sp &msg) { ATRACE_NAME("render"); // The client wants this buffer to be rendered. + // save buffers sent to the surface so we can get render time when they return + int64_t mediaTimeUs = -1; + info->mData->meta()->findInt64("timeUs", &mediaTimeUs); + if (mediaTimeUs >= 0) { + mCodec->mRenderTracker.onFrameQueued( + mediaTimeUs, info->mGraphicBuffer, new Fence(::dup(info->mFenceFd))); + } + int64_t timestampNs = 0; if (!msg->findInt64("timestampNs", ×tampNs)) { // TODO: it seems like we should use the timestamp @@ -5367,6 +5473,7 @@ bool ACodec::UninitializedState::onAllocateComponent(const sp &msg) { observer->setNotificationMessage(notify); mCodec->mComponentName = componentName; + mCodec->mRenderTracker.setComponentName(componentName); mCodec->mFlags = 0; if (componentName.endsWith(".secure")) { @@ -5969,6 +6076,7 @@ void ACodec::ExecutingState::resume() { void ACodec::ExecutingState::stateEntered() { ALOGV("[%s] Now Executing", mCodec->mComponentName.c_str()); + mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC)); mCodec->processDeferredMessages(); } @@ -6179,6 +6287,11 @@ void ACodec::onSignalEndOfInputStream() { notify->post(); } +bool ACodec::ExecutingState::onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) { + mCodec->onFrameRendered(mediaTimeUs, systemNano); + return true; +} + bool ACodec::ExecutingState::onOMXEvent( OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) { switch (event) { @@ -6266,6 +6379,12 @@ void ACodec::OutputPortSettingsChangedState::stateEntered() { mCodec->mComponentName.c_str()); } +bool ACodec::OutputPortSettingsChangedState::onOMXFrameRendered( + int64_t mediaTimeUs, nsecs_t systemNano) { + mCodec->onFrameRendered(mediaTimeUs, systemNano); + return true; +} + bool ACodec::OutputPortSettingsChangedState::onOMXEvent( OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) { switch (event) { @@ -6644,6 +6763,8 @@ void ACodec::FlushingState::changeStateIfWeOwnAllBuffers() { // the native window for rendering. Let's get those back as well. mCodec->waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs(); + mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC)); + sp notify = mCodec->mNotify->dup(); notify->setInt32("what", CodecBase::kWhatFlushCompleted); notify->post(); diff --git a/media/libstagefright/Android.mk b/media/libstagefright/Android.mk index 0dfa300..69128bd 100644 --- a/media/libstagefright/Android.mk +++ b/media/libstagefright/Android.mk @@ -23,6 +23,7 @@ LOCAL_SRC_FILES:= \ ESDS.cpp \ FileSource.cpp \ FLACExtractor.cpp \ + FrameRenderTracker.cpp \ HTTPBase.cpp \ JPEGSource.cpp \ MP3Extractor.cpp \ diff --git a/media/libstagefright/AwesomePlayer.cpp b/media/libstagefright/AwesomePlayer.cpp index df01e7c..4e6c2a6 100644 --- a/media/libstagefright/AwesomePlayer.cpp +++ b/media/libstagefright/AwesomePlayer.cpp @@ -113,11 +113,11 @@ struct AwesomeLocalRenderer : public AwesomeRenderer { CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs)); render((const uint8_t *)buffer->data() + buffer->range_offset(), - buffer->range_length(), timeUs * 1000); + buffer->range_length(), timeUs, timeUs * 1000); } - void render(const void *data, size_t size, int64_t timestampNs) { - mTarget->render(data, size, timestampNs, NULL, mFormat); + void render(const void *data, size_t size, int64_t mediaTimeUs, nsecs_t renderTimeNs) { + (void)mTarget->render(data, size, mediaTimeUs, renderTimeNs, NULL, mFormat); } protected: diff --git a/media/libstagefright/FrameRenderTracker.cpp b/media/libstagefright/FrameRenderTracker.cpp new file mode 100644 index 0000000..ebd2197 --- /dev/null +++ b/media/libstagefright/FrameRenderTracker.cpp @@ -0,0 +1,184 @@ +/* + * Copyright 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "FrameRenderTracker" + +#include +#include + +#include +#include + +namespace android { + +FrameRenderTracker::FrameRenderTracker() + : mLastRenderTimeNs(-1), + mComponentName("unknown component") { +} + +FrameRenderTracker::~FrameRenderTracker() { +} + +void FrameRenderTracker::setComponentName(const AString &componentName) { + mComponentName = componentName; +} + +void FrameRenderTracker::clear(nsecs_t lastRenderTimeNs) { + mRenderQueue.clear(); + mLastRenderTimeNs = lastRenderTimeNs; +} + +void FrameRenderTracker::onFrameQueued( + int64_t mediaTimeUs, const sp &graphicBuffer, const sp &fence) { + mRenderQueue.emplace_back(mediaTimeUs, graphicBuffer, fence); +} + +FrameRenderTracker::Info *FrameRenderTracker::updateInfoForDequeuedBuffer( + ANativeWindowBuffer *buf, int fenceFd, int index) { + if (index < 0) { + return NULL; + } + + // see if this is a buffer that was to be rendered + std::list::iterator renderInfo = mRenderQueue.end(); + for (std::list::iterator it = mRenderQueue.begin(); + it != mRenderQueue.end(); ++it) { + if (it->mGraphicBuffer->handle == buf->handle) { + renderInfo = it; + break; + } + } + if (renderInfo == mRenderQueue.end()) { + // could have been canceled after fence has signaled + return NULL; + } + + if (renderInfo->mIndex >= 0) { + // buffer has been dequeued before, so there is nothing to do + return NULL; + } + + // was this frame dropped (we could also infer this if the fence is invalid or a dup of + // the queued fence; however, there is no way to figure that out.) + if (fenceFd < 0) { + // frame is new or was dropped + mRenderQueue.erase(renderInfo); + return NULL; + } + + // store dequeue fence and buffer index + renderInfo->mFence = new Fence(::dup(fenceFd)); + renderInfo->mIndex = index; + return &*renderInfo; +} + +status_t FrameRenderTracker::onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) { + // ensure monotonic timestamps + if (mLastRenderTimeNs >= systemNano) { + ALOGW("[%s] Ignoring out of order/stale system nano %lld for media time %lld from codec.", + mComponentName.c_str(), (long long)systemNano, (long long)mediaTimeUs); + return BAD_VALUE; + } + + nsecs_t now = systemTime(SYSTEM_TIME_MONOTONIC); + if (systemNano > now) { + ALOGW("[%s] Ignoring system nano %lld in the future for media time %lld from codec.", + mComponentName.c_str(), (long long)systemNano, (long long)mediaTimeUs); + return OK; + } + + mRenderQueue.emplace_back(mediaTimeUs, systemNano); + mLastRenderTimeNs = systemNano; + return OK; +} + +std::list FrameRenderTracker::checkFencesAndGetRenderedFrames( + const FrameRenderTracker::Info *until, bool dropIncomplete) { + std::list done; + + // complete any frames queued prior to this and drop any incomplete ones if requested + for (std::list::iterator it = mRenderQueue.begin(); + it != mRenderQueue.end(); ) { + bool drop = false; // whether to drop each frame + if (it->mIndex < 0) { + // frame not yet dequeued (or already rendered on a tunneled surface) + drop = dropIncomplete; + } else if (it->mFence != NULL) { + // check if fence signaled + nsecs_t signalTime = it->mFence->getSignalTime(); + if (signalTime < 0) { // invalid fence + drop = true; + } else if (signalTime == INT64_MAX) { // unsignaled fence + drop = dropIncomplete; + } else { // signaled + // save render time + it->mFence.clear(); + it->mRenderTimeNs = signalTime; + } + } + bool foundFrame = (Info *)&*it == until; + + // Return frames with signaled fences at the start of the queue, as they are + // in submit order, and we don't have to wait for any in-between frames. + // Also return any dropped frames. + if (drop || (it->mFence == NULL && it == mRenderQueue.begin())) { + // (unrendered) dropped frames have their mRenderTimeNs still set to -1 + done.splice(done.end(), mRenderQueue, it++); + } else { + ++it; + } + if (foundFrame) { + break; + } + } + + return done; +} + +void FrameRenderTracker::untrackFrame(const FrameRenderTracker::Info *info) { + if (info != NULL) { + for (std::list::iterator it = mRenderQueue.begin(); + it != mRenderQueue.end(); ++it) { + if (&*it == info) { + mRenderQueue.erase(it); + return; + } + } + } +} + +void FrameRenderTracker::dumpRenderQueue() const { + ALOGI("[%s] Render Queue: (last render time: %lldns)", + mComponentName.c_str(), (long long)mLastRenderTimeNs); + for (std::list::const_iterator it = mRenderQueue.cbegin(); + it != mRenderQueue.cend(); ++it) { + if (it->mFence == NULL) { + ALOGI(" RENDERED: handle: %p, media time: %lldus, index: %zd, render time: %lldns", + it->mGraphicBuffer == NULL ? NULL : it->mGraphicBuffer->handle, + (long long)it->mMediaTimeUs, it->mIndex, (long long)it->mRenderTimeNs); + } else if (it->mIndex < 0) { + ALOGI(" QUEUED: handle: %p, media time: %lldus, fence: %s", + it->mGraphicBuffer->handle, (long long)it->mMediaTimeUs, + it->mFence->isValid() ? "YES" : "NO"); + } else { + ALOGI(" DEQUEUED: handle: %p, media time: %lldus, index: %zd", + it->mGraphicBuffer->handle, (long long)it->mMediaTimeUs, it->mIndex); + } + } +} + +} // namespace android diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp index f918d2d..7ee84a8 100644 --- a/media/libstagefright/MediaCodec.cpp +++ b/media/libstagefright/MediaCodec.cpp @@ -397,6 +397,12 @@ status_t MediaCodec::setCallback(const sp &callback) { return PostAndAwaitResponse(msg, &response); } +status_t MediaCodec::setOnFrameRenderedNotification(const sp ¬ify) { + sp msg = new AMessage(kWhatSetNotification, this); + msg->setMessage("on-frame-rendered", notify); + return msg->post(); +} + status_t MediaCodec::configure( const sp &format, const sp &surface, @@ -1333,6 +1339,18 @@ void MediaCodec::onMessageReceived(const sp &msg) { break; } + case CodecBase::kWhatOutputFramesRendered: + { + // ignore these in all states except running, and check that we have a + // notification set + if (mState == STARTED && mOnFrameRenderedNotification != NULL) { + sp notify = mOnFrameRenderedNotification->dup(); + notify->setMessage("data", msg); + notify->post(); + } + break; + } + case CodecBase::kWhatFillThisBuffer: { /* size_t index = */updateBuffers(kPortIndexInput, msg); @@ -1530,6 +1548,15 @@ void MediaCodec::onMessageReceived(const sp &msg) { break; } + case kWhatSetNotification: + { + sp notify; + if (msg->findMessage("on-frame-rendered", ¬ify)) { + mOnFrameRenderedNotification = notify; + } + break; + } + case kWhatSetCallback: { sp replyID; @@ -2372,6 +2399,23 @@ status_t MediaCodec::onQueueInputBuffer(const sp &msg) { return OK; } +//static +size_t MediaCodec::CreateFramesRenderedMessage( + std::list done, sp &msg) { + size_t index = 0; + + for (std::list::const_iterator it = done.cbegin(); + it != done.cend(); ++it) { + if (it->getRenderTimeNs() < 0) { + continue; // dropped frame from tracking + } + msg->setInt64(AStringPrintf("%zu-media-time-us", index).c_str(), it->getMediaTimeUs()); + msg->setInt64(AStringPrintf("%zu-system-nano", index).c_str(), it->getRenderTimeNs()); + ++index; + } + return index; +} + status_t MediaCodec::onReleaseOutputBuffer(const sp &msg) { size_t index; CHECK(msg->findSize("index", &index)); @@ -2404,26 +2448,37 @@ status_t MediaCodec::onReleaseOutputBuffer(const sp &msg) { if (render && info->mData != NULL && info->mData->size() != 0) { info->mNotify->setInt32("render", true); - int64_t timestampNs = 0; - if (msg->findInt64("timestampNs", ×tampNs)) { - info->mNotify->setInt64("timestampNs", timestampNs); + int64_t mediaTimeUs = -1; + info->mData->meta()->findInt64("timeUs", &mediaTimeUs); + + int64_t renderTimeNs = 0; + if (msg->findInt64("timestampNs", &renderTimeNs)) { + info->mNotify->setInt64("timestampNs", renderTimeNs); } else { // TODO: it seems like we should use the timestamp // in the (media)buffer as it potentially came from // an input surface, but we did not propagate it prior to // API 20. Perhaps check for target SDK version. #if 0 - if (info->mData->meta()->findInt64("timeUs", ×tampNs)) { - ALOGV("using buffer PTS of %" PRId64, timestampNs); - timestampNs *= 1000; - } + ALOGV("using buffer PTS of %" PRId64, timestampNs); + renderTimeNs = mediaTimeUs * 1000; #endif } if (mSoftRenderer != NULL) { - mSoftRenderer->render( + std::list doneFrames = mSoftRenderer->render( info->mData->data(), info->mData->size(), - timestampNs, NULL, info->mFormat); + mediaTimeUs, renderTimeNs, NULL, info->mFormat); + + // if we are running, notify rendered frames + if (!doneFrames.empty() && mState == STARTED && mOnFrameRenderedNotification != NULL) { + sp notify = mOnFrameRenderedNotification->dup(); + sp data = new AMessage; + if (CreateFramesRenderedMessage(doneFrames, data)) { + notify->setMessage("data", data); + notify->post(); + } + } } } diff --git a/media/libstagefright/colorconversion/SoftwareRenderer.cpp b/media/libstagefright/colorconversion/SoftwareRenderer.cpp index 695cfc8..d22451b 100644 --- a/media/libstagefright/colorconversion/SoftwareRenderer.cpp +++ b/media/libstagefright/colorconversion/SoftwareRenderer.cpp @@ -196,17 +196,29 @@ void SoftwareRenderer::resetFormatIfChanged(const sp &format) { mNativeWindow.get(), transform)); } -void SoftwareRenderer::render( - const void *data, size_t size, int64_t timestampNs, +void SoftwareRenderer::clearTracker() { + mRenderTracker.clear(-1 /* lastRenderTimeNs */); +} + +std::list SoftwareRenderer::render( + const void *data, size_t size, int64_t mediaTimeUs, nsecs_t renderTimeNs, void* /*platformPrivate*/, const sp& format) { resetFormatIfChanged(format); + FrameRenderTracker::Info *info = NULL; ANativeWindowBuffer *buf; - int err; - if ((err = native_window_dequeue_buffer_and_wait(mNativeWindow.get(), - &buf)) != 0) { + int fenceFd = -1; + int err = mNativeWindow->dequeueBuffer(mNativeWindow.get(), &buf, &fenceFd); + if (err == 0 && fenceFd >= 0) { + info = mRenderTracker.updateInfoForDequeuedBuffer(buf, fenceFd, 0); + sp fence = new Fence(fenceFd); + err = fence->waitForever("SoftwareRenderer::render"); + } + if (err != 0) { ALOGW("Surface::dequeueBuffer returned error %d", err); - return; + // complete (drop) dequeued frame if fence wait failed; otherwise, + // this returns an empty list as no frames should have rendered and not yet returned. + return mRenderTracker.checkFencesAndGetRenderedFrames(info, false /* dropIncomplete */); } GraphicBufferMapper &mapper = GraphicBufferMapper::get(); @@ -342,16 +354,21 @@ void SoftwareRenderer::render( skip_copying: CHECK_EQ(0, mapper.unlock(buf->handle)); - if ((err = native_window_set_buffers_timestamp(mNativeWindow.get(), - timestampNs)) != 0) { - ALOGW("Surface::set_buffers_timestamp returned error %d", err); + if (renderTimeNs >= 0) { + if ((err = native_window_set_buffers_timestamp(mNativeWindow.get(), + renderTimeNs)) != 0) { + ALOGW("Surface::set_buffers_timestamp returned error %d", err); + } } - if ((err = mNativeWindow->queueBuffer(mNativeWindow.get(), buf, - -1)) != 0) { + if ((err = mNativeWindow->queueBuffer(mNativeWindow.get(), buf, -1)) != 0) { ALOGW("Surface::queueBuffer returned error %d", err); + } else { + mRenderTracker.onFrameQueued(mediaTimeUs, (GraphicBuffer *)buf, Fence::NO_FENCE); } + buf = NULL; + return mRenderTracker.checkFencesAndGetRenderedFrames(info, info != NULL /* dropIncomplete */); } } // namespace android diff --git a/media/libstagefright/include/SoftwareRenderer.h b/media/libstagefright/include/SoftwareRenderer.h index fa3ea89..9e652d5 100644 --- a/media/libstagefright/include/SoftwareRenderer.h +++ b/media/libstagefright/include/SoftwareRenderer.h @@ -19,9 +19,12 @@ #define SOFTWARE_RENDERER_H_ #include +#include #include #include +#include + namespace android { struct AMessage; @@ -32,9 +35,10 @@ public: ~SoftwareRenderer(); - void render( - const void *data, size_t size, int64_t timestampNs, + std::list render( + const void *data, size_t size, int64_t mediaTimeUs, nsecs_t renderTimeNs, void *platformPrivate, const sp &format); + void clearTracker(); private: enum YUVMode { @@ -48,6 +52,7 @@ private: int32_t mWidth, mHeight; int32_t mCropLeft, mCropTop, mCropRight, mCropBottom; int32_t mCropWidth, mCropHeight; + FrameRenderTracker mRenderTracker; SoftwareRenderer(const SoftwareRenderer &); SoftwareRenderer &operator=(const SoftwareRenderer &); diff --git a/media/libstagefright/omx/OMX.cpp b/media/libstagefright/omx/OMX.cpp index e94adbd..cb7ab5e 100644 --- a/media/libstagefright/omx/OMX.cpp +++ b/media/libstagefright/omx/OMX.cpp @@ -34,6 +34,7 @@ #include #include +#include namespace android { @@ -455,12 +456,35 @@ OMX_ERRORTYPE OMX::OnEvent( OMX_IN OMX_EVENTTYPE eEvent, OMX_IN OMX_U32 nData1, OMX_IN OMX_U32 nData2, - OMX_IN OMX_PTR /* pEventData */) { + OMX_IN OMX_PTR pEventData) { ALOGV("OnEvent(%d, %" PRIu32", %" PRIu32 ")", eEvent, nData1, nData2); // Forward to OMXNodeInstance. findInstance(node)->onEvent(eEvent, nData1, nData2); + sp dispatcher = findDispatcher(node); + + // output rendered events are not processed as regular events until they hit the observer + if (eEvent == OMX_EventOutputRendered) { + if (pEventData == NULL) { + return OMX_ErrorBadParameter; + } + + // process data from array + OMX_VIDEO_RENDEREVENTTYPE *renderData = (OMX_VIDEO_RENDEREVENTTYPE *)pEventData; + for (size_t i = 0; i < nData1; ++i) { + omx_message msg; + msg.type = omx_message::FRAME_RENDERED; + msg.node = node; + msg.fenceFd = -1; + msg.u.render_data.timestamp = renderData[i].nMediaTimeUs; + msg.u.render_data.nanoTime = renderData[i].nSystemTimeNs; + + dispatcher->post(msg, false /* realTime */); + } + return OMX_ErrorNone; + } + omx_message msg; msg.type = omx_message::EVENT; msg.node = node; @@ -469,7 +493,7 @@ OMX_ERRORTYPE OMX::OnEvent( msg.u.event_data.data1 = nData1; msg.u.event_data.data2 = nData2; - findDispatcher(node)->post(msg); + dispatcher->post(msg, true /* realTime */); return OMX_ErrorNone; } -- cgit v1.1