From 90fcf68fd29f3cb695bd53a830ad984cb7d430c0 Mon Sep 17 00:00:00 2001 From: Lajos Molnar Date: Thu, 4 Jun 2015 10:29:19 -0700 Subject: stagefright: add support for output frame rendered callback - Added FRAME_RENDERED event in OMX, used by tunneled video decoders to signal rendered event timing - Track buffers sent for rendering in ACodec and in SoftwareRenderer, and determine when they have rendered - Propagate render times to MediaCodec Bug: 20503131 Change-Id: Idf0a8714d5368b237c2285dd39fa82db847c232f --- include/media/IOMX.h | 7 +- include/media/stagefright/ACodec.h | 20 ++++ include/media/stagefright/CodecBase.h | 1 + include/media/stagefright/FrameRenderTracker.h | 142 +++++++++++++++++++++++++ include/media/stagefright/MediaCodec.h | 12 +++ 5 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 include/media/stagefright/FrameRenderTracker.h (limited to 'include') diff --git a/include/media/IOMX.h b/include/media/IOMX.h index 7023453..3d29e4a 100644 --- a/include/media/IOMX.h +++ b/include/media/IOMX.h @@ -198,7 +198,7 @@ struct omx_message { EVENT, EMPTY_BUFFER_DONE, FILL_BUFFER_DONE, - + FRAME_RENDERED, } type; IOMX::node_id node; @@ -226,6 +226,11 @@ struct omx_message { OMX_TICKS timestamp; } extended_buffer_data; + // if type == FRAME_RENDERED + struct { + OMX_TICKS timestamp; + OMX_S64 nanoTime; + } render_data; } u; }; diff --git a/include/media/stagefright/ACodec.h b/include/media/stagefright/ACodec.h index d4891a1..f9ea38e 100644 --- a/include/media/stagefright/ACodec.h +++ b/include/media/stagefright/ACodec.h @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -162,6 +163,7 @@ private: sp mData; sp mGraphicBuffer; int mFenceFd; + FrameRenderTracker::Info *mRenderInfo; // The following field and 4 methods are used for debugging only bool mIsReadFence; @@ -214,6 +216,7 @@ private: sp mOutputFormat; sp mBaseOutputFormat; + FrameRenderTracker mRenderTracker; // render information for buffers rendered by ACodec Vector mBuffers[2]; bool mPortEOS[2]; status_t mInputEOSResult; @@ -375,6 +378,23 @@ private: void deferMessage(const sp &msg); void processDeferredMessages(); + void onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano); + // called when we have dequeued a buffer |buf| from the native window to track render info. + // |fenceFd| is the dequeue fence, and |info| points to the buffer info where this buffer is + // stored. + void updateRenderInfoForDequeuedBuffer( + ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info); + + // Checks to see if any frames have rendered up until |until|, and to notify client + // (MediaCodec) of rendered frames up-until the frame pointed to by |until| or the first + // unrendered frame. These frames are removed from the render queue. + // If |dropIncomplete| is true, unrendered frames up-until |until| will be dropped from the + // queue, allowing all rendered framed up till then to be notified of. + // (This will effectively clear the render queue up-until (and including) |until|.) + // If |until| is NULL, or is not in the rendered queue, this method will check all frames. + void notifyOfRenderedFrames( + bool dropIncomplete = false, FrameRenderTracker::Info *until = NULL); + void sendFormatChange(const sp &reply); status_t getPortFormat(OMX_U32 portIndex, sp ¬ify); diff --git a/include/media/stagefright/CodecBase.h b/include/media/stagefright/CodecBase.h index 989df4f..bb36052 100644 --- a/include/media/stagefright/CodecBase.h +++ b/include/media/stagefright/CodecBase.h @@ -43,6 +43,7 @@ struct CodecBase : public AHandler { kWhatInputSurfaceAccepted = 'isfa', kWhatSignaledInputEOS = 'seos', kWhatBuffersAllocated = 'allc', + kWhatOutputFramesRendered = 'outR', }; virtual void setNotificationMessage(const sp &msg) = 0; diff --git a/include/media/stagefright/FrameRenderTracker.h b/include/media/stagefright/FrameRenderTracker.h new file mode 100644 index 0000000..3b0db5a --- /dev/null +++ b/include/media/stagefright/FrameRenderTracker.h @@ -0,0 +1,142 @@ +/* + * Copyright 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FRAME_RENDER_TRACKER_H_ + +#define FRAME_RENDER_TRACKER_H_ + +#include +#include +#include + +#include +#include + +#include + +namespace android { + +class Fence; +class GraphicBuffer; + +struct FrameRenderTracker : public RefBase { + // Tracks the render information about a frame. Frames go through several states while + // the render information is tracked: + // + // 1. queued frame: mMediaTime and mGraphicBuffer are set for the frame. mFence is the + // queue fence (read fence). mIndex is negative, and mRenderTimeNs is invalid. + // Key characteristics: mFence is not NULL and mIndex is negative. + // + // 2. dequeued frame: mFence is updated with the dequeue fence (write fence). mIndex is set. + // Key characteristics: mFence is not NULL and mIndex is non-negative. mRenderTime is still + // invalid. + // + // 3. rendered frame or frame: mFence is cleared, mRenderTimeNs is set. + // Key characteristics: mFence is NULL. + // + struct Info { + // set by client during onFrameQueued or onFrameRendered + int64_t getMediaTimeUs() const { return mMediaTimeUs; } + + // -1 if frame is not yet rendered + nsecs_t getRenderTimeNs() const { return mRenderTimeNs; } + + // set by client during updateRenderInfoForDequeuedBuffer; -1 otherwise + ssize_t getIndex() const { return mIndex; } + + // creates information for a queued frame + Info(int64_t mediaTimeUs, const sp &graphicBuffer, const sp &fence) + : mMediaTimeUs(mediaTimeUs), + mRenderTimeNs(-1), + mIndex(-1), + mGraphicBuffer(graphicBuffer), + mFence(fence) { + } + + // creates information for a frame rendered on a tunneled surface + Info(int64_t mediaTimeUs, nsecs_t renderTimeNs) + : mMediaTimeUs(mediaTimeUs), + mRenderTimeNs(renderTimeNs), + mIndex(-1), + mGraphicBuffer(NULL), + mFence(NULL) { + } + + private: + int64_t mMediaTimeUs; + nsecs_t mRenderTimeNs; + ssize_t mIndex; // to be used by client + sp mGraphicBuffer; + sp mFence; + + friend class FrameRenderTracker; + }; + + FrameRenderTracker(); + + void setComponentName(const AString &componentName); + + // clears all tracked frames, and resets last render time + void clear(nsecs_t lastRenderTimeNs); + + // called when |graphicBuffer| corresponding to |mediaTimeUs| is + // queued to the output surface using |fence|. + void onFrameQueued( + int64_t mediaTimeUs, const sp &graphicBuffer, const sp &fence); + + // Called when we have dequeued a buffer |buf| from the native window to track render info. + // |fenceFd| is the dequeue fence, and |index| is a positive buffer ID to be usable by the + // client to track this render info among the dequeued buffers. + // Returns pointer to the tracked info, or NULL if buffer is not tracked or if |index| + // is negative. + Info *updateInfoForDequeuedBuffer(ANativeWindowBuffer *buf, int fenceFd, int index); + + // called when tunneled codec signals frame rendered event + // returns BAD_VALUE if systemNano is not monotonic. Otherwise, returns OK. + status_t onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano); + + // Checks to see if any frames have rendered up until |until|. If |until| is NULL or not a + // tracked info, this method searches the entire render queue. + // Returns list of rendered frames up-until the frame pointed to by |until| or the first + // unrendered frame, as well as any dropped frames (those with invalid fence) up-until |until|. + // These frames are removed from the render queue. + // If |dropIncomplete| is true, unrendered frames up-until |until| will also be dropped from the + // queue, allowing all rendered framed up till then to be notified of. + // (This will effectively clear the render queue up-until (and including) |until|.) + std::list checkFencesAndGetRenderedFrames(const Info *until, bool dropIncomplete); + + // Stop tracking a queued frame (e.g. if the frame has been discarded). If |info| is NULL or is + // not tracked, this method is a no-op. + void untrackFrame(const Info *info); + + void dumpRenderQueue() const; + + virtual ~FrameRenderTracker(); + +private: + + // Render information for buffers. Regular surface buffers are queued in the order of + // rendering. Tunneled buffers are queued in the order of receipt. + std::list mRenderQueue; + nsecs_t mLastRenderTimeNs; + AString mComponentName; + + DISALLOW_EVIL_CONSTRUCTORS(FrameRenderTracker); +}; + +} // namespace android + +#endif // FRAME_RENDER_TRACKER_H_ diff --git a/include/media/stagefright/MediaCodec.h b/include/media/stagefright/MediaCodec.h index d62b35d..09cbe8f 100644 --- a/include/media/stagefright/MediaCodec.h +++ b/include/media/stagefright/MediaCodec.h @@ -22,6 +22,7 @@ #include #include #include +#include #include namespace android { @@ -76,6 +77,8 @@ struct MediaCodec : public AHandler { status_t setCallback(const sp &callback); + status_t setOnFrameRenderedNotification(const sp ¬ify); + status_t createInputSurface(sp* bufferProducer); status_t setInputSurface(const sp &surface); @@ -157,6 +160,12 @@ struct MediaCodec : public AHandler { status_t setParameters(const sp ¶ms); + // Create a MediaCodec notification message from a list of rendered or dropped render infos + // by adding rendered frame information to a base notification message. Returns the number + // of frames that were rendered. + static size_t CreateFramesRenderedMessage( + std::list done, sp &msg); + protected: virtual ~MediaCodec(); virtual void onMessageReceived(const sp &msg); @@ -212,6 +221,7 @@ private: kWhatGetName = 'getN', kWhatSetParameters = 'setP', kWhatSetCallback = 'setC', + kWhatSetNotification = 'setN', }; enum { @@ -275,9 +285,11 @@ private: status_t mStickyError; sp mSurface; SoftwareRenderer *mSoftRenderer; + sp mOutputFormat; sp mInputFormat; sp mCallback; + sp mOnFrameRenderedNotification; sp mDealer; sp mResourceManagerClient; -- cgit v1.1