stagefright: add support for output frame rendered callback

- Added FRAME_RENDERED event in OMX, used by tunneled video decoders
  to signal rendered event timing
- Track buffers sent for rendering in ACodec and in SoftwareRenderer, and
  determine when they have rendered
- Propagate render times to MediaCodec

Bug: 20503131
Change-Id: Idf0a8714d5368b237c2285dd39fa82db847c232f

5 files changed, 181 insertions, 1 deletions

diff --git a/include/media/IOMX.h b/include/media/IOMX.h
index 7023453..3d29e4a 100644
--- a/include/media/IOMX.h
+++ b/include/media/IOMX.h
@@ -198,7 +198,7 @@ struct omx_message {
         EVENT,
         EMPTY_BUFFER_DONE,
         FILL_BUFFER_DONE,
-
+        FRAME_RENDERED,
     } type;
 
     IOMX::node_id node;
@@ -226,6 +226,11 @@ struct omx_message {
             OMX_TICKS timestamp;
         } extended_buffer_data;
 
+        // if type == FRAME_RENDERED
+        struct {
+            OMX_TICKS timestamp;
+            OMX_S64 nanoTime;
+        } render_data;
     } u;
 };
 
diff --git a/include/media/stagefright/ACodec.h b/include/media/stagefright/ACodec.h
index d4891a1..f9ea38e 100644
--- a/include/media/stagefright/ACodec.h
+++ b/include/media/stagefright/ACodec.h
@@ -24,6 +24,7 @@
 #include <media/IOMX.h>
 #include <media/stagefright/foundation/AHierarchicalStateMachine.h>
 #include <media/stagefright/CodecBase.h>
+#include <media/stagefright/FrameRenderTracker.h>
 #include <media/stagefright/SkipCutBuffer.h>
 #include <OMX_Audio.h>
 
@@ -162,6 +163,7 @@ private:
         sp<ABuffer> mData;
         sp<GraphicBuffer> mGraphicBuffer;
         int mFenceFd;
+        FrameRenderTracker::Info *mRenderInfo;
 
         // The following field and 4 methods are used for debugging only
         bool mIsReadFence;
@@ -214,6 +216,7 @@ private:
     sp<AMessage> mOutputFormat;
     sp<AMessage> mBaseOutputFormat;
 
+    FrameRenderTracker mRenderTracker; // render information for buffers rendered by ACodec
     Vector<BufferInfo> mBuffers[2];
     bool mPortEOS[2];
     status_t mInputEOSResult;
@@ -375,6 +378,23 @@ private:
     void deferMessage(const sp<AMessage> &msg);
     void processDeferredMessages();
 
+    void onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
+    // called when we have dequeued a buffer |buf| from the native window to track render info.
+    // |fenceFd| is the dequeue fence, and |info| points to the buffer info where this buffer is
+    // stored.
+    void updateRenderInfoForDequeuedBuffer(
+            ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info);
+
+    // Checks to see if any frames have rendered up until |until|, and to notify client
+    // (MediaCodec) of rendered frames up-until the frame pointed to by |until| or the first
+    // unrendered frame. These frames are removed from the render queue.
+    // If |dropIncomplete| is true, unrendered frames up-until |until| will be dropped from the
+    // queue, allowing all rendered framed up till then to be notified of.
+    // (This will effectively clear the render queue up-until (and including) |until|.)
+    // If |until| is NULL, or is not in the rendered queue, this method will check all frames.
+    void notifyOfRenderedFrames(
+            bool dropIncomplete = false, FrameRenderTracker::Info *until = NULL);
+
     void sendFormatChange(const sp<AMessage> &reply);
     status_t getPortFormat(OMX_U32 portIndex, sp<AMessage> &notify);
 
diff --git a/include/media/stagefright/CodecBase.h b/include/media/stagefright/CodecBase.h
index 989df4f..bb36052 100644
--- a/include/media/stagefright/CodecBase.h
+++ b/include/media/stagefright/CodecBase.h
@@ -43,6 +43,7 @@ struct CodecBase : public AHandler {
         kWhatInputSurfaceAccepted = 'isfa',
         kWhatSignaledInputEOS    = 'seos',
         kWhatBuffersAllocated    = 'allc',
+        kWhatOutputFramesRendered = 'outR',
     };
 
     virtual void setNotificationMessage(const sp<AMessage> &msg) = 0;
diff --git a/include/media/stagefright/FrameRenderTracker.h b/include/media/stagefright/FrameRenderTracker.h
new file mode 100644
index 0000000..3b0db5a
--- /dev/null
+++ b/include/media/stagefright/FrameRenderTracker.h
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FRAME_RENDER_TRACKER_H_
+
+#define FRAME_RENDER_TRACKER_H_
+
+#include <utils/RefBase.h>
+#include <utils/Timers.h>
+#include <system/window.h>
+
+#include <media/stagefright/foundation/ADebug.h>
+#include <media/stagefright/foundation/AString.h>
+
+#include <list>
+
+namespace android {
+
+class Fence;
+class GraphicBuffer;
+
+struct FrameRenderTracker : public RefBase {
+    // Tracks the render information about a frame. Frames go through several states while
+    // the render information is tracked:
+    //
+    // 1. queued frame: mMediaTime and mGraphicBuffer are set for the frame. mFence is the
+    // queue fence (read fence). mIndex is negative, and mRenderTimeNs is invalid.
+    // Key characteristics: mFence is not NULL and mIndex is negative.
+    //
+    // 2. dequeued frame: mFence is updated with the dequeue fence (write fence). mIndex is set.
+    // Key characteristics: mFence is not NULL and mIndex is non-negative. mRenderTime is still
+    // invalid.
+    //
+    // 3. rendered frame or frame: mFence is cleared, mRenderTimeNs is set.
+    // Key characteristics: mFence is NULL.
+    //
+    struct Info {
+        // set by client during onFrameQueued or onFrameRendered
+        int64_t getMediaTimeUs() const  { return mMediaTimeUs; }
+
+        // -1 if frame is not yet rendered
+        nsecs_t getRenderTimeNs() const { return mRenderTimeNs; }
+
+        // set by client during updateRenderInfoForDequeuedBuffer; -1 otherwise
+        ssize_t getIndex() const        { return mIndex; }
+
+        // creates information for a queued frame
+        Info(int64_t mediaTimeUs, const sp<GraphicBuffer> &graphicBuffer, const sp<Fence> &fence)
+            : mMediaTimeUs(mediaTimeUs),
+              mRenderTimeNs(-1),
+              mIndex(-1),
+              mGraphicBuffer(graphicBuffer),
+              mFence(fence) {
+        }
+
+        // creates information for a frame rendered on a tunneled surface
+        Info(int64_t mediaTimeUs, nsecs_t renderTimeNs)
+            : mMediaTimeUs(mediaTimeUs),
+              mRenderTimeNs(renderTimeNs),
+              mIndex(-1),
+              mGraphicBuffer(NULL),
+              mFence(NULL) {
+        }
+
+    private:
+        int64_t mMediaTimeUs;
+        nsecs_t mRenderTimeNs;
+        ssize_t mIndex;         // to be used by client
+        sp<GraphicBuffer> mGraphicBuffer;
+        sp<Fence> mFence;
+
+        friend class FrameRenderTracker;
+    };
+
+    FrameRenderTracker();
+
+    void setComponentName(const AString &componentName);
+
+    // clears all tracked frames, and resets last render time
+    void clear(nsecs_t lastRenderTimeNs);
+
+    // called when |graphicBuffer| corresponding to |mediaTimeUs| is
+    // queued to the output surface using |fence|.
+    void onFrameQueued(
+            int64_t mediaTimeUs, const sp<GraphicBuffer> &graphicBuffer, const sp<Fence> &fence);
+
+    // Called when we have dequeued a buffer |buf| from the native window to track render info.
+    // |fenceFd| is the dequeue fence, and |index| is a positive buffer ID to be usable by the
+    // client to track this render info among the dequeued buffers.
+    // Returns pointer to the tracked info, or NULL if buffer is not tracked or if |index|
+    // is negative.
+    Info *updateInfoForDequeuedBuffer(ANativeWindowBuffer *buf, int fenceFd, int index);
+
+    // called when tunneled codec signals frame rendered event
+    // returns BAD_VALUE if systemNano is not monotonic. Otherwise, returns OK.
+    status_t onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
+
+    // Checks to see if any frames have rendered up until |until|. If |until| is NULL or not a
+    // tracked info, this method searches the entire render queue.
+    // Returns list of rendered frames up-until the frame pointed to by |until| or the first
+    // unrendered frame, as well as any dropped frames (those with invalid fence) up-until |until|.
+    // These frames are removed from the render queue.
+    // If |dropIncomplete| is true, unrendered frames up-until |until| will also be dropped from the
+    // queue, allowing all rendered framed up till then to be notified of.
+    // (This will effectively clear the render queue up-until (and including) |until|.)
+    std::list<Info> checkFencesAndGetRenderedFrames(const Info *until, bool dropIncomplete);
+
+    // Stop tracking a queued frame (e.g. if the frame has been discarded). If |info| is NULL or is
+    // not tracked, this method is a no-op.
+    void untrackFrame(const Info *info);
+
+    void dumpRenderQueue() const;
+
+    virtual ~FrameRenderTracker();
+
+private:
+
+    // Render information for buffers. Regular surface buffers are queued in the order of
+    // rendering. Tunneled buffers are queued in the order of receipt.
+    std::list<Info> mRenderQueue;
+    nsecs_t mLastRenderTimeNs;
+    AString mComponentName;
+
+    DISALLOW_EVIL_CONSTRUCTORS(FrameRenderTracker);
+};
+
+}  // namespace android
+
+#endif  // FRAME_RENDER_TRACKER_H_
diff --git a/include/media/stagefright/MediaCodec.h b/include/media/stagefright/MediaCodec.h
index d62b35d..09cbe8f 100644
--- a/include/media/stagefright/MediaCodec.h
+++ b/include/media/stagefright/MediaCodec.h
@@ -22,6 +22,7 @@
 #include <media/hardware/CryptoAPI.h>
 #include <media/MediaResource.h>
 #include <media/stagefright/foundation/AHandler.h>
+#include <media/stagefright/FrameRenderTracker.h>
 #include <utils/Vector.h>
 
 namespace android {
@@ -76,6 +77,8 @@ struct MediaCodec : public AHandler {
 
     status_t setCallback(const sp<AMessage> &callback);
 
+    status_t setOnFrameRenderedNotification(const sp<AMessage> &notify);
+
     status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
 
     status_t setInputSurface(const sp<PersistentSurface> &surface);
@@ -157,6 +160,12 @@ struct MediaCodec : public AHandler {
 
     status_t setParameters(const sp<AMessage> &params);
 
+    // Create a MediaCodec notification message from a list of rendered or dropped render infos
+    // by adding rendered frame information to a base notification message. Returns the number
+    // of frames that were rendered.
+    static size_t CreateFramesRenderedMessage(
+            std::list<FrameRenderTracker::Info> done, sp<AMessage> &msg);
+
 protected:
     virtual ~MediaCodec();
     virtual void onMessageReceived(const sp<AMessage> &msg);
@@ -212,6 +221,7 @@ private:
         kWhatGetName                        = 'getN',
         kWhatSetParameters                  = 'setP',
         kWhatSetCallback                    = 'setC',
+        kWhatSetNotification                = 'setN',
     };
 
     enum {
@@ -275,9 +285,11 @@ private:
     status_t mStickyError;
     sp<Surface> mSurface;
     SoftwareRenderer *mSoftRenderer;
+
     sp<AMessage> mOutputFormat;
     sp<AMessage> mInputFormat;
     sp<AMessage> mCallback;
+    sp<AMessage> mOnFrameRenderedNotification;
     sp<MemoryDealer> mDealer;
 
     sp<IResourceManagerClient> mResourceManagerClient;