stagefright: add support for output frame rendered callback

- Added FRAME_RENDERED event in OMX, used by tunneled video decoders
  to signal rendered event timing
- Track buffers sent for rendering in ACodec and in SoftwareRenderer, and
  determine when they have rendered
- Propagate render times to MediaCodec

Bug: 20503131
Change-Id: Idf0a8714d5368b237c2285dd39fa82db847c232f

13 files changed, 616 insertions, 29 deletions

diff --git a/include/media/IOMX.h b/include/media/IOMX.h
index 7023453..3d29e4a 100644
--- a/include/media/IOMX.h
+++ b/include/media/IOMX.h
@@ -198,7 +198,7 @@ struct omx_message {
         EVENT,
         EMPTY_BUFFER_DONE,
         FILL_BUFFER_DONE,
-
+        FRAME_RENDERED,
     } type;
 
     IOMX::node_id node;
@@ -226,6 +226,11 @@ struct omx_message {
             OMX_TICKS timestamp;
         } extended_buffer_data;
 
+        // if type == FRAME_RENDERED
+        struct {
+            OMX_TICKS timestamp;
+            OMX_S64 nanoTime;
+        } render_data;
     } u;
 };
 
diff --git a/include/media/stagefright/ACodec.h b/include/media/stagefright/ACodec.h
index d4891a1..f9ea38e 100644
--- a/include/media/stagefright/ACodec.h
+++ b/include/media/stagefright/ACodec.h
@@ -24,6 +24,7 @@
 #include <media/IOMX.h>
 #include <media/stagefright/foundation/AHierarchicalStateMachine.h>
 #include <media/stagefright/CodecBase.h>
+#include <media/stagefright/FrameRenderTracker.h>
 #include <media/stagefright/SkipCutBuffer.h>
 #include <OMX_Audio.h>
 
@@ -162,6 +163,7 @@ private:
         sp<ABuffer> mData;
         sp<GraphicBuffer> mGraphicBuffer;
         int mFenceFd;
+        FrameRenderTracker::Info *mRenderInfo;
 
         // The following field and 4 methods are used for debugging only
         bool mIsReadFence;
@@ -214,6 +216,7 @@ private:
     sp<AMessage> mOutputFormat;
     sp<AMessage> mBaseOutputFormat;
 
+    FrameRenderTracker mRenderTracker; // render information for buffers rendered by ACodec
     Vector<BufferInfo> mBuffers[2];
     bool mPortEOS[2];
     status_t mInputEOSResult;
@@ -375,6 +378,23 @@ private:
     void deferMessage(const sp<AMessage> &msg);
     void processDeferredMessages();
 
+    void onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
+    // called when we have dequeued a buffer |buf| from the native window to track render info.
+    // |fenceFd| is the dequeue fence, and |info| points to the buffer info where this buffer is
+    // stored.
+    void updateRenderInfoForDequeuedBuffer(
+            ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info);
+
+    // Checks to see if any frames have rendered up until |until|, and to notify client
+    // (MediaCodec) of rendered frames up-until the frame pointed to by |until| or the first
+    // unrendered frame. These frames are removed from the render queue.
+    // If |dropIncomplete| is true, unrendered frames up-until |until| will be dropped from the
+    // queue, allowing all rendered framed up till then to be notified of.
+    // (This will effectively clear the render queue up-until (and including) |until|.)
+    // If |until| is NULL, or is not in the rendered queue, this method will check all frames.
+    void notifyOfRenderedFrames(
+            bool dropIncomplete = false, FrameRenderTracker::Info *until = NULL);
+
     void sendFormatChange(const sp<AMessage> &reply);
     status_t getPortFormat(OMX_U32 portIndex, sp<AMessage> &notify);
 
diff --git a/include/media/stagefright/CodecBase.h b/include/media/stagefright/CodecBase.h
index 989df4f..bb36052 100644
--- a/include/media/stagefright/CodecBase.h
+++ b/include/media/stagefright/CodecBase.h
@@ -43,6 +43,7 @@ struct CodecBase : public AHandler {
         kWhatInputSurfaceAccepted = 'isfa',
         kWhatSignaledInputEOS    = 'seos',
         kWhatBuffersAllocated    = 'allc',
+        kWhatOutputFramesRendered = 'outR',
     };
 
     virtual void setNotificationMessage(const sp<AMessage> &msg) = 0;
diff --git a/include/media/stagefright/FrameRenderTracker.h b/include/media/stagefright/FrameRenderTracker.h
new file mode 100644
index 0000000..3b0db5a
--- /dev/null
+++ b/include/media/stagefright/FrameRenderTracker.h
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FRAME_RENDER_TRACKER_H_
+
+#define FRAME_RENDER_TRACKER_H_
+
+#include <utils/RefBase.h>
+#include <utils/Timers.h>
+#include <system/window.h>
+
+#include <media/stagefright/foundation/ADebug.h>
+#include <media/stagefright/foundation/AString.h>
+
+#include <list>
+
+namespace android {
+
+class Fence;
+class GraphicBuffer;
+
+struct FrameRenderTracker : public RefBase {
+    // Tracks the render information about a frame. Frames go through several states while
+    // the render information is tracked:
+    //
+    // 1. queued frame: mMediaTime and mGraphicBuffer are set for the frame. mFence is the
+    // queue fence (read fence). mIndex is negative, and mRenderTimeNs is invalid.
+    // Key characteristics: mFence is not NULL and mIndex is negative.
+    //
+    // 2. dequeued frame: mFence is updated with the dequeue fence (write fence). mIndex is set.
+    // Key characteristics: mFence is not NULL and mIndex is non-negative. mRenderTime is still
+    // invalid.
+    //
+    // 3. rendered frame or frame: mFence is cleared, mRenderTimeNs is set.
+    // Key characteristics: mFence is NULL.
+    //
+    struct Info {
+        // set by client during onFrameQueued or onFrameRendered
+        int64_t getMediaTimeUs() const  { return mMediaTimeUs; }
+
+        // -1 if frame is not yet rendered
+        nsecs_t getRenderTimeNs() const { return mRenderTimeNs; }
+
+        // set by client during updateRenderInfoForDequeuedBuffer; -1 otherwise
+        ssize_t getIndex() const        { return mIndex; }
+
+        // creates information for a queued frame
+        Info(int64_t mediaTimeUs, const sp<GraphicBuffer> &graphicBuffer, const sp<Fence> &fence)
+            : mMediaTimeUs(mediaTimeUs),
+              mRenderTimeNs(-1),
+              mIndex(-1),
+              mGraphicBuffer(graphicBuffer),
+              mFence(fence) {
+        }
+
+        // creates information for a frame rendered on a tunneled surface
+        Info(int64_t mediaTimeUs, nsecs_t renderTimeNs)
+            : mMediaTimeUs(mediaTimeUs),
+              mRenderTimeNs(renderTimeNs),
+              mIndex(-1),
+              mGraphicBuffer(NULL),
+              mFence(NULL) {
+        }
+
+    private:
+        int64_t mMediaTimeUs;
+        nsecs_t mRenderTimeNs;
+        ssize_t mIndex;         // to be used by client
+        sp<GraphicBuffer> mGraphicBuffer;
+        sp<Fence> mFence;
+
+        friend class FrameRenderTracker;
+    };
+
+    FrameRenderTracker();
+
+    void setComponentName(const AString &componentName);
+
+    // clears all tracked frames, and resets last render time
+    void clear(nsecs_t lastRenderTimeNs);
+
+    // called when |graphicBuffer| corresponding to |mediaTimeUs| is
+    // queued to the output surface using |fence|.
+    void onFrameQueued(
+            int64_t mediaTimeUs, const sp<GraphicBuffer> &graphicBuffer, const sp<Fence> &fence);
+
+    // Called when we have dequeued a buffer |buf| from the native window to track render info.
+    // |fenceFd| is the dequeue fence, and |index| is a positive buffer ID to be usable by the
+    // client to track this render info among the dequeued buffers.
+    // Returns pointer to the tracked info, or NULL if buffer is not tracked or if |index|
+    // is negative.
+    Info *updateInfoForDequeuedBuffer(ANativeWindowBuffer *buf, int fenceFd, int index);
+
+    // called when tunneled codec signals frame rendered event
+    // returns BAD_VALUE if systemNano is not monotonic. Otherwise, returns OK.
+    status_t onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
+
+    // Checks to see if any frames have rendered up until |until|. If |until| is NULL or not a
+    // tracked info, this method searches the entire render queue.
+    // Returns list of rendered frames up-until the frame pointed to by |until| or the first
+    // unrendered frame, as well as any dropped frames (those with invalid fence) up-until |until|.
+    // These frames are removed from the render queue.
+    // If |dropIncomplete| is true, unrendered frames up-until |until| will also be dropped from the
+    // queue, allowing all rendered framed up till then to be notified of.
+    // (This will effectively clear the render queue up-until (and including) |until|.)
+    std::list<Info> checkFencesAndGetRenderedFrames(const Info *until, bool dropIncomplete);
+
+    // Stop tracking a queued frame (e.g. if the frame has been discarded). If |info| is NULL or is
+    // not tracked, this method is a no-op.
+    void untrackFrame(const Info *info);
+
+    void dumpRenderQueue() const;
+
+    virtual ~FrameRenderTracker();
+
+private:
+
+    // Render information for buffers. Regular surface buffers are queued in the order of
+    // rendering. Tunneled buffers are queued in the order of receipt.
+    std::list<Info> mRenderQueue;
+    nsecs_t mLastRenderTimeNs;
+    AString mComponentName;
+
+    DISALLOW_EVIL_CONSTRUCTORS(FrameRenderTracker);
+};
+
+}  // namespace android
+
+#endif  // FRAME_RENDER_TRACKER_H_
diff --git a/include/media/stagefright/MediaCodec.h b/include/media/stagefright/MediaCodec.h
index d62b35d..09cbe8f 100644
--- a/include/media/stagefright/MediaCodec.h
+++ b/include/media/stagefright/MediaCodec.h
@@ -22,6 +22,7 @@
 #include <media/hardware/CryptoAPI.h>
 #include <media/MediaResource.h>
 #include <media/stagefright/foundation/AHandler.h>
+#include <media/stagefright/FrameRenderTracker.h>
 #include <utils/Vector.h>
 
 namespace android {
@@ -76,6 +77,8 @@ struct MediaCodec : public AHandler {
 
     status_t setCallback(const sp<AMessage> &callback);
 
+    status_t setOnFrameRenderedNotification(const sp<AMessage> &notify);
+
     status_t createInputSurface(sp<IGraphicBufferProducer>* bufferProducer);
 
     status_t setInputSurface(const sp<PersistentSurface> &surface);
@@ -157,6 +160,12 @@ struct MediaCodec : public AHandler {
 
     status_t setParameters(const sp<AMessage> &params);
 
+    // Create a MediaCodec notification message from a list of rendered or dropped render infos
+    // by adding rendered frame information to a base notification message. Returns the number
+    // of frames that were rendered.
+    static size_t CreateFramesRenderedMessage(
+            std::list<FrameRenderTracker::Info> done, sp<AMessage> &msg);
+
 protected:
     virtual ~MediaCodec();
     virtual void onMessageReceived(const sp<AMessage> &msg);
@@ -212,6 +221,7 @@ private:
         kWhatGetName                        = 'getN',
         kWhatSetParameters                  = 'setP',
         kWhatSetCallback                    = 'setC',
+        kWhatSetNotification                = 'setN',
     };
 
     enum {
@@ -275,9 +285,11 @@ private:
     status_t mStickyError;
     sp<Surface> mSurface;
     SoftwareRenderer *mSoftRenderer;
+
     sp<AMessage> mOutputFormat;
     sp<AMessage> mInputFormat;
     sp<AMessage> mCallback;
+    sp<AMessage> mOnFrameRenderedNotification;
     sp<MemoryDealer> mDealer;
 
     sp<IResourceManagerClient> mResourceManagerClient;
diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index 172e19c..71cb194 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -37,6 +37,7 @@
 #include <media/stagefright/foundation/AUtils.h>
 
 #include <media/stagefright/BufferProducerWrapper.h>
+#include <media/stagefright/MediaCodec.h>
 #include <media/stagefright/MediaCodecList.h>
 #include <media/stagefright/MediaDefs.h>
 #include <media/stagefright/OMXClient.h>
@@ -175,6 +176,15 @@ struct CodecObserver : public BnOMXObserver {
                     break;
                 }
 
+                case omx_message::FRAME_RENDERED:
+                {
+                    msg->setInt64(
+                            "media_time_us", omx_msg.u.render_data.timestamp);
+                    msg->setInt64(
+                            "system_nano", omx_msg.u.render_data.nanoTime);
+                    break;
+                }
+
                 default:
                     ALOGE("Unrecognized message type: %d", omx_msg.type);
                     break;
@@ -238,6 +248,8 @@ private:
             int64_t timeUs,
             int fenceFd);
 
+    virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
+
     void getMoreInputDataIfPossible();
 
     DISALLOW_EVIL_CONSTRUCTORS(BaseState);
@@ -354,6 +366,7 @@ protected:
     virtual void stateEntered();
 
     virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2);
+    virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
 
 private:
     bool mActive;
@@ -372,6 +385,7 @@ protected:
     virtual void stateEntered();
 
     virtual bool onOMXEvent(OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2);
+    virtual bool onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano);
 
 private:
     DISALLOW_EVIL_CONSTRUCTORS(OutputPortSettingsChangedState);
@@ -786,6 +800,7 @@ status_t ACodec::allocateBuffersOnPort(OMX_U32 portIndex) {
                 BufferInfo info;
                 info.mStatus = BufferInfo::OWNED_BY_US;
                 info.mFenceFd = -1;
+                info.mRenderInfo = NULL;
 
                 uint32_t requiresAllocateBufferBit =
                     (portIndex == kPortIndexInput)
@@ -1002,6 +1017,7 @@ status_t ACodec::allocateOutputBuffersFromNativeWindow() {
         info.mStatus = BufferInfo::OWNED_BY_US;
         info.mFenceFd = fenceFd;
         info.mIsReadFence = false;
+        info.mRenderInfo = NULL;
         info.mData = new ABuffer(NULL /* data */, bufferSize /* capacity */);
         info.mGraphicBuffer = graphicBuffer;
         mBuffers[kPortIndexOutput].push(info);
@@ -1073,6 +1089,7 @@ status_t ACodec::allocateOutputMetadataBuffers() {
         BufferInfo info;
         info.mStatus = BufferInfo::OWNED_BY_NATIVE_WINDOW;
         info.mFenceFd = -1;
+        info.mRenderInfo = NULL;
         info.mGraphicBuffer = NULL;
         info.mDequeuedAt = mDequeueCounter;
 
@@ -1219,6 +1236,42 @@ status_t ACodec::cancelBufferToNativeWindow(BufferInfo *info) {
     return err;
 }
 
+void ACodec::updateRenderInfoForDequeuedBuffer(
+        ANativeWindowBuffer *buf, int fenceFd, BufferInfo *info) {
+
+    info->mRenderInfo =
+        mRenderTracker.updateInfoForDequeuedBuffer(
+                buf, fenceFd, info - &mBuffers[kPortIndexOutput][0]);
+
+    // check for any fences already signaled
+    notifyOfRenderedFrames(false /* dropIncomplete */, info->mRenderInfo);
+}
+
+void ACodec::onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) {
+    if (mRenderTracker.onFrameRendered(mediaTimeUs, systemNano) != OK) {
+        mRenderTracker.dumpRenderQueue();
+    }
+}
+
+void ACodec::notifyOfRenderedFrames(bool dropIncomplete, FrameRenderTracker::Info *until) {
+    sp<AMessage> msg = mNotify->dup();
+    msg->setInt32("what", CodecBase::kWhatOutputFramesRendered);
+    std::list<FrameRenderTracker::Info> done =
+        mRenderTracker.checkFencesAndGetRenderedFrames(until, dropIncomplete);
+
+    // unlink untracked frames
+    for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();
+            it != done.cend(); ++it) {
+        if (it->getIndex() >= 0) {
+            mBuffers[kPortIndexOutput].editItemAt(it->getIndex()).mRenderInfo = NULL;
+        }
+    }
+
+    if (MediaCodec::CreateFramesRenderedMessage(done, msg)) {
+        msg->post();
+    }
+}
+
 ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() {
     ANativeWindowBuffer *buf;
     CHECK(mNativeWindow.get() != NULL);
@@ -1242,7 +1295,7 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() {
             BufferInfo *info = &mBuffers[kPortIndexOutput].editItemAt(i);
 
             if (info->mGraphicBuffer != NULL &&
-                info->mGraphicBuffer->handle == buf->handle) {
+                    info->mGraphicBuffer->handle == buf->handle) {
                 // Since consumers can attach buffers to BufferQueues, it is possible
                 // that a known yet stale buffer can return from a surface that we
                 // once used.  We can simply ignore this as we have already dequeued
@@ -1255,9 +1308,11 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() {
                     stale = true;
                     break;
                 }
+
                 ALOGV("dequeued buffer %p", info->mGraphicBuffer->getNativeBuffer());
                 info->mStatus = BufferInfo::OWNED_BY_US;
                 info->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow");
+                updateRenderInfoForDequeuedBuffer(buf, fenceFd, info);
                 return info;
             }
         }
@@ -1301,6 +1356,8 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() {
     oldest->mGraphicBuffer = new GraphicBuffer(buf, false);
     oldest->mStatus = BufferInfo::OWNED_BY_US;
     oldest->setWriteFence(fenceFd, "dequeueBufferFromNativeWindow for oldest");
+    mRenderTracker.untrackFrame(oldest->mRenderInfo);
+    oldest->mRenderInfo = NULL;
 
     mOMX->updateGraphicBufferInMeta(
             mNode, kPortIndexOutput, oldest->mGraphicBuffer,
@@ -1324,6 +1381,7 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() {
                 oldest->mGraphicBuffer->getNativeBuffer(), oldest->mData->base());
     }
 
+    updateRenderInfoForDequeuedBuffer(buf, fenceFd, oldest);
     return oldest;
 }
 
@@ -1398,6 +1456,9 @@ status_t ACodec::freeBuffer(OMX_U32 portIndex, size_t i) {
         ::close(info->mFenceFd);
     }
 
+    mRenderTracker.untrackFrame(info->mRenderInfo);
+    info->mRenderInfo = NULL;
+
     // remove buffer even if mOMX->freeBuffer fails
     mBuffers[portIndex].removeAt(i);
 
@@ -4518,9 +4579,20 @@ bool ACodec::BaseState::onOMXMessageList(const sp<AMessage> &msg) {
     CHECK(msg->findObject("messages", &obj));
     sp<MessageList> msgList = static_cast<MessageList *>(obj.get());
 
+    bool receivedRenderedEvents = false;
     for (std::list<sp<AMessage>>::const_iterator it = msgList->getList().cbegin();
           it != msgList->getList().cend(); ++it) {
         onOMXMessage(*it);
+        int32_t type;
+        CHECK((*it)->findInt32("type", &type));
+        if (type == omx_message::FRAME_RENDERED) {
+            receivedRenderedEvents = true;
+        }
+    }
+
+    if (receivedRenderedEvents) {
+        // NOTE: all buffers are rendered in this case
+        mCodec->notifyOfRenderedFrames();
     }
     return true;
 }
@@ -4587,12 +4659,29 @@ bool ACodec::BaseState::onOMXMessage(const sp<AMessage> &msg) {
                     fenceFd);
         }
 
+        case omx_message::FRAME_RENDERED:
+        {
+            int64_t mediaTimeUs, systemNano;
+
+            CHECK(msg->findInt64("media_time_us", &mediaTimeUs));
+            CHECK(msg->findInt64("system_nano", &systemNano));
+
+            return onOMXFrameRendered(
+                    mediaTimeUs, systemNano);
+        }
+
         default:
             ALOGE("Unexpected message type: %d", type);
             return false;
     }
 }
 
+bool ACodec::BaseState::onOMXFrameRendered(
+        int64_t mediaTimeUs __unused, nsecs_t systemNano __unused) {
+    // ignore outside of Executing and PortSettingsChanged states
+    return true;
+}
+
 bool ACodec::BaseState::onOMXEvent(
         OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) {
     if (event != OMX_EventError) {
@@ -4949,6 +5038,15 @@ bool ACodec::BaseState::onOMXFillBufferDone(
     info->mDequeuedAt = ++mCodec->mDequeueCounter;
     info->mStatus = BufferInfo::OWNED_BY_US;
 
+    if (info->mRenderInfo != NULL) {
+        // The fence for an emptied buffer must have signaled, but there still could be queued
+        // or out-of-order dequeued buffers in the render queue prior to this buffer. Drop these,
+        // as we will soon requeue this buffer to the surface. While in theory we could still keep
+        // track of buffers that are requeued to the surface, it is better to add support to the
+        // buffer-queue to notify us of released buffers and their fences (in the future).
+        mCodec->notifyOfRenderedFrames(true /* dropIncomplete */);
+    }
+
     // byte buffers cannot take fences, so wait for any fence now
     if (mCodec->mNativeWindow == NULL) {
         (void)mCodec->waitForFence(fenceFd, "onOMXFillBufferDone");
@@ -5085,6 +5183,14 @@ void ACodec::BaseState::onOutputBufferDrained(const sp<AMessage> &msg) {
         ATRACE_NAME("render");
         // The client wants this buffer to be rendered.
 
+        // save buffers sent to the surface so we can get render time when they return
+        int64_t mediaTimeUs = -1;
+        info->mData->meta()->findInt64("timeUs", &mediaTimeUs);
+        if (mediaTimeUs >= 0) {
+            mCodec->mRenderTracker.onFrameQueued(
+                    mediaTimeUs, info->mGraphicBuffer, new Fence(::dup(info->mFenceFd)));
+        }
+
         int64_t timestampNs = 0;
         if (!msg->findInt64("timestampNs", &timestampNs)) {
             // TODO: it seems like we should use the timestamp
@@ -5367,6 +5473,7 @@ bool ACodec::UninitializedState::onAllocateComponent(const sp<AMessage> &msg) {
     observer->setNotificationMessage(notify);
 
     mCodec->mComponentName = componentName;
+    mCodec->mRenderTracker.setComponentName(componentName);
     mCodec->mFlags = 0;
 
     if (componentName.endsWith(".secure")) {
@@ -5969,6 +6076,7 @@ void ACodec::ExecutingState::resume() {
 void ACodec::ExecutingState::stateEntered() {
     ALOGV("[%s] Now Executing", mCodec->mComponentName.c_str());
 
+    mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC));
     mCodec->processDeferredMessages();
 }
 
@@ -6179,6 +6287,11 @@ void ACodec::onSignalEndOfInputStream() {
     notify->post();
 }
 
+bool ACodec::ExecutingState::onOMXFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) {
+    mCodec->onFrameRendered(mediaTimeUs, systemNano);
+    return true;
+}
+
 bool ACodec::ExecutingState::onOMXEvent(
         OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) {
     switch (event) {
@@ -6266,6 +6379,12 @@ void ACodec::OutputPortSettingsChangedState::stateEntered() {
          mCodec->mComponentName.c_str());
 }
 
+bool ACodec::OutputPortSettingsChangedState::onOMXFrameRendered(
+        int64_t mediaTimeUs, nsecs_t systemNano) {
+    mCodec->onFrameRendered(mediaTimeUs, systemNano);
+    return true;
+}
+
 bool ACodec::OutputPortSettingsChangedState::onOMXEvent(
         OMX_EVENTTYPE event, OMX_U32 data1, OMX_U32 data2) {
     switch (event) {
@@ -6644,6 +6763,8 @@ void ACodec::FlushingState::changeStateIfWeOwnAllBuffers() {
         // the native window for rendering. Let's get those back as well.
         mCodec->waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs();
 
+        mCodec->mRenderTracker.clear(systemTime(CLOCK_MONOTONIC));
+
         sp<AMessage> notify = mCodec->mNotify->dup();
         notify->setInt32("what", CodecBase::kWhatFlushCompleted);
         notify->post();
diff --git a/media/libstagefright/Android.mk b/media/libstagefright/Android.mk
index 0dfa300..69128bd 100644
--- a/media/libstagefright/Android.mk
+++ b/media/libstagefright/Android.mk
@@ -23,6 +23,7 @@ LOCAL_SRC_FILES:=                         \
         ESDS.cpp                          \
         FileSource.cpp                    \
         FLACExtractor.cpp                 \
+        FrameRenderTracker.cpp            \
         HTTPBase.cpp                      \
         JPEGSource.cpp                    \
         MP3Extractor.cpp                  \
diff --git a/media/libstagefright/AwesomePlayer.cpp b/media/libstagefright/AwesomePlayer.cpp
index df01e7c..4e6c2a6 100644
--- a/media/libstagefright/AwesomePlayer.cpp
+++ b/media/libstagefright/AwesomePlayer.cpp
@@ -113,11 +113,11 @@ struct AwesomeLocalRenderer : public AwesomeRenderer {
         CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs));
 
         render((const uint8_t *)buffer->data() + buffer->range_offset(),
-               buffer->range_length(), timeUs * 1000);
+               buffer->range_length(), timeUs, timeUs * 1000);
     }
 
-    void render(const void *data, size_t size, int64_t timestampNs) {
-        mTarget->render(data, size, timestampNs, NULL, mFormat);
+    void render(const void *data, size_t size, int64_t mediaTimeUs, nsecs_t renderTimeNs) {
+        (void)mTarget->render(data, size, mediaTimeUs, renderTimeNs, NULL, mFormat);
     }
 
 protected:
diff --git a/media/libstagefright/FrameRenderTracker.cpp b/media/libstagefright/FrameRenderTracker.cpp
new file mode 100644
index 0000000..ebd2197
--- /dev/null
+++ b/media/libstagefright/FrameRenderTracker.cpp
@@ -0,0 +1,184 @@
+/*
+ * Copyright 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "FrameRenderTracker"
+
+#include <inttypes.h>
+#include <gui/Surface.h>
+
+#include <media/stagefright/foundation/ADebug.h>
+#include <media/stagefright/FrameRenderTracker.h>
+
+namespace android {
+
+FrameRenderTracker::FrameRenderTracker()
+    : mLastRenderTimeNs(-1),
+      mComponentName("unknown component") {
+}
+
+FrameRenderTracker::~FrameRenderTracker() {
+}
+
+void FrameRenderTracker::setComponentName(const AString &componentName) {
+    mComponentName = componentName;
+}
+
+void FrameRenderTracker::clear(nsecs_t lastRenderTimeNs) {
+    mRenderQueue.clear();
+    mLastRenderTimeNs = lastRenderTimeNs;
+}
+
+void FrameRenderTracker::onFrameQueued(
+        int64_t mediaTimeUs, const sp<GraphicBuffer> &graphicBuffer, const sp<Fence> &fence) {
+    mRenderQueue.emplace_back(mediaTimeUs, graphicBuffer, fence);
+}
+
+FrameRenderTracker::Info *FrameRenderTracker::updateInfoForDequeuedBuffer(
+        ANativeWindowBuffer *buf, int fenceFd, int index) {
+    if (index < 0) {
+        return NULL;
+    }
+
+    // see if this is a buffer that was to be rendered
+    std::list<Info>::iterator renderInfo = mRenderQueue.end();
+    for (std::list<Info>::iterator it = mRenderQueue.begin();
+            it != mRenderQueue.end(); ++it) {
+        if (it->mGraphicBuffer->handle == buf->handle) {
+            renderInfo = it;
+            break;
+        }
+    }
+    if (renderInfo == mRenderQueue.end()) {
+        // could have been canceled after fence has signaled
+        return NULL;
+    }
+
+    if (renderInfo->mIndex >= 0) {
+        // buffer has been dequeued before, so there is nothing to do
+        return NULL;
+    }
+
+    // was this frame dropped (we could also infer this if the fence is invalid or a dup of
+    // the queued fence; however, there is no way to figure that out.)
+    if (fenceFd < 0) {
+        // frame is new or was dropped
+        mRenderQueue.erase(renderInfo);
+        return NULL;
+    }
+
+    // store dequeue fence and buffer index
+    renderInfo->mFence = new Fence(::dup(fenceFd));
+    renderInfo->mIndex = index;
+    return &*renderInfo;
+}
+
+status_t FrameRenderTracker::onFrameRendered(int64_t mediaTimeUs, nsecs_t systemNano) {
+    // ensure monotonic timestamps
+    if (mLastRenderTimeNs >= systemNano) {
+        ALOGW("[%s] Ignoring out of order/stale system nano %lld for media time %lld from codec.",
+                mComponentName.c_str(), (long long)systemNano, (long long)mediaTimeUs);
+        return BAD_VALUE;
+    }
+
+    nsecs_t now = systemTime(SYSTEM_TIME_MONOTONIC);
+    if (systemNano > now) {
+        ALOGW("[%s] Ignoring system nano %lld in the future for media time %lld from codec.",
+                mComponentName.c_str(), (long long)systemNano, (long long)mediaTimeUs);
+        return OK;
+    }
+
+    mRenderQueue.emplace_back(mediaTimeUs, systemNano);
+    mLastRenderTimeNs = systemNano;
+    return OK;
+}
+
+std::list<FrameRenderTracker::Info> FrameRenderTracker::checkFencesAndGetRenderedFrames(
+        const FrameRenderTracker::Info *until, bool dropIncomplete) {
+    std::list<Info> done;
+
+    // complete any frames queued prior to this and drop any incomplete ones if requested
+    for (std::list<Info>::iterator it = mRenderQueue.begin();
+            it != mRenderQueue.end(); ) {
+        bool drop = false; // whether to drop each frame
+        if (it->mIndex < 0) {
+            // frame not yet dequeued (or already rendered on a tunneled surface)
+            drop = dropIncomplete;
+        } else if (it->mFence != NULL) {
+            // check if fence signaled
+            nsecs_t signalTime = it->mFence->getSignalTime();
+            if (signalTime < 0) { // invalid fence
+                drop = true;
+            } else if (signalTime == INT64_MAX) { // unsignaled fence
+                drop = dropIncomplete;
+            } else { // signaled
+                // save render time
+                it->mFence.clear();
+                it->mRenderTimeNs = signalTime;
+            }
+        }
+        bool foundFrame = (Info *)&*it == until;
+
+        // Return frames with signaled fences at the start of the queue, as they are
+        // in submit order, and we don't have to wait for any in-between frames.
+        // Also return any dropped frames.
+        if (drop || (it->mFence == NULL && it == mRenderQueue.begin())) {
+            // (unrendered) dropped frames have their mRenderTimeNs still set to -1
+            done.splice(done.end(), mRenderQueue, it++);
+        } else {
+            ++it;
+        }
+        if (foundFrame) {
+            break;
+        }
+    }
+
+    return done;
+}
+
+void FrameRenderTracker::untrackFrame(const FrameRenderTracker::Info *info) {
+    if (info != NULL) {
+        for (std::list<Info>::iterator it = mRenderQueue.begin();
+                it != mRenderQueue.end(); ++it) {
+            if (&*it == info) {
+                mRenderQueue.erase(it);
+                return;
+            }
+        }
+    }
+}
+
+void FrameRenderTracker::dumpRenderQueue() const {
+    ALOGI("[%s] Render Queue: (last render time: %lldns)",
+            mComponentName.c_str(), (long long)mLastRenderTimeNs);
+    for (std::list<Info>::const_iterator it = mRenderQueue.cbegin();
+            it != mRenderQueue.cend(); ++it) {
+        if (it->mFence == NULL) {
+            ALOGI("  RENDERED: handle: %p, media time: %lldus, index: %zd, render time: %lldns",
+                    it->mGraphicBuffer == NULL ? NULL : it->mGraphicBuffer->handle,
+                    (long long)it->mMediaTimeUs, it->mIndex, (long long)it->mRenderTimeNs);
+        } else if (it->mIndex < 0) {
+            ALOGI("    QUEUED: handle: %p, media time: %lldus, fence: %s",
+                    it->mGraphicBuffer->handle, (long long)it->mMediaTimeUs,
+                    it->mFence->isValid() ? "YES" : "NO");
+        } else {
+            ALOGI("  DEQUEUED: handle: %p, media time: %lldus, index: %zd",
+                    it->mGraphicBuffer->handle, (long long)it->mMediaTimeUs, it->mIndex);
+        }
+    }
+}
+
+}  // namespace android
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index f918d2d..7ee84a8 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -397,6 +397,12 @@ status_t MediaCodec::setCallback(const sp<AMessage> &callback) {
     return PostAndAwaitResponse(msg, &response);
 }
 
+status_t MediaCodec::setOnFrameRenderedNotification(const sp<AMessage> &notify) {
+    sp<AMessage> msg = new AMessage(kWhatSetNotification, this);
+    msg->setMessage("on-frame-rendered", notify);
+    return msg->post();
+}
+
 status_t MediaCodec::configure(
         const sp<AMessage> &format,
         const sp<Surface> &surface,
@@ -1333,6 +1339,18 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
                     break;
                 }
 
+                case CodecBase::kWhatOutputFramesRendered:
+                {
+                    // ignore these in all states except running, and check that we have a
+                    // notification set
+                    if (mState == STARTED && mOnFrameRenderedNotification != NULL) {
+                        sp<AMessage> notify = mOnFrameRenderedNotification->dup();
+                        notify->setMessage("data", msg);
+                        notify->post();
+                    }
+                    break;
+                }
+
                 case CodecBase::kWhatFillThisBuffer:
                 {
                     /* size_t index = */updateBuffers(kPortIndexInput, msg);
@@ -1530,6 +1548,15 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
             break;
         }
 
+        case kWhatSetNotification:
+        {
+            sp<AMessage> notify;
+            if (msg->findMessage("on-frame-rendered", &notify)) {
+                mOnFrameRenderedNotification = notify;
+            }
+            break;
+        }
+
         case kWhatSetCallback:
         {
             sp<AReplyToken> replyID;
@@ -2372,6 +2399,23 @@ status_t MediaCodec::onQueueInputBuffer(const sp<AMessage> &msg) {
     return OK;
 }
 
+//static
+size_t MediaCodec::CreateFramesRenderedMessage(
+        std::list<FrameRenderTracker::Info> done, sp<AMessage> &msg) {
+    size_t index = 0;
+
+    for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();
+            it != done.cend(); ++it) {
+        if (it->getRenderTimeNs() < 0) {
+            continue; // dropped frame from tracking
+        }
+        msg->setInt64(AStringPrintf("%zu-media-time-us", index).c_str(), it->getMediaTimeUs());
+        msg->setInt64(AStringPrintf("%zu-system-nano", index).c_str(), it->getRenderTimeNs());
+        ++index;
+    }
+    return index;
+}
+
 status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {
     size_t index;
     CHECK(msg->findSize("index", &index));
@@ -2404,26 +2448,37 @@ status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {
     if (render && info->mData != NULL && info->mData->size() != 0) {
         info->mNotify->setInt32("render", true);
 
-        int64_t timestampNs = 0;
-        if (msg->findInt64("timestampNs", &timestampNs)) {
-            info->mNotify->setInt64("timestampNs", timestampNs);
+        int64_t mediaTimeUs = -1;
+        info->mData->meta()->findInt64("timeUs", &mediaTimeUs);
+
+        int64_t renderTimeNs = 0;
+        if (msg->findInt64("timestampNs", &renderTimeNs)) {
+            info->mNotify->setInt64("timestampNs", renderTimeNs);
         } else {
             // TODO: it seems like we should use the timestamp
             // in the (media)buffer as it potentially came from
             // an input surface, but we did not propagate it prior to
             // API 20.  Perhaps check for target SDK version.
 #if 0
-            if (info->mData->meta()->findInt64("timeUs", &timestampNs)) {
-                ALOGV("using buffer PTS of %" PRId64, timestampNs);
-                timestampNs *= 1000;
-            }
+            ALOGV("using buffer PTS of %" PRId64, timestampNs);
+            renderTimeNs = mediaTimeUs * 1000;
 #endif
         }
 
         if (mSoftRenderer != NULL) {
-            mSoftRenderer->render(
+            std::list<FrameRenderTracker::Info> doneFrames = mSoftRenderer->render(
                     info->mData->data(), info->mData->size(),
-                    timestampNs, NULL, info->mFormat);
+                    mediaTimeUs, renderTimeNs, NULL, info->mFormat);
+
+            // if we are running, notify rendered frames
+            if (!doneFrames.empty() && mState == STARTED && mOnFrameRenderedNotification != NULL) {
+                sp<AMessage> notify = mOnFrameRenderedNotification->dup();
+                sp<AMessage> data = new AMessage;
+                if (CreateFramesRenderedMessage(doneFrames, data)) {
+                    notify->setMessage("data", data);
+                    notify->post();
+                }
+            }
         }
     }
 
diff --git a/media/libstagefright/colorconversion/SoftwareRenderer.cpp b/media/libstagefright/colorconversion/SoftwareRenderer.cpp
index 695cfc8..d22451b 100644
--- a/media/libstagefright/colorconversion/SoftwareRenderer.cpp
+++ b/media/libstagefright/colorconversion/SoftwareRenderer.cpp
@@ -196,17 +196,29 @@ void SoftwareRenderer::resetFormatIfChanged(const sp<AMessage> &format) {
                 mNativeWindow.get(), transform));
 }
 
-void SoftwareRenderer::render(
-        const void *data, size_t size, int64_t timestampNs,
+void SoftwareRenderer::clearTracker() {
+    mRenderTracker.clear(-1 /* lastRenderTimeNs */);
+}
+
+std::list<FrameRenderTracker::Info> SoftwareRenderer::render(
+        const void *data, size_t size, int64_t mediaTimeUs, nsecs_t renderTimeNs,
         void* /*platformPrivate*/, const sp<AMessage>& format) {
     resetFormatIfChanged(format);
+    FrameRenderTracker::Info *info = NULL;
 
     ANativeWindowBuffer *buf;
-    int err;
-    if ((err = native_window_dequeue_buffer_and_wait(mNativeWindow.get(),
-            &buf)) != 0) {
+    int fenceFd = -1;
+    int err = mNativeWindow->dequeueBuffer(mNativeWindow.get(), &buf, &fenceFd);
+    if (err == 0 && fenceFd >= 0) {
+        info = mRenderTracker.updateInfoForDequeuedBuffer(buf, fenceFd, 0);
+        sp<Fence> fence = new Fence(fenceFd);
+        err = fence->waitForever("SoftwareRenderer::render");
+    }
+    if (err != 0) {
         ALOGW("Surface::dequeueBuffer returned error %d", err);
-        return;
+        // complete (drop) dequeued frame if fence wait failed; otherwise,
+        // this returns an empty list as no frames should have rendered and not yet returned.
+        return mRenderTracker.checkFencesAndGetRenderedFrames(info, false /* dropIncomplete */);
     }
 
     GraphicBufferMapper &mapper = GraphicBufferMapper::get();
@@ -342,16 +354,21 @@ void SoftwareRenderer::render(
 skip_copying:
     CHECK_EQ(0, mapper.unlock(buf->handle));
 
-    if ((err = native_window_set_buffers_timestamp(mNativeWindow.get(),
-            timestampNs)) != 0) {
-        ALOGW("Surface::set_buffers_timestamp returned error %d", err);
+    if (renderTimeNs >= 0) {
+        if ((err = native_window_set_buffers_timestamp(mNativeWindow.get(),
+                renderTimeNs)) != 0) {
+            ALOGW("Surface::set_buffers_timestamp returned error %d", err);
+        }
     }
 
-    if ((err = mNativeWindow->queueBuffer(mNativeWindow.get(), buf,
-            -1)) != 0) {
+    if ((err = mNativeWindow->queueBuffer(mNativeWindow.get(), buf, -1)) != 0) {
         ALOGW("Surface::queueBuffer returned error %d", err);
+    } else {
+        mRenderTracker.onFrameQueued(mediaTimeUs, (GraphicBuffer *)buf, Fence::NO_FENCE);
     }
+
     buf = NULL;
+    return mRenderTracker.checkFencesAndGetRenderedFrames(info, info != NULL /* dropIncomplete */);
 }
 
 }  // namespace android
diff --git a/media/libstagefright/include/SoftwareRenderer.h b/media/libstagefright/include/SoftwareRenderer.h
index fa3ea89..9e652d5 100644
--- a/media/libstagefright/include/SoftwareRenderer.h
+++ b/media/libstagefright/include/SoftwareRenderer.h
@@ -19,9 +19,12 @@
 #define SOFTWARE_RENDERER_H_
 
 #include <media/stagefright/ColorConverter.h>
+#include <media/stagefright/FrameRenderTracker.h>
 #include <utils/RefBase.h>
 #include <system/window.h>
 
+#include <list>
+
 namespace android {
 
 struct AMessage;
@@ -32,9 +35,10 @@ public:
 
     ~SoftwareRenderer();
 
-    void render(
-            const void *data, size_t size, int64_t timestampNs,
+    std::list<FrameRenderTracker::Info> render(
+            const void *data, size_t size, int64_t mediaTimeUs, nsecs_t renderTimeNs,
             void *platformPrivate, const sp<AMessage> &format);
+    void clearTracker();
 
 private:
     enum YUVMode {
@@ -48,6 +52,7 @@ private:
     int32_t mWidth, mHeight;
     int32_t mCropLeft, mCropTop, mCropRight, mCropBottom;
     int32_t mCropWidth, mCropHeight;
+    FrameRenderTracker mRenderTracker;
 
     SoftwareRenderer(const SoftwareRenderer &);
     SoftwareRenderer &operator=(const SoftwareRenderer &);
diff --git a/media/libstagefright/omx/OMX.cpp b/media/libstagefright/omx/OMX.cpp
index e94adbd..cb7ab5e 100644
--- a/media/libstagefright/omx/OMX.cpp
+++ b/media/libstagefright/omx/OMX.cpp
@@ -34,6 +34,7 @@
 
 #include <OMX_AsString.h>
 #include <OMX_Component.h>
+#include <OMX_VideoExt.h>
 
 namespace android {
 
@@ -455,12 +456,35 @@ OMX_ERRORTYPE OMX::OnEvent(
         OMX_IN OMX_EVENTTYPE eEvent,
         OMX_IN OMX_U32 nData1,
         OMX_IN OMX_U32 nData2,
-        OMX_IN OMX_PTR /* pEventData */) {
+        OMX_IN OMX_PTR pEventData) {
     ALOGV("OnEvent(%d, %" PRIu32", %" PRIu32 ")", eEvent, nData1, nData2);
 
     // Forward to OMXNodeInstance.
     findInstance(node)->onEvent(eEvent, nData1, nData2);
 
+    sp<OMX::CallbackDispatcher> dispatcher = findDispatcher(node);
+
+    // output rendered events are not processed as regular events until they hit the observer
+    if (eEvent == OMX_EventOutputRendered) {
+        if (pEventData == NULL) {
+            return OMX_ErrorBadParameter;
+        }
+
+        // process data from array
+        OMX_VIDEO_RENDEREVENTTYPE *renderData = (OMX_VIDEO_RENDEREVENTTYPE *)pEventData;
+        for (size_t i = 0; i < nData1; ++i) {
+            omx_message msg;
+            msg.type = omx_message::FRAME_RENDERED;
+            msg.node = node;
+            msg.fenceFd = -1;
+            msg.u.render_data.timestamp = renderData[i].nMediaTimeUs;
+            msg.u.render_data.nanoTime = renderData[i].nSystemTimeNs;
+
+            dispatcher->post(msg, false /* realTime */);
+        }
+        return OMX_ErrorNone;
+    }
+
     omx_message msg;
     msg.type = omx_message::EVENT;
     msg.node = node;
@@ -469,7 +493,7 @@ OMX_ERRORTYPE OMX::OnEvent(
     msg.u.event_data.data1 = nData1;
     msg.u.event_data.data2 = nData2;
 
-    findDispatcher(node)->post(msg);
+    dispatcher->post(msg, true /* realTime */);
 
     return OMX_ErrorNone;
 }