mediaplayer: schedule video frames in VSYNC valleys

Bug: 14659809
Change-Id: Ic340ac61ad4778b493625c79c2cb4f747ff54ede

4 files changed, 33 insertions, 14 deletions

diff --git a/media/libmediaplayerservice/nuplayer/Android.mk b/media/libmediaplayerservice/nuplayer/Android.mk
index 0dd2b61..676c0a6 100644
--- a/media/libmediaplayerservice/nuplayer/Android.mk
+++ b/media/libmediaplayerservice/nuplayer/Android.mk
@@ -19,6 +19,7 @@ LOCAL_C_INCLUDES := \
 	$(TOP)/frameworks/av/media/libstagefright/mpeg2ts             \
 	$(TOP)/frameworks/av/media/libstagefright/rtsp                \
 	$(TOP)/frameworks/av/media/libstagefright/timedtext           \
+	$(TOP)/frameworks/av/media/libmediaplayerservice              \
 	$(TOP)/frameworks/native/include/media/openmax
 
 LOCAL_MODULE:= libstagefright_nuplayer
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
index 87f85e7..915dd81 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
@@ -470,7 +470,9 @@ void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
     size_t bufferIx;
     CHECK(msg->findSize("buffer-ix", &bufferIx));
     if (msg->findInt32("render", &render) && render) {
-        err = mCodec->renderOutputBufferAndRelease(bufferIx);
+        int64_t timestampNs;
+        CHECK(msg->findInt64("timestampNs", &timestampNs));
+        err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
     } else {
         err = mCodec->releaseOutputBuffer(bufferIx);
     }
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
index 7674616..9934e06 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
@@ -26,6 +26,8 @@
 #include <media/stagefright/MediaErrors.h>
 #include <media/stagefright/MetaData.h>
 
+#include <VideoFrameScheduler.h>
+
 #include <inttypes.h>
 
 namespace android {
@@ -502,16 +504,20 @@ void NuPlayer::Renderer::postDrainVideoQueue() {
     sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id());
     msg->setInt32("generation", mVideoQueueGeneration);
 
-    int64_t delayUs;
-
     if (entry.mBuffer == NULL) {
         // EOS doesn't carry a timestamp.
-        delayUs = 0;
-    } else if (mFlags & FLAG_REAL_TIME) {
+        msg->post();
+        mDrainVideoQueuePending = true;
+        return;
+    }
+
+    int64_t delayUs;
+    int64_t nowUs = ALooper::GetNowUs();
+    int64_t realTimeUs;
+    if (mFlags & FLAG_REAL_TIME) {
         int64_t mediaTimeUs;
         CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
-
-        delayUs = mediaTimeUs - ALooper::GetNowUs();
+        realTimeUs = mediaTimeUs;
     } else {
         int64_t mediaTimeUs;
         CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
@@ -520,23 +526,26 @@ void NuPlayer::Renderer::postDrainVideoQueue() {
             mFirstAnchorTimeMediaUs = mediaTimeUs;
         }
         if (mAnchorTimeMediaUs < 0) {
-            delayUs = 0;
-
             if (!mHasAudio) {
                 mAnchorTimeMediaUs = mediaTimeUs;
-                mAnchorTimeRealUs = ALooper::GetNowUs();
+                mAnchorTimeRealUs = nowUs;
                 notifyPosition();
             }
+            realTimeUs = nowUs;
         } else {
-            int64_t realTimeUs =
+            realTimeUs =
                 (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs;
-
-            delayUs = realTimeUs - ALooper::GetNowUs();
         }
     }
 
+    realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
+    int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
+
+    delayUs = realTimeUs - nowUs;
+
     ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
-    msg->post(delayUs);
+    // post 2 display refreshes before rendering is due
+    msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
 
     mDrainVideoQueuePending = true;
 }
@@ -588,6 +597,7 @@ void NuPlayer::Renderer::onDrainVideoQueue() {
         mVideoLateByUs = 0ll;
     }
 
+    entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll);
     entry->mNotifyConsumed->setInt32("render", !tooLate);
     entry->mNotifyConsumed->post();
     mVideoQueue.erase(mVideoQueue.begin());
@@ -630,6 +640,10 @@ void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
         mHasAudio = true;
     } else {
         mHasVideo = true;
+        if (mVideoScheduler == NULL) {
+            mVideoScheduler = new VideoFrameScheduler();
+            mVideoScheduler->init();
+        }
     }
 
     if (dropBufferWhileFlushing(audio, msg)) {
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h
index 97fdae7..c5a6ec0 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h
@@ -23,6 +23,7 @@
 namespace android {
 
 struct ABuffer;
+struct VideoFrameScheduler;
 
 struct NuPlayer::Renderer : public AHandler {
     enum Flags {
@@ -100,6 +101,7 @@ private:
     List<QueueEntry> mAudioQueue;
     List<QueueEntry> mVideoQueue;
     uint32_t mNumFramesWritten;
+    sp<VideoFrameScheduler> mVideoScheduler;
 
     bool mDrainAudioQueuePending;
     bool mDrainVideoQueuePending;