mediaplayer: schedule video frames in VSYNC valleys

Bug: 14659809
Change-Id: Ic340ac61ad4778b493625c79c2cb4f747ff54ede

7 files changed, 224 insertions, 14 deletions

diff --git a/media/libmediaplayerservice/Android.mk b/media/libmediaplayerservice/Android.mk
index adc066d..2cf5710 100644
--- a/media/libmediaplayerservice/Android.mk
+++ b/media/libmediaplayerservice/Android.mk
@@ -22,6 +22,7 @@ LOCAL_SRC_FILES:=               \
     StagefrightPlayer.cpp       \
     StagefrightRecorder.cpp     \
     TestPlayerStub.cpp          \
+    VideoFrameScheduler.cpp     \
 
 LOCAL_SHARED_LIBRARIES :=       \
     libbinder                   \
diff --git a/media/libmediaplayerservice/VideoFrameScheduler.cpp b/media/libmediaplayerservice/VideoFrameScheduler.cpp
new file mode 100644
index 0000000..dd38d5d
--- /dev/null
+++ b/media/libmediaplayerservice/VideoFrameScheduler.cpp
@@ -0,0 +1,130 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "VideoFrameScheduler"
+#include <utils/Log.h>
+#define ATRACE_TAG ATRACE_TAG_VIDEO
+#include <utils/Trace.h>
+
+#include <sys/time.h>
+
+#include <binder/IServiceManager.h>
+#include <gui/ISurfaceComposer.h>
+#include <ui/DisplayStatInfo.h>
+
+#include <media/stagefright/foundation/ADebug.h>
+
+#include "VideoFrameScheduler.h"
+
+namespace android {
+
+static const nsecs_t kNanosIn1s = 1000000000;
+
+/* ======================================================================= */
+/*                             Frame Scheduler                             */
+/* ======================================================================= */
+
+static const nsecs_t kDefaultVsyncPeriod = kNanosIn1s / 60;  // 60Hz
+static const nsecs_t kVsyncRefreshPeriod = kNanosIn1s;       // 1 sec
+
+VideoFrameScheduler::VideoFrameScheduler()
+    : mVsyncTime(0),
+      mVsyncPeriod(0),
+      mVsyncRefreshAt(0) {
+}
+
+void VideoFrameScheduler::updateVsync() {
+    mVsyncRefreshAt = systemTime(SYSTEM_TIME_MONOTONIC) + kVsyncRefreshPeriod;
+    mVsyncPeriod = 0;
+    mVsyncTime = 0;
+
+    // TODO: schedule frames for the destination surface
+    // For now, surface flinger only schedules frames on the primary display
+    if (mComposer == NULL) {
+        String16 name("SurfaceFlinger");
+        sp<IServiceManager> sm = defaultServiceManager();
+        mComposer = interface_cast<ISurfaceComposer>(sm->checkService(name));
+    }
+    if (mComposer != NULL) {
+        DisplayStatInfo stats;
+        status_t res = mComposer->getDisplayStats(NULL /* display */, &stats);
+        if (res == OK) {
+            ALOGV("vsync time:%lld period:%lld",
+                    (long long)stats.vsyncTime, (long long)stats.vsyncPeriod);
+            mVsyncTime = stats.vsyncTime;
+            mVsyncPeriod = stats.vsyncPeriod;
+        } else {
+            ALOGW("getDisplayStats returned %d", res);
+        }
+    } else {
+        ALOGW("could not get surface mComposer service");
+    }
+}
+
+void VideoFrameScheduler::init() {
+    updateVsync();
+}
+
+nsecs_t VideoFrameScheduler::getVsyncPeriod() {
+    if (mVsyncPeriod > 0) {
+        return mVsyncPeriod;
+    }
+    return kDefaultVsyncPeriod;
+}
+
+nsecs_t VideoFrameScheduler::schedule(nsecs_t renderTime) {
+    nsecs_t origRenderTime = renderTime;
+
+    nsecs_t now = systemTime(SYSTEM_TIME_MONOTONIC);
+    if (now >= mVsyncRefreshAt) {
+        updateVsync();
+    }
+
+    // without VSYNC info, there is nothing to do
+    if (mVsyncPeriod == 0) {
+        ALOGV("no vsync: render=%lld", (long long)renderTime);
+        return renderTime;
+    }
+
+    // ensure vsync time is well before (corrected) render time
+    if (mVsyncTime > renderTime - 4 * mVsyncPeriod) {
+        mVsyncTime -=
+            ((mVsyncTime - renderTime) / mVsyncPeriod + 5) * mVsyncPeriod;
+    }
+
+    // Video presentation takes place at the VSYNC _after_ renderTime.  Adjust renderTime
+    // so this effectively becomes a rounding operation (to the _closest_ VSYNC.)
+    renderTime -= mVsyncPeriod / 2;
+
+    // align rendertime to the center between VSYNC edges
+    renderTime -= (renderTime - mVsyncTime) % mVsyncPeriod;
+    renderTime += mVsyncPeriod / 2;
+    ALOGV("adjusting render: %lld => %lld", (long long)origRenderTime, (long long)renderTime);
+    ATRACE_INT("FRAME_FLIP_IN(ms)", (renderTime - now) / 1000000);
+    return renderTime;
+}
+
+void VideoFrameScheduler::release() {
+    mComposer.clear();
+}
+
+VideoFrameScheduler::~VideoFrameScheduler() {
+    release();
+}
+
+} // namespace android
+
diff --git a/media/libmediaplayerservice/VideoFrameScheduler.h b/media/libmediaplayerservice/VideoFrameScheduler.h
new file mode 100644
index 0000000..9119e45
--- /dev/null
+++ b/media/libmediaplayerservice/VideoFrameScheduler.h
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2014, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef VIDEO_FRAME_SCHEDULER_H_
+#define VIDEO_FRAME_SCHEDULER_H_
+
+#include <utils/RefBase.h>
+#include <utils/Timers.h>
+
+#include <media/stagefright/foundation/ABase.h>
+
+namespace android {
+
+struct ISurfaceComposer;
+
+struct VideoFrameScheduler : public RefBase {
+    VideoFrameScheduler();
+
+    // (re)initialize scheduler
+    void init();
+    // get adjusted nanotime for a video frame render at renderTime
+    nsecs_t schedule(nsecs_t renderTime);
+
+    // returns the vsync period for the main display
+    nsecs_t getVsyncPeriod();
+
+    void release();
+
+protected:
+    virtual ~VideoFrameScheduler();
+
+private:
+    void updateVsync();
+
+    nsecs_t mVsyncTime;        // vsync timing from display
+    nsecs_t mVsyncPeriod;
+    nsecs_t mVsyncRefreshAt;   // next time to refresh timing info
+
+    sp<ISurfaceComposer> mComposer;
+
+    DISALLOW_EVIL_CONSTRUCTORS(VideoFrameScheduler);
+};
+
+}  // namespace android
+
+#endif  // VIDEO_FRAME_SCHEDULER_H_
+
diff --git a/media/libmediaplayerservice/nuplayer/Android.mk b/media/libmediaplayerservice/nuplayer/Android.mk
index 0dd2b61..676c0a6 100644
--- a/media/libmediaplayerservice/nuplayer/Android.mk
+++ b/media/libmediaplayerservice/nuplayer/Android.mk
@@ -19,6 +19,7 @@ LOCAL_C_INCLUDES := \
 	$(TOP)/frameworks/av/media/libstagefright/mpeg2ts             \
 	$(TOP)/frameworks/av/media/libstagefright/rtsp                \
 	$(TOP)/frameworks/av/media/libstagefright/timedtext           \
+	$(TOP)/frameworks/av/media/libmediaplayerservice              \
 	$(TOP)/frameworks/native/include/media/openmax
 
 LOCAL_MODULE:= libstagefright_nuplayer
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
index 87f85e7..915dd81 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
@@ -470,7 +470,9 @@ void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
     size_t bufferIx;
     CHECK(msg->findSize("buffer-ix", &bufferIx));
     if (msg->findInt32("render", &render) && render) {
-        err = mCodec->renderOutputBufferAndRelease(bufferIx);
+        int64_t timestampNs;
+        CHECK(msg->findInt64("timestampNs", &timestampNs));
+        err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs);
     } else {
         err = mCodec->releaseOutputBuffer(bufferIx);
     }
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
index 7674616..9934e06 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
@@ -26,6 +26,8 @@
 #include <media/stagefright/MediaErrors.h>
 #include <media/stagefright/MetaData.h>
 
+#include <VideoFrameScheduler.h>
+
 #include <inttypes.h>
 
 namespace android {
@@ -502,16 +504,20 @@ void NuPlayer::Renderer::postDrainVideoQueue() {
     sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id());
     msg->setInt32("generation", mVideoQueueGeneration);
 
-    int64_t delayUs;
-
     if (entry.mBuffer == NULL) {
         // EOS doesn't carry a timestamp.
-        delayUs = 0;
-    } else if (mFlags & FLAG_REAL_TIME) {
+        msg->post();
+        mDrainVideoQueuePending = true;
+        return;
+    }
+
+    int64_t delayUs;
+    int64_t nowUs = ALooper::GetNowUs();
+    int64_t realTimeUs;
+    if (mFlags & FLAG_REAL_TIME) {
         int64_t mediaTimeUs;
         CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
-
-        delayUs = mediaTimeUs - ALooper::GetNowUs();
+        realTimeUs = mediaTimeUs;
     } else {
         int64_t mediaTimeUs;
         CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
@@ -520,23 +526,26 @@ void NuPlayer::Renderer::postDrainVideoQueue() {
             mFirstAnchorTimeMediaUs = mediaTimeUs;
         }
         if (mAnchorTimeMediaUs < 0) {
-            delayUs = 0;
-
             if (!mHasAudio) {
                 mAnchorTimeMediaUs = mediaTimeUs;
-                mAnchorTimeRealUs = ALooper::GetNowUs();
+                mAnchorTimeRealUs = nowUs;
                 notifyPosition();
             }
+            realTimeUs = nowUs;
         } else {
-            int64_t realTimeUs =
+            realTimeUs =
                 (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs;
-
-            delayUs = realTimeUs - ALooper::GetNowUs();
         }
     }
 
+    realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
+    int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
+
+    delayUs = realTimeUs - nowUs;
+
     ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
-    msg->post(delayUs);
+    // post 2 display refreshes before rendering is due
+    msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
 
     mDrainVideoQueuePending = true;
 }
@@ -588,6 +597,7 @@ void NuPlayer::Renderer::onDrainVideoQueue() {
         mVideoLateByUs = 0ll;
     }
 
+    entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll);
     entry->mNotifyConsumed->setInt32("render", !tooLate);
     entry->mNotifyConsumed->post();
     mVideoQueue.erase(mVideoQueue.begin());
@@ -630,6 +640,10 @@ void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
         mHasAudio = true;
     } else {
         mHasVideo = true;
+        if (mVideoScheduler == NULL) {
+            mVideoScheduler = new VideoFrameScheduler();
+            mVideoScheduler->init();
+        }
     }
 
     if (dropBufferWhileFlushing(audio, msg)) {
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h
index 97fdae7..c5a6ec0 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h
@@ -23,6 +23,7 @@
 namespace android {
 
 struct ABuffer;
+struct VideoFrameScheduler;
 
 struct NuPlayer::Renderer : public AHandler {
     enum Flags {
@@ -100,6 +101,7 @@ private:
     List<QueueEntry> mAudioQueue;
     List<QueueEntry> mVideoQueue;
     uint32_t mNumFramesWritten;
+    sp<VideoFrameScheduler> mVideoScheduler;
 
     bool mDrainAudioQueuePending;
     bool mDrainVideoQueuePending;