stagefright: MediaSync: use VideoFrameScheduler

Move VideoFrameScheduler to libstagefright as part of this change.

Bug: 22234976
Change-Id: Ib23fb52399cb700a1dcf789e8486b94a3edf9d95

6 files changed, 114 insertions, 138 deletions

diff --git a/media/libmediaplayerservice/Android.mk b/media/libmediaplayerservice/Android.mk
index 7f0cca2..4d1b587 100644
--- a/media/libmediaplayerservice/Android.mk
+++ b/media/libmediaplayerservice/Android.mk
@@ -21,7 +21,6 @@ LOCAL_SRC_FILES:=               \
     StagefrightPlayer.cpp       \
     StagefrightRecorder.cpp     \
     TestPlayerStub.cpp          \
-    VideoFrameScheduler.cpp     \
 
 LOCAL_SHARED_LIBRARIES :=       \
     libbinder                   \
diff --git a/media/libmediaplayerservice/VideoFrameScheduler.h b/media/libmediaplayerservice/VideoFrameScheduler.h
deleted file mode 100644
index b1765c9..0000000
--- a/media/libmediaplayerservice/VideoFrameScheduler.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright 2014, The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef VIDEO_FRAME_SCHEDULER_H_
-#define VIDEO_FRAME_SCHEDULER_H_
-
-#include <utils/RefBase.h>
-#include <utils/Timers.h>
-
-#include <media/stagefright/foundation/ABase.h>
-
-namespace android {
-
-class ISurfaceComposer;
-
-struct VideoFrameScheduler : public RefBase {
-    VideoFrameScheduler();
-
-    // (re)initialize scheduler
-    void init(float videoFps = -1);
-    // use in case of video render-time discontinuity, e.g. seek
-    void restart();
-    // get adjusted nanotime for a video frame render at renderTime
-    nsecs_t schedule(nsecs_t renderTime);
-
-    // returns the vsync period for the main display
-    nsecs_t getVsyncPeriod();
-
-    void release();
-
-    static const size_t kHistorySize = 8;
-
-protected:
-    virtual ~VideoFrameScheduler();
-
-private:
-    struct PLL {
-        PLL();
-
-        // reset PLL to new PLL
-        void reset(float fps = -1);
-        // keep current estimate, but restart phase
-        void restart();
-        // returns period
-        nsecs_t addSample(nsecs_t time);
-
-    private:
-        nsecs_t mPeriod;
-        nsecs_t mPhase;
-
-        bool    mPrimed;        // have an estimate for the period
-        size_t  mSamplesUsedForPriming;
-
-        nsecs_t mLastTime;      // last input time
-        nsecs_t mRefitAt;       // next input time to fit at
-
-        size_t  mNumSamples;    // can go past kHistorySize
-        nsecs_t mTimes[kHistorySize];
-
-        void test();
-        // returns whether fit was successful
-        bool fit(nsecs_t phase, nsecs_t period, size_t numSamples,
-                int64_t *a, int64_t *b, int64_t *err);
-        void prime(size_t numSamples);
-    };
-
-    void updateVsync();
-
-    nsecs_t mVsyncTime;        // vsync timing from display
-    nsecs_t mVsyncPeriod;
-    nsecs_t mVsyncRefreshAt;   // next time to refresh timing info
-
-    nsecs_t mLastVsyncTime;    // estimated vsync time for last frame
-    nsecs_t mTimeCorrection;   // running adjustment
-
-    PLL mPll;                  // PLL for video frame rate based on render time
-
-    sp<ISurfaceComposer> mComposer;
-
-    DISALLOW_EVIL_CONSTRUCTORS(VideoFrameScheduler);
-};
-
-}  // namespace android
-
-#endif  // VIDEO_FRAME_SCHEDULER_H_
-
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
index 13a7d94..767417b 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
@@ -29,8 +29,7 @@
 #include <media/stagefright/MediaErrors.h>
 #include <media/stagefright/MetaData.h>
 #include <media/stagefright/Utils.h>
-
-#include <VideoFrameScheduler.h>
+#include <media/stagefright/VideoFrameScheduler.h>
 
 #include <inttypes.h>
 
diff --git a/media/libstagefright/Android.mk b/media/libstagefright/Android.mk
index 69128bd..b86c749 100644
--- a/media/libstagefright/Android.mk
+++ b/media/libstagefright/Android.mk
@@ -64,6 +64,7 @@ LOCAL_SRC_FILES:=                         \
         TimedEventQueue.cpp               \
         Utils.cpp                         \
         VBRISeeker.cpp                    \
+        VideoFrameScheduler.cpp           \
         WAVExtractor.cpp                  \
         WVMExtractor.cpp                  \
         XINGSeeker.cpp                    \
diff --git a/media/libstagefright/MediaSync.cpp b/media/libstagefright/MediaSync.cpp
index 52077a7..455db42 100644
--- a/media/libstagefright/MediaSync.cpp
+++ b/media/libstagefright/MediaSync.cpp
@@ -25,6 +25,7 @@
 #include <media/AudioTrack.h>
 #include <media/stagefright/MediaClock.h>
 #include <media/stagefright/MediaSync.h>
+#include <media/stagefright/VideoFrameScheduler.h>
 #include <media/stagefright/foundation/ADebug.h>
 #include <media/stagefright/foundation/ALooper.h>
 #include <media/stagefright/foundation/AMessage.h>
@@ -50,6 +51,7 @@ MediaSync::MediaSync()
         mReleaseCondition(),
         mNumOutstandingBuffers(0),
         mUsageFlagsFromOutput(0),
+        mMaxAcquiredBufferCount(1),
         mNativeSampleRateInHz(0),
         mNumFramesWritten(0),
         mHasAudio(false),
@@ -121,6 +123,11 @@ status_t MediaSync::setSurface(const sp<IGraphicBufferProducer> &output) {
             ALOGE("setSurface: failed to connect (%d)", status);
             return status;
         }
+
+        if (mFrameScheduler == NULL) {
+            mFrameScheduler = new VideoFrameScheduler();
+            mFrameScheduler->init();
+        }
     }
 
     if (mOutput != NULL) {
@@ -209,6 +216,12 @@ status_t MediaSync::createInputSurface(
         bufferConsumer->setConsumerUsageBits(mUsageFlagsFromOutput);
         *outBufferProducer = bufferProducer;
         mInput = bufferConsumer;
+
+        // set undequeued buffer count
+        int minUndequeuedBuffers;
+        mOutput->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBuffers);
+        mMaxAcquiredBufferCount = minUndequeuedBuffers;
+        bufferConsumer->setMaxAcquiredBufferCount(mMaxAcquiredBufferCount);
     }
     return status;
 }
@@ -326,12 +339,26 @@ void MediaSync::setName(const AString &name) {
 }
 
 status_t MediaSync::setVideoFrameRateHint(float rate) {
-    // ignored until we add the FrameScheduler
-    return rate >= 0.f ? OK : BAD_VALUE;
+    Mutex::Autolock lock(mMutex);
+    if (rate < 0.f) {
+        return BAD_VALUE;
+    }
+    if (mFrameScheduler != NULL) {
+        mFrameScheduler->init(rate);
+    }
+    return OK;
 }
 
 float MediaSync::getVideoFrameRate() {
-    // we don't know the frame rate
+    Mutex::Autolock lock(mMutex);
+    if (mFrameScheduler != NULL) {
+        float fps = mFrameScheduler->getFrameRate();
+        if (fps > 0.f) {
+            return fps;
+        }
+    }
+
+    // we don't have or know the frame rate
     return -1.f;
 }
 
@@ -470,7 +497,7 @@ int64_t MediaSync::getPlayedOutAudioDurationMedia_l(int64_t nowUs) {
         CHECK_EQ(res, (status_t)OK);
         numFramesPlayedAt = nowUs;
         numFramesPlayedAt += 1000LL * mAudioTrack->latency() / 2; /* XXX */
-        //ALOGD("getPosition: %d %lld", numFramesPlayed, numFramesPlayedAt);
+        //ALOGD("getPosition: %d %lld", numFramesPlayed, (long long)numFramesPlayedAt);
     }
 
     //can't be negative until 12.4 hrs, test.
@@ -510,18 +537,30 @@ void MediaSync::onDrainVideo_l() {
         int64_t itemMediaUs = bufferItem->mTimestamp / 1000;
         int64_t itemRealUs = getRealTime(itemMediaUs, nowUs);
 
-        if (itemRealUs <= nowUs) {
+        // adjust video frame PTS based on vsync
+        itemRealUs = mFrameScheduler->schedule(itemRealUs * 1000) / 1000;
+        int64_t oneVsyncUs = (mFrameScheduler->getVsyncPeriod() / 1000);
+        int64_t twoVsyncsUs = oneVsyncUs * 2;
+
+        // post 2 display refreshes before rendering is due
+        if (itemRealUs <= nowUs + twoVsyncsUs) {
+            ALOGV("adjusting PTS from %lld to %lld",
+                    (long long)bufferItem->mTimestamp / 1000, (long long)itemRealUs);
+            bufferItem->mTimestamp = itemRealUs * 1000;
+            bufferItem->mIsAutoTimestamp = false;
+
             if (mHasAudio) {
                 if (nowUs - itemRealUs <= kMaxAllowedVideoLateTimeUs) {
-                    renderOneBufferItem_l(*bufferItem);
+                    renderOneBufferItem_l(*bufferItem, nowUs + oneVsyncUs - itemRealUs);
                 } else {
                     // too late.
                     returnBufferToInput_l(
                             bufferItem->mGraphicBuffer, bufferItem->mFence);
+                    mFrameScheduler->restart();
                 }
             } else {
                 // always render video buffer in video-only mode.
-                renderOneBufferItem_l(*bufferItem);
+                renderOneBufferItem_l(*bufferItem, nowUs + oneVsyncUs - itemRealUs);
 
                 // smooth out videos >= 10fps
                 mMediaClock->updateAnchor(
@@ -534,7 +573,7 @@ void MediaSync::onDrainVideo_l() {
             if (mNextBufferItemMediaUs == -1
                     || mNextBufferItemMediaUs > itemMediaUs) {
                 sp<AMessage> msg = new AMessage(kWhatDrainVideo, this);
-                msg->post(itemRealUs - nowUs);
+                msg->post(itemRealUs - nowUs - twoVsyncsUs);
                 mNextBufferItemMediaUs = itemMediaUs;
             }
             break;
@@ -545,10 +584,15 @@ void MediaSync::onDrainVideo_l() {
 void MediaSync::onFrameAvailableFromInput() {
     Mutex::Autolock lock(mMutex);
 
+    const static nsecs_t kAcquireWaitTimeout = 2000000000; // 2 seconds
+
     // If there are too many outstanding buffers, wait until a buffer is
     // released back to the input in onBufferReleased.
-    while (mNumOutstandingBuffers >= MAX_OUTSTANDING_BUFFERS) {
-        mReleaseCondition.wait(mMutex);
+    // NOTE: BufferQueue allows dequeuing maxAcquiredBufferCount + 1 buffers
+    while (mNumOutstandingBuffers > mMaxAcquiredBufferCount && !mIsAbandoned) {
+        if (mReleaseCondition.waitRelative(mMutex, kAcquireWaitTimeout) != OK) {
+            ALOGI("still waiting to release a buffer before acquire");
+        }
 
         // If the sync is abandoned while we are waiting, the release
         // condition variable will be broadcast, and we should just return
@@ -582,6 +626,7 @@ void MediaSync::onFrameAvailableFromInput() {
 
     if (mBuffersFromInput.indexOfKey(bufferItem.mGraphicBuffer->getId()) >= 0) {
         // Something is wrong since this buffer should be at our hands, bail.
+        ALOGE("received buffer multiple times from input");
         mInput->consumerDisconnect();
         onAbandoned_l(true /* isInput */);
         return;
@@ -595,7 +640,7 @@ void MediaSync::onFrameAvailableFromInput() {
     }
 }
 
-void MediaSync::renderOneBufferItem_l( const BufferItem &bufferItem) {
+void MediaSync::renderOneBufferItem_l(const BufferItem &bufferItem, int64_t checkInUs) {
     IGraphicBufferProducer::QueueBufferInput queueInput(
             bufferItem.mTimestamp,
             bufferItem.mIsAutoTimestamp,
@@ -635,6 +680,12 @@ void MediaSync::renderOneBufferItem_l( const BufferItem &bufferItem) {
     mBuffersSentToOutput.add(bufferItem.mGraphicBuffer->getId(), bufferItem.mGraphicBuffer);
 
     ALOGV("queued buffer %#llx to output", (long long)bufferItem.mGraphicBuffer->getId());
+
+    // If we have already queued more than one buffer, check for any free buffers in case
+    // one of them were dropped - as BQ does not signal onBufferReleased in that case.
+    if (mBuffersSentToOutput.size() > 1) {
+        (new AMessage(kWhatCheckFrameAvailable, this))->post(checkInUs);
+    }
 }
 
 void MediaSync::onBufferReleasedByOutput(sp<IGraphicBufferProducer> &output) {
@@ -646,32 +697,38 @@ void MediaSync::onBufferReleasedByOutput(sp<IGraphicBufferProducer> &output) {
 
     sp<GraphicBuffer> buffer;
     sp<Fence> fence;
-    status_t status = mOutput->detachNextBuffer(&buffer, &fence);
-    ALOGE_IF(status != NO_ERROR, "detaching buffer from output failed (%d)", status);
+    status_t status;
+    // NOTE: This is a workaround for a BufferQueue bug where onBufferReleased is
+    // called only for released buffers, but not for buffers that were dropped during
+    // acquire. Dropped buffers can still be detached as they are on the free list.
+    // TODO: remove if released callback happens also for dropped buffers
+    while ((status = mOutput->detachNextBuffer(&buffer, &fence)) != NO_MEMORY) {
+        ALOGE_IF(status != NO_ERROR, "detaching buffer from output failed (%d)", status);
 
-    if (status == NO_INIT) {
-        // If the output has been abandoned, we can't do anything else,
-        // since buffer is invalid.
-        onAbandoned_l(false /* isInput */);
-        return;
-    }
+        if (status == NO_INIT) {
+            // If the output has been abandoned, we can't do anything else,
+            // since buffer is invalid.
+            onAbandoned_l(false /* isInput */);
+            return;
+        }
 
-    ALOGV("detached buffer %#llx from output", (long long)buffer->getId());
+        ALOGV("detached buffer %#llx from output", (long long)buffer->getId());
 
-    // If we've been abandoned, we can't return the buffer to the input, so just
-    // move on.
-    if (mIsAbandoned) {
-        return;
-    }
+        // If we've been abandoned, we can't return the buffer to the input, so just
+        // move on.
+        if (mIsAbandoned) {
+            return;
+        }
 
-    ssize_t ix = mBuffersSentToOutput.indexOfKey(buffer->getId());
-    if (ix < 0) {
-        // The buffer is unknown, maybe leftover, ignore.
-        return;
-    }
-    mBuffersSentToOutput.removeItemsAt(ix);
+        ssize_t ix = mBuffersSentToOutput.indexOfKey(buffer->getId());
+        if (ix < 0) {
+            // The buffer is unknown, maybe leftover, ignore.
+            return;
+        }
+        mBuffersSentToOutput.removeItemsAt(ix);
 
-    returnBufferToInput_l(buffer, fence);
+        returnBufferToInput_l(buffer, fence);
+    }
 }
 
 void MediaSync::returnBufferToInput_l(
@@ -679,6 +736,7 @@ void MediaSync::returnBufferToInput_l(
     ssize_t ix = mBuffersFromInput.indexOfKey(buffer->getId());
     if (ix < 0) {
         // The buffer is unknown, something is wrong, bail.
+        ALOGE("output returned unknown buffer");
         mOutput->disconnect(NATIVE_WINDOW_API_MEDIA);
         onAbandoned_l(false /* isInput */);
         return;
@@ -741,6 +799,12 @@ void MediaSync::onMessageReceived(const sp<AMessage> &msg) {
             break;
         }
 
+        case kWhatCheckFrameAvailable:
+        {
+            onBufferReleasedByOutput(mOutput);
+            break;
+        }
+
         default:
             TRESPASS();
             break;
diff --git a/media/libmediaplayerservice/VideoFrameScheduler.cpp b/media/libstagefright/VideoFrameScheduler.cpp
index ce5f5fe..5fe9bf9 100644
--- a/media/libmediaplayerservice/VideoFrameScheduler.cpp
+++ b/media/libstagefright/VideoFrameScheduler.cpp
@@ -28,8 +28,7 @@
 
 #include <media/stagefright/foundation/ADebug.h>
 #include <media/stagefright/foundation/AUtils.h>
-
-#include "VideoFrameScheduler.h"
+#include <media/stagefright/VideoFrameScheduler.h>
 
 namespace android {
 
@@ -56,7 +55,7 @@ static const size_t kMinSamplesToEstimatePeriod = 3;
 static const size_t kMaxSamplesToEstimatePeriod = VideoFrameScheduler::kHistorySize;
 
 static const size_t kPrecision = 12;
-static const size_t kErrorThreshold = (1 << (kPrecision * 2)) / 10;
+static const int64_t kErrorThreshold = (1 << (kPrecision * 2)) / 10;
 static const int64_t kMultiplesThresholdDiv = 4;            // 25%
 static const int64_t kReFitThresholdDiv = 100;              // 1%
 static const nsecs_t kMaxAllowedFrameSkip = kNanosIn1s;     // 1 sec
@@ -258,7 +257,8 @@ void VideoFrameScheduler::PLL::prime(size_t numSamplesToUse) {
             mPhase = firstTime;
         }
     }
-    ALOGV("priming[%zu] phase:%lld period:%lld", numSamplesToUse, mPhase, mPeriod);
+    ALOGV("priming[%zu] phase:%lld period:%lld",
+            numSamplesToUse, (long long)mPhase, (long long)mPeriod);
 }
 
 nsecs_t VideoFrameScheduler::PLL::addSample(nsecs_t time) {
@@ -316,6 +316,10 @@ nsecs_t VideoFrameScheduler::PLL::addSample(nsecs_t time) {
     return mPeriod;
 }
 
+nsecs_t VideoFrameScheduler::PLL::getPeriod() const {
+    return mPrimed ? mPeriod : 0;
+}
+
 /* ======================================================================= */
 /*                             Frame Scheduler                             */
 /* ======================================================================= */
@@ -382,6 +386,14 @@ nsecs_t VideoFrameScheduler::getVsyncPeriod() {
     return kDefaultVsyncPeriod;
 }
 
+float VideoFrameScheduler::getFrameRate() {
+    nsecs_t videoPeriod = mPll.getPeriod();
+    if (videoPeriod > 0) {
+        return 1e9 / videoPeriod;
+    }
+    return 0.f;
+}
+
 nsecs_t VideoFrameScheduler::schedule(nsecs_t renderTime) {
     nsecs_t origRenderTime = renderTime;