stagefright: MediaSync: use VideoFrameScheduler

Move VideoFrameScheduler to libstagefright as part of this change.

Bug: 22234976
Change-Id: Ib23fb52399cb700a1dcf789e8486b94a3edf9d95

4 files changed, 1 insertions, 584 deletions

diff --git a/media/libmediaplayerservice/Android.mk b/media/libmediaplayerservice/Android.mk
index 7f0cca2..4d1b587 100644
--- a/media/libmediaplayerservice/Android.mk
+++ b/media/libmediaplayerservice/Android.mk
@@ -21,7 +21,6 @@ LOCAL_SRC_FILES:=               \
     StagefrightPlayer.cpp       \
     StagefrightRecorder.cpp     \
     TestPlayerStub.cpp          \
-    VideoFrameScheduler.cpp     \
 
 LOCAL_SHARED_LIBRARIES :=       \
     libbinder                   \
diff --git a/media/libmediaplayerservice/VideoFrameScheduler.cpp b/media/libmediaplayerservice/VideoFrameScheduler.cpp
deleted file mode 100644
index ce5f5fe..0000000
--- a/media/libmediaplayerservice/VideoFrameScheduler.cpp
+++ /dev/null
@@ -1,482 +0,0 @@
-/*
- * Copyright (C) 2014 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-//#define LOG_NDEBUG 0
-#define LOG_TAG "VideoFrameScheduler"
-#include <utils/Log.h>
-#define ATRACE_TAG ATRACE_TAG_VIDEO
-#include <utils/Trace.h>
-
-#include <sys/time.h>
-
-#include <binder/IServiceManager.h>
-#include <gui/ISurfaceComposer.h>
-#include <ui/DisplayStatInfo.h>
-
-#include <media/stagefright/foundation/ADebug.h>
-#include <media/stagefright/foundation/AUtils.h>
-
-#include "VideoFrameScheduler.h"
-
-namespace android {
-
-static const nsecs_t kNanosIn1s = 1000000000;
-
-template<class T>
-static int compare(const T *lhs, const T *rhs) {
-    if (*lhs < *rhs) {
-        return -1;
-    } else if (*lhs > *rhs) {
-        return 1;
-    } else {
-        return 0;
-    }
-}
-
-/* ======================================================================= */
-/*                                   PLL                                   */
-/* ======================================================================= */
-
-static const size_t kMinSamplesToStartPrime = 3;
-static const size_t kMinSamplesToStopPrime = VideoFrameScheduler::kHistorySize;
-static const size_t kMinSamplesToEstimatePeriod = 3;
-static const size_t kMaxSamplesToEstimatePeriod = VideoFrameScheduler::kHistorySize;
-
-static const size_t kPrecision = 12;
-static const size_t kErrorThreshold = (1 << (kPrecision * 2)) / 10;
-static const int64_t kMultiplesThresholdDiv = 4;            // 25%
-static const int64_t kReFitThresholdDiv = 100;              // 1%
-static const nsecs_t kMaxAllowedFrameSkip = kNanosIn1s;     // 1 sec
-static const nsecs_t kMinPeriod = kNanosIn1s / 120;         // 120Hz
-static const nsecs_t kRefitRefreshPeriod = 10 * kNanosIn1s; // 10 sec
-
-VideoFrameScheduler::PLL::PLL()
-    : mPeriod(-1),
-      mPhase(0),
-      mPrimed(false),
-      mSamplesUsedForPriming(0),
-      mLastTime(-1),
-      mNumSamples(0) {
-}
-
-void VideoFrameScheduler::PLL::reset(float fps) {
-    //test();
-
-    mSamplesUsedForPriming = 0;
-    mLastTime = -1;
-
-    // set up or reset video PLL
-    if (fps <= 0.f) {
-        mPeriod = -1;
-        mPrimed = false;
-    } else {
-        ALOGV("reset at %.1f fps", fps);
-        mPeriod = (nsecs_t)(1e9 / fps + 0.5);
-        mPrimed = true;
-    }
-
-    restart();
-}
-
-// reset PLL but keep previous period estimate
-void VideoFrameScheduler::PLL::restart() {
-    mNumSamples = 0;
-    mPhase = -1;
-}
-
-#if 0
-
-void VideoFrameScheduler::PLL::test() {
-    nsecs_t period = kNanosIn1s / 60;
-    mTimes[0] = 0;
-    mTimes[1] = period;
-    mTimes[2] = period * 3;
-    mTimes[3] = period * 4;
-    mTimes[4] = period * 7;
-    mTimes[5] = period * 8;
-    mTimes[6] = period * 10;
-    mTimes[7] = period * 12;
-    mNumSamples = 8;
-    int64_t a, b, err;
-    fit(0, period * 12 / 7, 8, &a, &b, &err);
-    // a = 0.8(5)+
-    // b = -0.14097(2)+
-    // err = 0.2750578(703)+
-    ALOGD("a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)",
-            (long long)a, (a / (float)(1 << kPrecision)),
-            (long long)b, (b / (float)(1 << kPrecision)),
-            (long long)err, (err / (float)(1 << (kPrecision * 2))));
-}
-
-#endif
-
-bool VideoFrameScheduler::PLL::fit(
-        nsecs_t phase, nsecs_t period, size_t numSamplesToUse,
-        int64_t *a, int64_t *b, int64_t *err) {
-    if (numSamplesToUse > mNumSamples) {
-        numSamplesToUse = mNumSamples;
-    }
-
-    int64_t sumX = 0;
-    int64_t sumXX = 0;
-    int64_t sumXY = 0;
-    int64_t sumYY = 0;
-    int64_t sumY = 0;
-
-    int64_t x = 0; // x usually is in [0..numSamplesToUse)
-    nsecs_t lastTime;
-    for (size_t i = 0; i < numSamplesToUse; i++) {
-        size_t ix = (mNumSamples - numSamplesToUse + i) % kHistorySize;
-        nsecs_t time = mTimes[ix];
-        if (i > 0) {
-            x += divRound(time - lastTime, period);
-        }
-        // y is usually in [-numSamplesToUse..numSamplesToUse+kRefitRefreshPeriod/kMinPeriod) << kPrecision
-        //   ideally in [0..numSamplesToUse), but shifted by -numSamplesToUse during
-        //   priming, and possibly shifted by up to kRefitRefreshPeriod/kMinPeriod
-        //   while we are not refitting.
-        int64_t y = divRound(time - phase, period >> kPrecision);
-        sumX += x;
-        sumY += y;
-        sumXX += x * x;
-        sumXY += x * y;
-        sumYY += y * y;
-        lastTime = time;
-    }
-
-    int64_t div   = numSamplesToUse * sumXX - sumX * sumX;
-    if (div == 0) {
-        return false;
-    }
-
-    int64_t a_nom = numSamplesToUse * sumXY - sumX * sumY;
-    int64_t b_nom = sumXX * sumY            - sumX * sumXY;
-    *a = divRound(a_nom, div);
-    *b = divRound(b_nom, div);
-    // don't use a and b directly as the rounding error is significant
-    *err = sumYY - divRound(a_nom * sumXY + b_nom * sumY, div);
-    ALOGV("fitting[%zu] a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)",
-            numSamplesToUse,
-            (long long)*a,   (*a / (float)(1 << kPrecision)),
-            (long long)*b,   (*b / (float)(1 << kPrecision)),
-            (long long)*err, (*err / (float)(1 << (kPrecision * 2))));
-    return true;
-}
-
-void VideoFrameScheduler::PLL::prime(size_t numSamplesToUse) {
-    if (numSamplesToUse > mNumSamples) {
-        numSamplesToUse = mNumSamples;
-    }
-    CHECK(numSamplesToUse >= 3);  // must have at least 3 samples
-
-    // estimate video framerate from deltas between timestamps, and
-    // 2nd order deltas
-    Vector<nsecs_t> deltas;
-    nsecs_t lastTime, firstTime;
-    for (size_t i = 0; i < numSamplesToUse; ++i) {
-        size_t index = (mNumSamples - numSamplesToUse + i) % kHistorySize;
-        nsecs_t time = mTimes[index];
-        if (i > 0) {
-            if (time - lastTime > kMinPeriod) {
-                //ALOGV("delta: %lld", (long long)(time - lastTime));
-                deltas.push(time - lastTime);
-            }
-        } else {
-            firstTime = time;
-        }
-        lastTime = time;
-    }
-    deltas.sort(compare<nsecs_t>);
-    size_t numDeltas = deltas.size();
-    if (numDeltas > 1) {
-        nsecs_t deltaMinLimit = max(deltas[0] / kMultiplesThresholdDiv, kMinPeriod);
-        nsecs_t deltaMaxLimit = deltas[numDeltas / 2] * kMultiplesThresholdDiv;
-        for (size_t i = numDeltas / 2 + 1; i < numDeltas; ++i) {
-            if (deltas[i] > deltaMaxLimit) {
-                deltas.resize(i);
-                numDeltas = i;
-                break;
-            }
-        }
-        for (size_t i = 1; i < numDeltas; ++i) {
-            nsecs_t delta2nd = deltas[i] - deltas[i - 1];
-            if (delta2nd >= deltaMinLimit) {
-                //ALOGV("delta2: %lld", (long long)(delta2nd));
-                deltas.push(delta2nd);
-            }
-        }
-    }
-
-    // use the one that yields the best match
-    int64_t bestScore;
-    for (size_t i = 0; i < deltas.size(); ++i) {
-        nsecs_t delta = deltas[i];
-        int64_t score = 0;
-#if 1
-        // simplest score: number of deltas that are near multiples
-        size_t matches = 0;
-        for (size_t j = 0; j < deltas.size(); ++j) {
-            nsecs_t err = periodicError(deltas[j], delta);
-            if (err < delta / kMultiplesThresholdDiv) {
-                ++matches;
-            }
-        }
-        score = matches;
-#if 0
-        // could be weighed by the (1 - normalized error)
-        if (numSamplesToUse >= kMinSamplesToEstimatePeriod) {
-            int64_t a, b, err;
-            fit(firstTime, delta, numSamplesToUse, &a, &b, &err);
-            err = (1 << (2 * kPrecision)) - err;
-            score *= max(0, err);
-        }
-#endif
-#else
-        // or use the error as a negative score
-        if (numSamplesToUse >= kMinSamplesToEstimatePeriod) {
-            int64_t a, b, err;
-            fit(firstTime, delta, numSamplesToUse, &a, &b, &err);
-            score = -delta * err;
-        }
-#endif
-        if (i == 0 || score > bestScore) {
-            bestScore = score;
-            mPeriod = delta;
-            mPhase = firstTime;
-        }
-    }
-    ALOGV("priming[%zu] phase:%lld period:%lld", numSamplesToUse, mPhase, mPeriod);
-}
-
-nsecs_t VideoFrameScheduler::PLL::addSample(nsecs_t time) {
-    if (mLastTime >= 0
-            // if time goes backward, or we skipped rendering
-            && (time > mLastTime + kMaxAllowedFrameSkip || time < mLastTime)) {
-        restart();
-    }
-
-    mLastTime = time;
-    mTimes[mNumSamples % kHistorySize] = time;
-    ++mNumSamples;
-
-    bool doFit = time > mRefitAt;
-    if ((mPeriod <= 0 || !mPrimed) && mNumSamples >= kMinSamplesToStartPrime) {
-        prime(kMinSamplesToStopPrime);
-        ++mSamplesUsedForPriming;
-        doFit = true;
-    }
-    if (mPeriod > 0 && mNumSamples >= kMinSamplesToEstimatePeriod) {
-        if (mPhase < 0) {
-            // initialize phase to the current render time
-            mPhase = time;
-            doFit = true;
-        } else if (!doFit) {
-            int64_t err = periodicError(time - mPhase, mPeriod);
-            doFit = err > mPeriod / kReFitThresholdDiv;
-        }
-
-        if (doFit) {
-            int64_t a, b, err;
-            if (!fit(mPhase, mPeriod, kMaxSamplesToEstimatePeriod, &a, &b, &err)) {
-                // samples are not suitable for fitting.  this means they are
-                // also not suitable for priming.
-                ALOGV("could not fit - keeping old period:%lld", (long long)mPeriod);
-                return mPeriod;
-            }
-
-            mRefitAt = time + kRefitRefreshPeriod;
-
-            mPhase += (mPeriod * b) >> kPrecision;
-            mPeriod = (mPeriod * a) >> kPrecision;
-            ALOGV("new phase:%lld period:%lld", (long long)mPhase, (long long)mPeriod);
-
-            if (err < kErrorThreshold) {
-                if (!mPrimed && mSamplesUsedForPriming >= kMinSamplesToStopPrime) {
-                    mPrimed = true;
-                }
-            } else {
-                mPrimed = false;
-                mSamplesUsedForPriming = 0;
-            }
-        }
-    }
-    return mPeriod;
-}
-
-/* ======================================================================= */
-/*                             Frame Scheduler                             */
-/* ======================================================================= */
-
-static const nsecs_t kDefaultVsyncPeriod = kNanosIn1s / 60;  // 60Hz
-static const nsecs_t kVsyncRefreshPeriod = kNanosIn1s;       // 1 sec
-
-VideoFrameScheduler::VideoFrameScheduler()
-    : mVsyncTime(0),
-      mVsyncPeriod(0),
-      mVsyncRefreshAt(0),
-      mLastVsyncTime(-1),
-      mTimeCorrection(0) {
-}
-
-void VideoFrameScheduler::updateVsync() {
-    mVsyncRefreshAt = systemTime(SYSTEM_TIME_MONOTONIC) + kVsyncRefreshPeriod;
-    mVsyncPeriod = 0;
-    mVsyncTime = 0;
-
-    // TODO: schedule frames for the destination surface
-    // For now, surface flinger only schedules frames on the primary display
-    if (mComposer == NULL) {
-        String16 name("SurfaceFlinger");
-        sp<IServiceManager> sm = defaultServiceManager();
-        mComposer = interface_cast<ISurfaceComposer>(sm->checkService(name));
-    }
-    if (mComposer != NULL) {
-        DisplayStatInfo stats;
-        status_t res = mComposer->getDisplayStats(NULL /* display */, &stats);
-        if (res == OK) {
-            ALOGV("vsync time:%lld period:%lld",
-                    (long long)stats.vsyncTime, (long long)stats.vsyncPeriod);
-            mVsyncTime = stats.vsyncTime;
-            mVsyncPeriod = stats.vsyncPeriod;
-        } else {
-            ALOGW("getDisplayStats returned %d", res);
-        }
-    } else {
-        ALOGW("could not get surface mComposer service");
-    }
-}
-
-void VideoFrameScheduler::init(float videoFps) {
-    updateVsync();
-
-    mLastVsyncTime = -1;
-    mTimeCorrection = 0;
-
-    mPll.reset(videoFps);
-}
-
-void VideoFrameScheduler::restart() {
-    mLastVsyncTime = -1;
-    mTimeCorrection = 0;
-
-    mPll.restart();
-}
-
-nsecs_t VideoFrameScheduler::getVsyncPeriod() {
-    if (mVsyncPeriod > 0) {
-        return mVsyncPeriod;
-    }
-    return kDefaultVsyncPeriod;
-}
-
-nsecs_t VideoFrameScheduler::schedule(nsecs_t renderTime) {
-    nsecs_t origRenderTime = renderTime;
-
-    nsecs_t now = systemTime(SYSTEM_TIME_MONOTONIC);
-    if (now >= mVsyncRefreshAt) {
-        updateVsync();
-    }
-
-    // without VSYNC info, there is nothing to do
-    if (mVsyncPeriod == 0) {
-        ALOGV("no vsync: render=%lld", (long long)renderTime);
-        return renderTime;
-    }
-
-    // ensure vsync time is well before (corrected) render time
-    if (mVsyncTime > renderTime - 4 * mVsyncPeriod) {
-        mVsyncTime -=
-            ((mVsyncTime - renderTime) / mVsyncPeriod + 5) * mVsyncPeriod;
-    }
-
-    // Video presentation takes place at the VSYNC _after_ renderTime.  Adjust renderTime
-    // so this effectively becomes a rounding operation (to the _closest_ VSYNC.)
-    renderTime -= mVsyncPeriod / 2;
-
-    const nsecs_t videoPeriod = mPll.addSample(origRenderTime);
-    if (videoPeriod > 0) {
-        // Smooth out rendering
-        size_t N = 12;
-        nsecs_t fiveSixthDev =
-            abs(((videoPeriod * 5 + mVsyncPeriod) % (mVsyncPeriod * 6)) - mVsyncPeriod)
-                    / (mVsyncPeriod / 100);
-        // use 20 samples if we are doing 5:6 ratio +- 1% (e.g. playing 50Hz on 60Hz)
-        if (fiveSixthDev < 12) {  /* 12% / 6 = 2% */
-            N = 20;
-        }
-
-        nsecs_t offset = 0;
-        nsecs_t edgeRemainder = 0;
-        for (size_t i = 1; i <= N; i++) {
-            offset +=
-                (renderTime + mTimeCorrection + videoPeriod * i - mVsyncTime) % mVsyncPeriod;
-            edgeRemainder += (videoPeriod * i) % mVsyncPeriod;
-        }
-        mTimeCorrection += mVsyncPeriod / 2 - offset / N;
-        renderTime += mTimeCorrection;
-        nsecs_t correctionLimit = mVsyncPeriod * 3 / 5;
-        edgeRemainder = abs(edgeRemainder / N - mVsyncPeriod / 2);
-        if (edgeRemainder <= mVsyncPeriod / 3) {
-            correctionLimit /= 2;
-        }
-
-        // estimate how many VSYNCs a frame will spend on the display
-        nsecs_t nextVsyncTime =
-            renderTime + mVsyncPeriod - ((renderTime - mVsyncTime) % mVsyncPeriod);
-        if (mLastVsyncTime >= 0) {
-            size_t minVsyncsPerFrame = videoPeriod / mVsyncPeriod;
-            size_t vsyncsForLastFrame = divRound(nextVsyncTime - mLastVsyncTime, mVsyncPeriod);
-            bool vsyncsPerFrameAreNearlyConstant =
-                periodicError(videoPeriod, mVsyncPeriod) / (mVsyncPeriod / 20) == 0;
-
-            if (mTimeCorrection > correctionLimit &&
-                    (vsyncsPerFrameAreNearlyConstant || vsyncsForLastFrame > minVsyncsPerFrame)) {
-                // remove a VSYNC
-                mTimeCorrection -= mVsyncPeriod / 2;
-                renderTime -= mVsyncPeriod / 2;
-                nextVsyncTime -= mVsyncPeriod;
-                --vsyncsForLastFrame;
-            } else if (mTimeCorrection < -correctionLimit &&
-                    (vsyncsPerFrameAreNearlyConstant || vsyncsForLastFrame == minVsyncsPerFrame)) {
-                // add a VSYNC
-                mTimeCorrection += mVsyncPeriod / 2;
-                renderTime += mVsyncPeriod / 2;
-                nextVsyncTime += mVsyncPeriod;
-                ++vsyncsForLastFrame;
-            }
-            ATRACE_INT("FRAME_VSYNCS", vsyncsForLastFrame);
-        }
-        mLastVsyncTime = nextVsyncTime;
-    }
-
-    // align rendertime to the center between VSYNC edges
-    renderTime -= (renderTime - mVsyncTime) % mVsyncPeriod;
-    renderTime += mVsyncPeriod / 2;
-    ALOGV("adjusting render: %lld => %lld", (long long)origRenderTime, (long long)renderTime);
-    ATRACE_INT("FRAME_FLIP_IN(ms)", (renderTime - now) / 1000000);
-    return renderTime;
-}
-
-void VideoFrameScheduler::release() {
-    mComposer.clear();
-}
-
-VideoFrameScheduler::~VideoFrameScheduler() {
-    release();
-}
-
-} // namespace android
-
diff --git a/media/libmediaplayerservice/VideoFrameScheduler.h b/media/libmediaplayerservice/VideoFrameScheduler.h
deleted file mode 100644
index b1765c9..0000000
--- a/media/libmediaplayerservice/VideoFrameScheduler.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Copyright 2014, The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef VIDEO_FRAME_SCHEDULER_H_
-#define VIDEO_FRAME_SCHEDULER_H_
-
-#include <utils/RefBase.h>
-#include <utils/Timers.h>
-
-#include <media/stagefright/foundation/ABase.h>
-
-namespace android {
-
-class ISurfaceComposer;
-
-struct VideoFrameScheduler : public RefBase {
-    VideoFrameScheduler();
-
-    // (re)initialize scheduler
-    void init(float videoFps = -1);
-    // use in case of video render-time discontinuity, e.g. seek
-    void restart();
-    // get adjusted nanotime for a video frame render at renderTime
-    nsecs_t schedule(nsecs_t renderTime);
-
-    // returns the vsync period for the main display
-    nsecs_t getVsyncPeriod();
-
-    void release();
-
-    static const size_t kHistorySize = 8;
-
-protected:
-    virtual ~VideoFrameScheduler();
-
-private:
-    struct PLL {
-        PLL();
-
-        // reset PLL to new PLL
-        void reset(float fps = -1);
-        // keep current estimate, but restart phase
-        void restart();
-        // returns period
-        nsecs_t addSample(nsecs_t time);
-
-    private:
-        nsecs_t mPeriod;
-        nsecs_t mPhase;
-
-        bool    mPrimed;        // have an estimate for the period
-        size_t  mSamplesUsedForPriming;
-
-        nsecs_t mLastTime;      // last input time
-        nsecs_t mRefitAt;       // next input time to fit at
-
-        size_t  mNumSamples;    // can go past kHistorySize
-        nsecs_t mTimes[kHistorySize];
-
-        void test();
-        // returns whether fit was successful
-        bool fit(nsecs_t phase, nsecs_t period, size_t numSamples,
-                int64_t *a, int64_t *b, int64_t *err);
-        void prime(size_t numSamples);
-    };
-
-    void updateVsync();
-
-    nsecs_t mVsyncTime;        // vsync timing from display
-    nsecs_t mVsyncPeriod;
-    nsecs_t mVsyncRefreshAt;   // next time to refresh timing info
-
-    nsecs_t mLastVsyncTime;    // estimated vsync time for last frame
-    nsecs_t mTimeCorrection;   // running adjustment
-
-    PLL mPll;                  // PLL for video frame rate based on render time
-
-    sp<ISurfaceComposer> mComposer;
-
-    DISALLOW_EVIL_CONSTRUCTORS(VideoFrameScheduler);
-};
-
-}  // namespace android
-
-#endif  // VIDEO_FRAME_SCHEDULER_H_
-
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
index 13a7d94..767417b 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
@@ -29,8 +29,7 @@
 #include <media/stagefright/MediaErrors.h>
 #include <media/stagefright/MetaData.h>
 #include <media/stagefright/Utils.h>
-
-#include <VideoFrameScheduler.h>
+#include <media/stagefright/VideoFrameScheduler.h>
 
 #include <inttypes.h>