From a3725d7b0cb79ddb49f81cba00a0164d8e645acd Mon Sep 17 00:00:00 2001 From: Lajos Molnar Date: Wed, 1 Jul 2015 16:47:22 -0700 Subject: stagefright: MediaSync: use VideoFrameScheduler Move VideoFrameScheduler to libstagefright as part of this change. Bug: 22234976 Change-Id: Ib23fb52399cb700a1dcf789e8486b94a3edf9d95 --- media/libstagefright/VideoFrameScheduler.cpp | 494 +++++++++++++++++++++++++++ 1 file changed, 494 insertions(+) create mode 100644 media/libstagefright/VideoFrameScheduler.cpp (limited to 'media/libstagefright/VideoFrameScheduler.cpp') diff --git a/media/libstagefright/VideoFrameScheduler.cpp b/media/libstagefright/VideoFrameScheduler.cpp new file mode 100644 index 0000000..5fe9bf9 --- /dev/null +++ b/media/libstagefright/VideoFrameScheduler.cpp @@ -0,0 +1,494 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "VideoFrameScheduler" +#include +#define ATRACE_TAG ATRACE_TAG_VIDEO +#include + +#include + +#include +#include +#include + +#include +#include +#include + +namespace android { + +static const nsecs_t kNanosIn1s = 1000000000; + +template +static int compare(const T *lhs, const T *rhs) { + if (*lhs < *rhs) { + return -1; + } else if (*lhs > *rhs) { + return 1; + } else { + return 0; + } +} + +/* ======================================================================= */ +/* PLL */ +/* ======================================================================= */ + +static const size_t kMinSamplesToStartPrime = 3; +static const size_t kMinSamplesToStopPrime = VideoFrameScheduler::kHistorySize; +static const size_t kMinSamplesToEstimatePeriod = 3; +static const size_t kMaxSamplesToEstimatePeriod = VideoFrameScheduler::kHistorySize; + +static const size_t kPrecision = 12; +static const int64_t kErrorThreshold = (1 << (kPrecision * 2)) / 10; +static const int64_t kMultiplesThresholdDiv = 4; // 25% +static const int64_t kReFitThresholdDiv = 100; // 1% +static const nsecs_t kMaxAllowedFrameSkip = kNanosIn1s; // 1 sec +static const nsecs_t kMinPeriod = kNanosIn1s / 120; // 120Hz +static const nsecs_t kRefitRefreshPeriod = 10 * kNanosIn1s; // 10 sec + +VideoFrameScheduler::PLL::PLL() + : mPeriod(-1), + mPhase(0), + mPrimed(false), + mSamplesUsedForPriming(0), + mLastTime(-1), + mNumSamples(0) { +} + +void VideoFrameScheduler::PLL::reset(float fps) { + //test(); + + mSamplesUsedForPriming = 0; + mLastTime = -1; + + // set up or reset video PLL + if (fps <= 0.f) { + mPeriod = -1; + mPrimed = false; + } else { + ALOGV("reset at %.1f fps", fps); + mPeriod = (nsecs_t)(1e9 / fps + 0.5); + mPrimed = true; + } + + restart(); +} + +// reset PLL but keep previous period estimate +void VideoFrameScheduler::PLL::restart() { + mNumSamples = 0; + mPhase = -1; +} + +#if 0 + +void VideoFrameScheduler::PLL::test() { + nsecs_t period = kNanosIn1s / 60; + mTimes[0] = 0; + mTimes[1] = period; + mTimes[2] = period * 3; + mTimes[3] = period * 4; + mTimes[4] = period * 7; + mTimes[5] = period * 8; + mTimes[6] = period * 10; + mTimes[7] = period * 12; + mNumSamples = 8; + int64_t a, b, err; + fit(0, period * 12 / 7, 8, &a, &b, &err); + // a = 0.8(5)+ + // b = -0.14097(2)+ + // err = 0.2750578(703)+ + ALOGD("a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)", + (long long)a, (a / (float)(1 << kPrecision)), + (long long)b, (b / (float)(1 << kPrecision)), + (long long)err, (err / (float)(1 << (kPrecision * 2)))); +} + +#endif + +bool VideoFrameScheduler::PLL::fit( + nsecs_t phase, nsecs_t period, size_t numSamplesToUse, + int64_t *a, int64_t *b, int64_t *err) { + if (numSamplesToUse > mNumSamples) { + numSamplesToUse = mNumSamples; + } + + int64_t sumX = 0; + int64_t sumXX = 0; + int64_t sumXY = 0; + int64_t sumYY = 0; + int64_t sumY = 0; + + int64_t x = 0; // x usually is in [0..numSamplesToUse) + nsecs_t lastTime; + for (size_t i = 0; i < numSamplesToUse; i++) { + size_t ix = (mNumSamples - numSamplesToUse + i) % kHistorySize; + nsecs_t time = mTimes[ix]; + if (i > 0) { + x += divRound(time - lastTime, period); + } + // y is usually in [-numSamplesToUse..numSamplesToUse+kRefitRefreshPeriod/kMinPeriod) << kPrecision + // ideally in [0..numSamplesToUse), but shifted by -numSamplesToUse during + // priming, and possibly shifted by up to kRefitRefreshPeriod/kMinPeriod + // while we are not refitting. + int64_t y = divRound(time - phase, period >> kPrecision); + sumX += x; + sumY += y; + sumXX += x * x; + sumXY += x * y; + sumYY += y * y; + lastTime = time; + } + + int64_t div = numSamplesToUse * sumXX - sumX * sumX; + if (div == 0) { + return false; + } + + int64_t a_nom = numSamplesToUse * sumXY - sumX * sumY; + int64_t b_nom = sumXX * sumY - sumX * sumXY; + *a = divRound(a_nom, div); + *b = divRound(b_nom, div); + // don't use a and b directly as the rounding error is significant + *err = sumYY - divRound(a_nom * sumXY + b_nom * sumY, div); + ALOGV("fitting[%zu] a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)", + numSamplesToUse, + (long long)*a, (*a / (float)(1 << kPrecision)), + (long long)*b, (*b / (float)(1 << kPrecision)), + (long long)*err, (*err / (float)(1 << (kPrecision * 2)))); + return true; +} + +void VideoFrameScheduler::PLL::prime(size_t numSamplesToUse) { + if (numSamplesToUse > mNumSamples) { + numSamplesToUse = mNumSamples; + } + CHECK(numSamplesToUse >= 3); // must have at least 3 samples + + // estimate video framerate from deltas between timestamps, and + // 2nd order deltas + Vector deltas; + nsecs_t lastTime, firstTime; + for (size_t i = 0; i < numSamplesToUse; ++i) { + size_t index = (mNumSamples - numSamplesToUse + i) % kHistorySize; + nsecs_t time = mTimes[index]; + if (i > 0) { + if (time - lastTime > kMinPeriod) { + //ALOGV("delta: %lld", (long long)(time - lastTime)); + deltas.push(time - lastTime); + } + } else { + firstTime = time; + } + lastTime = time; + } + deltas.sort(compare); + size_t numDeltas = deltas.size(); + if (numDeltas > 1) { + nsecs_t deltaMinLimit = max(deltas[0] / kMultiplesThresholdDiv, kMinPeriod); + nsecs_t deltaMaxLimit = deltas[numDeltas / 2] * kMultiplesThresholdDiv; + for (size_t i = numDeltas / 2 + 1; i < numDeltas; ++i) { + if (deltas[i] > deltaMaxLimit) { + deltas.resize(i); + numDeltas = i; + break; + } + } + for (size_t i = 1; i < numDeltas; ++i) { + nsecs_t delta2nd = deltas[i] - deltas[i - 1]; + if (delta2nd >= deltaMinLimit) { + //ALOGV("delta2: %lld", (long long)(delta2nd)); + deltas.push(delta2nd); + } + } + } + + // use the one that yields the best match + int64_t bestScore; + for (size_t i = 0; i < deltas.size(); ++i) { + nsecs_t delta = deltas[i]; + int64_t score = 0; +#if 1 + // simplest score: number of deltas that are near multiples + size_t matches = 0; + for (size_t j = 0; j < deltas.size(); ++j) { + nsecs_t err = periodicError(deltas[j], delta); + if (err < delta / kMultiplesThresholdDiv) { + ++matches; + } + } + score = matches; +#if 0 + // could be weighed by the (1 - normalized error) + if (numSamplesToUse >= kMinSamplesToEstimatePeriod) { + int64_t a, b, err; + fit(firstTime, delta, numSamplesToUse, &a, &b, &err); + err = (1 << (2 * kPrecision)) - err; + score *= max(0, err); + } +#endif +#else + // or use the error as a negative score + if (numSamplesToUse >= kMinSamplesToEstimatePeriod) { + int64_t a, b, err; + fit(firstTime, delta, numSamplesToUse, &a, &b, &err); + score = -delta * err; + } +#endif + if (i == 0 || score > bestScore) { + bestScore = score; + mPeriod = delta; + mPhase = firstTime; + } + } + ALOGV("priming[%zu] phase:%lld period:%lld", + numSamplesToUse, (long long)mPhase, (long long)mPeriod); +} + +nsecs_t VideoFrameScheduler::PLL::addSample(nsecs_t time) { + if (mLastTime >= 0 + // if time goes backward, or we skipped rendering + && (time > mLastTime + kMaxAllowedFrameSkip || time < mLastTime)) { + restart(); + } + + mLastTime = time; + mTimes[mNumSamples % kHistorySize] = time; + ++mNumSamples; + + bool doFit = time > mRefitAt; + if ((mPeriod <= 0 || !mPrimed) && mNumSamples >= kMinSamplesToStartPrime) { + prime(kMinSamplesToStopPrime); + ++mSamplesUsedForPriming; + doFit = true; + } + if (mPeriod > 0 && mNumSamples >= kMinSamplesToEstimatePeriod) { + if (mPhase < 0) { + // initialize phase to the current render time + mPhase = time; + doFit = true; + } else if (!doFit) { + int64_t err = periodicError(time - mPhase, mPeriod); + doFit = err > mPeriod / kReFitThresholdDiv; + } + + if (doFit) { + int64_t a, b, err; + if (!fit(mPhase, mPeriod, kMaxSamplesToEstimatePeriod, &a, &b, &err)) { + // samples are not suitable for fitting. this means they are + // also not suitable for priming. + ALOGV("could not fit - keeping old period:%lld", (long long)mPeriod); + return mPeriod; + } + + mRefitAt = time + kRefitRefreshPeriod; + + mPhase += (mPeriod * b) >> kPrecision; + mPeriod = (mPeriod * a) >> kPrecision; + ALOGV("new phase:%lld period:%lld", (long long)mPhase, (long long)mPeriod); + + if (err < kErrorThreshold) { + if (!mPrimed && mSamplesUsedForPriming >= kMinSamplesToStopPrime) { + mPrimed = true; + } + } else { + mPrimed = false; + mSamplesUsedForPriming = 0; + } + } + } + return mPeriod; +} + +nsecs_t VideoFrameScheduler::PLL::getPeriod() const { + return mPrimed ? mPeriod : 0; +} + +/* ======================================================================= */ +/* Frame Scheduler */ +/* ======================================================================= */ + +static const nsecs_t kDefaultVsyncPeriod = kNanosIn1s / 60; // 60Hz +static const nsecs_t kVsyncRefreshPeriod = kNanosIn1s; // 1 sec + +VideoFrameScheduler::VideoFrameScheduler() + : mVsyncTime(0), + mVsyncPeriod(0), + mVsyncRefreshAt(0), + mLastVsyncTime(-1), + mTimeCorrection(0) { +} + +void VideoFrameScheduler::updateVsync() { + mVsyncRefreshAt = systemTime(SYSTEM_TIME_MONOTONIC) + kVsyncRefreshPeriod; + mVsyncPeriod = 0; + mVsyncTime = 0; + + // TODO: schedule frames for the destination surface + // For now, surface flinger only schedules frames on the primary display + if (mComposer == NULL) { + String16 name("SurfaceFlinger"); + sp sm = defaultServiceManager(); + mComposer = interface_cast(sm->checkService(name)); + } + if (mComposer != NULL) { + DisplayStatInfo stats; + status_t res = mComposer->getDisplayStats(NULL /* display */, &stats); + if (res == OK) { + ALOGV("vsync time:%lld period:%lld", + (long long)stats.vsyncTime, (long long)stats.vsyncPeriod); + mVsyncTime = stats.vsyncTime; + mVsyncPeriod = stats.vsyncPeriod; + } else { + ALOGW("getDisplayStats returned %d", res); + } + } else { + ALOGW("could not get surface mComposer service"); + } +} + +void VideoFrameScheduler::init(float videoFps) { + updateVsync(); + + mLastVsyncTime = -1; + mTimeCorrection = 0; + + mPll.reset(videoFps); +} + +void VideoFrameScheduler::restart() { + mLastVsyncTime = -1; + mTimeCorrection = 0; + + mPll.restart(); +} + +nsecs_t VideoFrameScheduler::getVsyncPeriod() { + if (mVsyncPeriod > 0) { + return mVsyncPeriod; + } + return kDefaultVsyncPeriod; +} + +float VideoFrameScheduler::getFrameRate() { + nsecs_t videoPeriod = mPll.getPeriod(); + if (videoPeriod > 0) { + return 1e9 / videoPeriod; + } + return 0.f; +} + +nsecs_t VideoFrameScheduler::schedule(nsecs_t renderTime) { + nsecs_t origRenderTime = renderTime; + + nsecs_t now = systemTime(SYSTEM_TIME_MONOTONIC); + if (now >= mVsyncRefreshAt) { + updateVsync(); + } + + // without VSYNC info, there is nothing to do + if (mVsyncPeriod == 0) { + ALOGV("no vsync: render=%lld", (long long)renderTime); + return renderTime; + } + + // ensure vsync time is well before (corrected) render time + if (mVsyncTime > renderTime - 4 * mVsyncPeriod) { + mVsyncTime -= + ((mVsyncTime - renderTime) / mVsyncPeriod + 5) * mVsyncPeriod; + } + + // Video presentation takes place at the VSYNC _after_ renderTime. Adjust renderTime + // so this effectively becomes a rounding operation (to the _closest_ VSYNC.) + renderTime -= mVsyncPeriod / 2; + + const nsecs_t videoPeriod = mPll.addSample(origRenderTime); + if (videoPeriod > 0) { + // Smooth out rendering + size_t N = 12; + nsecs_t fiveSixthDev = + abs(((videoPeriod * 5 + mVsyncPeriod) % (mVsyncPeriod * 6)) - mVsyncPeriod) + / (mVsyncPeriod / 100); + // use 20 samples if we are doing 5:6 ratio +- 1% (e.g. playing 50Hz on 60Hz) + if (fiveSixthDev < 12) { /* 12% / 6 = 2% */ + N = 20; + } + + nsecs_t offset = 0; + nsecs_t edgeRemainder = 0; + for (size_t i = 1; i <= N; i++) { + offset += + (renderTime + mTimeCorrection + videoPeriod * i - mVsyncTime) % mVsyncPeriod; + edgeRemainder += (videoPeriod * i) % mVsyncPeriod; + } + mTimeCorrection += mVsyncPeriod / 2 - offset / N; + renderTime += mTimeCorrection; + nsecs_t correctionLimit = mVsyncPeriod * 3 / 5; + edgeRemainder = abs(edgeRemainder / N - mVsyncPeriod / 2); + if (edgeRemainder <= mVsyncPeriod / 3) { + correctionLimit /= 2; + } + + // estimate how many VSYNCs a frame will spend on the display + nsecs_t nextVsyncTime = + renderTime + mVsyncPeriod - ((renderTime - mVsyncTime) % mVsyncPeriod); + if (mLastVsyncTime >= 0) { + size_t minVsyncsPerFrame = videoPeriod / mVsyncPeriod; + size_t vsyncsForLastFrame = divRound(nextVsyncTime - mLastVsyncTime, mVsyncPeriod); + bool vsyncsPerFrameAreNearlyConstant = + periodicError(videoPeriod, mVsyncPeriod) / (mVsyncPeriod / 20) == 0; + + if (mTimeCorrection > correctionLimit && + (vsyncsPerFrameAreNearlyConstant || vsyncsForLastFrame > minVsyncsPerFrame)) { + // remove a VSYNC + mTimeCorrection -= mVsyncPeriod / 2; + renderTime -= mVsyncPeriod / 2; + nextVsyncTime -= mVsyncPeriod; + --vsyncsForLastFrame; + } else if (mTimeCorrection < -correctionLimit && + (vsyncsPerFrameAreNearlyConstant || vsyncsForLastFrame == minVsyncsPerFrame)) { + // add a VSYNC + mTimeCorrection += mVsyncPeriod / 2; + renderTime += mVsyncPeriod / 2; + nextVsyncTime += mVsyncPeriod; + ++vsyncsForLastFrame; + } + ATRACE_INT("FRAME_VSYNCS", vsyncsForLastFrame); + } + mLastVsyncTime = nextVsyncTime; + } + + // align rendertime to the center between VSYNC edges + renderTime -= (renderTime - mVsyncTime) % mVsyncPeriod; + renderTime += mVsyncPeriod / 2; + ALOGV("adjusting render: %lld => %lld", (long long)origRenderTime, (long long)renderTime); + ATRACE_INT("FRAME_FLIP_IN(ms)", (renderTime - now) / 1000000); + return renderTime; +} + +void VideoFrameScheduler::release() { + mComposer.clear(); +} + +VideoFrameScheduler::~VideoFrameScheduler() { + release(); +} + +} // namespace android + -- cgit v1.1