diff options
5 files changed, 446 insertions, 3 deletions
diff --git a/media/libmediaplayerservice/VideoFrameScheduler.cpp b/media/libmediaplayerservice/VideoFrameScheduler.cpp index dd38d5d..4251c4e 100644 --- a/media/libmediaplayerservice/VideoFrameScheduler.cpp +++ b/media/libmediaplayerservice/VideoFrameScheduler.cpp @@ -34,6 +34,305 @@ namespace android { static const nsecs_t kNanosIn1s = 1000000000; +template<class T> +inline static const T divRound(const T &nom, const T &den) { + if ((nom >= 0) ^ (den >= 0)) { + return (nom - den / 2) / den; + } else { + return (nom + den / 2) / den; + } +} + +template<class T> +inline static T abs(const T &a) { + return a < 0 ? -a : a; +} + +template<class T> +inline static const T &min(const T &a, const T &b) { + return a < b ? a : b; +} + +template<class T> +inline static const T &max(const T &a, const T &b) { + return a > b ? a : b; +} + +template<class T> +inline static T periodicError(const T &val, const T &period) { + T err = abs(val) % period; + return (err < (period / 2)) ? err : (period - err); +} + +template<class T> +static int compare(const T *lhs, const T *rhs) { + if (*lhs < *rhs) { + return -1; + } else if (*lhs > *rhs) { + return 1; + } else { + return 0; + } +} + +/* ======================================================================= */ +/* PLL */ +/* ======================================================================= */ + +static const size_t kMinSamplesToStartPrime = 3; +static const size_t kMinSamplesToStopPrime = VideoFrameScheduler::kHistorySize; +static const size_t kMinSamplesToEstimatePeriod = 3; +static const size_t kMaxSamplesToEstimatePeriod = VideoFrameScheduler::kHistorySize; + +static const size_t kPrecision = 12; +static const size_t kErrorThreshold = (1 << (kPrecision * 2)) / 10; +static const int64_t kMultiplesThresholdDiv = 4; // 25% +static const int64_t kReFitThresholdDiv = 100; // 1% +static const nsecs_t kMaxAllowedFrameSkip = kNanosIn1s; // 1 sec +static const nsecs_t kMinPeriod = kNanosIn1s / 120; // 120Hz +static const nsecs_t kRefitRefreshPeriod = 10 * kNanosIn1s; // 10 sec + +VideoFrameScheduler::PLL::PLL() + : mPeriod(-1), + mPhase(0), + mPrimed(false), + mSamplesUsedForPriming(0), + mLastTime(-1), + mNumSamples(0) { +} + +void VideoFrameScheduler::PLL::reset(float fps) { + //test(); + + mSamplesUsedForPriming = 0; + mLastTime = -1; + + // set up or reset video PLL + if (fps <= 0.f) { + mPeriod = -1; + mPrimed = false; + } else { + ALOGV("reset at %.1f fps", fps); + mPeriod = (nsecs_t)(1e9 / fps + 0.5); + mPrimed = true; + } + + restart(); +} + +// reset PLL but keep previous period estimate +void VideoFrameScheduler::PLL::restart() { + mNumSamples = 0; + mPhase = -1; +} + +#if 0 + +void VideoFrameScheduler::PLL::test() { + nsecs_t period = kNanosIn1s / 60; + mTimes[0] = 0; + mTimes[1] = period; + mTimes[2] = period * 3; + mTimes[3] = period * 4; + mTimes[4] = period * 7; + mTimes[5] = period * 8; + mTimes[6] = period * 10; + mTimes[7] = period * 12; + mNumSamples = 8; + int64_t a, b, err; + fit(0, period * 12 / 7, 8, &a, &b, &err); + // a = 0.8(5)+ + // b = -0.14097(2)+ + // err = 0.2750578(703)+ + ALOGD("a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)", + (long long)a, (a / (float)(1 << kPrecision)), + (long long)b, (b / (float)(1 << kPrecision)), + (long long)err, (err / (float)(1 << (kPrecision * 2)))); +} + +#endif + +void VideoFrameScheduler::PLL::fit( + nsecs_t phase, nsecs_t period, size_t numSamplesToUse, + int64_t *a, int64_t *b, int64_t *err) { + if (numSamplesToUse > mNumSamples) { + numSamplesToUse = mNumSamples; + } + + int64_t sumX = 0; + int64_t sumXX = 0; + int64_t sumXY = 0; + int64_t sumYY = 0; + int64_t sumY = 0; + + int64_t x = 0; // x usually is in [0..numSamplesToUse) + nsecs_t lastTime; + for (size_t i = 0; i < numSamplesToUse; i++) { + size_t ix = (mNumSamples - numSamplesToUse + i) % kHistorySize; + nsecs_t time = mTimes[ix]; + if (i > 0) { + x += divRound(time - lastTime, period); + } + // y is usually in [-numSamplesToUse..numSamplesToUse+kRefitRefreshPeriod/kMinPeriod) << kPrecision + // ideally in [0..numSamplesToUse), but shifted by -numSamplesToUse during + // priming, and possibly shifted by up to kRefitRefreshPeriod/kMinPeriod + // while we are not refitting. + int64_t y = divRound(time - phase, period >> kPrecision); + sumX += x; + sumY += y; + sumXX += x * x; + sumXY += x * y; + sumYY += y * y; + lastTime = time; + } + + int64_t div = numSamplesToUse * sumXX - sumX * sumX; + int64_t a_nom = numSamplesToUse * sumXY - sumX * sumY; + int64_t b_nom = sumXX * sumY - sumX * sumXY; + *a = divRound(a_nom, div); + *b = divRound(b_nom, div); + // don't use a and b directly as the rounding error is significant + *err = sumYY - divRound(a_nom * sumXY + b_nom * sumY, div); + ALOGV("fitting[%zu] a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)", + numSamplesToUse, + (long long)*a, (*a / (float)(1 << kPrecision)), + (long long)*b, (*b / (float)(1 << kPrecision)), + (long long)*err, (*err / (float)(1 << (kPrecision * 2)))); +} + +void VideoFrameScheduler::PLL::prime(size_t numSamplesToUse) { + if (numSamplesToUse > mNumSamples) { + numSamplesToUse = mNumSamples; + } + CHECK(numSamplesToUse >= 3); // must have at least 3 samples + + // estimate video framerate from deltas between timestamps, and + // 2nd order deltas + Vector<nsecs_t> deltas; + nsecs_t lastTime, firstTime; + for (size_t i = 0; i < numSamplesToUse; ++i) { + size_t index = (mNumSamples - numSamplesToUse + i) % kHistorySize; + nsecs_t time = mTimes[index]; + if (i > 0) { + if (time - lastTime > kMinPeriod) { + //ALOGV("delta: %lld", (long long)(time - lastTime)); + deltas.push(time - lastTime); + } + } else { + firstTime = time; + } + lastTime = time; + } + deltas.sort(compare<nsecs_t>); + size_t numDeltas = deltas.size(); + if (numDeltas > 1) { + nsecs_t deltaMinLimit = min(deltas[0] / kMultiplesThresholdDiv, kMinPeriod); + nsecs_t deltaMaxLimit = deltas[numDeltas / 2] * kMultiplesThresholdDiv; + for (size_t i = numDeltas / 2 + 1; i < numDeltas; ++i) { + if (deltas[i] > deltaMaxLimit) { + deltas.resize(i); + numDeltas = i; + break; + } + } + for (size_t i = 1; i < numDeltas; ++i) { + nsecs_t delta2nd = deltas[i] - deltas[i - 1]; + if (delta2nd >= deltaMinLimit) { + //ALOGV("delta2: %lld", (long long)(delta2nd)); + deltas.push(delta2nd); + } + } + } + + // use the one that yields the best match + int64_t bestScore; + for (size_t i = 0; i < deltas.size(); ++i) { + nsecs_t delta = deltas[i]; + int64_t score = 0; +#if 1 + // simplest score: number of deltas that are near multiples + size_t matches = 0; + for (size_t j = 0; j < deltas.size(); ++j) { + nsecs_t err = periodicError(deltas[j], delta); + if (err < delta / kMultiplesThresholdDiv) { + ++matches; + } + } + score = matches; +#if 0 + // could be weighed by the (1 - normalized error) + if (numSamplesToUse >= kMinSamplesToEstimatePeriod) { + int64_t a, b, err; + fit(firstTime, delta, numSamplesToUse, &a, &b, &err); + err = (1 << (2 * kPrecision)) - err; + score *= max(0, err); + } +#endif +#else + // or use the error as a negative score + if (numSamplesToUse >= kMinSamplesToEstimatePeriod) { + int64_t a, b, err; + fit(firstTime, delta, numSamplesToUse, &a, &b, &err); + score = -delta * err; + } +#endif + if (i == 0 || score > bestScore) { + bestScore = score; + mPeriod = delta; + mPhase = firstTime; + } + } + ALOGV("priming[%zu] phase:%lld period:%lld", numSamplesToUse, mPhase, mPeriod); +} + +nsecs_t VideoFrameScheduler::PLL::addSample(nsecs_t time) { + if (mLastTime >= 0 + // if time goes backward, or we skipped rendering + && (time > mLastTime + kMaxAllowedFrameSkip || time < mLastTime)) { + restart(); + } + + mLastTime = time; + mTimes[mNumSamples % kHistorySize] = time; + ++mNumSamples; + + bool doFit = time > mRefitAt; + if ((mPeriod <= 0 || !mPrimed) && mNumSamples >= kMinSamplesToStartPrime) { + prime(kMinSamplesToStopPrime); + ++mSamplesUsedForPriming; + doFit = true; + } + if (mPeriod > 0 && mNumSamples >= kMinSamplesToEstimatePeriod) { + if (mPhase < 0) { + // initialize phase to the current render time + mPhase = time; + doFit = true; + } else if (!doFit) { + int64_t err = periodicError(time - mPhase, mPeriod); + doFit = err > mPeriod / kReFitThresholdDiv; + } + + if (doFit) { + int64_t a, b, err; + mRefitAt = time + kRefitRefreshPeriod; + fit(mPhase, mPeriod, kMaxSamplesToEstimatePeriod, &a, &b, &err); + mPhase += (mPeriod * b) >> kPrecision; + mPeriod = (mPeriod * a) >> kPrecision; + ALOGV("new phase:%lld period:%lld", (long long)mPhase, (long long)mPeriod); + + if (err < kErrorThreshold) { + if (!mPrimed && mSamplesUsedForPriming >= kMinSamplesToStopPrime) { + mPrimed = true; + } + } else { + mPrimed = false; + mSamplesUsedForPriming = 0; + } + } + } + return mPeriod; +} + /* ======================================================================= */ /* Frame Scheduler */ /* ======================================================================= */ @@ -44,7 +343,9 @@ static const nsecs_t kVsyncRefreshPeriod = kNanosIn1s; // 1 sec VideoFrameScheduler::VideoFrameScheduler() : mVsyncTime(0), mVsyncPeriod(0), - mVsyncRefreshAt(0) { + mVsyncRefreshAt(0), + mLastVsyncTime(-1), + mTimeCorrection(0) { } void VideoFrameScheduler::updateVsync() { @@ -75,8 +376,20 @@ void VideoFrameScheduler::updateVsync() { } } -void VideoFrameScheduler::init() { +void VideoFrameScheduler::init(float videoFps) { updateVsync(); + + mLastVsyncTime = -1; + mTimeCorrection = 0; + + mPll.reset(videoFps); +} + +void VideoFrameScheduler::restart() { + mLastVsyncTime = -1; + mTimeCorrection = 0; + + mPll.restart(); } nsecs_t VideoFrameScheduler::getVsyncPeriod() { @@ -110,6 +423,62 @@ nsecs_t VideoFrameScheduler::schedule(nsecs_t renderTime) { // so this effectively becomes a rounding operation (to the _closest_ VSYNC.) renderTime -= mVsyncPeriod / 2; + const nsecs_t videoPeriod = mPll.addSample(origRenderTime); + if (videoPeriod > 0) { + // Smooth out rendering + size_t N = 12; + nsecs_t fiveSixthDev = + abs(((videoPeriod * 5 + mVsyncPeriod) % (mVsyncPeriod * 6)) - mVsyncPeriod) + / (mVsyncPeriod / 100); + // use 20 samples if we are doing 5:6 ratio +- 1% (e.g. playing 50Hz on 60Hz) + if (fiveSixthDev < 12) { /* 12% / 6 = 2% */ + N = 20; + } + + nsecs_t offset = 0; + nsecs_t edgeRemainder = 0; + for (size_t i = 1; i <= N; i++) { + offset += + (renderTime + mTimeCorrection + videoPeriod * i - mVsyncTime) % mVsyncPeriod; + edgeRemainder += (videoPeriod * i) % mVsyncPeriod; + } + mTimeCorrection += mVsyncPeriod / 2 - offset / N; + renderTime += mTimeCorrection; + nsecs_t correctionLimit = mVsyncPeriod * 3 / 5; + edgeRemainder = abs(edgeRemainder / N - mVsyncPeriod / 2); + if (edgeRemainder <= mVsyncPeriod / 3) { + correctionLimit /= 2; + } + + // estimate how many VSYNCs a frame will spend on the display + nsecs_t nextVsyncTime = + renderTime + mVsyncPeriod - ((renderTime - mVsyncTime) % mVsyncPeriod); + if (mLastVsyncTime >= 0) { + size_t minVsyncsPerFrame = videoPeriod / mVsyncPeriod; + size_t vsyncsForLastFrame = divRound(nextVsyncTime - mLastVsyncTime, mVsyncPeriod); + bool vsyncsPerFrameAreNearlyConstant = + periodicError(videoPeriod, mVsyncPeriod) / (mVsyncPeriod / 20) == 0; + + if (mTimeCorrection > correctionLimit && + (vsyncsPerFrameAreNearlyConstant || vsyncsForLastFrame > minVsyncsPerFrame)) { + // remove a VSYNC + mTimeCorrection -= mVsyncPeriod / 2; + renderTime -= mVsyncPeriod / 2; + nextVsyncTime -= mVsyncPeriod; + --vsyncsForLastFrame; + } else if (mTimeCorrection < -correctionLimit && + (vsyncsPerFrameAreNearlyConstant || vsyncsForLastFrame == minVsyncsPerFrame)) { + // add a VSYNC + mTimeCorrection += mVsyncPeriod / 2; + renderTime += mVsyncPeriod / 2; + nextVsyncTime += mVsyncPeriod; + ++vsyncsForLastFrame; + } + ATRACE_INT("FRAME_VSYNCS", vsyncsForLastFrame); + } + mLastVsyncTime = nextVsyncTime; + } + // align rendertime to the center between VSYNC edges renderTime -= (renderTime - mVsyncTime) % mVsyncPeriod; renderTime += mVsyncPeriod / 2; diff --git a/media/libmediaplayerservice/VideoFrameScheduler.h b/media/libmediaplayerservice/VideoFrameScheduler.h index 9119e45..19f0787 100644 --- a/media/libmediaplayerservice/VideoFrameScheduler.h +++ b/media/libmediaplayerservice/VideoFrameScheduler.h @@ -30,7 +30,9 @@ struct VideoFrameScheduler : public RefBase { VideoFrameScheduler(); // (re)initialize scheduler - void init(); + void init(float videoFps = -1); + // use in case of video render-time discontinuity, e.g. seek + void restart(); // get adjusted nanotime for a video frame render at renderTime nsecs_t schedule(nsecs_t renderTime); @@ -39,16 +41,52 @@ struct VideoFrameScheduler : public RefBase { void release(); + static const size_t kHistorySize = 8; + protected: virtual ~VideoFrameScheduler(); private: + struct PLL { + PLL(); + + // reset PLL to new PLL + void reset(float fps = -1); + // keep current estimate, but restart phase + void restart(); + // returns period + nsecs_t addSample(nsecs_t time); + + private: + nsecs_t mPeriod; + nsecs_t mPhase; + + bool mPrimed; // have an estimate for the period + size_t mSamplesUsedForPriming; + + nsecs_t mLastTime; // last input time + nsecs_t mRefitAt; // next input time to fit at + + size_t mNumSamples; // can go past kHistorySize + nsecs_t mTimes[kHistorySize]; + + void test(); + void fit(nsecs_t phase, nsecs_t period, size_t numSamples, + int64_t *a, int64_t *b, int64_t *err); + void prime(size_t numSamples); + }; + void updateVsync(); nsecs_t mVsyncTime; // vsync timing from display nsecs_t mVsyncPeriod; nsecs_t mVsyncRefreshAt; // next time to refresh timing info + nsecs_t mLastVsyncTime; // estimated vsync time for last frame + nsecs_t mTimeCorrection; // running adjustment + + PLL mPll; // PLL for video frame rate based on render time + sp<ISurfaceComposer> mComposer; DISALLOW_EVIL_CONSTRUCTORS(VideoFrameScheduler); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp index cdbea9f..dad480d 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp @@ -643,6 +643,13 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO); mRendererLooper->registerHandler(mRenderer); + sp<MetaData> meta = getFileMeta(); + int32_t rate; + if (meta != NULL + && meta->findInt32(kKeyFrameRate, &rate) && rate > 0) { + mRenderer->setVideoFrameRate(rate); + } + postScanSources(); break; } diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp index 9934e06..a8c8818 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp @@ -139,6 +139,12 @@ void NuPlayer::Renderer::resume() { (new AMessage(kWhatResume, id()))->post(); } +void NuPlayer::Renderer::setVideoFrameRate(float fps) { + sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, id()); + msg->setFloat("frame-rate", fps); + msg->post(); +} + void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { switch (msg->what()) { case kWhatStopAudioSink: @@ -239,6 +245,14 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatSetVideoFrameRate: + { + float fps; + CHECK(msg->findFloat("frame-rate", &fps)); + onSetVideoFrameRate(fps); + break; + } + case kWhatAudioOffloadTearDown: { onAudioOffloadTearDown(); @@ -809,6 +823,10 @@ void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { mDrainVideoQueuePending = false; ++mVideoQueueGeneration; + if (mVideoScheduler != NULL) { + mVideoScheduler->restart(); + } + prepareForMediaRenderingStart(); } @@ -960,6 +978,13 @@ void NuPlayer::Renderer::onResume() { } } +void NuPlayer::Renderer::onSetVideoFrameRate(float fps) { + if (mVideoScheduler == NULL) { + mVideoScheduler = new VideoFrameScheduler(); + } + mVideoScheduler->init(fps); +} + // TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs() // as it acquires locks and may query the audio driver. // diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h index c5a6ec0..e28071f 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h @@ -57,6 +57,8 @@ struct NuPlayer::Renderer : public AHandler { void pause(); void resume(); + void setVideoFrameRate(float fps); + enum { kWhatEOS = 'eos ', kWhatFlushComplete = 'fluC', @@ -83,6 +85,7 @@ private: kWhatResume = 'resm', kWhatStopAudioSink = 'stpA', kWhatDisableOffloadAudio = 'noOA', + kWhatSetVideoFrameRate = 'sVFR', }; struct QueueEntry { @@ -149,6 +152,7 @@ private: void onDisableOffloadAudio(); void onPause(); void onResume(); + void onSetVideoFrameRate(float fps); void onAudioOffloadTearDown(); void notifyEOS(bool audio, status_t finalResult, int64_t delayUs = 0); |