diff options
Diffstat (limited to 'media/libmediaplayerservice')
10 files changed, 697 insertions, 41 deletions
diff --git a/media/libmediaplayerservice/Android.mk b/media/libmediaplayerservice/Android.mk index adc066d..2cf5710 100644 --- a/media/libmediaplayerservice/Android.mk +++ b/media/libmediaplayerservice/Android.mk @@ -22,6 +22,7 @@ LOCAL_SRC_FILES:= \ StagefrightPlayer.cpp \ StagefrightRecorder.cpp \ TestPlayerStub.cpp \ + VideoFrameScheduler.cpp \ LOCAL_SHARED_LIBRARIES := \ libbinder \ diff --git a/media/libmediaplayerservice/VideoFrameScheduler.cpp b/media/libmediaplayerservice/VideoFrameScheduler.cpp new file mode 100644 index 0000000..4251c4e --- /dev/null +++ b/media/libmediaplayerservice/VideoFrameScheduler.cpp @@ -0,0 +1,499 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "VideoFrameScheduler" +#include <utils/Log.h> +#define ATRACE_TAG ATRACE_TAG_VIDEO +#include <utils/Trace.h> + +#include <sys/time.h> + +#include <binder/IServiceManager.h> +#include <gui/ISurfaceComposer.h> +#include <ui/DisplayStatInfo.h> + +#include <media/stagefright/foundation/ADebug.h> + +#include "VideoFrameScheduler.h" + +namespace android { + +static const nsecs_t kNanosIn1s = 1000000000; + +template<class T> +inline static const T divRound(const T &nom, const T &den) { + if ((nom >= 0) ^ (den >= 0)) { + return (nom - den / 2) / den; + } else { + return (nom + den / 2) / den; + } +} + +template<class T> +inline static T abs(const T &a) { + return a < 0 ? -a : a; +} + +template<class T> +inline static const T &min(const T &a, const T &b) { + return a < b ? a : b; +} + +template<class T> +inline static const T &max(const T &a, const T &b) { + return a > b ? a : b; +} + +template<class T> +inline static T periodicError(const T &val, const T &period) { + T err = abs(val) % period; + return (err < (period / 2)) ? err : (period - err); +} + +template<class T> +static int compare(const T *lhs, const T *rhs) { + if (*lhs < *rhs) { + return -1; + } else if (*lhs > *rhs) { + return 1; + } else { + return 0; + } +} + +/* ======================================================================= */ +/* PLL */ +/* ======================================================================= */ + +static const size_t kMinSamplesToStartPrime = 3; +static const size_t kMinSamplesToStopPrime = VideoFrameScheduler::kHistorySize; +static const size_t kMinSamplesToEstimatePeriod = 3; +static const size_t kMaxSamplesToEstimatePeriod = VideoFrameScheduler::kHistorySize; + +static const size_t kPrecision = 12; +static const size_t kErrorThreshold = (1 << (kPrecision * 2)) / 10; +static const int64_t kMultiplesThresholdDiv = 4; // 25% +static const int64_t kReFitThresholdDiv = 100; // 1% +static const nsecs_t kMaxAllowedFrameSkip = kNanosIn1s; // 1 sec +static const nsecs_t kMinPeriod = kNanosIn1s / 120; // 120Hz +static const nsecs_t kRefitRefreshPeriod = 10 * kNanosIn1s; // 10 sec + +VideoFrameScheduler::PLL::PLL() + : mPeriod(-1), + mPhase(0), + mPrimed(false), + mSamplesUsedForPriming(0), + mLastTime(-1), + mNumSamples(0) { +} + +void VideoFrameScheduler::PLL::reset(float fps) { + //test(); + + mSamplesUsedForPriming = 0; + mLastTime = -1; + + // set up or reset video PLL + if (fps <= 0.f) { + mPeriod = -1; + mPrimed = false; + } else { + ALOGV("reset at %.1f fps", fps); + mPeriod = (nsecs_t)(1e9 / fps + 0.5); + mPrimed = true; + } + + restart(); +} + +// reset PLL but keep previous period estimate +void VideoFrameScheduler::PLL::restart() { + mNumSamples = 0; + mPhase = -1; +} + +#if 0 + +void VideoFrameScheduler::PLL::test() { + nsecs_t period = kNanosIn1s / 60; + mTimes[0] = 0; + mTimes[1] = period; + mTimes[2] = period * 3; + mTimes[3] = period * 4; + mTimes[4] = period * 7; + mTimes[5] = period * 8; + mTimes[6] = period * 10; + mTimes[7] = period * 12; + mNumSamples = 8; + int64_t a, b, err; + fit(0, period * 12 / 7, 8, &a, &b, &err); + // a = 0.8(5)+ + // b = -0.14097(2)+ + // err = 0.2750578(703)+ + ALOGD("a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)", + (long long)a, (a / (float)(1 << kPrecision)), + (long long)b, (b / (float)(1 << kPrecision)), + (long long)err, (err / (float)(1 << (kPrecision * 2)))); +} + +#endif + +void VideoFrameScheduler::PLL::fit( + nsecs_t phase, nsecs_t period, size_t numSamplesToUse, + int64_t *a, int64_t *b, int64_t *err) { + if (numSamplesToUse > mNumSamples) { + numSamplesToUse = mNumSamples; + } + + int64_t sumX = 0; + int64_t sumXX = 0; + int64_t sumXY = 0; + int64_t sumYY = 0; + int64_t sumY = 0; + + int64_t x = 0; // x usually is in [0..numSamplesToUse) + nsecs_t lastTime; + for (size_t i = 0; i < numSamplesToUse; i++) { + size_t ix = (mNumSamples - numSamplesToUse + i) % kHistorySize; + nsecs_t time = mTimes[ix]; + if (i > 0) { + x += divRound(time - lastTime, period); + } + // y is usually in [-numSamplesToUse..numSamplesToUse+kRefitRefreshPeriod/kMinPeriod) << kPrecision + // ideally in [0..numSamplesToUse), but shifted by -numSamplesToUse during + // priming, and possibly shifted by up to kRefitRefreshPeriod/kMinPeriod + // while we are not refitting. + int64_t y = divRound(time - phase, period >> kPrecision); + sumX += x; + sumY += y; + sumXX += x * x; + sumXY += x * y; + sumYY += y * y; + lastTime = time; + } + + int64_t div = numSamplesToUse * sumXX - sumX * sumX; + int64_t a_nom = numSamplesToUse * sumXY - sumX * sumY; + int64_t b_nom = sumXX * sumY - sumX * sumXY; + *a = divRound(a_nom, div); + *b = divRound(b_nom, div); + // don't use a and b directly as the rounding error is significant + *err = sumYY - divRound(a_nom * sumXY + b_nom * sumY, div); + ALOGV("fitting[%zu] a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)", + numSamplesToUse, + (long long)*a, (*a / (float)(1 << kPrecision)), + (long long)*b, (*b / (float)(1 << kPrecision)), + (long long)*err, (*err / (float)(1 << (kPrecision * 2)))); +} + +void VideoFrameScheduler::PLL::prime(size_t numSamplesToUse) { + if (numSamplesToUse > mNumSamples) { + numSamplesToUse = mNumSamples; + } + CHECK(numSamplesToUse >= 3); // must have at least 3 samples + + // estimate video framerate from deltas between timestamps, and + // 2nd order deltas + Vector<nsecs_t> deltas; + nsecs_t lastTime, firstTime; + for (size_t i = 0; i < numSamplesToUse; ++i) { + size_t index = (mNumSamples - numSamplesToUse + i) % kHistorySize; + nsecs_t time = mTimes[index]; + if (i > 0) { + if (time - lastTime > kMinPeriod) { + //ALOGV("delta: %lld", (long long)(time - lastTime)); + deltas.push(time - lastTime); + } + } else { + firstTime = time; + } + lastTime = time; + } + deltas.sort(compare<nsecs_t>); + size_t numDeltas = deltas.size(); + if (numDeltas > 1) { + nsecs_t deltaMinLimit = min(deltas[0] / kMultiplesThresholdDiv, kMinPeriod); + nsecs_t deltaMaxLimit = deltas[numDeltas / 2] * kMultiplesThresholdDiv; + for (size_t i = numDeltas / 2 + 1; i < numDeltas; ++i) { + if (deltas[i] > deltaMaxLimit) { + deltas.resize(i); + numDeltas = i; + break; + } + } + for (size_t i = 1; i < numDeltas; ++i) { + nsecs_t delta2nd = deltas[i] - deltas[i - 1]; + if (delta2nd >= deltaMinLimit) { + //ALOGV("delta2: %lld", (long long)(delta2nd)); + deltas.push(delta2nd); + } + } + } + + // use the one that yields the best match + int64_t bestScore; + for (size_t i = 0; i < deltas.size(); ++i) { + nsecs_t delta = deltas[i]; + int64_t score = 0; +#if 1 + // simplest score: number of deltas that are near multiples + size_t matches = 0; + for (size_t j = 0; j < deltas.size(); ++j) { + nsecs_t err = periodicError(deltas[j], delta); + if (err < delta / kMultiplesThresholdDiv) { + ++matches; + } + } + score = matches; +#if 0 + // could be weighed by the (1 - normalized error) + if (numSamplesToUse >= kMinSamplesToEstimatePeriod) { + int64_t a, b, err; + fit(firstTime, delta, numSamplesToUse, &a, &b, &err); + err = (1 << (2 * kPrecision)) - err; + score *= max(0, err); + } +#endif +#else + // or use the error as a negative score + if (numSamplesToUse >= kMinSamplesToEstimatePeriod) { + int64_t a, b, err; + fit(firstTime, delta, numSamplesToUse, &a, &b, &err); + score = -delta * err; + } +#endif + if (i == 0 || score > bestScore) { + bestScore = score; + mPeriod = delta; + mPhase = firstTime; + } + } + ALOGV("priming[%zu] phase:%lld period:%lld", numSamplesToUse, mPhase, mPeriod); +} + +nsecs_t VideoFrameScheduler::PLL::addSample(nsecs_t time) { + if (mLastTime >= 0 + // if time goes backward, or we skipped rendering + && (time > mLastTime + kMaxAllowedFrameSkip || time < mLastTime)) { + restart(); + } + + mLastTime = time; + mTimes[mNumSamples % kHistorySize] = time; + ++mNumSamples; + + bool doFit = time > mRefitAt; + if ((mPeriod <= 0 || !mPrimed) && mNumSamples >= kMinSamplesToStartPrime) { + prime(kMinSamplesToStopPrime); + ++mSamplesUsedForPriming; + doFit = true; + } + if (mPeriod > 0 && mNumSamples >= kMinSamplesToEstimatePeriod) { + if (mPhase < 0) { + // initialize phase to the current render time + mPhase = time; + doFit = true; + } else if (!doFit) { + int64_t err = periodicError(time - mPhase, mPeriod); + doFit = err > mPeriod / kReFitThresholdDiv; + } + + if (doFit) { + int64_t a, b, err; + mRefitAt = time + kRefitRefreshPeriod; + fit(mPhase, mPeriod, kMaxSamplesToEstimatePeriod, &a, &b, &err); + mPhase += (mPeriod * b) >> kPrecision; + mPeriod = (mPeriod * a) >> kPrecision; + ALOGV("new phase:%lld period:%lld", (long long)mPhase, (long long)mPeriod); + + if (err < kErrorThreshold) { + if (!mPrimed && mSamplesUsedForPriming >= kMinSamplesToStopPrime) { + mPrimed = true; + } + } else { + mPrimed = false; + mSamplesUsedForPriming = 0; + } + } + } + return mPeriod; +} + +/* ======================================================================= */ +/* Frame Scheduler */ +/* ======================================================================= */ + +static const nsecs_t kDefaultVsyncPeriod = kNanosIn1s / 60; // 60Hz +static const nsecs_t kVsyncRefreshPeriod = kNanosIn1s; // 1 sec + +VideoFrameScheduler::VideoFrameScheduler() + : mVsyncTime(0), + mVsyncPeriod(0), + mVsyncRefreshAt(0), + mLastVsyncTime(-1), + mTimeCorrection(0) { +} + +void VideoFrameScheduler::updateVsync() { + mVsyncRefreshAt = systemTime(SYSTEM_TIME_MONOTONIC) + kVsyncRefreshPeriod; + mVsyncPeriod = 0; + mVsyncTime = 0; + + // TODO: schedule frames for the destination surface + // For now, surface flinger only schedules frames on the primary display + if (mComposer == NULL) { + String16 name("SurfaceFlinger"); + sp<IServiceManager> sm = defaultServiceManager(); + mComposer = interface_cast<ISurfaceComposer>(sm->checkService(name)); + } + if (mComposer != NULL) { + DisplayStatInfo stats; + status_t res = mComposer->getDisplayStats(NULL /* display */, &stats); + if (res == OK) { + ALOGV("vsync time:%lld period:%lld", + (long long)stats.vsyncTime, (long long)stats.vsyncPeriod); + mVsyncTime = stats.vsyncTime; + mVsyncPeriod = stats.vsyncPeriod; + } else { + ALOGW("getDisplayStats returned %d", res); + } + } else { + ALOGW("could not get surface mComposer service"); + } +} + +void VideoFrameScheduler::init(float videoFps) { + updateVsync(); + + mLastVsyncTime = -1; + mTimeCorrection = 0; + + mPll.reset(videoFps); +} + +void VideoFrameScheduler::restart() { + mLastVsyncTime = -1; + mTimeCorrection = 0; + + mPll.restart(); +} + +nsecs_t VideoFrameScheduler::getVsyncPeriod() { + if (mVsyncPeriod > 0) { + return mVsyncPeriod; + } + return kDefaultVsyncPeriod; +} + +nsecs_t VideoFrameScheduler::schedule(nsecs_t renderTime) { + nsecs_t origRenderTime = renderTime; + + nsecs_t now = systemTime(SYSTEM_TIME_MONOTONIC); + if (now >= mVsyncRefreshAt) { + updateVsync(); + } + + // without VSYNC info, there is nothing to do + if (mVsyncPeriod == 0) { + ALOGV("no vsync: render=%lld", (long long)renderTime); + return renderTime; + } + + // ensure vsync time is well before (corrected) render time + if (mVsyncTime > renderTime - 4 * mVsyncPeriod) { + mVsyncTime -= + ((mVsyncTime - renderTime) / mVsyncPeriod + 5) * mVsyncPeriod; + } + + // Video presentation takes place at the VSYNC _after_ renderTime. Adjust renderTime + // so this effectively becomes a rounding operation (to the _closest_ VSYNC.) + renderTime -= mVsyncPeriod / 2; + + const nsecs_t videoPeriod = mPll.addSample(origRenderTime); + if (videoPeriod > 0) { + // Smooth out rendering + size_t N = 12; + nsecs_t fiveSixthDev = + abs(((videoPeriod * 5 + mVsyncPeriod) % (mVsyncPeriod * 6)) - mVsyncPeriod) + / (mVsyncPeriod / 100); + // use 20 samples if we are doing 5:6 ratio +- 1% (e.g. playing 50Hz on 60Hz) + if (fiveSixthDev < 12) { /* 12% / 6 = 2% */ + N = 20; + } + + nsecs_t offset = 0; + nsecs_t edgeRemainder = 0; + for (size_t i = 1; i <= N; i++) { + offset += + (renderTime + mTimeCorrection + videoPeriod * i - mVsyncTime) % mVsyncPeriod; + edgeRemainder += (videoPeriod * i) % mVsyncPeriod; + } + mTimeCorrection += mVsyncPeriod / 2 - offset / N; + renderTime += mTimeCorrection; + nsecs_t correctionLimit = mVsyncPeriod * 3 / 5; + edgeRemainder = abs(edgeRemainder / N - mVsyncPeriod / 2); + if (edgeRemainder <= mVsyncPeriod / 3) { + correctionLimit /= 2; + } + + // estimate how many VSYNCs a frame will spend on the display + nsecs_t nextVsyncTime = + renderTime + mVsyncPeriod - ((renderTime - mVsyncTime) % mVsyncPeriod); + if (mLastVsyncTime >= 0) { + size_t minVsyncsPerFrame = videoPeriod / mVsyncPeriod; + size_t vsyncsForLastFrame = divRound(nextVsyncTime - mLastVsyncTime, mVsyncPeriod); + bool vsyncsPerFrameAreNearlyConstant = + periodicError(videoPeriod, mVsyncPeriod) / (mVsyncPeriod / 20) == 0; + + if (mTimeCorrection > correctionLimit && + (vsyncsPerFrameAreNearlyConstant || vsyncsForLastFrame > minVsyncsPerFrame)) { + // remove a VSYNC + mTimeCorrection -= mVsyncPeriod / 2; + renderTime -= mVsyncPeriod / 2; + nextVsyncTime -= mVsyncPeriod; + --vsyncsForLastFrame; + } else if (mTimeCorrection < -correctionLimit && + (vsyncsPerFrameAreNearlyConstant || vsyncsForLastFrame == minVsyncsPerFrame)) { + // add a VSYNC + mTimeCorrection += mVsyncPeriod / 2; + renderTime += mVsyncPeriod / 2; + nextVsyncTime += mVsyncPeriod; + ++vsyncsForLastFrame; + } + ATRACE_INT("FRAME_VSYNCS", vsyncsForLastFrame); + } + mLastVsyncTime = nextVsyncTime; + } + + // align rendertime to the center between VSYNC edges + renderTime -= (renderTime - mVsyncTime) % mVsyncPeriod; + renderTime += mVsyncPeriod / 2; + ALOGV("adjusting render: %lld => %lld", (long long)origRenderTime, (long long)renderTime); + ATRACE_INT("FRAME_FLIP_IN(ms)", (renderTime - now) / 1000000); + return renderTime; +} + +void VideoFrameScheduler::release() { + mComposer.clear(); +} + +VideoFrameScheduler::~VideoFrameScheduler() { + release(); +} + +} // namespace android + diff --git a/media/libmediaplayerservice/VideoFrameScheduler.h b/media/libmediaplayerservice/VideoFrameScheduler.h new file mode 100644 index 0000000..19f0787 --- /dev/null +++ b/media/libmediaplayerservice/VideoFrameScheduler.h @@ -0,0 +1,98 @@ +/* + * Copyright 2014, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef VIDEO_FRAME_SCHEDULER_H_ +#define VIDEO_FRAME_SCHEDULER_H_ + +#include <utils/RefBase.h> +#include <utils/Timers.h> + +#include <media/stagefright/foundation/ABase.h> + +namespace android { + +struct ISurfaceComposer; + +struct VideoFrameScheduler : public RefBase { + VideoFrameScheduler(); + + // (re)initialize scheduler + void init(float videoFps = -1); + // use in case of video render-time discontinuity, e.g. seek + void restart(); + // get adjusted nanotime for a video frame render at renderTime + nsecs_t schedule(nsecs_t renderTime); + + // returns the vsync period for the main display + nsecs_t getVsyncPeriod(); + + void release(); + + static const size_t kHistorySize = 8; + +protected: + virtual ~VideoFrameScheduler(); + +private: + struct PLL { + PLL(); + + // reset PLL to new PLL + void reset(float fps = -1); + // keep current estimate, but restart phase + void restart(); + // returns period + nsecs_t addSample(nsecs_t time); + + private: + nsecs_t mPeriod; + nsecs_t mPhase; + + bool mPrimed; // have an estimate for the period + size_t mSamplesUsedForPriming; + + nsecs_t mLastTime; // last input time + nsecs_t mRefitAt; // next input time to fit at + + size_t mNumSamples; // can go past kHistorySize + nsecs_t mTimes[kHistorySize]; + + void test(); + void fit(nsecs_t phase, nsecs_t period, size_t numSamples, + int64_t *a, int64_t *b, int64_t *err); + void prime(size_t numSamples); + }; + + void updateVsync(); + + nsecs_t mVsyncTime; // vsync timing from display + nsecs_t mVsyncPeriod; + nsecs_t mVsyncRefreshAt; // next time to refresh timing info + + nsecs_t mLastVsyncTime; // estimated vsync time for last frame + nsecs_t mTimeCorrection; // running adjustment + + PLL mPll; // PLL for video frame rate based on render time + + sp<ISurfaceComposer> mComposer; + + DISALLOW_EVIL_CONSTRUCTORS(VideoFrameScheduler); +}; + +} // namespace android + +#endif // VIDEO_FRAME_SCHEDULER_H_ + diff --git a/media/libmediaplayerservice/nuplayer/Android.mk b/media/libmediaplayerservice/nuplayer/Android.mk index 0dd2b61..676c0a6 100644 --- a/media/libmediaplayerservice/nuplayer/Android.mk +++ b/media/libmediaplayerservice/nuplayer/Android.mk @@ -19,6 +19,7 @@ LOCAL_C_INCLUDES := \ $(TOP)/frameworks/av/media/libstagefright/mpeg2ts \ $(TOP)/frameworks/av/media/libstagefright/rtsp \ $(TOP)/frameworks/av/media/libstagefright/timedtext \ + $(TOP)/frameworks/av/media/libmediaplayerservice \ $(TOP)/frameworks/native/include/media/openmax LOCAL_MODULE:= libstagefright_nuplayer diff --git a/media/libmediaplayerservice/nuplayer/GenericSource.cpp b/media/libmediaplayerservice/nuplayer/GenericSource.cpp index 142107d..bd75034 100644 --- a/media/libmediaplayerservice/nuplayer/GenericSource.cpp +++ b/media/libmediaplayerservice/nuplayer/GenericSource.cpp @@ -158,7 +158,12 @@ status_t NuPlayer::GenericSource::initFromDataSource() { int32_t totalBitrate = 0; - for (size_t i = 0; i < extractor->countTracks(); ++i) { + size_t numtracks = extractor->countTracks(); + if (numtracks == 0) { + return UNKNOWN_ERROR; + } + + for (size_t i = 0; i < numtracks; ++i) { sp<MediaSource> track = extractor->getTrack(i); sp<MetaData> meta = extractor->getTrackMetaData(i); @@ -1110,8 +1115,8 @@ sp<ABuffer> NuPlayer::GenericSource::mediaBufferToABuffer( if (mIsWidevine && !audio) { // data is already provided in the buffer ab = new ABuffer(NULL, mb->range_length()); - ab->meta()->setPointer("mediaBuffer", mb); mb->add_ref(); + ab->setMediaBufferBase(mb); } else { ab = new ABuffer(outLength); memcpy(ab->data(), diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp index cdbea9f..b589909 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp @@ -155,6 +155,7 @@ NuPlayer::NuPlayer() mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER), mAudioDecoderGeneration(0), mVideoDecoderGeneration(0), + mRendererGeneration(0), mAudioEOS(false), mVideoEOS(false), mScanSourcesPending(false), @@ -633,16 +634,23 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { flags |= Renderer::FLAG_OFFLOAD_AUDIO; } - mRenderer = new Renderer( - mAudioSink, - new AMessage(kWhatRendererNotify, id()), - flags); + sp<AMessage> notify = new AMessage(kWhatRendererNotify, id()); + ++mRendererGeneration; + notify->setInt32("generation", mRendererGeneration); + mRenderer = new Renderer(mAudioSink, notify, flags); mRendererLooper = new ALooper; mRendererLooper->setName("NuPlayerRenderer"); mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO); mRendererLooper->registerHandler(mRenderer); + sp<MetaData> meta = getFileMeta(); + int32_t rate; + if (meta != NULL + && meta->findInt32(kKeyFrameRate, &rate) && rate > 0) { + mRenderer->setVideoFrameRate(rate); + } + postScanSources(); break; } @@ -806,11 +814,13 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { ALOGV("%s shutdown completed", audio ? "audio" : "video"); if (audio) { mAudioDecoder.clear(); + ++mAudioDecoderGeneration; CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER); mFlushingAudio = SHUT_DOWN; } else { mVideoDecoder.clear(); + ++mVideoDecoderGeneration; CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER); mFlushingVideo = SHUT_DOWN; @@ -828,9 +838,11 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { mRenderer->queueEOS(audio, err); if (audio && mFlushingAudio != NONE) { mAudioDecoder.clear(); + ++mAudioDecoderGeneration; mFlushingAudio = SHUT_DOWN; } else if (!audio && mFlushingVideo != NONE){ mVideoDecoder.clear(); + ++mVideoDecoderGeneration; mFlushingVideo = SHUT_DOWN; } finishFlushIfPossible(); @@ -850,6 +862,14 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { case kWhatRendererNotify: { + int32_t requesterGeneration = mRendererGeneration - 1; + CHECK(msg->findInt32("generation", &requesterGeneration)); + if (requesterGeneration != mRendererGeneration) { + ALOGV("got message from old renderer, generation(%d:%d)", + requesterGeneration, mRendererGeneration); + return; + } + int32_t what; CHECK(msg->findInt32("what", &what)); @@ -912,6 +932,7 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { CHECK(msg->findInt64("positionUs", &positionUs)); closeAudioSink(); mAudioDecoder.clear(); + ++mAudioDecoderGeneration; mRenderer->flush(true /* audio */); if (mVideoDecoder != NULL) { mRenderer->flush(false /* audio */); @@ -1842,9 +1863,6 @@ void NuPlayer::performReset() { ++mScanSourcesGeneration; mScanSourcesPending = false; - ++mAudioDecoderGeneration; - ++mVideoDecoderGeneration; - if (mRendererLooper != NULL) { if (mRenderer != NULL) { mRendererLooper->unregisterHandler(mRenderer->id()); @@ -1853,6 +1871,7 @@ void NuPlayer::performReset() { mRendererLooper.clear(); } mRenderer.clear(); + ++mRendererGeneration; if (mSource != NULL) { mSource->stop(); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h index 7197e5f..eee96ca 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h @@ -137,6 +137,7 @@ private: sp<ALooper> mRendererLooper; int32_t mAudioDecoderGeneration; int32_t mVideoDecoderGeneration; + int32_t mRendererGeneration; List<sp<Action> > mDeferredActions; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp index 87f85e7..cdb860c 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp @@ -283,14 +283,9 @@ void android::NuPlayer::Decoder::onInputBufferFilled(const sp<AMessage> &msg) { // handle widevine classic source - that fills an arbitrary input buffer MediaBuffer *mediaBuffer = NULL; - if (hasBuffer && buffer->meta()->findPointer( - "mediaBuffer", (void **)&mediaBuffer)) { - if (mediaBuffer == NULL) { - // received no actual buffer - ALOGW("[%s] received null MediaBuffer %s", - mComponentName.c_str(), msg->debugString().c_str()); - buffer = NULL; - } else { + if (hasBuffer) { + mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase()); + if (mediaBuffer != NULL) { // likely filled another buffer than we requested: adjust buffer index size_t ix; for (ix = 0; ix < mInputBuffers.size(); ix++) { @@ -470,7 +465,9 @@ void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) { size_t bufferIx; CHECK(msg->findSize("buffer-ix", &bufferIx)); if (msg->findInt32("render", &render) && render) { - err = mCodec->renderOutputBufferAndRelease(bufferIx); + int64_t timestampNs; + CHECK(msg->findInt64("timestampNs", ×tampNs)); + err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs); } else { err = mCodec->releaseOutputBuffer(bufferIx); } @@ -598,16 +595,6 @@ void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) { { if (!isStaleReply(msg)) { onInputBufferFilled(msg); - } else { - /* release any MediaBuffer passed in the stale buffer */ - sp<ABuffer> buffer; - MediaBuffer *mediaBuffer = NULL; - if (msg->findBuffer("buffer", &buffer) && - buffer->meta()->findPointer( - "mediaBuffer", (void **)&mediaBuffer) && - mediaBuffer != NULL) { - mediaBuffer->release(); - } } break; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp index 4ffcea6..73ac057 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp @@ -26,6 +26,8 @@ #include <media/stagefright/MediaErrors.h> #include <media/stagefright/MetaData.h> +#include <VideoFrameScheduler.h> + #include <inttypes.h> namespace android { @@ -143,6 +145,12 @@ void NuPlayer::Renderer::resume() { (new AMessage(kWhatResume, id()))->post(); } +void NuPlayer::Renderer::setVideoFrameRate(float fps) { + sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, id()); + msg->setFloat("frame-rate", fps); + msg->post(); +} + void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { switch (msg->what()) { case kWhatStopAudioSink: @@ -243,6 +251,14 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatSetVideoFrameRate: + { + float fps; + CHECK(msg->findFloat("frame-rate", &fps)); + onSetVideoFrameRate(fps); + break; + } + case kWhatAudioOffloadTearDown: { onAudioOffloadTearDown(); @@ -519,16 +535,20 @@ void NuPlayer::Renderer::postDrainVideoQueue() { sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id()); msg->setInt32("generation", mVideoQueueGeneration); - int64_t delayUs; - if (entry.mBuffer == NULL) { // EOS doesn't carry a timestamp. - delayUs = 0; - } else if (mFlags & FLAG_REAL_TIME) { + msg->post(); + mDrainVideoQueuePending = true; + return; + } + + int64_t delayUs; + int64_t nowUs = ALooper::GetNowUs(); + int64_t realTimeUs; + if (mFlags & FLAG_REAL_TIME) { int64_t mediaTimeUs; CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); - - delayUs = mediaTimeUs - ALooper::GetNowUs(); + realTimeUs = mediaTimeUs; } else { int64_t mediaTimeUs; CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); @@ -537,23 +557,26 @@ void NuPlayer::Renderer::postDrainVideoQueue() { mFirstAnchorTimeMediaUs = mediaTimeUs; } if (mAnchorTimeMediaUs < 0) { - delayUs = 0; - if (!mHasAudio) { mAnchorTimeMediaUs = mediaTimeUs; - mAnchorTimeRealUs = ALooper::GetNowUs(); + mAnchorTimeRealUs = nowUs; notifyPosition(); } + realTimeUs = nowUs; } else { - int64_t realTimeUs = + realTimeUs = (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs; - - delayUs = realTimeUs - ALooper::GetNowUs(); } } + realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000; + int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000); + + delayUs = realTimeUs - nowUs; + ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs); - msg->post(delayUs); + // post 2 display refreshes before rendering is due + msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0); mDrainVideoQueuePending = true; } @@ -605,6 +628,7 @@ void NuPlayer::Renderer::onDrainVideoQueue() { mVideoLateByUs = 0ll; } + entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll); entry->mNotifyConsumed->setInt32("render", !tooLate); entry->mNotifyConsumed->post(); mVideoQueue.erase(mVideoQueue.begin()); @@ -647,6 +671,10 @@ void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) { mHasAudio = true; } else { mHasVideo = true; + if (mVideoScheduler == NULL) { + mVideoScheduler = new VideoFrameScheduler(); + mVideoScheduler->init(); + } } if (dropBufferWhileFlushing(audio, msg)) { @@ -812,6 +840,10 @@ void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { mDrainVideoQueuePending = false; ++mVideoQueueGeneration; + if (mVideoScheduler != NULL) { + mVideoScheduler->restart(); + } + prepareForMediaRenderingStart(); } @@ -965,6 +997,13 @@ void NuPlayer::Renderer::onResume() { } } +void NuPlayer::Renderer::onSetVideoFrameRate(float fps) { + if (mVideoScheduler == NULL) { + mVideoScheduler = new VideoFrameScheduler(); + } + mVideoScheduler->init(fps); +} + // TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs() // as it acquires locks and may query the audio driver. // diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h index aba3c81..8e6112b 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h @@ -23,6 +23,7 @@ namespace android { struct ABuffer; +struct VideoFrameScheduler; struct NuPlayer::Renderer : public AHandler { enum Flags { @@ -56,6 +57,8 @@ struct NuPlayer::Renderer : public AHandler { void pause(); void resume(); + void setVideoFrameRate(float fps); + enum { kWhatEOS = 'eos ', kWhatFlushComplete = 'fluC', @@ -83,6 +86,7 @@ private: kWhatResume = 'resm', kWhatStopAudioSink = 'stpA', kWhatDisableOffloadAudio = 'noOA', + kWhatSetVideoFrameRate = 'sVFR', }; struct QueueEntry { @@ -101,6 +105,7 @@ private: List<QueueEntry> mAudioQueue; List<QueueEntry> mVideoQueue; uint32_t mNumFramesWritten; + sp<VideoFrameScheduler> mVideoScheduler; bool mDrainAudioQueuePending; bool mDrainVideoQueuePending; @@ -151,6 +156,7 @@ private: void onDisableOffloadAudio(); void onPause(); void onResume(); + void onSetVideoFrameRate(float fps); void onAudioOffloadTearDown(); void notifyEOS(bool audio, status_t finalResult, int64_t delayUs = 0); |