diff options
Diffstat (limited to 'media')
26 files changed, 888 insertions, 158 deletions
diff --git a/media/libmedia/AudioTrack.cpp b/media/libmedia/AudioTrack.cpp index 762dca5..ea7b279 100644 --- a/media/libmedia/AudioTrack.cpp +++ b/media/libmedia/AudioTrack.cpp @@ -1935,7 +1935,7 @@ status_t AudioTrack::getTimestamp(AudioTimestamp& timestamp) // To avoid a race, read the presented frames first. This ensures that presented <= consumed. status_t status = mAudioTrack->getTimestamp(timestamp); if (status != NO_ERROR) { - ALOGW_IF(status != WOULD_BLOCK, "getTimestamp error:%#x", status); + ALOGV_IF(status != WOULD_BLOCK, "getTimestamp error:%#x", status); return status; } if (isOffloadedOrDirect_l()) { diff --git a/media/libmediaplayerservice/Android.mk b/media/libmediaplayerservice/Android.mk index adc066d..2cf5710 100644 --- a/media/libmediaplayerservice/Android.mk +++ b/media/libmediaplayerservice/Android.mk @@ -22,6 +22,7 @@ LOCAL_SRC_FILES:= \ StagefrightPlayer.cpp \ StagefrightRecorder.cpp \ TestPlayerStub.cpp \ + VideoFrameScheduler.cpp \ LOCAL_SHARED_LIBRARIES := \ libbinder \ diff --git a/media/libmediaplayerservice/VideoFrameScheduler.cpp b/media/libmediaplayerservice/VideoFrameScheduler.cpp new file mode 100644 index 0000000..4251c4e --- /dev/null +++ b/media/libmediaplayerservice/VideoFrameScheduler.cpp @@ -0,0 +1,499 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "VideoFrameScheduler" +#include <utils/Log.h> +#define ATRACE_TAG ATRACE_TAG_VIDEO +#include <utils/Trace.h> + +#include <sys/time.h> + +#include <binder/IServiceManager.h> +#include <gui/ISurfaceComposer.h> +#include <ui/DisplayStatInfo.h> + +#include <media/stagefright/foundation/ADebug.h> + +#include "VideoFrameScheduler.h" + +namespace android { + +static const nsecs_t kNanosIn1s = 1000000000; + +template<class T> +inline static const T divRound(const T &nom, const T &den) { + if ((nom >= 0) ^ (den >= 0)) { + return (nom - den / 2) / den; + } else { + return (nom + den / 2) / den; + } +} + +template<class T> +inline static T abs(const T &a) { + return a < 0 ? -a : a; +} + +template<class T> +inline static const T &min(const T &a, const T &b) { + return a < b ? a : b; +} + +template<class T> +inline static const T &max(const T &a, const T &b) { + return a > b ? a : b; +} + +template<class T> +inline static T periodicError(const T &val, const T &period) { + T err = abs(val) % period; + return (err < (period / 2)) ? err : (period - err); +} + +template<class T> +static int compare(const T *lhs, const T *rhs) { + if (*lhs < *rhs) { + return -1; + } else if (*lhs > *rhs) { + return 1; + } else { + return 0; + } +} + +/* ======================================================================= */ +/* PLL */ +/* ======================================================================= */ + +static const size_t kMinSamplesToStartPrime = 3; +static const size_t kMinSamplesToStopPrime = VideoFrameScheduler::kHistorySize; +static const size_t kMinSamplesToEstimatePeriod = 3; +static const size_t kMaxSamplesToEstimatePeriod = VideoFrameScheduler::kHistorySize; + +static const size_t kPrecision = 12; +static const size_t kErrorThreshold = (1 << (kPrecision * 2)) / 10; +static const int64_t kMultiplesThresholdDiv = 4; // 25% +static const int64_t kReFitThresholdDiv = 100; // 1% +static const nsecs_t kMaxAllowedFrameSkip = kNanosIn1s; // 1 sec +static const nsecs_t kMinPeriod = kNanosIn1s / 120; // 120Hz +static const nsecs_t kRefitRefreshPeriod = 10 * kNanosIn1s; // 10 sec + +VideoFrameScheduler::PLL::PLL() + : mPeriod(-1), + mPhase(0), + mPrimed(false), + mSamplesUsedForPriming(0), + mLastTime(-1), + mNumSamples(0) { +} + +void VideoFrameScheduler::PLL::reset(float fps) { + //test(); + + mSamplesUsedForPriming = 0; + mLastTime = -1; + + // set up or reset video PLL + if (fps <= 0.f) { + mPeriod = -1; + mPrimed = false; + } else { + ALOGV("reset at %.1f fps", fps); + mPeriod = (nsecs_t)(1e9 / fps + 0.5); + mPrimed = true; + } + + restart(); +} + +// reset PLL but keep previous period estimate +void VideoFrameScheduler::PLL::restart() { + mNumSamples = 0; + mPhase = -1; +} + +#if 0 + +void VideoFrameScheduler::PLL::test() { + nsecs_t period = kNanosIn1s / 60; + mTimes[0] = 0; + mTimes[1] = period; + mTimes[2] = period * 3; + mTimes[3] = period * 4; + mTimes[4] = period * 7; + mTimes[5] = period * 8; + mTimes[6] = period * 10; + mTimes[7] = period * 12; + mNumSamples = 8; + int64_t a, b, err; + fit(0, period * 12 / 7, 8, &a, &b, &err); + // a = 0.8(5)+ + // b = -0.14097(2)+ + // err = 0.2750578(703)+ + ALOGD("a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)", + (long long)a, (a / (float)(1 << kPrecision)), + (long long)b, (b / (float)(1 << kPrecision)), + (long long)err, (err / (float)(1 << (kPrecision * 2)))); +} + +#endif + +void VideoFrameScheduler::PLL::fit( + nsecs_t phase, nsecs_t period, size_t numSamplesToUse, + int64_t *a, int64_t *b, int64_t *err) { + if (numSamplesToUse > mNumSamples) { + numSamplesToUse = mNumSamples; + } + + int64_t sumX = 0; + int64_t sumXX = 0; + int64_t sumXY = 0; + int64_t sumYY = 0; + int64_t sumY = 0; + + int64_t x = 0; // x usually is in [0..numSamplesToUse) + nsecs_t lastTime; + for (size_t i = 0; i < numSamplesToUse; i++) { + size_t ix = (mNumSamples - numSamplesToUse + i) % kHistorySize; + nsecs_t time = mTimes[ix]; + if (i > 0) { + x += divRound(time - lastTime, period); + } + // y is usually in [-numSamplesToUse..numSamplesToUse+kRefitRefreshPeriod/kMinPeriod) << kPrecision + // ideally in [0..numSamplesToUse), but shifted by -numSamplesToUse during + // priming, and possibly shifted by up to kRefitRefreshPeriod/kMinPeriod + // while we are not refitting. + int64_t y = divRound(time - phase, period >> kPrecision); + sumX += x; + sumY += y; + sumXX += x * x; + sumXY += x * y; + sumYY += y * y; + lastTime = time; + } + + int64_t div = numSamplesToUse * sumXX - sumX * sumX; + int64_t a_nom = numSamplesToUse * sumXY - sumX * sumY; + int64_t b_nom = sumXX * sumY - sumX * sumXY; + *a = divRound(a_nom, div); + *b = divRound(b_nom, div); + // don't use a and b directly as the rounding error is significant + *err = sumYY - divRound(a_nom * sumXY + b_nom * sumY, div); + ALOGV("fitting[%zu] a=%lld (%.6f), b=%lld (%.6f), err=%lld (%.6f)", + numSamplesToUse, + (long long)*a, (*a / (float)(1 << kPrecision)), + (long long)*b, (*b / (float)(1 << kPrecision)), + (long long)*err, (*err / (float)(1 << (kPrecision * 2)))); +} + +void VideoFrameScheduler::PLL::prime(size_t numSamplesToUse) { + if (numSamplesToUse > mNumSamples) { + numSamplesToUse = mNumSamples; + } + CHECK(numSamplesToUse >= 3); // must have at least 3 samples + + // estimate video framerate from deltas between timestamps, and + // 2nd order deltas + Vector<nsecs_t> deltas; + nsecs_t lastTime, firstTime; + for (size_t i = 0; i < numSamplesToUse; ++i) { + size_t index = (mNumSamples - numSamplesToUse + i) % kHistorySize; + nsecs_t time = mTimes[index]; + if (i > 0) { + if (time - lastTime > kMinPeriod) { + //ALOGV("delta: %lld", (long long)(time - lastTime)); + deltas.push(time - lastTime); + } + } else { + firstTime = time; + } + lastTime = time; + } + deltas.sort(compare<nsecs_t>); + size_t numDeltas = deltas.size(); + if (numDeltas > 1) { + nsecs_t deltaMinLimit = min(deltas[0] / kMultiplesThresholdDiv, kMinPeriod); + nsecs_t deltaMaxLimit = deltas[numDeltas / 2] * kMultiplesThresholdDiv; + for (size_t i = numDeltas / 2 + 1; i < numDeltas; ++i) { + if (deltas[i] > deltaMaxLimit) { + deltas.resize(i); + numDeltas = i; + break; + } + } + for (size_t i = 1; i < numDeltas; ++i) { + nsecs_t delta2nd = deltas[i] - deltas[i - 1]; + if (delta2nd >= deltaMinLimit) { + //ALOGV("delta2: %lld", (long long)(delta2nd)); + deltas.push(delta2nd); + } + } + } + + // use the one that yields the best match + int64_t bestScore; + for (size_t i = 0; i < deltas.size(); ++i) { + nsecs_t delta = deltas[i]; + int64_t score = 0; +#if 1 + // simplest score: number of deltas that are near multiples + size_t matches = 0; + for (size_t j = 0; j < deltas.size(); ++j) { + nsecs_t err = periodicError(deltas[j], delta); + if (err < delta / kMultiplesThresholdDiv) { + ++matches; + } + } + score = matches; +#if 0 + // could be weighed by the (1 - normalized error) + if (numSamplesToUse >= kMinSamplesToEstimatePeriod) { + int64_t a, b, err; + fit(firstTime, delta, numSamplesToUse, &a, &b, &err); + err = (1 << (2 * kPrecision)) - err; + score *= max(0, err); + } +#endif +#else + // or use the error as a negative score + if (numSamplesToUse >= kMinSamplesToEstimatePeriod) { + int64_t a, b, err; + fit(firstTime, delta, numSamplesToUse, &a, &b, &err); + score = -delta * err; + } +#endif + if (i == 0 || score > bestScore) { + bestScore = score; + mPeriod = delta; + mPhase = firstTime; + } + } + ALOGV("priming[%zu] phase:%lld period:%lld", numSamplesToUse, mPhase, mPeriod); +} + +nsecs_t VideoFrameScheduler::PLL::addSample(nsecs_t time) { + if (mLastTime >= 0 + // if time goes backward, or we skipped rendering + && (time > mLastTime + kMaxAllowedFrameSkip || time < mLastTime)) { + restart(); + } + + mLastTime = time; + mTimes[mNumSamples % kHistorySize] = time; + ++mNumSamples; + + bool doFit = time > mRefitAt; + if ((mPeriod <= 0 || !mPrimed) && mNumSamples >= kMinSamplesToStartPrime) { + prime(kMinSamplesToStopPrime); + ++mSamplesUsedForPriming; + doFit = true; + } + if (mPeriod > 0 && mNumSamples >= kMinSamplesToEstimatePeriod) { + if (mPhase < 0) { + // initialize phase to the current render time + mPhase = time; + doFit = true; + } else if (!doFit) { + int64_t err = periodicError(time - mPhase, mPeriod); + doFit = err > mPeriod / kReFitThresholdDiv; + } + + if (doFit) { + int64_t a, b, err; + mRefitAt = time + kRefitRefreshPeriod; + fit(mPhase, mPeriod, kMaxSamplesToEstimatePeriod, &a, &b, &err); + mPhase += (mPeriod * b) >> kPrecision; + mPeriod = (mPeriod * a) >> kPrecision; + ALOGV("new phase:%lld period:%lld", (long long)mPhase, (long long)mPeriod); + + if (err < kErrorThreshold) { + if (!mPrimed && mSamplesUsedForPriming >= kMinSamplesToStopPrime) { + mPrimed = true; + } + } else { + mPrimed = false; + mSamplesUsedForPriming = 0; + } + } + } + return mPeriod; +} + +/* ======================================================================= */ +/* Frame Scheduler */ +/* ======================================================================= */ + +static const nsecs_t kDefaultVsyncPeriod = kNanosIn1s / 60; // 60Hz +static const nsecs_t kVsyncRefreshPeriod = kNanosIn1s; // 1 sec + +VideoFrameScheduler::VideoFrameScheduler() + : mVsyncTime(0), + mVsyncPeriod(0), + mVsyncRefreshAt(0), + mLastVsyncTime(-1), + mTimeCorrection(0) { +} + +void VideoFrameScheduler::updateVsync() { + mVsyncRefreshAt = systemTime(SYSTEM_TIME_MONOTONIC) + kVsyncRefreshPeriod; + mVsyncPeriod = 0; + mVsyncTime = 0; + + // TODO: schedule frames for the destination surface + // For now, surface flinger only schedules frames on the primary display + if (mComposer == NULL) { + String16 name("SurfaceFlinger"); + sp<IServiceManager> sm = defaultServiceManager(); + mComposer = interface_cast<ISurfaceComposer>(sm->checkService(name)); + } + if (mComposer != NULL) { + DisplayStatInfo stats; + status_t res = mComposer->getDisplayStats(NULL /* display */, &stats); + if (res == OK) { + ALOGV("vsync time:%lld period:%lld", + (long long)stats.vsyncTime, (long long)stats.vsyncPeriod); + mVsyncTime = stats.vsyncTime; + mVsyncPeriod = stats.vsyncPeriod; + } else { + ALOGW("getDisplayStats returned %d", res); + } + } else { + ALOGW("could not get surface mComposer service"); + } +} + +void VideoFrameScheduler::init(float videoFps) { + updateVsync(); + + mLastVsyncTime = -1; + mTimeCorrection = 0; + + mPll.reset(videoFps); +} + +void VideoFrameScheduler::restart() { + mLastVsyncTime = -1; + mTimeCorrection = 0; + + mPll.restart(); +} + +nsecs_t VideoFrameScheduler::getVsyncPeriod() { + if (mVsyncPeriod > 0) { + return mVsyncPeriod; + } + return kDefaultVsyncPeriod; +} + +nsecs_t VideoFrameScheduler::schedule(nsecs_t renderTime) { + nsecs_t origRenderTime = renderTime; + + nsecs_t now = systemTime(SYSTEM_TIME_MONOTONIC); + if (now >= mVsyncRefreshAt) { + updateVsync(); + } + + // without VSYNC info, there is nothing to do + if (mVsyncPeriod == 0) { + ALOGV("no vsync: render=%lld", (long long)renderTime); + return renderTime; + } + + // ensure vsync time is well before (corrected) render time + if (mVsyncTime > renderTime - 4 * mVsyncPeriod) { + mVsyncTime -= + ((mVsyncTime - renderTime) / mVsyncPeriod + 5) * mVsyncPeriod; + } + + // Video presentation takes place at the VSYNC _after_ renderTime. Adjust renderTime + // so this effectively becomes a rounding operation (to the _closest_ VSYNC.) + renderTime -= mVsyncPeriod / 2; + + const nsecs_t videoPeriod = mPll.addSample(origRenderTime); + if (videoPeriod > 0) { + // Smooth out rendering + size_t N = 12; + nsecs_t fiveSixthDev = + abs(((videoPeriod * 5 + mVsyncPeriod) % (mVsyncPeriod * 6)) - mVsyncPeriod) + / (mVsyncPeriod / 100); + // use 20 samples if we are doing 5:6 ratio +- 1% (e.g. playing 50Hz on 60Hz) + if (fiveSixthDev < 12) { /* 12% / 6 = 2% */ + N = 20; + } + + nsecs_t offset = 0; + nsecs_t edgeRemainder = 0; + for (size_t i = 1; i <= N; i++) { + offset += + (renderTime + mTimeCorrection + videoPeriod * i - mVsyncTime) % mVsyncPeriod; + edgeRemainder += (videoPeriod * i) % mVsyncPeriod; + } + mTimeCorrection += mVsyncPeriod / 2 - offset / N; + renderTime += mTimeCorrection; + nsecs_t correctionLimit = mVsyncPeriod * 3 / 5; + edgeRemainder = abs(edgeRemainder / N - mVsyncPeriod / 2); + if (edgeRemainder <= mVsyncPeriod / 3) { + correctionLimit /= 2; + } + + // estimate how many VSYNCs a frame will spend on the display + nsecs_t nextVsyncTime = + renderTime + mVsyncPeriod - ((renderTime - mVsyncTime) % mVsyncPeriod); + if (mLastVsyncTime >= 0) { + size_t minVsyncsPerFrame = videoPeriod / mVsyncPeriod; + size_t vsyncsForLastFrame = divRound(nextVsyncTime - mLastVsyncTime, mVsyncPeriod); + bool vsyncsPerFrameAreNearlyConstant = + periodicError(videoPeriod, mVsyncPeriod) / (mVsyncPeriod / 20) == 0; + + if (mTimeCorrection > correctionLimit && + (vsyncsPerFrameAreNearlyConstant || vsyncsForLastFrame > minVsyncsPerFrame)) { + // remove a VSYNC + mTimeCorrection -= mVsyncPeriod / 2; + renderTime -= mVsyncPeriod / 2; + nextVsyncTime -= mVsyncPeriod; + --vsyncsForLastFrame; + } else if (mTimeCorrection < -correctionLimit && + (vsyncsPerFrameAreNearlyConstant || vsyncsForLastFrame == minVsyncsPerFrame)) { + // add a VSYNC + mTimeCorrection += mVsyncPeriod / 2; + renderTime += mVsyncPeriod / 2; + nextVsyncTime += mVsyncPeriod; + ++vsyncsForLastFrame; + } + ATRACE_INT("FRAME_VSYNCS", vsyncsForLastFrame); + } + mLastVsyncTime = nextVsyncTime; + } + + // align rendertime to the center between VSYNC edges + renderTime -= (renderTime - mVsyncTime) % mVsyncPeriod; + renderTime += mVsyncPeriod / 2; + ALOGV("adjusting render: %lld => %lld", (long long)origRenderTime, (long long)renderTime); + ATRACE_INT("FRAME_FLIP_IN(ms)", (renderTime - now) / 1000000); + return renderTime; +} + +void VideoFrameScheduler::release() { + mComposer.clear(); +} + +VideoFrameScheduler::~VideoFrameScheduler() { + release(); +} + +} // namespace android + diff --git a/media/libmediaplayerservice/VideoFrameScheduler.h b/media/libmediaplayerservice/VideoFrameScheduler.h new file mode 100644 index 0000000..19f0787 --- /dev/null +++ b/media/libmediaplayerservice/VideoFrameScheduler.h @@ -0,0 +1,98 @@ +/* + * Copyright 2014, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef VIDEO_FRAME_SCHEDULER_H_ +#define VIDEO_FRAME_SCHEDULER_H_ + +#include <utils/RefBase.h> +#include <utils/Timers.h> + +#include <media/stagefright/foundation/ABase.h> + +namespace android { + +struct ISurfaceComposer; + +struct VideoFrameScheduler : public RefBase { + VideoFrameScheduler(); + + // (re)initialize scheduler + void init(float videoFps = -1); + // use in case of video render-time discontinuity, e.g. seek + void restart(); + // get adjusted nanotime for a video frame render at renderTime + nsecs_t schedule(nsecs_t renderTime); + + // returns the vsync period for the main display + nsecs_t getVsyncPeriod(); + + void release(); + + static const size_t kHistorySize = 8; + +protected: + virtual ~VideoFrameScheduler(); + +private: + struct PLL { + PLL(); + + // reset PLL to new PLL + void reset(float fps = -1); + // keep current estimate, but restart phase + void restart(); + // returns period + nsecs_t addSample(nsecs_t time); + + private: + nsecs_t mPeriod; + nsecs_t mPhase; + + bool mPrimed; // have an estimate for the period + size_t mSamplesUsedForPriming; + + nsecs_t mLastTime; // last input time + nsecs_t mRefitAt; // next input time to fit at + + size_t mNumSamples; // can go past kHistorySize + nsecs_t mTimes[kHistorySize]; + + void test(); + void fit(nsecs_t phase, nsecs_t period, size_t numSamples, + int64_t *a, int64_t *b, int64_t *err); + void prime(size_t numSamples); + }; + + void updateVsync(); + + nsecs_t mVsyncTime; // vsync timing from display + nsecs_t mVsyncPeriod; + nsecs_t mVsyncRefreshAt; // next time to refresh timing info + + nsecs_t mLastVsyncTime; // estimated vsync time for last frame + nsecs_t mTimeCorrection; // running adjustment + + PLL mPll; // PLL for video frame rate based on render time + + sp<ISurfaceComposer> mComposer; + + DISALLOW_EVIL_CONSTRUCTORS(VideoFrameScheduler); +}; + +} // namespace android + +#endif // VIDEO_FRAME_SCHEDULER_H_ + diff --git a/media/libmediaplayerservice/nuplayer/Android.mk b/media/libmediaplayerservice/nuplayer/Android.mk index 0dd2b61..676c0a6 100644 --- a/media/libmediaplayerservice/nuplayer/Android.mk +++ b/media/libmediaplayerservice/nuplayer/Android.mk @@ -19,6 +19,7 @@ LOCAL_C_INCLUDES := \ $(TOP)/frameworks/av/media/libstagefright/mpeg2ts \ $(TOP)/frameworks/av/media/libstagefright/rtsp \ $(TOP)/frameworks/av/media/libstagefright/timedtext \ + $(TOP)/frameworks/av/media/libmediaplayerservice \ $(TOP)/frameworks/native/include/media/openmax LOCAL_MODULE:= libstagefright_nuplayer diff --git a/media/libmediaplayerservice/nuplayer/GenericSource.cpp b/media/libmediaplayerservice/nuplayer/GenericSource.cpp index 142107d..bd75034 100644 --- a/media/libmediaplayerservice/nuplayer/GenericSource.cpp +++ b/media/libmediaplayerservice/nuplayer/GenericSource.cpp @@ -158,7 +158,12 @@ status_t NuPlayer::GenericSource::initFromDataSource() { int32_t totalBitrate = 0; - for (size_t i = 0; i < extractor->countTracks(); ++i) { + size_t numtracks = extractor->countTracks(); + if (numtracks == 0) { + return UNKNOWN_ERROR; + } + + for (size_t i = 0; i < numtracks; ++i) { sp<MediaSource> track = extractor->getTrack(i); sp<MetaData> meta = extractor->getTrackMetaData(i); @@ -1110,8 +1115,8 @@ sp<ABuffer> NuPlayer::GenericSource::mediaBufferToABuffer( if (mIsWidevine && !audio) { // data is already provided in the buffer ab = new ABuffer(NULL, mb->range_length()); - ab->meta()->setPointer("mediaBuffer", mb); mb->add_ref(); + ab->setMediaBufferBase(mb); } else { ab = new ABuffer(outLength); memcpy(ab->data(), diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp index cdbea9f..2ea12ae 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp @@ -155,6 +155,7 @@ NuPlayer::NuPlayer() mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER), mAudioDecoderGeneration(0), mVideoDecoderGeneration(0), + mRendererGeneration(0), mAudioEOS(false), mVideoEOS(false), mScanSourcesPending(false), @@ -633,16 +634,23 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { flags |= Renderer::FLAG_OFFLOAD_AUDIO; } - mRenderer = new Renderer( - mAudioSink, - new AMessage(kWhatRendererNotify, id()), - flags); + sp<AMessage> notify = new AMessage(kWhatRendererNotify, id()); + ++mRendererGeneration; + notify->setInt32("generation", mRendererGeneration); + mRenderer = new Renderer(mAudioSink, notify, flags); mRendererLooper = new ALooper; mRendererLooper->setName("NuPlayerRenderer"); mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO); mRendererLooper->registerHandler(mRenderer); + sp<MetaData> meta = getFileMeta(); + int32_t rate; + if (meta != NULL + && meta->findInt32(kKeyFrameRate, &rate) && rate > 0) { + mRenderer->setVideoFrameRate(rate); + } + postScanSources(); break; } @@ -806,11 +814,13 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { ALOGV("%s shutdown completed", audio ? "audio" : "video"); if (audio) { mAudioDecoder.clear(); + ++mAudioDecoderGeneration; CHECK_EQ((int)mFlushingAudio, (int)SHUTTING_DOWN_DECODER); mFlushingAudio = SHUT_DOWN; } else { mVideoDecoder.clear(); + ++mVideoDecoderGeneration; CHECK_EQ((int)mFlushingVideo, (int)SHUTTING_DOWN_DECODER); mFlushingVideo = SHUT_DOWN; @@ -828,9 +838,11 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { mRenderer->queueEOS(audio, err); if (audio && mFlushingAudio != NONE) { mAudioDecoder.clear(); + ++mAudioDecoderGeneration; mFlushingAudio = SHUT_DOWN; } else if (!audio && mFlushingVideo != NONE){ mVideoDecoder.clear(); + ++mVideoDecoderGeneration; mFlushingVideo = SHUT_DOWN; } finishFlushIfPossible(); @@ -850,6 +862,14 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { case kWhatRendererNotify: { + int32_t requesterGeneration = mRendererGeneration - 1; + CHECK(msg->findInt32("generation", &requesterGeneration)); + if (requesterGeneration != mRendererGeneration) { + ALOGV("got message from old renderer, generation(%d:%d)", + requesterGeneration, mRendererGeneration); + return; + } + int32_t what; CHECK(msg->findInt32("what", &what)); @@ -912,6 +932,7 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { CHECK(msg->findInt64("positionUs", &positionUs)); closeAudioSink(); mAudioDecoder.clear(); + ++mAudioDecoderGeneration; mRenderer->flush(true /* audio */); if (mVideoDecoder != NULL) { mRenderer->flush(false /* audio */); @@ -963,17 +984,31 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { case kWhatPause: { - CHECK(mRenderer != NULL); - mSource->pause(); - mRenderer->pause(); + if (mSource != NULL) { + mSource->pause(); + } else { + ALOGW("pause called when source is gone or not set"); + } + if (mRenderer != NULL) { + mRenderer->pause(); + } else { + ALOGW("pause called when renderer is gone or not set"); + } break; } case kWhatResume: { - CHECK(mRenderer != NULL); - mSource->resume(); - mRenderer->resume(); + if (mSource != NULL) { + mSource->resume(); + } else { + ALOGW("resume called when source is gone or not set"); + } + if (mRenderer != NULL) { + mRenderer->resume(); + } else { + ALOGW("resume called when renderer is gone or not set"); + } break; } @@ -1842,9 +1877,6 @@ void NuPlayer::performReset() { ++mScanSourcesGeneration; mScanSourcesPending = false; - ++mAudioDecoderGeneration; - ++mVideoDecoderGeneration; - if (mRendererLooper != NULL) { if (mRenderer != NULL) { mRendererLooper->unregisterHandler(mRenderer->id()); @@ -1853,6 +1885,7 @@ void NuPlayer::performReset() { mRendererLooper.clear(); } mRenderer.clear(); + ++mRendererGeneration; if (mSource != NULL) { mSource->stop(); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h index 7197e5f..eee96ca 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h @@ -137,6 +137,7 @@ private: sp<ALooper> mRendererLooper; int32_t mAudioDecoderGeneration; int32_t mVideoDecoderGeneration; + int32_t mRendererGeneration; List<sp<Action> > mDeferredActions; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp index 87f85e7..cdb860c 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp @@ -283,14 +283,9 @@ void android::NuPlayer::Decoder::onInputBufferFilled(const sp<AMessage> &msg) { // handle widevine classic source - that fills an arbitrary input buffer MediaBuffer *mediaBuffer = NULL; - if (hasBuffer && buffer->meta()->findPointer( - "mediaBuffer", (void **)&mediaBuffer)) { - if (mediaBuffer == NULL) { - // received no actual buffer - ALOGW("[%s] received null MediaBuffer %s", - mComponentName.c_str(), msg->debugString().c_str()); - buffer = NULL; - } else { + if (hasBuffer) { + mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase()); + if (mediaBuffer != NULL) { // likely filled another buffer than we requested: adjust buffer index size_t ix; for (ix = 0; ix < mInputBuffers.size(); ix++) { @@ -470,7 +465,9 @@ void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) { size_t bufferIx; CHECK(msg->findSize("buffer-ix", &bufferIx)); if (msg->findInt32("render", &render) && render) { - err = mCodec->renderOutputBufferAndRelease(bufferIx); + int64_t timestampNs; + CHECK(msg->findInt64("timestampNs", ×tampNs)); + err = mCodec->renderOutputBufferAndRelease(bufferIx, timestampNs); } else { err = mCodec->releaseOutputBuffer(bufferIx); } @@ -598,16 +595,6 @@ void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) { { if (!isStaleReply(msg)) { onInputBufferFilled(msg); - } else { - /* release any MediaBuffer passed in the stale buffer */ - sp<ABuffer> buffer; - MediaBuffer *mediaBuffer = NULL; - if (msg->findBuffer("buffer", &buffer) && - buffer->meta()->findPointer( - "mediaBuffer", (void **)&mediaBuffer) && - mediaBuffer != NULL) { - mediaBuffer->release(); - } } break; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp index 7674616..73ac057 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp @@ -26,10 +26,16 @@ #include <media/stagefright/MediaErrors.h> #include <media/stagefright/MetaData.h> +#include <VideoFrameScheduler.h> + #include <inttypes.h> namespace android { +// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink +// is closed to allow the audio DSP to power down. +static const int64_t kOffloadPauseMaxUs = 60000000ll; + // static const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll; @@ -59,7 +65,9 @@ NuPlayer::Renderer::Renderer( mVideoRenderingStartGeneration(0), mAudioRenderingStartGeneration(0), mLastPositionUpdateUs(-1ll), - mVideoLateByUs(0ll) { + mVideoLateByUs(0ll), + mAudioOffloadPauseTimeoutGeneration(0), + mAudioOffloadTornDown(false) { } NuPlayer::Renderer::~Renderer() { @@ -137,6 +145,12 @@ void NuPlayer::Renderer::resume() { (new AMessage(kWhatResume, id()))->post(); } +void NuPlayer::Renderer::setVideoFrameRate(float fps) { + sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, id()); + msg->setFloat("frame-rate", fps); + msg->post(); +} + void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { switch (msg->what()) { case kWhatStopAudioSink: @@ -237,12 +251,31 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatSetVideoFrameRate: + { + float fps; + CHECK(msg->findFloat("frame-rate", &fps)); + onSetVideoFrameRate(fps); + break; + } + case kWhatAudioOffloadTearDown: { onAudioOffloadTearDown(); break; } + case kWhatAudioOffloadPauseTimeout: + { + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + if (generation != mAudioOffloadPauseTimeoutGeneration) { + break; + } + onAudioOffloadTearDown(); + break; + } + default: TRESPASS(); break; @@ -502,16 +535,20 @@ void NuPlayer::Renderer::postDrainVideoQueue() { sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id()); msg->setInt32("generation", mVideoQueueGeneration); - int64_t delayUs; - if (entry.mBuffer == NULL) { // EOS doesn't carry a timestamp. - delayUs = 0; - } else if (mFlags & FLAG_REAL_TIME) { + msg->post(); + mDrainVideoQueuePending = true; + return; + } + + int64_t delayUs; + int64_t nowUs = ALooper::GetNowUs(); + int64_t realTimeUs; + if (mFlags & FLAG_REAL_TIME) { int64_t mediaTimeUs; CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); - - delayUs = mediaTimeUs - ALooper::GetNowUs(); + realTimeUs = mediaTimeUs; } else { int64_t mediaTimeUs; CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); @@ -520,23 +557,26 @@ void NuPlayer::Renderer::postDrainVideoQueue() { mFirstAnchorTimeMediaUs = mediaTimeUs; } if (mAnchorTimeMediaUs < 0) { - delayUs = 0; - if (!mHasAudio) { mAnchorTimeMediaUs = mediaTimeUs; - mAnchorTimeRealUs = ALooper::GetNowUs(); + mAnchorTimeRealUs = nowUs; notifyPosition(); } + realTimeUs = nowUs; } else { - int64_t realTimeUs = + realTimeUs = (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs; - - delayUs = realTimeUs - ALooper::GetNowUs(); } } + realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000; + int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000); + + delayUs = realTimeUs - nowUs; + ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs); - msg->post(delayUs); + // post 2 display refreshes before rendering is due + msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0); mDrainVideoQueuePending = true; } @@ -588,6 +628,7 @@ void NuPlayer::Renderer::onDrainVideoQueue() { mVideoLateByUs = 0ll; } + entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll); entry->mNotifyConsumed->setInt32("render", !tooLate); entry->mNotifyConsumed->post(); mVideoQueue.erase(mVideoQueue.begin()); @@ -630,6 +671,10 @@ void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) { mHasAudio = true; } else { mHasVideo = true; + if (mVideoScheduler == NULL) { + mVideoScheduler = new VideoFrameScheduler(); + mVideoScheduler->init(); + } } if (dropBufferWhileFlushing(audio, msg)) { @@ -795,6 +840,10 @@ void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { mDrainVideoQueuePending = false; ++mVideoQueueGeneration; + if (mVideoScheduler != NULL) { + mVideoScheduler->restart(); + } + prepareForMediaRenderingStart(); } @@ -919,6 +968,7 @@ void NuPlayer::Renderer::onPause() { if (mHasAudio) { mAudioSink->pause(); + startAudioOffloadPauseTimeout(); } ALOGV("now paused audio queue has %d entries, video has %d entries", @@ -931,6 +981,7 @@ void NuPlayer::Renderer::onResume() { } if (mHasAudio) { + cancelAudioOffloadPauseTimeout(); mAudioSink->start(); } @@ -946,6 +997,13 @@ void NuPlayer::Renderer::onResume() { } } +void NuPlayer::Renderer::onSetVideoFrameRate(float fps) { + if (mVideoScheduler == NULL) { + mVideoScheduler = new VideoFrameScheduler(); + } + mVideoScheduler->init(fps); +} + // TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs() // as it acquires locks and may query the audio driver. // @@ -1012,6 +1070,11 @@ int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) { } void NuPlayer::Renderer::onAudioOffloadTearDown() { + if (mAudioOffloadTornDown) { + return; + } + mAudioOffloadTornDown = true; + int64_t firstAudioTimeUs; { Mutex::Autolock autoLock(mLock); @@ -1030,5 +1093,19 @@ void NuPlayer::Renderer::onAudioOffloadTearDown() { notify->post(); } +void NuPlayer::Renderer::startAudioOffloadPauseTimeout() { + if (offloadingAudio()) { + sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, id()); + msg->setInt32("generation", mAudioOffloadPauseTimeoutGeneration); + msg->post(kOffloadPauseMaxUs); + } +} + +void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() { + if (offloadingAudio()) { + ++mAudioOffloadPauseTimeoutGeneration; + } +} + } // namespace android diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h index 97fdae7..8e6112b 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h @@ -23,6 +23,7 @@ namespace android { struct ABuffer; +struct VideoFrameScheduler; struct NuPlayer::Renderer : public AHandler { enum Flags { @@ -56,6 +57,8 @@ struct NuPlayer::Renderer : public AHandler { void pause(); void resume(); + void setVideoFrameRate(float fps); + enum { kWhatEOS = 'eos ', kWhatFlushComplete = 'fluC', @@ -63,6 +66,7 @@ struct NuPlayer::Renderer : public AHandler { kWhatVideoRenderingStart = 'vdrd', kWhatMediaRenderingStart = 'mdrd', kWhatAudioOffloadTearDown = 'aOTD', + kWhatAudioOffloadPauseTimeout = 'aOPT', }; protected: @@ -82,6 +86,7 @@ private: kWhatResume = 'resm', kWhatStopAudioSink = 'stpA', kWhatDisableOffloadAudio = 'noOA', + kWhatSetVideoFrameRate = 'sVFR', }; struct QueueEntry { @@ -100,6 +105,7 @@ private: List<QueueEntry> mAudioQueue; List<QueueEntry> mVideoQueue; uint32_t mNumFramesWritten; + sp<VideoFrameScheduler> mVideoScheduler; bool mDrainAudioQueuePending; bool mDrainVideoQueuePending; @@ -127,6 +133,9 @@ private: int64_t mLastPositionUpdateUs; int64_t mVideoLateByUs; + int32_t mAudioOffloadPauseTimeoutGeneration; + bool mAudioOffloadTornDown; + size_t fillAudioBuffer(void *buffer, size_t size); bool onDrainAudioQueue(); @@ -147,6 +156,7 @@ private: void onDisableOffloadAudio(); void onPause(); void onResume(); + void onSetVideoFrameRate(float fps); void onAudioOffloadTearDown(); void notifyEOS(bool audio, status_t finalResult, int64_t delayUs = 0); @@ -162,6 +172,9 @@ private: bool offloadingAudio() const { return (mFlags & FLAG_OFFLOAD_AUDIO) != 0; } + void startAudioOffloadPauseTimeout(); + void cancelAudioOffloadPauseTimeout(); + DISALLOW_EVIL_CONSTRUCTORS(Renderer); }; diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp index 9b03b71..3c04859 100644 --- a/media/libstagefright/ACodec.cpp +++ b/media/libstagefright/ACodec.cpp @@ -3782,23 +3782,12 @@ bool ACodec::BaseState::onOMXEmptyBufferDone(IOMX::buffer_id bufferID) { CHECK_EQ((int)info->mStatus, (int)BufferInfo::OWNED_BY_COMPONENT); info->mStatus = BufferInfo::OWNED_BY_US; - const sp<AMessage> &bufferMeta = info->mData->meta(); - void *mediaBuffer; - if (bufferMeta->findPointer("mediaBuffer", &mediaBuffer) - && mediaBuffer != NULL) { - // We're in "store-metadata-in-buffers" mode, the underlying - // OMX component had access to data that's implicitly refcounted - // by this "mediaBuffer" object. Now that the OMX component has - // told us that it's done with the input buffer, we can decrement - // the mediaBuffer's reference count. - - ALOGV("releasing mbuf %p", mediaBuffer); - - ((MediaBuffer *)mediaBuffer)->release(); - mediaBuffer = NULL; - - bufferMeta->setPointer("mediaBuffer", NULL); - } + // We're in "store-metadata-in-buffers" mode, the underlying + // OMX component had access to data that's implicitly refcounted + // by this "MediaBuffer" object. Now that the OMX component has + // told us that it's done with the input buffer, we can decrement + // the mediaBuffer's reference count. + info->mData->setMediaBufferBase(NULL); PortMode mode = getPortMode(kPortIndexInput); diff --git a/media/libstagefright/DataSource.cpp b/media/libstagefright/DataSource.cpp index 9d6fd78..a72cbd5 100644 --- a/media/libstagefright/DataSource.cpp +++ b/media/libstagefright/DataSource.cpp @@ -199,7 +199,18 @@ sp<DataSource> DataSource::CreateFromURI( } else if (!strncasecmp("http://", uri, 7) || !strncasecmp("https://", uri, 8) || isWidevine) { - sp<HTTPBase> httpSource = new MediaHTTP(httpService->makeHTTPConnection()); + if (httpService == NULL) { + ALOGE("Invalid http service!"); + return NULL; + } + + sp<IMediaHTTPConnection> conn = httpService->makeHTTPConnection(); + if (conn == NULL) { + ALOGE("Failed to make http connection from http service!"); + return NULL; + } + + sp<HTTPBase> httpSource = new MediaHTTP(conn); String8 tmp; if (isWidevine) { diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp index 0bfc6e4..6c98c52 100644 --- a/media/libstagefright/MediaCodec.cpp +++ b/media/libstagefright/MediaCodec.cpp @@ -179,7 +179,7 @@ void MediaCodec::PostReplyWithError(int32_t replyID, int32_t err) { response->postReply(replyID); } -status_t MediaCodec::init(const char *name, bool nameIsType, bool encoder) { +status_t MediaCodec::init(const AString &name, bool nameIsType, bool encoder) { // save init parameters for reset mInitName = name; mInitNameIsType = nameIsType; @@ -191,7 +191,7 @@ status_t MediaCodec::init(const char *name, bool nameIsType, bool encoder) { // queue. mCodec = new ACodec; bool needDedicatedLooper = false; - if (nameIsType && !strncasecmp(name, "video/", 6)) { + if (nameIsType && !strncasecmp(name.c_str(), "video/", 6)) { needDedicatedLooper = true; } else { AString tmp = name; @@ -357,7 +357,7 @@ status_t MediaCodec::reset() { mHaveInputSurface = false; if (err == OK) { - err = init(mInitName.c_str(), mInitNameIsType, mInitIsEncoder); + err = init(mInitName, mInitNameIsType, mInitIsEncoder); } return err; } diff --git a/media/libstagefright/MediaCodecSource.cpp b/media/libstagefright/MediaCodecSource.cpp index 1a80dcc..27cd231 100644 --- a/media/libstagefright/MediaCodecSource.cpp +++ b/media/libstagefright/MediaCodecSource.cpp @@ -37,19 +37,6 @@ namespace android { -static void ReleaseMediaBufferReference(const sp<ABuffer> &accessUnit) { - void *mbuf; - if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf) - && mbuf != NULL) { - ALOGV("releasing mbuf %p", mbuf); - - accessUnit->meta()->setPointer("mediaBuffer", NULL); - - static_cast<MediaBuffer *>(mbuf)->release(); - mbuf = NULL; - } -} - struct MediaCodecSource::Puller : public AHandler { Puller(const sp<MediaSource> &source); @@ -477,7 +464,7 @@ void MediaCodecSource::releaseEncoder() { for (size_t i = 0; i < mEncoderInputBuffers.size(); ++i) { sp<ABuffer> accessUnit = mEncoderInputBuffers.itemAt(i); - ReleaseMediaBufferReference(accessUnit); + accessUnit->setMediaBufferBase(NULL); } mEncoderInputBuffers.clear(); @@ -608,8 +595,8 @@ status_t MediaCodecSource::feedEncoderInputBuffers() { if (mIsVideo) { // video encoder will release MediaBuffer when done // with underlying data. - mEncoderInputBuffers.itemAt(bufferIndex)->meta() - ->setPointer("mediaBuffer", mbuf); + mEncoderInputBuffers.itemAt(bufferIndex)->setMediaBufferBase( + mbuf); } else { mbuf->release(); } diff --git a/media/libstagefright/codecs/aacdec/SoftAAC2.cpp b/media/libstagefright/codecs/aacdec/SoftAAC2.cpp index 863b908..fb27dca 100644 --- a/media/libstagefright/codecs/aacdec/SoftAAC2.cpp +++ b/media/libstagefright/codecs/aacdec/SoftAAC2.cpp @@ -137,6 +137,7 @@ status_t SoftAAC2::initDecoder() { mOutputDelayRingBuffer = new short[mOutputDelayRingBufferSize]; mOutputDelayRingBufferWritePos = 0; mOutputDelayRingBufferReadPos = 0; + mOutputDelayRingBufferFilled = 0; if (mAACDecoder == NULL) { ALOGE("AAC decoder is null. TODO: Can not call aacDecoder_SetParam in the following code"); @@ -411,6 +412,10 @@ bool SoftAAC2::outputDelayRingBufferPutSamples(INT_PCM *samples, int32_t numSamp if (numSamples == 0) { return true; } + if (outputDelayRingBufferSpaceLeft() < numSamples) { + ALOGE("RING BUFFER WOULD OVERFLOW"); + return false; + } if (mOutputDelayRingBufferWritePos + numSamples <= mOutputDelayRingBufferSize && (mOutputDelayRingBufferReadPos <= mOutputDelayRingBufferWritePos || mOutputDelayRingBufferReadPos > mOutputDelayRingBufferWritePos + numSamples)) { @@ -422,10 +427,6 @@ bool SoftAAC2::outputDelayRingBufferPutSamples(INT_PCM *samples, int32_t numSamp if (mOutputDelayRingBufferWritePos >= mOutputDelayRingBufferSize) { mOutputDelayRingBufferWritePos -= mOutputDelayRingBufferSize; } - if (mOutputDelayRingBufferWritePos == mOutputDelayRingBufferReadPos) { - ALOGE("RING BUFFER OVERFLOW"); - return false; - } } else { ALOGV("slow SoftAAC2::outputDelayRingBufferPutSamples()"); @@ -435,16 +436,19 @@ bool SoftAAC2::outputDelayRingBufferPutSamples(INT_PCM *samples, int32_t numSamp if (mOutputDelayRingBufferWritePos >= mOutputDelayRingBufferSize) { mOutputDelayRingBufferWritePos -= mOutputDelayRingBufferSize; } - if (mOutputDelayRingBufferWritePos == mOutputDelayRingBufferReadPos) { - ALOGE("RING BUFFER OVERFLOW"); - return false; - } } } + mOutputDelayRingBufferFilled += numSamples; return true; } int32_t SoftAAC2::outputDelayRingBufferGetSamples(INT_PCM *samples, int32_t numSamples) { + + if (numSamples > mOutputDelayRingBufferFilled) { + ALOGE("RING BUFFER WOULD UNDERRUN"); + return -1; + } + if (mOutputDelayRingBufferReadPos + numSamples <= mOutputDelayRingBufferSize && (mOutputDelayRingBufferWritePos < mOutputDelayRingBufferReadPos || mOutputDelayRingBufferWritePos >= mOutputDelayRingBufferReadPos + numSamples)) { @@ -463,10 +467,6 @@ int32_t SoftAAC2::outputDelayRingBufferGetSamples(INT_PCM *samples, int32_t numS ALOGV("slow SoftAAC2::outputDelayRingBufferGetSamples()"); for (int32_t i = 0; i < numSamples; i++) { - if (mOutputDelayRingBufferWritePos == mOutputDelayRingBufferReadPos) { - ALOGE("RING BUFFER UNDERRUN"); - return -1; - } if (samples != 0) { samples[i] = mOutputDelayRingBuffer[mOutputDelayRingBufferReadPos]; } @@ -476,22 +476,15 @@ int32_t SoftAAC2::outputDelayRingBufferGetSamples(INT_PCM *samples, int32_t numS } } } + mOutputDelayRingBufferFilled -= numSamples; return numSamples; } int32_t SoftAAC2::outputDelayRingBufferSamplesAvailable() { - int32_t available = mOutputDelayRingBufferWritePos - mOutputDelayRingBufferReadPos; - if (available < 0) { - available += mOutputDelayRingBufferSize; - } - if (available < 0) { - ALOGE("FATAL RING BUFFER ERROR"); - return 0; - } - return available; + return mOutputDelayRingBufferFilled; } -int32_t SoftAAC2::outputDelayRingBufferSamplesLeft() { +int32_t SoftAAC2::outputDelayRingBufferSpaceLeft() { return mOutputDelayRingBufferSize - outputDelayRingBufferSamplesAvailable(); } @@ -658,7 +651,7 @@ void SoftAAC2::onQueueFilled(OMX_U32 /* portIndex */) { AAC_DECODER_ERROR decoderErr; do { - if (outputDelayRingBufferSamplesLeft() < + if (outputDelayRingBufferSpaceLeft() < (mStreamInfo->frameSize * mStreamInfo->numChannels)) { ALOGV("skipping decode: not enough space left in ringbuffer"); break; @@ -1032,6 +1025,7 @@ void SoftAAC2::onReset() { mOutputDelayCompensated = 0; mOutputDelayRingBufferWritePos = 0; mOutputDelayRingBufferReadPos = 0; + mOutputDelayRingBufferFilled = 0; mEndOfInput = false; mEndOfOutput = false; mBufferTimestamps.clear(); diff --git a/media/libstagefright/codecs/aacdec/SoftAAC2.h b/media/libstagefright/codecs/aacdec/SoftAAC2.h index 9fcb598..c3e4459 100644 --- a/media/libstagefright/codecs/aacdec/SoftAAC2.h +++ b/media/libstagefright/codecs/aacdec/SoftAAC2.h @@ -85,10 +85,11 @@ private: short *mOutputDelayRingBuffer; int32_t mOutputDelayRingBufferWritePos; int32_t mOutputDelayRingBufferReadPos; + int32_t mOutputDelayRingBufferFilled; bool outputDelayRingBufferPutSamples(INT_PCM *samples, int numSamples); int32_t outputDelayRingBufferGetSamples(INT_PCM *samples, int numSamples); int32_t outputDelayRingBufferSamplesAvailable(); - int32_t outputDelayRingBufferSamplesLeft(); + int32_t outputDelayRingBufferSpaceLeft(); DISALLOW_EVIL_CONSTRUCTORS(SoftAAC2); }; diff --git a/media/libstagefright/foundation/ABuffer.cpp b/media/libstagefright/foundation/ABuffer.cpp index c93c7e8..b214870 100644 --- a/media/libstagefright/foundation/ABuffer.cpp +++ b/media/libstagefright/foundation/ABuffer.cpp @@ -19,11 +19,13 @@ #include "ADebug.h" #include "ALooper.h" #include "AMessage.h" +#include "MediaBufferBase.h" namespace android { ABuffer::ABuffer(size_t capacity) - : mData(malloc(capacity)), + : mMediaBufferBase(NULL), + mData(malloc(capacity)), mCapacity(capacity), mRangeOffset(0), mRangeLength(capacity), @@ -32,7 +34,8 @@ ABuffer::ABuffer(size_t capacity) } ABuffer::ABuffer(void *data, size_t capacity) - : mData(data), + : mMediaBufferBase(NULL), + mData(data), mCapacity(capacity), mRangeOffset(0), mRangeLength(capacity), @@ -59,6 +62,8 @@ ABuffer::~ABuffer() { if (mFarewell != NULL) { mFarewell->post(); } + + setMediaBufferBase(NULL); } void ABuffer::setRange(size_t offset, size_t size) { @@ -80,5 +85,19 @@ sp<AMessage> ABuffer::meta() { return mMeta; } +MediaBufferBase *ABuffer::getMediaBufferBase() { + if (mMediaBufferBase != NULL) { + mMediaBufferBase->add_ref(); + } + return mMediaBufferBase; +} + +void ABuffer::setMediaBufferBase(MediaBufferBase *mediaBuffer) { + if (mMediaBufferBase != NULL) { + mMediaBufferBase->release(); + } + mMediaBufferBase = mediaBuffer; +} + } // namespace android diff --git a/media/libstagefright/httplive/LiveSession.cpp b/media/libstagefright/httplive/LiveSession.cpp index b465566..a289637 100644 --- a/media/libstagefright/httplive/LiveSession.cpp +++ b/media/libstagefright/httplive/LiveSession.cpp @@ -63,6 +63,7 @@ LiveSession::LiveSession( mSwapMask(0), mCheckBandwidthGeneration(0), mSwitchGeneration(0), + mSubtitleGeneration(0), mLastDequeuedTimeUs(0ll), mRealTimeBaseUs(0ll), mReconfigurationInProgress(false), @@ -289,6 +290,11 @@ status_t LiveSession::dequeueAccessUnit( mLastDequeuedTimeUs = timeUs; mRealTimeBaseUs = ALooper::GetNowUs() - timeUs; } else if (stream == STREAMTYPE_SUBTITLES) { + int32_t subtitleGeneration; + if ((*accessUnit)->meta()->findInt32("subtitleGeneration", &subtitleGeneration) + && subtitleGeneration != mSubtitleGeneration) { + return -EAGAIN; + }; (*accessUnit)->meta()->setInt32( "trackIndex", mPlaylist->getSelectedIndex()); (*accessUnit)->meta()->setInt64("baseUs", mRealTimeBaseUs); @@ -759,7 +765,7 @@ sp<PlaylistFetcher> LiveSession::addFetcher(const char *uri) { notify->setInt32("switchGeneration", mSwitchGeneration); FetcherInfo info; - info.mFetcher = new PlaylistFetcher(notify, this, uri); + info.mFetcher = new PlaylistFetcher(notify, this, uri, mSubtitleGeneration); info.mDurationUs = -1ll; info.mIsPrepared = false; info.mToBeRemoved = false; @@ -1065,6 +1071,24 @@ size_t LiveSession::getBandwidthIndex() { return index; } +int64_t LiveSession::latestMediaSegmentStartTimeUs() { + sp<AMessage> audioMeta = mPacketSources.valueFor(STREAMTYPE_AUDIO)->getLatestDequeuedMeta(); + int64_t minSegmentStartTimeUs = -1, videoSegmentStartTimeUs = -1; + if (audioMeta != NULL) { + audioMeta->findInt64("segmentStartTimeUs", &minSegmentStartTimeUs); + } + + sp<AMessage> videoMeta = mPacketSources.valueFor(STREAMTYPE_VIDEO)->getLatestDequeuedMeta(); + if (videoMeta != NULL + && videoMeta->findInt64("segmentStartTimeUs", &videoSegmentStartTimeUs)) { + if (minSegmentStartTimeUs < 0 || videoSegmentStartTimeUs < minSegmentStartTimeUs) { + minSegmentStartTimeUs = videoSegmentStartTimeUs; + } + + } + return minSegmentStartTimeUs; +} + status_t LiveSession::onSeek(const sp<AMessage> &msg) { int64_t timeUs; CHECK(msg->findInt64("timeUs", &timeUs)); @@ -1117,6 +1141,11 @@ sp<AMessage> LiveSession::getTrackInfo(size_t trackIndex) const { } status_t LiveSession::selectTrack(size_t index, bool select) { + if (mPlaylist == NULL) { + return INVALID_OPERATION; + } + + ++mSubtitleGeneration; status_t err = mPlaylist->selectTrack(index, select); if (err == OK) { sp<AMessage> msg = new AMessage(kWhatChangeConfiguration, id()); @@ -1399,6 +1428,10 @@ void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { int32_t discontinuitySeq = -1; sp<AnotherPacketSource> sources[kMaxStreams]; + if (i == kSubtitleIndex) { + segmentStartTimeUs = latestMediaSegmentStartTimeUs(); + } + // TRICKY: looping from i as earlier streams are already removed from streamMask for (size_t j = i; j < kMaxStreams; ++j) { const AString &streamUri = switching ? mStreams[j].mNewUri : mStreams[j].mUri; diff --git a/media/libstagefright/httplive/LiveSession.h b/media/libstagefright/httplive/LiveSession.h index 6be86cf..7aacca6 100644 --- a/media/libstagefright/httplive/LiveSession.h +++ b/media/libstagefright/httplive/LiveSession.h @@ -189,6 +189,7 @@ private: int32_t mCheckBandwidthGeneration; int32_t mSwitchGeneration; + int32_t mSubtitleGeneration; size_t mContinuationCounter; sp<AMessage> mContinuation; @@ -240,6 +241,7 @@ private: const char *url, uint8_t *curPlaylistHash, bool *unchanged); size_t getBandwidthIndex(); + int64_t latestMediaSegmentStartTimeUs(); static int SortByBandwidth(const BandwidthItem *, const BandwidthItem *); static StreamType indexToType(int idx); diff --git a/media/libstagefright/httplive/PlaylistFetcher.cpp b/media/libstagefright/httplive/PlaylistFetcher.cpp index f78f8b4..30fa868 100644 --- a/media/libstagefright/httplive/PlaylistFetcher.cpp +++ b/media/libstagefright/httplive/PlaylistFetcher.cpp @@ -55,7 +55,8 @@ const int32_t PlaylistFetcher::kNumSkipFrames = 10; PlaylistFetcher::PlaylistFetcher( const sp<AMessage> ¬ify, const sp<LiveSession> &session, - const char *uri) + const char *uri, + int32_t subtitleGeneration) : mNotify(notify), mStartTimeUsNotify(notify->dup()), mSession(session), @@ -73,6 +74,7 @@ PlaylistFetcher::PlaylistFetcher( mPrepared(false), mNextPTSTimeUs(-1ll), mMonitorQueueGeneration(0), + mSubtitleGeneration(subtitleGeneration), mRefreshState(INITIAL_MINIMUM_RELOAD_DELAY), mFirstPTSValid(false), mAbsoluteTimeAnchorUs(0ll), @@ -1407,6 +1409,7 @@ status_t PlaylistFetcher::extractAndQueueAccessUnits( buffer->meta()->setInt64("durationUs", durationUs); buffer->meta()->setInt64("segmentStartTimeUs", getSegmentStartTimeUs(mSeqNumber)); buffer->meta()->setInt32("discontinuitySeq", mDiscontinuitySeq); + buffer->meta()->setInt32("subtitleGeneration", mSubtitleGeneration); packetSource->queueAccessUnit(buffer); return OK; diff --git a/media/libstagefright/httplive/PlaylistFetcher.h b/media/libstagefright/httplive/PlaylistFetcher.h index 4b09df1..78c358f 100644 --- a/media/libstagefright/httplive/PlaylistFetcher.h +++ b/media/libstagefright/httplive/PlaylistFetcher.h @@ -49,7 +49,8 @@ struct PlaylistFetcher : public AHandler { PlaylistFetcher( const sp<AMessage> ¬ify, const sp<LiveSession> &session, - const char *uri); + const char *uri, + int32_t subtitleGeneration); sp<DataSource> getDataSource(); @@ -133,6 +134,7 @@ private: int64_t mNextPTSTimeUs; int32_t mMonitorQueueGeneration; + const int32_t mSubtitleGeneration; enum RefreshState { INITIAL_MINIMUM_RELOAD_DELAY, diff --git a/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp b/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp index c74c3e7..a03f6f9 100644 --- a/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp +++ b/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp @@ -254,11 +254,6 @@ void AnotherPacketSource::queueDiscontinuity( int32_t oldDiscontinuityType; if (!oldBuffer->meta()->findInt32( "discontinuity", &oldDiscontinuityType)) { - MediaBuffer *mbuf = NULL; - oldBuffer->meta()->findPointer("mediaBuffer", (void**)&mbuf); - if (mbuf != NULL) { - mbuf->release(); - } it = mBuffers.erase(it); continue; } diff --git a/media/libstagefright/rtsp/MyHandler.h b/media/libstagefright/rtsp/MyHandler.h index f3dfc59..423a420 100644 --- a/media/libstagefright/rtsp/MyHandler.h +++ b/media/libstagefright/rtsp/MyHandler.h @@ -254,7 +254,9 @@ struct MyHandler : public AHandler { static void addSDES(int s, const sp<ABuffer> &buffer) { struct sockaddr_in addr; socklen_t addrSize = sizeof(addr); - CHECK_EQ(0, getsockname(s, (sockaddr *)&addr, &addrSize)); + if (getsockname(s, (sockaddr *)&addr, &addrSize) != 0) { + inet_aton("0.0.0.0", &(addr.sin_addr)); + } uint8_t *data = buffer->data() + buffer->size(); data[0] = 0x80 | 1; diff --git a/media/libstagefright/wifi-display/source/Converter.cpp b/media/libstagefright/wifi-display/source/Converter.cpp index 753b3ec..2834a66 100644 --- a/media/libstagefright/wifi-display/source/Converter.cpp +++ b/media/libstagefright/wifi-display/source/Converter.cpp @@ -74,19 +74,6 @@ Converter::Converter( } } -static void ReleaseMediaBufferReference(const sp<ABuffer> &accessUnit) { - void *mbuf; - if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf) - && mbuf != NULL) { - ALOGV("releasing mbuf %p", mbuf); - - accessUnit->meta()->setPointer("mediaBuffer", NULL); - - static_cast<MediaBuffer *>(mbuf)->release(); - mbuf = NULL; - } -} - void Converter::releaseEncoder() { if (mEncoder == NULL) { return; @@ -95,18 +82,7 @@ void Converter::releaseEncoder() { mEncoder->release(); mEncoder.clear(); - while (!mInputBufferQueue.empty()) { - sp<ABuffer> accessUnit = *mInputBufferQueue.begin(); - mInputBufferQueue.erase(mInputBufferQueue.begin()); - - ReleaseMediaBufferReference(accessUnit); - } - - for (size_t i = 0; i < mEncoderInputBuffers.size(); ++i) { - sp<ABuffer> accessUnit = mEncoderInputBuffers.itemAt(i); - ReleaseMediaBufferReference(accessUnit); - } - + mInputBufferQueue.clear(); mEncoderInputBuffers.clear(); mEncoderOutputBuffers.clear(); } @@ -328,7 +304,7 @@ void Converter::onMessageReceived(const sp<AMessage> &msg) { sp<ABuffer> accessUnit; CHECK(msg->findBuffer("accessUnit", &accessUnit)); - ReleaseMediaBufferReference(accessUnit); + accessUnit->setMediaBufferBase(NULL); } break; } @@ -351,15 +327,16 @@ void Converter::onMessageReceived(const sp<AMessage> &msg) { ALOGI("dropping frame."); } - ReleaseMediaBufferReference(accessUnit); + accessUnit->setMediaBufferBase(NULL); break; } #if 0 - void *mbuf; - if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf) - && mbuf != NULL) { + MediaBuffer *mbuf = + (MediaBuffer *)(accessUnit->getMediaBufferBase()); + if (mbuf != NULL) { ALOGI("queueing mbuf %p", mbuf); + mbuf->release(); } #endif @@ -647,13 +624,13 @@ status_t Converter::feedEncoderInputBuffers() { buffer->data(), buffer->size()); - void *mediaBuffer; - if (buffer->meta()->findPointer("mediaBuffer", &mediaBuffer) - && mediaBuffer != NULL) { - mEncoderInputBuffers.itemAt(bufferIndex)->meta() - ->setPointer("mediaBuffer", mediaBuffer); + MediaBuffer *mediaBuffer = + (MediaBuffer *)(buffer->getMediaBufferBase()); + if (mediaBuffer != NULL) { + mEncoderInputBuffers.itemAt(bufferIndex)->setMediaBufferBase( + mediaBuffer); - buffer->meta()->setPointer("mediaBuffer", NULL); + buffer->setMediaBufferBase(NULL); } } else { flags = MediaCodec::BUFFER_FLAG_EOS; diff --git a/media/libstagefright/wifi-display/source/MediaPuller.cpp b/media/libstagefright/wifi-display/source/MediaPuller.cpp index 7e8891d..86b918f 100644 --- a/media/libstagefright/wifi-display/source/MediaPuller.cpp +++ b/media/libstagefright/wifi-display/source/MediaPuller.cpp @@ -179,7 +179,7 @@ void MediaPuller::onMessageReceived(const sp<AMessage> &msg) { } else { // video encoder will release MediaBuffer when done // with underlying data. - accessUnit->meta()->setPointer("mediaBuffer", mbuf); + accessUnit->setMediaBufferBase(mbuf); } sp<AMessage> notify = mNotify->dup(); |