diff options
Diffstat (limited to 'media/libmediaplayerservice/nuplayer')
21 files changed, 647 insertions, 141 deletions
diff --git a/media/libmediaplayerservice/nuplayer/Android.mk b/media/libmediaplayerservice/nuplayer/Android.mk index cd20837..ff2a202 100644 --- a/media/libmediaplayerservice/nuplayer/Android.mk +++ b/media/libmediaplayerservice/nuplayer/Android.mk @@ -23,15 +23,25 @@ LOCAL_C_INCLUDES := \ $(TOP)/frameworks/av/media/libstagefright/rtsp \ $(TOP)/frameworks/av/media/libstagefright/timedtext \ $(TOP)/frameworks/av/media/libmediaplayerservice \ - $(TOP)/frameworks/native/include/media/openmax + $(TOP)/frameworks/native/include/media/openmax \ + $(TOP)/frameworks/av/media/libavextensions \ + $(TOP)/frameworks/av/include/media \ -LOCAL_CFLAGS += -Werror -Wall +LOCAL_CFLAGS += -Werror -Wall #-DLOG_NDEBUG=0 # enable experiments only in userdebug and eng builds ifneq (,$(filter userdebug eng,$(TARGET_BUILD_VARIANT))) LOCAL_CFLAGS += -DENABLE_STAGEFRIGHT_EXPERIMENTS endif +ifeq ($(TARGET_BOARD_PLATFORM),msm8974) +LOCAL_CFLAGS += -DTARGET_8974 +endif + +ifeq ($(TARGET_NUPLAYER_CANNOT_SET_SURFACE_WITHOUT_A_FLUSH),true) +LOCAL_CFLAGS += -DCANNOT_SET_SURFACE_WITHOUT_A_FLUSH +endif + LOCAL_CLANG := true LOCAL_MODULE:= libstagefright_nuplayer diff --git a/media/libmediaplayerservice/nuplayer/GenericSource.cpp b/media/libmediaplayerservice/nuplayer/GenericSource.cpp index 0598254..949c12f 100644 --- a/media/libmediaplayerservice/nuplayer/GenericSource.cpp +++ b/media/libmediaplayerservice/nuplayer/GenericSource.cpp @@ -37,6 +37,7 @@ #include "../../libstagefright/include/NuCachedSource2.h" #include "../../libstagefright/include/WVMExtractor.h" #include "../../libstagefright/include/HTTPBase.h" +#include "mediaplayerservice/AVNuExtensions.h" namespace android { @@ -60,6 +61,7 @@ NuPlayer::GenericSource::GenericSource( mAudioIsVorbis(false), mIsWidevine(false), mIsSecure(false), + mUseSetBuffers(false), mIsStreaming(false), mUIDValid(uidValid), mUID(uid), @@ -126,6 +128,7 @@ status_t NuPlayer::GenericSource::setDataSource( status_t NuPlayer::GenericSource::setDataSource(const sp<DataSource>& source) { resetDataSource(); + Mutex::Autolock _l(mSourceLock); mDataSource = source; return OK; } @@ -138,14 +141,14 @@ status_t NuPlayer::GenericSource::initFromDataSource() { sp<MediaExtractor> extractor; String8 mimeType; float confidence; - sp<AMessage> dummy; + sp<AMessage> meta; bool isWidevineStreaming = false; CHECK(mDataSource != NULL); if (mIsWidevine) { isWidevineStreaming = SniffWVM( - mDataSource, &mimeType, &confidence, &dummy); + mDataSource, &mimeType, &confidence, &meta); if (!isWidevineStreaming || strcasecmp( mimeType.string(), MEDIA_MIMETYPE_CONTAINER_WVM)) { @@ -153,7 +156,12 @@ status_t NuPlayer::GenericSource::initFromDataSource() { return UNKNOWN_ERROR; } } else if (mIsStreaming) { - if (!mDataSource->sniff(&mimeType, &confidence, &dummy)) { + sp<DataSource> dataSource; + { + Mutex::Autolock _l(mSourceLock); + dataSource = mDataSource; + } + if (!dataSource->sniff(&mimeType, &confidence, &meta)) { return UNKNOWN_ERROR; } isWidevineStreaming = !strcasecmp( @@ -171,8 +179,14 @@ status_t NuPlayer::GenericSource::initFromDataSource() { } extractor = mWVMExtractor; } else { +#ifndef TARGET_8974 + int32_t flags = AVNuUtils::get()->getFlags(); +#else + int32_t flags = 0; +#endif extractor = MediaExtractor::Create(mDataSource, - mimeType.isEmpty() ? NULL : mimeType.string()); + mimeType.isEmpty() ? NULL : mimeType.string(), + mIsStreaming ? 0 : flags, &meta); } if (extractor == NULL) { @@ -202,6 +216,13 @@ status_t NuPlayer::GenericSource::initFromDataSource() { } } +#ifndef TARGET_8974 + if (AVNuUtils::get()->canUseSetBuffers(mFileMeta)) { + mUseSetBuffers = true; + ALOGI("setBuffers mode enabled"); + } +#endif + int32_t totalBitrate = 0; size_t numtracks = extractor->countTracks(); @@ -324,7 +345,7 @@ int64_t NuPlayer::GenericSource::getLastReadPosition() { status_t NuPlayer::GenericSource::setBuffers( bool audio, Vector<MediaBuffer *> &buffers) { - if (mIsWidevine && !audio && mVideoTrack.mSource != NULL) { + if (mIsSecure && !audio && mVideoTrack.mSource != NULL) { return mVideoTrack.mSource->setBuffers(buffers); } return INVALID_OPERATION; @@ -346,7 +367,7 @@ void NuPlayer::GenericSource::prepareAsync() { if (mLooper == NULL) { mLooper = new ALooper; mLooper->setName("generic"); - mLooper->start(); + mLooper->start(false, false, PRIORITY_AUDIO); mLooper->registerHandler(this); } @@ -378,13 +399,20 @@ void NuPlayer::GenericSource::onPrepareAsync() { } } - mDataSource = DataSource::CreateFromURI( + sp<DataSource> dataSource; + dataSource = DataSource::CreateFromURI( mHTTPService, uri, &mUriHeaders, &contentType, - static_cast<HTTPBase *>(mHttpSource.get())); + static_cast<HTTPBase *>(mHttpSource.get()), + true /*use extended cache*/); + Mutex::Autolock _l(mSourceLock); + mDataSource = dataSource; } else { mIsWidevine = false; - mDataSource = new FileSource(mFd, mOffset, mLength); + sp<DataSource> dataSource; + dataSource = new FileSource(mFd, mOffset, mLength); + Mutex::Autolock _l(mSourceLock); + mDataSource = dataSource; mFd = -1; } @@ -433,7 +461,8 @@ void NuPlayer::GenericSource::onPrepareAsync() { | FLAG_CAN_PAUSE | FLAG_CAN_SEEK_BACKWARD | FLAG_CAN_SEEK_FORWARD - | FLAG_CAN_SEEK); + | FLAG_CAN_SEEK + | (mUseSetBuffers ? FLAG_USE_SET_BUFFERS : 0)); if (mIsSecure) { // secure decoders must be instantiated before starting widevine source @@ -1030,7 +1059,8 @@ status_t NuPlayer::GenericSource::dequeueAccessUnit( // start pulling in more buffers if we only have one (or no) buffer left // so that decoder has less chance of being starved - if (track->mPackets->getAvailableBufferCount(&finalResult) < 2) { + if ((track->mPackets->getAvailableBufferCount(&finalResult) < 2) + && !mUseSetBuffers) { postReadBuffer(audio? MEDIA_TRACK_TYPE_AUDIO : MEDIA_TRACK_TYPE_VIDEO); } @@ -1374,7 +1404,7 @@ sp<ABuffer> NuPlayer::GenericSource::mediaBufferToABuffer( } sp<ABuffer> ab; - if (mIsSecure && !audio) { + if ((mIsSecure || mUseSetBuffers) && !audio) { // data is already provided in the buffer ab = new ABuffer(NULL, mb->range_length()); mb->add_ref(); @@ -1489,7 +1519,9 @@ void NuPlayer::GenericSource::readBuffer( break; case MEDIA_TRACK_TYPE_AUDIO: track = &mAudioTrack; - if (mIsWidevine) { + if (mHttpSource != NULL && getTrackCount() == 1) { + maxBuffers = 16; + } else if (mIsWidevine || (mHttpSource != NULL)) { maxBuffers = 8; } else { maxBuffers = 64; @@ -1520,9 +1552,10 @@ void NuPlayer::GenericSource::readBuffer( if (seekTimeUs >= 0) { options.setSeekTo(seekTimeUs, MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC); seeking = true; + track->mPackets->clear(); } - if (mIsWidevine) { + if (mIsWidevine || mUseSetBuffers) { options.setNonBlocking(); } @@ -1544,7 +1577,8 @@ void NuPlayer::GenericSource::readBuffer( queueDiscontinuityIfNeeded(seeking, formatChange, trackType, track); sp<ABuffer> buffer = mediaBufferToABuffer( - mbuf, trackType, seekTimeUs, actualTimeUs); + mbuf, trackType, seekTimeUs, + numBuffers == 0 ? actualTimeUs : NULL); track->mPackets->queueAccessUnit(buffer); formatChange = false; seeking = false; diff --git a/media/libmediaplayerservice/nuplayer/GenericSource.h b/media/libmediaplayerservice/nuplayer/GenericSource.h index ac980ef..5deb61e 100644 --- a/media/libmediaplayerservice/nuplayer/GenericSource.h +++ b/media/libmediaplayerservice/nuplayer/GenericSource.h @@ -84,7 +84,7 @@ protected: virtual sp<MetaData> getFormatMeta(bool audio); -private: +protected: enum { kWhatPrepareAsync, kWhatFetchSubtitleData, @@ -126,6 +126,7 @@ private: bool mAudioIsVorbis; bool mIsWidevine; bool mIsSecure; + bool mUseSetBuffers; bool mIsStreaming; bool mUIDValid; uid_t mUID; @@ -136,6 +137,7 @@ private: int64_t mOffset; int64_t mLength; + Mutex mSourceLock; sp<DataSource> mDataSource; sp<NuCachedSource2> mCachedSource; sp<DataSource> mHttpSource; @@ -164,7 +166,7 @@ private: int64_t getLastReadPosition(); void setDrmPlaybackStatusIfNeeded(int playbackStatus, int64_t position); - void notifyPreparedAndCleanup(status_t err); + virtual void notifyPreparedAndCleanup(status_t err); void onSecureDecodersInstantiated(status_t err); void finishPrepareAsync(); status_t startSources(); @@ -181,7 +183,7 @@ private: void onSeek(sp<AMessage> msg); status_t doSeek(int64_t seekTimeUs); - void onPrepareAsync(); + virtual void onPrepareAsync(); void fetchTextData( uint32_t what, media_track_type type, diff --git a/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp b/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp index 126625a..a57fdc1 100644 --- a/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp +++ b/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp @@ -30,6 +30,8 @@ #include <media/stagefright/MediaErrors.h> #include <media/stagefright/MetaData.h> #include <media/stagefright/MediaDefs.h> +#include <media/stagefright/Utils.h> + namespace android { @@ -118,6 +120,19 @@ sp<AMessage> NuPlayer::HTTPLiveSource::getFormat(bool audio) { return format; } +sp<MetaData> NuPlayer::HTTPLiveSource::getFormatMeta(bool audio) { + sp<AMessage> format = getFormat(audio); + + if (format == NULL) { + return NULL; + } + + sp<MetaData> meta = new MetaData; + convertMessageToMetaData(format, meta); + return meta; +} + + status_t NuPlayer::HTTPLiveSource::feedMoreTSData() { return OK; } @@ -197,7 +212,11 @@ status_t NuPlayer::HTTPLiveSource::selectTrack(size_t trackIndex, bool select, i } status_t NuPlayer::HTTPLiveSource::seekTo(int64_t seekTimeUs) { - return mLiveSession->seekTo(seekTimeUs); + if (mLiveSession->isSeekable()) { + return mLiveSession->seekTo(seekTimeUs); + } else { + return INVALID_OPERATION; + } } void NuPlayer::HTTPLiveSource::pollForRawData( diff --git a/media/libmediaplayerservice/nuplayer/HTTPLiveSource.h b/media/libmediaplayerservice/nuplayer/HTTPLiveSource.h index 9e0ec2f..388156c 100644 --- a/media/libmediaplayerservice/nuplayer/HTTPLiveSource.h +++ b/media/libmediaplayerservice/nuplayer/HTTPLiveSource.h @@ -39,6 +39,7 @@ struct NuPlayer::HTTPLiveSource : public NuPlayer::Source { virtual status_t dequeueAccessUnit(bool audio, sp<ABuffer> *accessUnit); virtual sp<AMessage> getFormat(bool audio); + virtual sp<MetaData> getFormatMeta(bool audio); virtual status_t feedMoreTSData(); virtual status_t getDuration(int64_t *durationUs); @@ -53,7 +54,6 @@ protected: virtual void onMessageReceived(const sp<AMessage> &msg); -private: enum Flags { // Don't log any URLs. kFlagIncognito = 1, @@ -78,7 +78,7 @@ private: bool mHasMetadata; bool mMetadataSelected; - void onSessionNotify(const sp<AMessage> &msg); + virtual void onSessionNotify(const sp<AMessage> &msg); void pollForRawData( const sp<AMessage> &msg, int32_t currentGeneration, LiveSession::StreamType fetchType, int32_t pushWhat); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp index 26532d7..ef2e6ec 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp @@ -56,6 +56,7 @@ #include "ESDS.h" #include <media/stagefright/Utils.h> +#include "mediaplayerservice/AVNuExtensions.h" namespace android { @@ -130,6 +131,23 @@ private: DISALLOW_EVIL_CONSTRUCTORS(FlushDecoderAction); }; +struct NuPlayer::InstantiateDecoderAction : public Action { + InstantiateDecoderAction(bool audio, sp<DecoderBase> *decoder) + : mAudio(audio), + mdecoder(decoder) { + } + + virtual void execute(NuPlayer *player) { + player->instantiateDecoder(mAudio, mdecoder); + } + +private: + bool mAudio; + sp<DecoderBase> *mdecoder; + + DISALLOW_EVIL_CONSTRUCTORS(InstantiateDecoderAction); +}; + struct NuPlayer::PostMessageAction : public Action { PostMessageAction(const sp<AMessage> &msg) : mMessage(msg) { @@ -171,6 +189,7 @@ NuPlayer::NuPlayer(pid_t pid) mPID(pid), mSourceFlags(0), mOffloadAudio(false), + mOffloadDecodedPCM(false), mAudioDecoderGeneration(0), mVideoDecoderGeneration(0), mRendererGeneration(0), @@ -188,6 +207,7 @@ NuPlayer::NuPlayer(pid_t pid) mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT), mVideoFpsHint(-1.f), mStarted(false), + mResetting(false), mSourceStarted(false), mPaused(false), mPausedByClient(false), @@ -216,7 +236,7 @@ void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) { msg->post(); } -static bool IsHTTPLiveURL(const char *url) { +bool NuPlayer::IsHTTPLiveURL(const char *url) { if (!strncasecmp("http://", url, 7) || !strncasecmp("https://", url, 8) || !strncasecmp("file://", url, 7)) { @@ -640,7 +660,10 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { // When mStarted is true, mSource must have been set. if (mSource == NULL || !mStarted || mSource->getFormat(false /* audio */) == NULL // NOTE: mVideoDecoder's mSurface is always non-null - || (mVideoDecoder != NULL && mVideoDecoder->setVideoSurface(surface) == OK)) { +#ifndef CANNOT_SET_SURFACE_WITHOUT_A_FLUSH + || (mVideoDecoder != NULL && mVideoDecoder->setVideoSurface(surface) == OK) +#endif + ) { performSetSurface(surface); break; } @@ -658,10 +681,9 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { // If the video decoder is not set (perhaps audio only in this case) // do not perform a seek as it is not needed. int64_t currentPositionUs = 0; - if (getCurrentPosition(¤tPositionUs) == OK) { - mDeferredActions.push_back( - new SeekAction(currentPositionUs)); - } + getCurrentPosition(¤tPositionUs); + mDeferredActions.push_back( + new SeekAction(currentPositionUs)); } // If there is a new surface texture, instantiate decoders @@ -912,10 +934,21 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { audio ? "audio" : "video", formatChange); if (formatChange) { - mDeferredActions.push_back( - new FlushDecoderAction( + int32_t seamlessChange = 0; + if (msg->findInt32("video-seamlessChange", &seamlessChange) && seamlessChange) { + ALOGE("video decoder seamlessChange in smooth streaming mode, " + "flush the video decoder"); + mDeferredActions.push_back( + new FlushDecoderAction(FLUSH_CMD_NONE, FLUSH_CMD_FLUSH)); + mDeferredActions.push_back(new ResumeDecoderAction(false)); + processDeferredActions(); + break; + } else { + mDeferredActions.push_back( + new FlushDecoderAction( audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE, audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN)); + } } mDeferredActions.push_back( @@ -1098,6 +1131,7 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { int32_t reason; CHECK(msg->findInt32("reason", &reason)); ALOGV("Tear down audio with reason %d.", reason); + mAudioDecoder->pause(); mAudioDecoder.clear(); ++mAudioDecoderGeneration; bool needsToCreateAudioDecoder = true; @@ -1122,6 +1156,7 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { mRenderer->flush( false /* audio */, false /* notifyComplete */); } + mRenderer->signalAudioTearDownComplete(); int64_t positionUs; if (!msg->findInt64("positionUs", &positionUs)) { @@ -1145,10 +1180,17 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { { ALOGV("kWhatReset"); + mResetting = true; + + if (mAudioDecoder != NULL && mFlushingAudio == NONE) { + mDeferredActions.push_back( + new FlushDecoderAction( + FLUSH_CMD_SHUTDOWN /* audio */, + FLUSH_CMD_SHUTDOWN /* video */)); + } + mDeferredActions.push_back( - new FlushDecoderAction( - FLUSH_CMD_SHUTDOWN /* audio */, - FLUSH_CMD_SHUTDOWN /* video */)); + new SimpleAction(&NuPlayer::closeAudioSink)); mDeferredActions.push_back( new SimpleAction(&NuPlayer::performReset)); @@ -1227,7 +1269,8 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { } void NuPlayer::onResume() { - if (!mPaused) { + if (!mPaused || mResetting) { + ALOGD_IF(mResetting, "resetting, onResume discarded"); return; } mPaused = false; @@ -1236,6 +1279,13 @@ void NuPlayer::onResume() { } else { ALOGW("resume called when source is gone or not set"); } + if (mOffloadAudio && !mOffloadDecodedPCM) { + // Resuming after a pause timed out event, check if can continue with offload + sp<AMessage> videoFormat = mSource->getFormat(false /* audio */); + sp<AMessage> format = mSource->getFormat(true /*audio*/); + const bool hasVideo = (videoFormat != NULL); + tryOpenAudioSinkForOffload(format, hasVideo); + } // |mAudioDecoder| may have been released due to the pause timeout, so re-create it if // needed. if (audioDecoderStillNeeded() && mAudioDecoder == NULL) { @@ -1290,6 +1340,7 @@ void NuPlayer::onStart(int64_t startPositionUs) { } mOffloadAudio = false; + mOffloadDecodedPCM = false; mAudioEOS = false; mVideoEOS = false; mStarted = true; @@ -1300,7 +1351,10 @@ void NuPlayer::onStart(int64_t startPositionUs) { flags |= Renderer::FLAG_REAL_TIME; } + ALOGV("onStart"); sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */); + AVNuUtils::get()->setSourcePCMFormat(audioMeta); + audio_stream_type_t streamType = AUDIO_STREAM_MUSIC; if (mAudioSink != NULL) { streamType = mAudioSink->getAudioStreamType(); @@ -1310,6 +1364,10 @@ void NuPlayer::onStart(int64_t startPositionUs) { mOffloadAudio = canOffloadStream(audioMeta, (videoFormat != NULL), mSource->isStreaming(), streamType); + if (!mOffloadAudio && (audioMeta != NULL)) { + mOffloadDecodedPCM = mOffloadAudio = canOffloadDecodedPCMStream(audioMeta, (videoFormat != NULL), mSource->isStreaming(), streamType); + } + if (mOffloadAudio) { flags |= Renderer::FLAG_OFFLOAD_AUDIO; } @@ -1317,7 +1375,7 @@ void NuPlayer::onStart(int64_t startPositionUs) { sp<AMessage> notify = new AMessage(kWhatRendererNotify, this); ++mRendererGeneration; notify->setInt32("generation", mRendererGeneration); - mRenderer = new Renderer(mAudioSink, notify, flags); + mRenderer = AVNuFactory::get()->createRenderer(mAudioSink, notify, flags); mRendererLooper = new ALooper; mRendererLooper->setName("NuPlayerRenderer"); mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO); @@ -1444,15 +1502,22 @@ void NuPlayer::postScanSources() { void NuPlayer::tryOpenAudioSinkForOffload(const sp<AMessage> &format, bool hasVideo) { // Note: This is called early in NuPlayer to determine whether offloading // is possible; otherwise the decoders call the renderer openAudioSink directly. - + sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */); + sp<AMessage> pcmFormat; + if (mOffloadDecodedPCM) { + sp<MetaData> pcm = AVNuUtils::get()->createPCMMetaFromSource(audioMeta); + audioMeta = pcm; + convertMetaDataToMessage(pcm, &pcmFormat); + } status_t err = mRenderer->openAudioSink( - format, true /* offloadOnly */, hasVideo, AUDIO_OUTPUT_FLAG_NONE, &mOffloadAudio); + mOffloadDecodedPCM ? pcmFormat : format, + true /* offloadOnly */, hasVideo, AUDIO_OUTPUT_FLAG_NONE, + &mOffloadAudio, mSource->isStreaming()); if (err != OK) { // Any failure we turn off mOffloadAudio. mOffloadAudio = false; + mOffloadDecodedPCM = false; } else if (mOffloadAudio) { - sp<MetaData> audioMeta = - mSource->getFormatMeta(true /* audio */); sendMetaDataToHal(mAudioSink, audioMeta); } } @@ -1469,6 +1534,7 @@ void NuPlayer::determineAudioModeChange() { if (mRenderer == NULL) { ALOGW("No renderer can be used to determine audio mode. Use non-offload for safety."); mOffloadAudio = false; + mOffloadDecodedPCM = false; return; } @@ -1476,8 +1542,11 @@ void NuPlayer::determineAudioModeChange() { sp<AMessage> videoFormat = mSource->getFormat(false /* audio */); audio_stream_type_t streamType = mAudioSink->getAudioStreamType(); const bool hasVideo = (videoFormat != NULL); - const bool canOffload = canOffloadStream( + bool canOffload = canOffloadStream( audioMeta, hasVideo, mSource->isStreaming(), streamType); + if (!canOffload) { + mOffloadDecodedPCM = canOffload = canOffloadDecodedPCMStream(audioMeta, (videoFormat != NULL), mSource->isStreaming(), streamType); + } if (canOffload) { if (!mOffloadAudio) { mRenderer->signalEnableOffloadAudio(); @@ -1499,7 +1568,10 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<DecoderBase> *decoder) { if (*decoder != NULL || (audio && mFlushingAudio == SHUT_DOWN)) { return OK; } - + if (mSource == NULL) { + ALOGD("%s Ignore instantiate decoder after clearing source", __func__); + return INVALID_OPERATION; + } sp<AMessage> format = mSource->getFormat(audio); if (format == NULL) { @@ -1537,12 +1609,17 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<DecoderBase> *decoder) { notify->setInt32("generation", mAudioDecoderGeneration); determineAudioModeChange(); - if (mOffloadAudio) { + + if (AVNuUtils::get()->isRAWFormat(format)) { + AVNuUtils::get()->setPCMFormat(format, + AVNuUtils::get()->getKeyPCMFormat(mSource->getFormatMeta(true /* audio */))); + } + if (mOffloadAudio && !ifDecodedPCMOffload()) { const bool hasVideo = (mSource->getFormat(false /*audio */) != NULL); format->setInt32("has-video", hasVideo); - *decoder = new DecoderPassThrough(notify, mSource, mRenderer); + *decoder = AVNuFactory::get()->createPassThruDecoder(notify, mSource, mRenderer); } else { - *decoder = new Decoder(notify, mSource, mPID, mRenderer); + *decoder = AVNuFactory::get()->createDecoder(notify, mSource, mPID, mRenderer); } } else { sp<AMessage> notify = new AMessage(kWhatVideoNotify, this); @@ -1567,7 +1644,8 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<DecoderBase> *decoder) { (*decoder)->configure(format); // allocate buffers to decrypt widevine source buffers - if (!audio && (mSourceFlags & Source::FLAG_SECURE)) { + if (!audio && ((mSourceFlags & Source::FLAG_SECURE) || + (mSourceFlags & Source::FLAG_USE_SET_BUFFERS))) { Vector<sp<ABuffer> > inputBufs; CHECK_EQ((*decoder)->getInputBuffers(&inputBufs), (status_t)OK); @@ -1680,6 +1758,18 @@ void NuPlayer::flushDecoder(bool audio, bool needShutdown) { return; } + FlushStatus *state = audio ? &mFlushingAudio : &mFlushingVideo; + + bool inShutdown = *state != NONE && + *state != FLUSHING_DECODER && + *state != FLUSHED; + + // Reject flush if the decoder state is not one of the above + if (inShutdown) { + ALOGI("flush %s called while in shutdown", audio ? "audio" : "video"); + return; + } + // Make sure we don't continue to scan sources until we finish flushing. ++mScanSourcesGeneration; if (mScanSourcesPending) { @@ -1898,9 +1988,6 @@ void NuPlayer::performDecoderFlush(FlushCommand audio, FlushCommand video) { void NuPlayer::performReset() { ALOGV("performReset"); - CHECK(mAudioDecoder == NULL); - CHECK(mVideoDecoder == NULL); - cancelPollDuration(); ++mScanSourcesGeneration; @@ -1930,6 +2017,7 @@ void NuPlayer::performReset() { } mStarted = false; + mResetting = false; mSourceStarted = false; } @@ -2190,7 +2278,7 @@ void NuPlayer::onSourceNotify(const sp<AMessage> &msg) { int posMs; int64_t timeUs, posUs; driver->getCurrentPosition(&posMs); - posUs = posMs * 1000; + posUs = (int64_t) posMs * 1000ll; CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); if (posUs < timeUs) { @@ -2223,6 +2311,13 @@ void NuPlayer::onSourceNotify(const sp<AMessage> &msg) { break; } + case Source::kWhatRTCPByeReceived: + { + ALOGV("notify the client that Bye message is received"); + notifyListener(MEDIA_INFO, 2000, 0); + break; + } + default: TRESPASS(); } @@ -2368,4 +2463,32 @@ void NuPlayer::Source::onMessageReceived(const sp<AMessage> & /* msg */) { TRESPASS(); } +bool NuPlayer::ifDecodedPCMOffload() { + return mOffloadDecodedPCM; +} + +void NuPlayer::setDecodedPcmOffload(bool decodePcmOffload) { + mOffloadDecodedPCM = decodePcmOffload; +} + +bool NuPlayer::canOffloadDecodedPCMStream(const sp<MetaData> audioMeta, + bool hasVideo, bool isStreaming, audio_stream_type_t streamType) { + const char *mime = NULL; + + //For offloading decoded content + if (!mOffloadAudio && (audioMeta != NULL)) { + audioMeta->findCString(kKeyMIMEType, &mime); + sp<MetaData> audioPCMMeta = + AVNuUtils::get()->createPCMMetaFromSource(audioMeta); + + ALOGI("canOffloadDecodedPCMStream"); + audioPCMMeta->dumpToLog(); + mOffloadDecodedPCM = + ((mime && !AVNuUtils::get()->pcmOffloadException(audioMeta)) && + canOffloadStream(audioPCMMeta, hasVideo, isStreaming, streamType)); + ALOGI("PCM offload decided: %d", mOffloadDecodedPCM); + } + return mOffloadDecodedPCM; +} + } // namespace android diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h index c9f0bbd..725a1b2 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h @@ -41,7 +41,7 @@ struct NuPlayer : public AHandler { void setDataSourceAsync(const sp<IStreamSource> &source); - void setDataSourceAsync( + virtual void setDataSourceAsync( const sp<IMediaHTTPService> &httpService, const char *url, const KeyedVector<String8, String8> *headers); @@ -86,12 +86,16 @@ protected: virtual ~NuPlayer(); virtual void onMessageReceived(const sp<AMessage> &msg); + virtual bool ifDecodedPCMOffload(); + virtual void setDecodedPcmOffload(bool decodePcmOffload); + virtual bool canOffloadDecodedPCMStream(const sp<MetaData> meta, + bool hasVideo, bool isStreaming, audio_stream_type_t streamType); + static bool IsHTTPLiveURL(const char *url); public: struct NuPlayerStreamListener; struct Source; -private: struct Decoder; struct DecoderBase; struct DecoderPassThrough; @@ -106,9 +110,11 @@ private: struct SetSurfaceAction; struct ResumeDecoderAction; struct FlushDecoderAction; + struct InstantiateDecoderAction; struct PostMessageAction; struct SimpleAction; +protected: enum { kWhatSetDataSource = '=DaS', kWhatPrepare = 'prep', @@ -146,6 +152,7 @@ private: sp<MediaPlayerBase::AudioSink> mAudioSink; sp<DecoderBase> mVideoDecoder; bool mOffloadAudio; + bool mOffloadDecodedPCM; sp<DecoderBase> mAudioDecoder; sp<CCDecoder> mCCDecoder; sp<Renderer> mRenderer; @@ -197,6 +204,7 @@ private: AVSyncSettings mSyncSettings; float mVideoFpsHint; bool mStarted; + bool mResetting; bool mSourceStarted; // Actual pause state, either as requested by client or due to buffering. @@ -221,11 +229,11 @@ private: mFlushComplete[1][1] = false; } - void tryOpenAudioSinkForOffload(const sp<AMessage> &format, bool hasVideo); + virtual void tryOpenAudioSinkForOffload(const sp<AMessage> &format, bool hasVideo); void closeAudioSink(); void determineAudioModeChange(); - status_t instantiateDecoder(bool audio, sp<DecoderBase> *decoder); + virtual status_t instantiateDecoder(bool audio, sp<DecoderBase> *decoder); status_t onInstantiateSecureDecoders(); @@ -233,13 +241,13 @@ private: const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat = NULL); - void notifyListener(int msg, int ext1, int ext2, const Parcel *in = NULL); + virtual void notifyListener(int msg, int ext1, int ext2, const Parcel *in = NULL); void handleFlushComplete(bool audio, bool isDecoder); void finishFlushIfPossible(); void onStart(int64_t startPositionUs = -1); - void onResume(); + virtual void onResume(); void onPause(); bool audioDecoderStillNeeded(); @@ -256,14 +264,14 @@ private: void processDeferredActions(); - void performSeek(int64_t seekTimeUs); + virtual void performSeek(int64_t seekTimeUs); void performDecoderFlush(FlushCommand audio, FlushCommand video); void performReset(); void performScanSources(); void performSetSurface(const sp<Surface> &wrapper); void performResumeDecoders(bool needNotify); - void onSourceNotify(const sp<AMessage> &msg); + virtual void onSourceNotify(const sp<AMessage> &msg); void onClosedCaptionNotify(const sp<AMessage> &msg); void queueDecoderShutdown( diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.cpp index ac3c6b6..2c07f28 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.cpp @@ -235,6 +235,12 @@ bool NuPlayer::CCDecoder::parseSEINalUnit( payload_size += last_byte; } while (last_byte == 0xFF); + if (payload_size > SIZE_MAX / 8 + || !br.atLeastNumBitsLeft(payload_size * 8)) { + ALOGV("Malformed SEI payload"); + break; + } + // sei_payload() if (payload_type == 4) { bool isCC = false; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp index c005f3f..a18e1da 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp @@ -34,10 +34,14 @@ #include <media/stagefright/MediaDefs.h> #include <media/stagefright/MediaErrors.h> +#include <stagefright/AVExtensions.h> +#include <stagefright/FFMPEGSoftCodec.h> #include <gui/Surface.h> #include "avc_utils.h" #include "ATSParser.h" +#include "mediaplayerservice/AVNuExtensions.h" + namespace android { @@ -69,7 +73,7 @@ NuPlayer::Decoder::Decoder( mIsSecure(false), mFormatChangePending(false), mTimeChangePending(false), - mPaused(true), + mVideoFormatChangeDoFlushOnly(false), mResumePending(false), mComponentName("decoder") { mCodecLooper = new ALooper; @@ -78,7 +82,9 @@ NuPlayer::Decoder::Decoder( } NuPlayer::Decoder::~Decoder() { - mCodec->release(); + if (mCodec != NULL) { + mCodec->release(); + } releaseAndResetMediaBuffers(); } @@ -238,6 +244,7 @@ void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) { mFormatChangePending = false; mTimeChangePending = false; + mVideoFormatChangeDoFlushOnly = false; ++mBufferGeneration; @@ -251,8 +258,17 @@ void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) { mComponentName.append(" decoder"); ALOGV("[%s] onConfigure (surface=%p)", mComponentName.c_str(), mSurface.get()); - mCodec = MediaCodec::CreateByType( - mCodecLooper, mime.c_str(), false /* encoder */, NULL /* err */, mPid); + mCodec = AVUtils::get()->createCustomComponentByName(mCodecLooper, mime.c_str(), false /* encoder */, format); + FFMPEGSoftCodec::overrideComponentName(0, format, &mComponentName, &mime, false); + + if (mCodec == NULL) { + if (!mComponentName.startsWith(mime.c_str())) { + mCodec = MediaCodec::CreateByComponentName(mCodecLooper, mComponentName.c_str()); + } else { + mCodec = MediaCodec::CreateByType(mCodecLooper, mime.c_str(), false /* encoder */); + } + } + int32_t secure = 0; if (format->findInt32("secure", &secure) && secure != 0) { if (mCodec != NULL) { @@ -357,7 +373,14 @@ void NuPlayer::Decoder::onResume(bool notifyComplete) { if (notifyComplete) { mResumePending = true; } - mCodec->start(); + + if (mCodec != NULL) { + mCodec->start(); + } else { + ALOGW("Decoder %s onResume without a valid codec object", + mComponentName.c_str()); + handleError(NO_INIT); + } } void NuPlayer::Decoder::doFlush(bool notifyComplete) { @@ -558,6 +581,11 @@ bool NuPlayer::Decoder::handleAnOutputBuffer( sp<ABuffer> buffer; mCodec->getOutputBuffer(index, &buffer); + if (buffer == NULL) { + handleError(UNKNOWN_ERROR); + return false; + } + if (index >= mOutputBuffers.size()) { for (size_t i = mOutputBuffers.size(); i <= index; ++i) { mOutputBuffers.add(); @@ -569,6 +597,10 @@ bool NuPlayer::Decoder::handleAnOutputBuffer( buffer->setRange(offset, size); buffer->meta()->clear(); buffer->meta()->setInt64("timeUs", timeUs); + setPcmFormat(buffer->meta()); +#ifdef TARGET_8974 + AVNuUtils::get()->addFlagsInMeta(buffer, flags, mIsAudio); +#endif bool eos = flags & MediaCodec::BUFFER_FLAG_EOS; // we do not expect CODECCONFIG or SYNCFRAME for decoder @@ -592,6 +624,12 @@ bool NuPlayer::Decoder::handleAnOutputBuffer( } mSkipRenderingUntilMediaTimeUs = -1; + } else if ((flags & MediaCodec::BUFFER_FLAG_DATACORRUPT) && + AVNuUtils::get()->dropCorruptFrame()) { + ALOGV("[%s] dropping corrupt buffer at time %lld as requested.", + mComponentName.c_str(), (long long)timeUs); + reply->post(); + return true; } mNumFramesTotal += !mIsAudio; @@ -636,7 +674,7 @@ void NuPlayer::Decoder::handleOutputFormatChange(const sp<AMessage> &format) { } status_t err = mRenderer->openAudioSink( - format, false /* offloadOnly */, hasVideo, flags, NULL /* isOffloaed */); + format, false /* offloadOnly */, hasVideo, flags, NULL /* isOffloaed */, mSource->isStreaming()); if (err != OK) { handleError(err); } @@ -711,6 +749,7 @@ status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) { // treat seamless format change separately formatChange = !seamlessFormatChange; } + AVNuUtils::get()->checkFormatChange(&formatChange, accessUnit); // For format or time change, return EOS to queue EOS input, // then wait for EOS on output. @@ -722,9 +761,20 @@ status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) { mTimeChangePending = true; err = ERROR_END_OF_STREAM; } else if (seamlessFormatChange) { - // reuse existing decoder and don't flush - rememberCodecSpecificData(newFormat); - continue; + if (!mIsAudio && + newFormat != NULL && + newFormat->contains("prefer-adaptive-playback")) { + ALOGV("in smooth streaming mode, " + "do video flush in video seamless format change"); + mFormatChangePending = true; + mVideoFormatChangeDoFlushOnly = true; + err = ERROR_END_OF_STREAM; + } else { + // reuse existing decoder and don't flush + rememberCodecSpecificData(newFormat); + continue; + } + } else { // This stream is unaffected by the discontinuity return -EWOULDBLOCK; @@ -952,10 +1002,14 @@ void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) { sp<AMessage> msg = mNotify->dup(); msg->setInt32("what", kWhatInputDiscontinuity); msg->setInt32("formatChange", mFormatChangePending); + if (mVideoFormatChangeDoFlushOnly) { + msg->setInt32("video-seamlessChange", mVideoFormatChangeDoFlushOnly); + } msg->post(); mFormatChangePending = false; mTimeChangePending = false; + mVideoFormatChangeDoFlushOnly = false; } bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange( diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h index eeb4af4..1fbefda 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h @@ -17,9 +17,13 @@ #ifndef NUPLAYER_DECODER_H_ #define NUPLAYER_DECODER_H_ -#include "NuPlayer.h" +#include <media/stagefright/foundation/AMessage.h> +#include "NuPlayer.h" #include "NuPlayerDecoderBase.h" +#include "NuPlayerSource.h" + +#include "mediaplayerservice/AVNuExtensions.h" namespace android { @@ -49,8 +53,9 @@ protected: virtual void onFlush(); virtual void onShutdown(bool notifyComplete); virtual bool doRequestBuffers(); + virtual void setPcmFormat(const sp<AMessage> &format) { format->setInt32("pcm-format", + AVNuUtils::get()->getKeyPCMFormat(mSource->getFormatMeta(true))); } -private: enum { kWhatCodecNotify = 'cdcN', kWhatRenderBuffer = 'rndr', @@ -90,8 +95,8 @@ private: bool mIsSecure; bool mFormatChangePending; bool mTimeChangePending; + bool mVideoFormatChangeDoFlushOnly; - bool mPaused; bool mResumePending; AString mComponentName; @@ -103,7 +108,7 @@ private: size_t size, int64_t timeUs, int32_t flags); - void handleOutputFormatChange(const sp<AMessage> &format); + virtual void handleOutputFormatChange(const sp<AMessage> &format); void releaseAndResetMediaBuffers(); void requestCodecNotification(); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.cpp index 7e76842..04bb61c 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.cpp @@ -31,6 +31,7 @@ namespace android { NuPlayer::DecoderBase::DecoderBase(const sp<AMessage> ¬ify) : mNotify(notify), mBufferGeneration(0), + mPaused(false), mStats(new AMessage), mRequestInputBuffersPending(false) { // Every decoder has its own looper because MediaCodec operations @@ -83,6 +84,13 @@ void NuPlayer::DecoderBase::setRenderer(const sp<Renderer> &renderer) { msg->post(); } +void NuPlayer::DecoderBase::pause() { + sp<AMessage> msg = new AMessage(kWhatPause, this); + + sp<AMessage> response; + PostAndAwaitResponse(msg, &response); +} + status_t NuPlayer::DecoderBase::getInputBuffers(Vector<sp<ABuffer> > *buffers) const { sp<AMessage> msg = new AMessage(kWhatGetInputBuffers, this); msg->setPointer("buffers", buffers); @@ -146,6 +154,17 @@ void NuPlayer::DecoderBase::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatPause: + { + sp<AReplyToken> replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + + mPaused = true; + + (new AMessage)->postReply(replyID); + break; + } + case kWhatGetInputBuffers: { sp<AReplyToken> replyID; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.h index b0dc01d..a334ec5 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.h @@ -36,6 +36,9 @@ struct NuPlayer::DecoderBase : public AHandler { void init(); void setParameters(const sp<AMessage> ¶ms); + // Synchronous call to ensure decoder will not request or send out data. + void pause(); + void setRenderer(const sp<Renderer> &renderer); virtual status_t setVideoSurface(const sp<Surface> &) { return INVALID_OPERATION; } @@ -78,6 +81,7 @@ protected: sp<AMessage> mNotify; int32_t mBufferGeneration; + bool mPaused; sp<AMessage> mStats; private: @@ -85,6 +89,7 @@ private: kWhatConfigure = 'conf', kWhatSetParameters = 'setP', kWhatSetRenderer = 'setR', + kWhatPause = 'paus', kWhatGetInputBuffers = 'gInB', kWhatRequestInputBuffers = 'reqB', kWhatFlush = 'flus', diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp index 30146c4..b8b0505 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp @@ -29,14 +29,12 @@ #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> #include <media/stagefright/MediaErrors.h> +#include "mediaplayerservice/AVNuExtensions.h" #include "ATSParser.h" namespace android { -// TODO optimize buffer size for power consumption -// The offload read buffer size is 32 KB but 24 KB uses less power. -static const size_t kAggregateBufferSizeBytes = 24 * 1024; static const size_t kMaxCachedBytes = 200000; NuPlayer::DecoderPassThrough::DecoderPassThrough( @@ -46,13 +44,16 @@ NuPlayer::DecoderPassThrough::DecoderPassThrough( : DecoderBase(notify), mSource(source), mRenderer(renderer), + // TODO optimize buffer size for power consumption + // The offload read buffer size is 32 KB but 24 KB uses less power. + mAggregateBufferSizeBytes(24 * 1024), mSkipRenderingUntilMediaTimeUs(-1ll), - mPaused(false), mReachedEOS(true), mPendingAudioErr(OK), mPendingBuffersToDrain(0), mCachedBytes(0), - mComponentName("pass through decoder") { + mComponentName("pass through decoder"), + mPCMFormat(AUDIO_FORMAT_INVALID) { ALOGW_IF(renderer == NULL, "expect a non-NULL renderer"); } @@ -74,9 +75,18 @@ void NuPlayer::DecoderPassThrough::onConfigure(const sp<AMessage> &format) { // The audio sink is already opened before the PassThrough decoder is created. // Opening again might be relevant if decoder is instantiated after shutdown and // format is different. + sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */); + if (AVNuUtils::get()->isRAWFormat(audioMeta)) { + mPCMFormat = AVNuUtils::get()->getKeyPCMFormat(audioMeta); + if (mPCMFormat != AUDIO_FORMAT_INVALID) { + AVNuUtils::get()->setPCMFormat(format, mPCMFormat); + AVNuUtils::get()->updateAudioBitWidth(mPCMFormat, format); + } + } + status_t err = mRenderer->openAudioSink( format, true /* offloadOnly */, hasVideo, - AUDIO_OUTPUT_FLAG_NONE /* flags */, NULL /* isOffloaded */); + AUDIO_OUTPUT_FLAG_NONE /* flags */, NULL /* isOffloaded */, mSource->isStreaming()); if (err != OK) { handleError(err); } @@ -173,9 +183,9 @@ sp<ABuffer> NuPlayer::DecoderPassThrough::aggregateBuffer( size_t smallSize = accessUnit->size(); if ((mAggregateBuffer == NULL) // Don't bother if only room for a few small buffers. - && (smallSize < (kAggregateBufferSizeBytes / 3))) { + && (smallSize < (mAggregateBufferSizeBytes / 3))) { // Create a larger buffer for combining smaller buffers from the extractor. - mAggregateBuffer = new ABuffer(kAggregateBufferSizeBytes); + mAggregateBuffer = new ABuffer(mAggregateBufferSizeBytes); mAggregateBuffer->setRange(0, 0); // start empty } @@ -201,6 +211,7 @@ sp<ABuffer> NuPlayer::DecoderPassThrough::aggregateBuffer( if ((bigSize == 0) && smallTimestampValid) { mAggregateBuffer->meta()->setInt64("timeUs", timeUs); } + setPcmFormat(mAggregateBuffer->meta()); // Append small buffer to the bigger buffer. memcpy(mAggregateBuffer->base() + bigSize, accessUnit->data(), smallSize); bigSize += smallSize; @@ -212,6 +223,7 @@ sp<ABuffer> NuPlayer::DecoderPassThrough::aggregateBuffer( } else { // decided not to aggregate aggregate = accessUnit; + setPcmFormat(aggregate->meta()); } return aggregate; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h index db33e87..91da1e1 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h @@ -18,6 +18,8 @@ #define NUPLAYER_DECODER_PASS_THROUGH_H_ +#include <media/stagefright/foundation/AMessage.h> + #include "NuPlayer.h" #include "NuPlayerDecoderBase.h" @@ -43,36 +45,37 @@ protected: virtual void onFlush(); virtual void onShutdown(bool notifyComplete); virtual bool doRequestBuffers(); + virtual void setPcmFormat(const sp<AMessage> &format) { format->setInt32("pcm-format", mPCMFormat); } + virtual sp<ABuffer> aggregateBuffer(const sp<ABuffer> &accessUnit); -private: enum { kWhatBufferConsumed = 'bufC', }; sp<Source> mSource; sp<Renderer> mRenderer; + size_t mAggregateBufferSizeBytes; int64_t mSkipRenderingUntilMediaTimeUs; - bool mPaused; - - bool mReachedEOS; + bool mReachedEOS; // Used by feedDecoderInputData to aggregate small buffers into // one large buffer. + status_t mPendingAudioErr; sp<ABuffer> mPendingAudioAccessUnit; - status_t mPendingAudioErr; sp<ABuffer> mAggregateBuffer; +private: // mPendingBuffersToDrain are only for debugging. It can be removed // when the power investigation is done. size_t mPendingBuffersToDrain; size_t mCachedBytes; AString mComponentName; + audio_format_t mPCMFormat; bool isStaleReply(const sp<AMessage> &msg); bool isDoneFetching() const; status_t dequeueAccessUnit(sp<ABuffer> *accessUnit); - sp<ABuffer> aggregateBuffer(const sp<ABuffer> &accessUnit); status_t fetchInputData(sp<AMessage> &reply); void doFlush(bool notifyComplete); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp index f288c36..4383fce 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp @@ -31,6 +31,9 @@ #include <media/stagefright/MetaData.h> #include <media/stagefright/Utils.h> +#include "mediaplayerservice/AVNuExtensions.h" +#include "mediaplayerservice/AVMediaServiceExtensions.h" + namespace android { NuPlayerDriver::NuPlayerDriver(pid_t pid) @@ -55,7 +58,7 @@ NuPlayerDriver::NuPlayerDriver(pid_t pid) true, /* canCallJava */ PRIORITY_AUDIO); - mPlayer = new NuPlayer(pid); + mPlayer = AVNuFactory::get()->createNuPlayer(pid); mLooper->registerHandler(mPlayer); mPlayer->setDriver(this); @@ -114,6 +117,7 @@ status_t NuPlayerDriver::setDataSource(int fd, int64_t offset, int64_t length) { mCondition.wait(mLock); } + AVNuUtils::get()->printFileName(fd); return mAsyncResult; } @@ -405,6 +409,9 @@ status_t NuPlayerDriver::seekTo(int msec) { { mAtEOS = false; mSeekInProgress = true; + if (mState == STATE_PAUSED) { + mStartupSeekTimeUs = seekTimeUs; + } // seeks can take a while, so we essentially paused notifyListener_l(MEDIA_PAUSED); mPlayer->seekToAsync(seekTimeUs, true /* needNotify */); @@ -607,6 +614,8 @@ status_t NuPlayerDriver::getMetadata( Metadata::kSeekAvailable, mPlayerFlags & NuPlayer::Source::FLAG_CAN_SEEK); + AVMediaServiceUtils::get()->appendMeta(&meta); + return OK; } @@ -741,12 +750,19 @@ void NuPlayerDriver::notifyListener_l( } } if (mLooping || mAutoLoop) { - mPlayer->seekToAsync(0); - if (mAudioSink != NULL) { - // The renderer has stopped the sink at the end in order to play out - // the last little bit of audio. If we're looping, we need to restart it. - mAudioSink->start(); + if (mState == STATE_RUNNING) { + mPlayer->seekToAsync(0); + if (mAudioSink != NULL) { + // The renderer has stopped the sink at the end in order to play out + // the last little bit of audio. If we're looping, we need to restart it. + mAudioSink->start(); + } + } else { + mPlayer->pause(); + mState = STATE_PAUSED; + mAtEOS = true; } + // don't send completion event when looping return; } diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp index 4d25294..8afdefe 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp @@ -26,12 +26,15 @@ #include <media/stagefright/foundation/AUtils.h> #include <media/stagefright/foundation/AWakeLock.h> #include <media/stagefright/MediaClock.h> +#include <media/stagefright/MediaDefs.h> #include <media/stagefright/MediaErrors.h> #include <media/stagefright/MetaData.h> #include <media/stagefright/Utils.h> #include <media/stagefright/VideoFrameScheduler.h> #include <inttypes.h> +#include "mediaplayerservice/AVNuExtensions.h" +#include "stagefright/AVExtensions.h" namespace android { @@ -81,6 +84,16 @@ const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER // static const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll; +static bool sFrameAccurateAVsync = false; + +static void readProperties() { + char value[PROPERTY_VALUE_MAX]; + if (property_get("persist.sys.media.avsync", value, NULL)) { + sFrameAccurateAVsync = + !strcmp("1", value) || !strcasecmp("true", value); + } +} + NuPlayer::Renderer::Renderer( const sp<MediaPlayerBase::AudioSink> &sink, const sp<AMessage> ¬ify, @@ -102,6 +115,7 @@ NuPlayer::Renderer::Renderer( mVideoLateByUs(0ll), mHasAudio(false), mHasVideo(false), + mFoundAudioEOS(false), mNotifyCompleteAudio(false), mNotifyCompleteVideo(false), mSyncQueues(false), @@ -113,7 +127,7 @@ NuPlayer::Renderer::Renderer( mAudioRenderingStartGeneration(0), mRenderingDataDelivered(false), mAudioOffloadPauseTimeoutGeneration(0), - mAudioTornDown(false), + mAudioTearingDown(false), mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER), mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER), mTotalBuffersQueued(0), @@ -123,6 +137,7 @@ NuPlayer::Renderer::Renderer( mMediaClock = new MediaClock; mPlaybackRate = mPlaybackSettings.mSpeed; mMediaClock->setPlaybackRate(mPlaybackRate); + readProperties(); } NuPlayer::Renderer::~Renderer() { @@ -313,7 +328,8 @@ void NuPlayer::Renderer::setVideoFrameRate(float fps) { // Called on any threads. status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) { - return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs); + return mMediaClock->getMediaTime( + ALooper::GetNowUs(), mediaUs, (mHasAudio && mFoundAudioEOS)); } void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() { @@ -349,18 +365,20 @@ status_t NuPlayer::Renderer::openAudioSink( bool offloadOnly, bool hasVideo, uint32_t flags, - bool *isOffloaded) { + bool *isOffloaded, + bool isStreaming) { sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this); msg->setMessage("format", format); msg->setInt32("offload-only", offloadOnly); msg->setInt32("has-video", hasVideo); msg->setInt32("flags", flags); + msg->setInt32("isStreaming", isStreaming); sp<AMessage> response; - msg->postAndAwaitResponse(&response); + status_t postStatus = msg->postAndAwaitResponse(&response); int32_t err; - if (!response->findInt32("err", &err)) { + if (postStatus != OK || !response->findInt32("err", &err)) { err = INVALID_OPERATION; } else if (err == OK && isOffloaded != NULL) { int32_t offload; @@ -393,7 +411,10 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { uint32_t flags; CHECK(msg->findInt32("flags", (int32_t *)&flags)); - status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags); + uint32_t isStreaming; + CHECK(msg->findInt32("isStreaming", (int32_t *)&isStreaming)); + + status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags, isStreaming); sp<AMessage> response = new AMessage; response->setInt32("err", err); @@ -436,9 +457,10 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { if (onDrainAudioQueue()) { uint32_t numFramesPlayed; - CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), - (status_t)OK); - + if (mAudioSink->getPosition(&numFramesPlayed) != OK) { + ALOGW("mAudioSink->getPosition failed"); + break; + } uint32_t numFramesPendingPlayout = mNumFramesWritten - numFramesPlayed; @@ -604,6 +626,12 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatAudioTearDownComplete: + { + onAudioTearDownComplete(); + break; + } + case kWhatAudioOffloadPauseTimeout: { int32_t generation; @@ -758,6 +786,7 @@ size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) { mAudioFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs); // we don't know how much data we are queueing for offloaded tracks. mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX); + mAnchorTimeMediaUs = nowMediaUs; } // for non-offloaded audio, we need to compute the frames written because @@ -814,7 +843,7 @@ void NuPlayer::Renderer::drainAudioQueueUntilLastEOS() { bool NuPlayer::Renderer::onDrainAudioQueue() { // do not drain audio during teardown as queued buffers may be invalid. - if (mAudioTornDown) { + if (mAudioTearingDown) { return false; } // TODO: This call to getPosition checks if AudioTrack has been created @@ -940,7 +969,17 @@ bool NuPlayer::Renderer::onDrainAudioQueue() { // (Case 1) // Must be a multiple of the frame size. If it is not a multiple of a frame size, it // needs to fail, as we should not carry over fractional frames between calls. - CHECK_EQ(copy % mAudioSink->frameSize(), 0); + + if (copy % mAudioSink->frameSize()) { + // CHECK_EQ(copy % mAudioSink->frameSize(), 0); + ALOGE("CHECK_EQ(copy % mAudioSink->frameSize(), 0) failed b/25372978"); + ALOGE("mAudioSink->frameSize() %zu", mAudioSink->frameSize()); + ALOGE("bytes to copy %zu", copy); + ALOGE("entry size %zu, entry offset %zu", entry->mBuffer->size(), + entry->mOffset - written); + notifyEOS(true /*audio*/, UNKNOWN_ERROR); + return false; + } // (Case 2, 3, 4) // Return early to the caller. @@ -1042,6 +1081,9 @@ void NuPlayer::Renderer::postDrainVideoQueue() { mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs); mAnchorTimeMediaUs = mediaTimeUs; realTimeUs = nowUs; + } else if (!mVideoSampleReceived) { + // Always render the first video frame. + realTimeUs = nowUs; } else { realTimeUs = getRealTimeUs(mediaTimeUs, nowUs); } @@ -1078,6 +1120,11 @@ void NuPlayer::Renderer::postDrainVideoQueue() { ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs); // post 2 display refreshes before rendering is due + // FIXME currently this increases power consumption, so unless frame-accurate + // AV sync is requested, post closer to required render time (at 0.63 vsyncs) + if (!sFrameAccurateAVsync) { + twoVsyncsUs >>= 4; + } msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0); mDrainVideoQueuePending = true; @@ -1102,7 +1149,7 @@ void NuPlayer::Renderer::onDrainVideoQueue() { return; } - int64_t nowUs = -1; + int64_t nowUs = ALooper::GetNowUs(); int64_t realTimeUs; if (mFlags & FLAG_REAL_TIME) { CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs)); @@ -1110,16 +1157,12 @@ void NuPlayer::Renderer::onDrainVideoQueue() { int64_t mediaTimeUs; CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); - nowUs = ALooper::GetNowUs(); realTimeUs = getRealTimeUs(mediaTimeUs, nowUs); } bool tooLate = false; if (!mPaused) { - if (nowUs == -1) { - nowUs = ALooper::GetNowUs(); - } setVideoLateByUs(nowUs - realTimeUs); tooLate = (mVideoLateByUs > 40000); @@ -1143,6 +1186,12 @@ void NuPlayer::Renderer::onDrainVideoQueue() { } } + // Always render the first video frame while keeping stats on A/V sync. + if (!mVideoSampleReceived) { + realTimeUs = nowUs; + tooLate = false; + } + entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll); entry->mNotifyConsumed->setInt32("render", !tooLate); entry->mNotifyConsumed->post(); @@ -1168,6 +1217,9 @@ void NuPlayer::Renderer::notifyVideoRenderingStart() { } void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) { + if (audio) { + mFoundAudioEOS = true; + } sp<AMessage> notify = mNotify->dup(); notify->setInt32("what", kWhatEOS); notify->setInt32("audio", static_cast<int32_t>(audio)); @@ -1215,6 +1267,30 @@ void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) { if (audio) { Mutex::Autolock autoLock(mLock); +#if 1 + sp<ABuffer> newBuffer; + status_t err = AVNuUtils::get()->convertToSinkFormatIfNeeded( + buffer, newBuffer, + (offloadingAudio() ? mCurrentOffloadInfo.format : mCurrentPcmInfo.mFormat), + offloadingAudio()); + switch (err) { + case NO_INIT: + // passthru decoder pushes some buffers before the audio sink + // is opened. Since the offload format is known only when the sink + // is opened, pcm conversions cannot take place. So, retry. + ALOGI("init pending, retrying in 10ms, this shouldn't happen"); + msg->post(10000LL); + return; + case OK: + break; + default: + ALOGW("error 0x%x in converting to sink format, drop buffer", err); + notifyConsumed->post(); + return; + } + CHECK(newBuffer != NULL); + entry.mBuffer = newBuffer; +#endif mAudioQueue.push_back(entry); postDrainAudioQueue_l(); } else { @@ -1483,6 +1559,7 @@ void NuPlayer::Renderer::onPause() { mDrainAudioQueuePending = false; mDrainVideoQueuePending = false; + mVideoRenderingStarted = false; // force-notify NOTE_INFO MEDIA_INFO_RENDERING_START after resume if (mHasAudio) { mAudioSink->pause(); @@ -1494,17 +1571,27 @@ void NuPlayer::Renderer::onPause() { } void NuPlayer::Renderer::onResume() { + readProperties(); + if (!mPaused) { return; } if (mHasAudio) { + status_t status = NO_ERROR; cancelAudioOffloadPauseTimeout(); - status_t err = mAudioSink->start(); - if (err != OK) { - ALOGE("cannot start AudioSink err %d", err); + status = mAudioSink->start(); + if (offloadingAudio() && status != NO_ERROR && status != INVALID_OPERATION) { + ALOGD("received error :%d on resume for offload track posting TEAR_DOWN event",status); notifyAudioTearDown(); } + //Update anchor time after resuming playback. + if (offloadingAudio() && status == NO_ERROR) { + int64_t nowUs = ALooper::GetNowUs(); + int64_t nowMediaUs = + mAudioFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs); + mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX); + } } { @@ -1566,6 +1653,7 @@ int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) { int64_t numFramesPlayedAt; AudioTimestamp ts; static const int64_t kStaleTimestamp100ms = 100000; + int64_t durationUs; status_t res = mAudioSink->getTimestamp(ts); if (res == OK) { // case 1: mixing audio tracks and offloaded tracks. @@ -1592,14 +1680,20 @@ int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) { // numFramesPlayed, (long long)numFramesPlayedAt); } else { // case 3: transitory at new track or audio fast tracks. res = mAudioSink->getPosition(&numFramesPlayed); - CHECK_EQ(res, (status_t)OK); - numFramesPlayedAt = nowUs; - numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */ + if (res != OK) { + //query to getPosition fails, use media clock to simulate render position + getCurrentPosition(&durationUs); + durationUs = durationUs - mAnchorTimeMediaUs; + return durationUs; + } else { + numFramesPlayedAt = nowUs; + numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */ + } //ALOGD("getPosition: %u %lld", numFramesPlayed, (long long)numFramesPlayedAt); } //CHECK_EQ(numFramesPlayed & (1 << 31), 0); // can't be negative until 12.4 hrs, test - int64_t durationUs = getDurationUsIfPlayedAtSampleRate(numFramesPlayed) + durationUs = getDurationUsIfPlayedAtSampleRate(numFramesPlayed) + nowUs - numFramesPlayedAt; if (durationUs < 0) { // Occurs when numFramesPlayed position is very small and the following: @@ -1618,10 +1712,10 @@ int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) { } void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) { - if (mAudioTornDown) { + if (mAudioTearingDown) { return; } - mAudioTornDown = true; + mAudioTearingDown = true; int64_t currentPositionUs; sp<AMessage> notify = mNotify->dup(); @@ -1657,9 +1751,15 @@ status_t NuPlayer::Renderer::onOpenAudioSink( const sp<AMessage> &format, bool offloadOnly, bool hasVideo, - uint32_t flags) { + uint32_t flags, + bool isStreaming) { ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)", offloadOnly, offloadingAudio()); + + if (mAudioTearingDown) { + ALOGW("openAudioSink: not opening now!, would happen after teardown"); + return OK; + } bool audioSinkChanged = false; int32_t numChannels; @@ -1671,13 +1771,17 @@ status_t NuPlayer::Renderer::onOpenAudioSink( channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER; } + int32_t bitWidth = 16; + format->findInt32("bits-per-sample", &bitWidth); + int32_t sampleRate; CHECK(format->findInt32("sample-rate", &sampleRate)); + AString mime; + CHECK(format->findString("mime", &mime)); + if (offloadingAudio()) { audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT; - AString mime; - CHECK(format->findString("mime", &mime)); status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str()); if (err != OK) { @@ -1685,22 +1789,38 @@ status_t NuPlayer::Renderer::onOpenAudioSink( "audio_format", mime.c_str()); onDisableOffloadAudio(); } else { + audioFormat = AVUtils::get()->updateAudioFormat(audioFormat, format); + bitWidth = AVUtils::get()->getAudioSampleBits(format); ALOGV("Mime \"%s\" mapped to audio_format 0x%x", mime.c_str(), audioFormat); int avgBitRate = -1; - format->findInt32("bit-rate", &avgBitRate); + format->findInt32("bitrate", &avgBitRate); int32_t aacProfile = -1; if (audioFormat == AUDIO_FORMAT_AAC && format->findInt32("aac-profile", &aacProfile)) { // Redefine AAC format as per aac profile - mapAACProfileToAudioFormat( - audioFormat, - aacProfile); + int32_t isADTSSupported; + isADTSSupported = AVUtils::get()->mapAACProfileToAudioFormat(format, + audioFormat, + aacProfile); + if (!isADTSSupported) { + mapAACProfileToAudioFormat(audioFormat, + aacProfile); + } else { + ALOGV("Format is AAC ADTS\n"); + } } + ALOGV("onOpenAudioSink: %s", format->debugString().c_str()); + + int32_t offloadBufferSize = + AVUtils::get()->getAudioMaxInputBufferSize( + audioFormat, + format); audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER; + offloadInfo.duration_us = -1; format->findInt64( "durationUs", &offloadInfo.duration_us); @@ -1710,7 +1830,9 @@ status_t NuPlayer::Renderer::onOpenAudioSink( offloadInfo.stream_type = AUDIO_STREAM_MUSIC; offloadInfo.bit_rate = avgBitRate; offloadInfo.has_video = hasVideo; - offloadInfo.is_streaming = true; + offloadInfo.is_streaming = isStreaming; + offloadInfo.bit_width = bitWidth; + offloadInfo.offload_buffer_size = offloadBufferSize; if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) { ALOGV("openAudioSink: no change in offload mode"); @@ -1763,6 +1885,7 @@ status_t NuPlayer::Renderer::onOpenAudioSink( } else { mUseAudioCallback = true; // offload mode transfers data through callback ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message. + mFlags |= FLAG_OFFLOAD_AUDIO; } } } @@ -1774,7 +1897,7 @@ status_t NuPlayer::Renderer::onOpenAudioSink( const PcmInfo info = { (audio_channel_mask_t)channelMask, (audio_output_flags_t)pcmFlags, - AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat + AVNuUtils::get()->getPCMFormat(format), numChannels, sampleRate }; @@ -1809,7 +1932,7 @@ status_t NuPlayer::Renderer::onOpenAudioSink( sampleRate, numChannels, (audio_channel_mask_t)channelMask, - AUDIO_FORMAT_PCM_16_BIT, + AVNuUtils::get()->getPCMFormat(format), 0 /* bufferCount - unused */, mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL, mUseAudioCallback ? this : NULL, @@ -1834,7 +1957,6 @@ status_t NuPlayer::Renderer::onOpenAudioSink( if (audioSinkChanged) { onAudioSinkChanged(); } - mAudioTornDown = false; return OK; } @@ -1844,5 +1966,13 @@ void NuPlayer::Renderer::onCloseAudioSink() { mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER; } +void NuPlayer::Renderer::signalAudioTearDownComplete() { + (new AMessage(kWhatAudioTearDownComplete, this))->post(); +} + +void NuPlayer::Renderer::onAudioTearDownComplete() { + mAudioTearingDown = false; +} + } // namespace android diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h index 9479c31..a84e673 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h @@ -73,13 +73,17 @@ struct NuPlayer::Renderer : public AHandler { status_t getCurrentPosition(int64_t *mediaUs); int64_t getVideoLateByUs(); + virtual audio_stream_type_t getAudioStreamType(){return AUDIO_STREAM_DEFAULT;} + status_t openAudioSink( const sp<AMessage> &format, bool offloadOnly, bool hasVideo, uint32_t flags, - bool *isOffloaded); + bool *isOffloaded, + bool isStreaming); void closeAudioSink(); + void signalAudioTearDownComplete(); enum { kWhatEOS = 'eos ', @@ -89,6 +93,7 @@ struct NuPlayer::Renderer : public AHandler { kWhatMediaRenderingStart = 'mdrd', kWhatAudioTearDown = 'adTD', kWhatAudioOffloadPauseTimeout = 'aOPT', + kWhatAudioTearDownComplete = 'aTDC', }; enum AudioTearDownReason { @@ -101,7 +106,6 @@ protected: virtual void onMessageReceived(const sp<AMessage> &msg); -private: enum { kWhatDrainAudioQueue = 'draA', kWhatDrainVideoQueue = 'draV', @@ -162,6 +166,7 @@ private: int64_t mVideoLateByUs; bool mHasAudio; bool mHasVideo; + bool mFoundAudioEOS; bool mNotifyCompleteAudio; bool mNotifyCompleteVideo; @@ -181,7 +186,7 @@ private: int64_t mLastPositionUpdateUs; int32_t mAudioOffloadPauseTimeoutGeneration; - bool mAudioTornDown; + bool mAudioTearingDown; audio_offload_info_t mCurrentOffloadInfo; struct PcmInfo { @@ -229,7 +234,7 @@ private: void prepareForMediaRenderingStart_l(); void notifyIfMediaRenderingStarted_l(); - void onQueueBuffer(const sp<AMessage> &msg); + virtual void onQueueBuffer(const sp<AMessage> &msg); void onQueueEOS(const sp<AMessage> &msg); void onFlush(const sp<AMessage> &msg); void onAudioSinkChanged(); @@ -251,8 +256,10 @@ private: const sp<AMessage> &format, bool offloadOnly, bool hasVideo, - uint32_t flags); + uint32_t flags, + bool isStreaming); void onCloseAudioSink(); + void onAudioTearDownComplete(); void notifyEOS(bool audio, status_t finalResult, int64_t delayUs = 0); void notifyFlushComplete(bool audio); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerSource.h b/media/libmediaplayerservice/nuplayer/NuPlayerSource.h index 11a6a9f..b248316 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerSource.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerSource.h @@ -39,6 +39,7 @@ struct NuPlayer::Source : public AHandler { FLAG_DYNAMIC_DURATION = 16, FLAG_SECURE = 32, FLAG_PROTECTED = 64, + FLAG_USE_SET_BUFFERS = 128, }; enum { @@ -57,6 +58,7 @@ struct NuPlayer::Source : public AHandler { kWhatQueueDecoderShutdown, kWhatDrmNoLicense, kWhatInstantiateSecureDecoders, + kWhatRTCPByeReceived, }; // The provides message is used to notify the player about various @@ -132,10 +134,10 @@ protected: void notifyFlagsChanged(uint32_t flags); void notifyVideoSizeChanged(const sp<AMessage> &format = NULL); void notifyInstantiateSecureDecoders(const sp<AMessage> &reply); - void notifyPrepared(status_t err = OK); + virtual void notifyPrepared(status_t err = OK); -private: sp<AMessage> mNotify; +private: DISALLOW_EVIL_CONSTRUCTORS(Source); }; diff --git a/media/libmediaplayerservice/nuplayer/RTSPSource.cpp b/media/libmediaplayerservice/nuplayer/RTSPSource.cpp index af0351e..4962520 100644 --- a/media/libmediaplayerservice/nuplayer/RTSPSource.cpp +++ b/media/libmediaplayerservice/nuplayer/RTSPSource.cpp @@ -24,13 +24,16 @@ #include "MyHandler.h" #include "SDPLoader.h" +#include <cutils/properties.h> #include <media/IMediaHTTPService.h> #include <media/stagefright/MediaDefs.h> #include <media/stagefright/MetaData.h> +#include <mediaplayerservice/AVMediaServiceExtensions.h> namespace android { const int64_t kNearEOSTimeoutUs = 2000000ll; // 2 secs +const uint32_t kMaxNumKeepDamagedAccessUnits = 30; NuPlayer::RTSPSource::RTSPSource( const sp<AMessage> ¬ify, @@ -53,7 +56,10 @@ NuPlayer::RTSPSource::RTSPSource( mBuffering(false), mSeekGeneration(0), mEOSTimeoutAudio(0), - mEOSTimeoutVideo(0) { + mEOSTimeoutVideo(0), + mVideoTrackIndex(-1), + mKeepDamagedAccessUnits(false), + mNumKeepDamagedAccessUnits(0) { if (headers) { mExtraHeaders = *headers; @@ -131,6 +137,10 @@ void NuPlayer::RTSPSource::pause() { // Check if EOS or ERROR is received if (source != NULL && source->isFinished(mediaDurationUs)) { + if (mHandler != NULL) { + ALOGI("Nearing EOS...No Pause is issued"); + mHandler->cancelTimeoutCheck(); + } return; } } @@ -428,11 +438,22 @@ void NuPlayer::RTSPSource::onMessageReceived(const sp<AMessage> &msg) { sp<ABuffer> accessUnit; CHECK(msg->findBuffer("accessUnit", &accessUnit)); + bool isVideo = trackIndex == (size_t)mVideoTrackIndex; int32_t damaged; if (accessUnit->meta()->findInt32("damaged", &damaged) && damaged) { - ALOGI("dropping damaged access unit."); - break; + if (isVideo && mKeepDamagedAccessUnits + && mNumKeepDamagedAccessUnits < kMaxNumKeepDamagedAccessUnits) { + ALOGI("keep a damaged access unit."); + ++mNumKeepDamagedAccessUnits; + } else { + ALOGI("dropping damaged access unit."); + break; + } + } else { + if (isVideo) { + mNumKeepDamagedAccessUnits = 0; + } } if (mTSParser != NULL) { @@ -476,8 +497,11 @@ void NuPlayer::RTSPSource::onMessageReceived(const sp<AMessage> &msg) { if (!info->mNPTMappingValid) { // This is a live stream, we didn't receive any normal // playtime mapping. We won't map to npt time. - source->queueAccessUnit(accessUnit); - break; + if (!AVMediaServiceUtils::get()->checkNPTMapping(&info->mRTPTime, + &info->mNormalPlaytimeUs, &info->mNPTMappingValid, rtpTime)) { + source->queueAccessUnit(accessUnit); + break; + } } int64_t nptUs = @@ -563,6 +587,14 @@ void NuPlayer::RTSPSource::onMessageReceived(const sp<AMessage> &msg) { break; } + case MyHandler::kWhatByeReceived: + { + sp<AMessage> msg = dupNotify(); + msg->setInt32("what", kWhatRTCPByeReceived); + msg->post(); + break; + } + case SDPLoader::kWhatSDPLoaded: { onSDPLoaded(msg); @@ -597,6 +629,16 @@ void NuPlayer::RTSPSource::onConnected() { bool isAudio = !strncasecmp(mime, "audio/", 6); bool isVideo = !strncasecmp(mime, "video/", 6); + if (isVideo) { + mVideoTrackIndex = i; + char value[PROPERTY_VALUE_MAX]; + if (property_get("rtsp.video.keep-damaged-au", value, NULL) + && !strcasecmp(mime, value)) { + ALOGV("enable to keep damaged au for %s", mime); + mKeepDamagedAccessUnits = true; + } + } + TrackInfo info; info.mTimeScale = timeScale; info.mRTPTime = 0; diff --git a/media/libmediaplayerservice/nuplayer/RTSPSource.h b/media/libmediaplayerservice/nuplayer/RTSPSource.h index 6438a1e..c431174 100644 --- a/media/libmediaplayerservice/nuplayer/RTSPSource.h +++ b/media/libmediaplayerservice/nuplayer/RTSPSource.h @@ -118,6 +118,10 @@ private: sp<AReplyToken> mSeekReplyID; + int32_t mVideoTrackIndex; + bool mKeepDamagedAccessUnits; + uint32_t mNumKeepDamagedAccessUnits; + sp<AnotherPacketSource> getSource(bool audio); void onConnected(); diff --git a/media/libmediaplayerservice/nuplayer/StreamingSource.cpp b/media/libmediaplayerservice/nuplayer/StreamingSource.cpp index 0246b59..b9c915e 100644 --- a/media/libmediaplayerservice/nuplayer/StreamingSource.cpp +++ b/media/libmediaplayerservice/nuplayer/StreamingSource.cpp @@ -18,6 +18,8 @@ #define LOG_TAG "StreamingSource" #include <utils/Log.h> +#include <inttypes.h> + #include "StreamingSource.h" #include "ATSParser.h" @@ -29,9 +31,12 @@ #include <media/stagefright/foundation/AMessage.h> #include <media/stagefright/MediaSource.h> #include <media/stagefright/MetaData.h> +#include <inttypes.h> namespace android { +const int32_t kNumListenerQueuePackets = 80; + NuPlayer::StreamingSource::StreamingSource( const sp<AMessage> ¬ify, const sp<IStreamSource> &source) @@ -84,7 +89,7 @@ status_t NuPlayer::StreamingSource::feedMoreTSData() { } void NuPlayer::StreamingSource::onReadBuffer() { - for (int32_t i = 0; i < 50; ++i) { + for (int32_t i = 0; i < kNumListenerQueuePackets; ++i) { char buffer[188]; sp<AMessage> extra; ssize_t n = mStreamListener->read(buffer, sizeof(buffer), &extra); @@ -248,7 +253,7 @@ status_t NuPlayer::StreamingSource::dequeueAccessUnit( if (err == OK) { int64_t timeUs; CHECK((*accessUnit)->meta()->findInt64("timeUs", &timeUs)); - ALOGV("dequeueAccessUnit timeUs=%lld us", timeUs); + ALOGV("dequeueAccessUnit timeUs=%" PRId64 " us", timeUs); } #endif |