diff options
Diffstat (limited to 'media/libmediaplayerservice/nuplayer')
23 files changed, 2359 insertions, 1663 deletions
diff --git a/media/libmediaplayerservice/nuplayer/Android.mk b/media/libmediaplayerservice/nuplayer/Android.mk index 676c0a6..6609874 100644 --- a/media/libmediaplayerservice/nuplayer/Android.mk +++ b/media/libmediaplayerservice/nuplayer/Android.mk @@ -5,7 +5,9 @@ LOCAL_SRC_FILES:= \ GenericSource.cpp \ HTTPLiveSource.cpp \ NuPlayer.cpp \ + NuPlayerCCDecoder.cpp \ NuPlayerDecoder.cpp \ + NuPlayerDecoderBase.cpp \ NuPlayerDecoderPassThrough.cpp \ NuPlayerDriver.cpp \ NuPlayerRenderer.cpp \ diff --git a/media/libmediaplayerservice/nuplayer/GenericSource.cpp b/media/libmediaplayerservice/nuplayer/GenericSource.cpp index 6859a1a..1af2713 100644 --- a/media/libmediaplayerservice/nuplayer/GenericSource.cpp +++ b/media/libmediaplayerservice/nuplayer/GenericSource.cpp @@ -45,13 +45,19 @@ NuPlayer::GenericSource::GenericSource( bool uidValid, uid_t uid) : Source(notify), + mAudioTimeUs(0), + mAudioLastDequeueTimeUs(0), + mVideoTimeUs(0), + mVideoLastDequeueTimeUs(0), mFetchSubtitleDataGeneration(0), mFetchTimedTextDataGeneration(0), mDurationUs(0ll), mAudioIsVorbis(false), mIsWidevine(false), + mIsSecure(false), mUIDValid(uidValid), mUID(uid), + mFd(-1), mDrmManagerClient(NULL), mMetaDataSize(-1ll), mBitrate(-1ll), @@ -62,13 +68,14 @@ NuPlayer::GenericSource::GenericSource( } void NuPlayer::GenericSource::resetDataSource() { - mAudioTimeUs = 0; - mVideoTimeUs = 0; mHTTPService.clear(); mHttpSource.clear(); mUri.clear(); mUriHeaders.clear(); - mFd = -1; + if (mFd >= 0) { + close(mFd); + mFd = -1; + } mOffset = 0; mLength = 0; setDrmPlaybackStatusIfNeeded(Playback::STOP, 0); @@ -157,6 +164,17 @@ status_t NuPlayer::GenericSource::initFromDataSource() { if (mFileMeta->findInt64(kKeyDuration, &duration)) { mDurationUs = duration; } + + if (!mIsWidevine) { + // Check mime to see if we actually have a widevine source. + // If the data source is not URL-type (eg. file source), we + // won't be able to tell until now. + const char *fileMime; + if (mFileMeta->findCString(kKeyMIMEType, &fileMime) + && !strncasecmp(fileMime, "video/wvm", 9)) { + mIsWidevine = true; + } + } } int32_t totalBitrate = 0; @@ -202,7 +220,7 @@ status_t NuPlayer::GenericSource::initFromDataSource() { int32_t secure; if (meta->findInt32(kKeyRequiresSecureBuffers, &secure) && secure) { - mIsWidevine = true; + mIsSecure = true; if (mUIDValid) { extractor->setUID(mUID); } @@ -257,7 +275,7 @@ int64_t NuPlayer::GenericSource::getLastReadPosition() { status_t NuPlayer::GenericSource::setBuffers( bool audio, Vector<MediaBuffer *> &buffers) { - if (mIsWidevine && !audio) { + if (mIsSecure && !audio) { return mVideoTrack.mSource->setBuffers(buffers); } return INVALID_OPERATION; @@ -268,6 +286,7 @@ NuPlayer::GenericSource::~GenericSource() { mLooper->unregisterHandler(id()); mLooper->stop(); } + resetDataSource(); } void NuPlayer::GenericSource::prepareAsync() { @@ -286,6 +305,10 @@ void NuPlayer::GenericSource::prepareAsync() { void NuPlayer::GenericSource::onPrepareAsync() { // delayed data source creation if (mDataSource == NULL) { + // set to false first, if the extractor + // comes back as secure, set it to true then. + mIsSecure = false; + if (!mUri.empty()) { const char* uri = mUri.c_str(); mIsWidevine = !strncasecmp(uri, "widevine://", 11); @@ -305,11 +328,10 @@ void NuPlayer::GenericSource::onPrepareAsync() { mHTTPService, uri, &mUriHeaders, &mContentType, static_cast<HTTPBase *>(mHttpSource.get())); } else { - // set to false first, if the extractor - // comes back as secure, set it to true then. mIsWidevine = false; mDataSource = new FileSource(mFd, mOffset, mLength); + mFd = -1; } if (mDataSource == NULL) { @@ -360,7 +382,7 @@ void NuPlayer::GenericSource::onPrepareAsync() { } notifyFlagsChanged( - (mIsWidevine ? FLAG_SECURE : 0) + (mIsSecure ? FLAG_SECURE : 0) | FLAG_CAN_PAUSE | FLAG_CAN_SEEK_BACKWARD | FLAG_CAN_SEEK_FORWARD @@ -477,8 +499,8 @@ void NuPlayer::GenericSource::stop() { // nothing to do, just account for DRM playback status setDrmPlaybackStatusIfNeeded(Playback::STOP, 0); mStarted = false; - if (mIsWidevine) { - // For a widevine source we need to prevent any further reads. + if (mIsWidevine || mIsSecure) { + // For widevine or secure sources we need to prevent any further reads. sp<AMessage> msg = new AMessage(kWhatStopWidevine, id()); sp<AMessage> response; (void) msg->postAndAwaitResponse(&response); @@ -644,17 +666,13 @@ void NuPlayer::GenericSource::onMessageReceived(const sp<AMessage> &msg) { track->mSource->start(); track->mIndex = trackIndex; - status_t avail; - if (!track->mPackets->hasBufferAvailable(&avail)) { - // sync from other source - TRESPASS(); - break; - } - int64_t timeUs, actualTimeUs; const bool formatChange = true; - sp<AMessage> latestMeta = track->mPackets->getLatestEnqueuedMeta(); - CHECK(latestMeta != NULL && latestMeta->findInt64("timeUs", &timeUs)); + if (trackType == MEDIA_TRACK_TYPE_AUDIO) { + timeUs = mAudioLastDequeueTimeUs; + } else { + timeUs = mVideoLastDequeueTimeUs; + } readBuffer(trackType, timeUs, &actualTimeUs, formatChange); readBuffer(counterpartType, -1, NULL, formatChange); ALOGV("timeUs %lld actualTimeUs %lld", timeUs, actualTimeUs); @@ -842,7 +860,12 @@ status_t NuPlayer::GenericSource::dequeueAccessUnit( status_t finalResult; if (!track->mPackets->hasBufferAvailable(&finalResult)) { - return (finalResult == OK ? -EWOULDBLOCK : finalResult); + if (finalResult == OK) { + postReadBuffer( + audio ? MEDIA_TRACK_TYPE_AUDIO : MEDIA_TRACK_TYPE_VIDEO); + return -EWOULDBLOCK; + } + return finalResult; } status_t result = track->mPackets->dequeueAccessUnit(accessUnit); @@ -866,6 +889,11 @@ status_t NuPlayer::GenericSource::dequeueAccessUnit( int64_t timeUs; status_t eosResult; // ignored CHECK((*accessUnit)->meta()->findInt64("timeUs", &timeUs)); + if (audio) { + mAudioLastDequeueTimeUs = timeUs; + } else { + mVideoLastDequeueTimeUs = timeUs; + } if (mSubtitleTrack.mSource != NULL && !mSubtitleTrack.mPackets->hasBufferAvailable(&eosResult)) { @@ -996,11 +1024,12 @@ ssize_t NuPlayer::GenericSource::doGetSelectedTrack(media_track_type type) const return -1; } -status_t NuPlayer::GenericSource::selectTrack(size_t trackIndex, bool select) { +status_t NuPlayer::GenericSource::selectTrack(size_t trackIndex, bool select, int64_t timeUs) { ALOGV("%s track: %zu", select ? "select" : "deselect", trackIndex); sp<AMessage> msg = new AMessage(kWhatSelectTrack, id()); msg->setInt32("trackIndex", trackIndex); msg->setInt32("select", select); + msg->setInt64("timeUs", timeUs); sp<AMessage> response; status_t err = msg->postAndAwaitResponse(&response); @@ -1013,11 +1042,13 @@ status_t NuPlayer::GenericSource::selectTrack(size_t trackIndex, bool select) { void NuPlayer::GenericSource::onSelectTrack(sp<AMessage> msg) { int32_t trackIndex, select; + int64_t timeUs; CHECK(msg->findInt32("trackIndex", &trackIndex)); CHECK(msg->findInt32("select", &select)); + CHECK(msg->findInt64("timeUs", &timeUs)); sp<AMessage> response = new AMessage; - status_t err = doSelectTrack(trackIndex, select); + status_t err = doSelectTrack(trackIndex, select, timeUs); response->setInt32("err", err); uint32_t replyID; @@ -1025,7 +1056,7 @@ void NuPlayer::GenericSource::onSelectTrack(sp<AMessage> msg) { response->postReply(replyID); } -status_t NuPlayer::GenericSource::doSelectTrack(size_t trackIndex, bool select) { +status_t NuPlayer::GenericSource::doSelectTrack(size_t trackIndex, bool select, int64_t timeUs) { if (trackIndex >= mSources.size()) { return BAD_INDEX; } @@ -1078,6 +1109,23 @@ status_t NuPlayer::GenericSource::doSelectTrack(size_t trackIndex, bool select) mFetchTimedTextDataGeneration++; } + status_t eosResult; // ignored + if (mSubtitleTrack.mSource != NULL + && !mSubtitleTrack.mPackets->hasBufferAvailable(&eosResult)) { + sp<AMessage> msg = new AMessage(kWhatFetchSubtitleData, id()); + msg->setInt64("timeUs", timeUs); + msg->setInt32("generation", mFetchSubtitleDataGeneration); + msg->post(); + } + + if (mTimedTextTrack.mSource != NULL + && !mTimedTextTrack.mPackets->hasBufferAvailable(&eosResult)) { + sp<AMessage> msg = new AMessage(kWhatFetchTimedTextData, id()); + msg->setInt64("timeUs", timeUs); + msg->setInt32("generation", mFetchTimedTextDataGeneration); + msg->post(); + } + return OK; } else if (!strncasecmp(mime, "audio/", 6) || !strncasecmp(mime, "video/", 6)) { bool audio = !strncasecmp(mime, "audio/", 6); @@ -1132,10 +1180,12 @@ status_t NuPlayer::GenericSource::doSeek(int64_t seekTimeUs) { readBuffer(MEDIA_TRACK_TYPE_VIDEO, seekTimeUs, &actualTimeUs); seekTimeUs = actualTimeUs; + mVideoLastDequeueTimeUs = seekTimeUs; } if (mAudioTrack.mSource != NULL) { readBuffer(MEDIA_TRACK_TYPE_AUDIO, seekTimeUs); + mAudioLastDequeueTimeUs = seekTimeUs; } setDrmPlaybackStatusIfNeeded(Playback::START, seekTimeUs / 1000); @@ -1148,6 +1198,7 @@ status_t NuPlayer::GenericSource::doSeek(int64_t seekTimeUs) { sp<ABuffer> NuPlayer::GenericSource::mediaBufferToABuffer( MediaBuffer* mb, media_track_type trackType, + int64_t /* seekTimeUs */, int64_t *actualTimeUs) { bool audio = trackType == MEDIA_TRACK_TYPE_AUDIO; size_t outLength = mb->range_length(); @@ -1157,7 +1208,7 @@ sp<ABuffer> NuPlayer::GenericSource::mediaBufferToABuffer( } sp<ABuffer> ab; - if (mIsWidevine && !audio) { + if (mIsSecure && !audio) { // data is already provided in the buffer ab = new ABuffer(NULL, mb->range_length()); mb->add_ref(); @@ -1185,6 +1236,16 @@ sp<ABuffer> NuPlayer::GenericSource::mediaBufferToABuffer( CHECK(mb->meta_data()->findInt64(kKeyTime, &timeUs)); meta->setInt64("timeUs", timeUs); +#if 0 + // Temporarily disable pre-roll till we have a full solution to handle + // both single seek and continous seek gracefully. + if (seekTimeUs > timeUs) { + sp<AMessage> extra = new AMessage; + extra->setInt64("resume-at-mediaTimeUs", seekTimeUs); + meta->setMessage("extra", extra); + } +#endif + if (trackType == MEDIA_TRACK_TYPE_TIMEDTEXT) { const char *mime; CHECK(mTimedTextTrack.mSource != NULL @@ -1226,14 +1287,13 @@ void NuPlayer::GenericSource::onReadBuffer(sp<AMessage> msg) { int32_t tmpType; CHECK(msg->findInt32("trackType", &tmpType)); media_track_type trackType = (media_track_type)tmpType; + readBuffer(trackType); { // only protect the variable change, as readBuffer may - // take considerable time. This may result in one extra - // read being processed, but that is benign. + // take considerable time. Mutex::Autolock _l(mReadBufferLock); mPendingReadBufferTypes &= ~(1 << trackType); } - readBuffer(trackType); } void NuPlayer::GenericSource::readBuffer( @@ -1286,7 +1346,7 @@ void NuPlayer::GenericSource::readBuffer( seeking = true; } - if (mIsWidevine && trackType != MEDIA_TRACK_TYPE_AUDIO) { + if (mIsWidevine) { options.setNonBlocking(); } @@ -1311,15 +1371,14 @@ void NuPlayer::GenericSource::readBuffer( if ((seeking || formatChange) && (trackType == MEDIA_TRACK_TYPE_AUDIO || trackType == MEDIA_TRACK_TYPE_VIDEO)) { - ATSParser::DiscontinuityType type = formatChange - ? (seeking - ? ATSParser::DISCONTINUITY_FORMATCHANGE - : ATSParser::DISCONTINUITY_NONE) - : ATSParser::DISCONTINUITY_SEEK; + ATSParser::DiscontinuityType type = (formatChange && seeking) + ? ATSParser::DISCONTINUITY_FORMATCHANGE + : ATSParser::DISCONTINUITY_NONE; track->mPackets->queueDiscontinuity( type, NULL, true /* discard */); } - sp<ABuffer> buffer = mediaBufferToABuffer(mbuf, trackType, actualTimeUs); + sp<ABuffer> buffer = mediaBufferToABuffer( + mbuf, trackType, seekTimeUs, actualTimeUs); track->mPackets->queueAccessUnit(buffer); formatChange = false; seeking = false; diff --git a/media/libmediaplayerservice/nuplayer/GenericSource.h b/media/libmediaplayerservice/nuplayer/GenericSource.h index f8601ea..1b63a1f 100644 --- a/media/libmediaplayerservice/nuplayer/GenericSource.h +++ b/media/libmediaplayerservice/nuplayer/GenericSource.h @@ -67,7 +67,7 @@ struct NuPlayer::GenericSource : public NuPlayer::Source { virtual size_t getTrackCount() const; virtual sp<AMessage> getTrackInfo(size_t trackIndex) const; virtual ssize_t getSelectedTrack(media_track_type type) const; - virtual status_t selectTrack(size_t trackIndex, bool select); + virtual status_t selectTrack(size_t trackIndex, bool select, int64_t timeUs); virtual status_t seekTo(int64_t seekTimeUs); virtual status_t setBuffers(bool audio, Vector<MediaBuffer *> &buffers); @@ -106,8 +106,10 @@ private: Track mAudioTrack; int64_t mAudioTimeUs; + int64_t mAudioLastDequeueTimeUs; Track mVideoTrack; int64_t mVideoTimeUs; + int64_t mVideoLastDequeueTimeUs; Track mSubtitleTrack; Track mTimedTextTrack; @@ -116,6 +118,7 @@ private: int64_t mDurationUs; bool mAudioIsVorbis; bool mIsWidevine; + bool mIsSecure; bool mUIDValid; uid_t mUID; sp<IMediaHTTPService> mHTTPService; @@ -162,7 +165,7 @@ private: ssize_t doGetSelectedTrack(media_track_type type) const; void onSelectTrack(sp<AMessage> msg); - status_t doSelectTrack(size_t trackIndex, bool select); + status_t doSelectTrack(size_t trackIndex, bool select, int64_t timeUs); void onSeek(sp<AMessage> msg); status_t doSeek(int64_t seekTimeUs); @@ -180,6 +183,7 @@ private: sp<ABuffer> mediaBufferToABuffer( MediaBuffer *mbuf, media_track_type trackType, + int64_t seekTimeUs, int64_t *actualTimeUs = NULL); void postReadBuffer(media_track_type trackType); diff --git a/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp b/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp index a003c81..a26ef9e 100644 --- a/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp +++ b/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp @@ -98,6 +98,10 @@ void NuPlayer::HTTPLiveSource::start() { } sp<AMessage> NuPlayer::HTTPLiveSource::getFormat(bool audio) { + if (mLiveSession == NULL) { + return NULL; + } + sp<AMessage> format; status_t err = mLiveSession->getStreamFormat( audio ? LiveSession::STREAMTYPE_AUDIO @@ -135,7 +139,15 @@ sp<AMessage> NuPlayer::HTTPLiveSource::getTrackInfo(size_t trackIndex) const { return mLiveSession->getTrackInfo(trackIndex); } -status_t NuPlayer::HTTPLiveSource::selectTrack(size_t trackIndex, bool select) { +ssize_t NuPlayer::HTTPLiveSource::getSelectedTrack(media_track_type type) const { + if (mLiveSession == NULL) { + return -1; + } else { + return mLiveSession->getSelectedTrack(type); + } +} + +status_t NuPlayer::HTTPLiveSource::selectTrack(size_t trackIndex, bool select, int64_t /*timeUs*/) { status_t err = mLiveSession->selectTrack(trackIndex, select); if (err == OK) { diff --git a/media/libmediaplayerservice/nuplayer/HTTPLiveSource.h b/media/libmediaplayerservice/nuplayer/HTTPLiveSource.h index 6b5f6af..bbb8981 100644 --- a/media/libmediaplayerservice/nuplayer/HTTPLiveSource.h +++ b/media/libmediaplayerservice/nuplayer/HTTPLiveSource.h @@ -42,7 +42,8 @@ struct NuPlayer::HTTPLiveSource : public NuPlayer::Source { virtual status_t getDuration(int64_t *durationUs); virtual size_t getTrackCount() const; virtual sp<AMessage> getTrackInfo(size_t trackIndex) const; - virtual status_t selectTrack(size_t trackIndex, bool select); + virtual ssize_t getSelectedTrack(media_track_type /* type */) const; + virtual status_t selectTrack(size_t trackIndex, bool select, int64_t timeUs); virtual status_t seekTo(int64_t seekTimeUs); protected: diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp index a63a940..080cd52 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp @@ -21,7 +21,9 @@ #include "NuPlayer.h" #include "HTTPLiveSource.h" +#include "NuPlayerCCDecoder.h" #include "NuPlayerDecoder.h" +#include "NuPlayerDecoderBase.h" #include "NuPlayerDecoderPassThrough.h" #include "NuPlayerDriver.h" #include "NuPlayerRenderer.h" @@ -33,6 +35,8 @@ #include "ATSParser.h" +#include <cutils/properties.h> + #include <media/stagefright/foundation/hexdump.h> #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> @@ -50,10 +54,6 @@ namespace android { -// TODO optimize buffer size for power consumption -// The offload read buffer size is 32 KB but 24 KB uses less power. -const size_t NuPlayer::kAggregateBufferSizeBytes = 24 * 1024; - struct NuPlayer::Action : public RefBase { Action() {} @@ -80,6 +80,21 @@ private: DISALLOW_EVIL_CONSTRUCTORS(SeekAction); }; +struct NuPlayer::ResumeDecoderAction : public Action { + ResumeDecoderAction(bool needNotify) + : mNeedNotify(needNotify) { + } + + virtual void execute(NuPlayer *player) { + player->performResumeDecoders(mNeedNotify); + } + +private: + bool mNeedNotify; + + DISALLOW_EVIL_CONSTRUCTORS(ResumeDecoderAction); +}; + struct NuPlayer::SetSurfaceAction : public Action { SetSurfaceAction(const sp<NativeWindowWrapper> &wrapper) : mWrapper(wrapper) { @@ -95,21 +110,21 @@ private: DISALLOW_EVIL_CONSTRUCTORS(SetSurfaceAction); }; -struct NuPlayer::ShutdownDecoderAction : public Action { - ShutdownDecoderAction(bool audio, bool video) +struct NuPlayer::FlushDecoderAction : public Action { + FlushDecoderAction(FlushCommand audio, FlushCommand video) : mAudio(audio), mVideo(video) { } virtual void execute(NuPlayer *player) { - player->performDecoderShutdown(mAudio, mVideo); + player->performDecoderFlush(mAudio, mVideo); } private: - bool mAudio; - bool mVideo; + FlushCommand mAudio; + FlushCommand mVideo; - DISALLOW_EVIL_CONSTRUCTORS(ShutdownDecoderAction); + DISALLOW_EVIL_CONSTRUCTORS(FlushDecoderAction); }; struct NuPlayer::PostMessageAction : public Action { @@ -151,7 +166,6 @@ private: NuPlayer::NuPlayer() : mUIDValid(false), mSourceFlags(0), - mVideoIsAVC(false), mOffloadAudio(false), mAudioDecoderGeneration(0), mVideoDecoderGeneration(0), @@ -162,13 +176,9 @@ NuPlayer::NuPlayer() mScanSourcesGeneration(0), mPollDurationGeneration(0), mTimedTextGeneration(0), - mTimeDiscontinuityPending(false), mFlushingAudio(NONE), mFlushingVideo(NONE), - mSkipRenderingAudioUntilMediaTimeUs(-1ll), - mSkipRenderingVideoUntilMediaTimeUs(-1ll), - mNumFramesTotal(0ll), - mNumFramesDropped(0ll), + mResumePending(false), mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW), mStarted(false) { clearFlushComplete(); @@ -306,10 +316,6 @@ void NuPlayer::pause() { (new AMessage(kWhatPause, id()))->post(); } -void NuPlayer::resume() { - (new AMessage(kWhatResume, id()))->post(); -} - void NuPlayer::resetAsync() { if (mSource != NULL) { // During a reset, the data source might be unresponsive already, we need to @@ -459,8 +465,10 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { size_t trackIndex; int32_t select; + int64_t timeUs; CHECK(msg->findSize("trackIndex", &trackIndex)); CHECK(msg->findInt32("select", &select)); + CHECK(msg->findInt64("timeUs", &timeUs)); status_t err = INVALID_OPERATION; @@ -474,7 +482,7 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { } if (trackIndex < inbandTracks) { - err = mSource->selectTrack(trackIndex, select); + err = mSource->selectTrack(trackIndex, select, timeUs); if (!select && err == OK) { int32_t type; @@ -526,19 +534,24 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { { ALOGV("kWhatSetVideoNativeWindow"); - mDeferredActions.push_back( - new ShutdownDecoderAction( - false /* audio */, true /* video */)); - sp<RefBase> obj; CHECK(msg->findObject("native-window", &obj)); + if (mSource == NULL || mSource->getFormat(false /* audio */) == NULL) { + performSetSurface(static_cast<NativeWindowWrapper *>(obj.get())); + break; + } + + mDeferredActions.push_back( + new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */, + FLUSH_CMD_SHUTDOWN /* video */)); + mDeferredActions.push_back( new SetSurfaceAction( static_cast<NativeWindowWrapper *>(obj.get()))); if (obj != NULL) { - if (mStarted && mSource->getFormat(false /* audio */) != NULL) { + if (mStarted) { // Issue a seek to refresh the video screen only if started otherwise // the extractor may not yet be started and will assert. // If the video decoder is not set (perhaps audio only in this case) @@ -556,6 +569,12 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { new SimpleAction(&NuPlayer::performScanSources)); } + // After a flush without shutdown, decoder is paused. + // Don't resume it until source seek is done, otherwise it could + // start pulling stale data too soon. + mDeferredActions.push_back( + new ResumeDecoderAction(false /* needNotify */)); + processDeferredActions(); break; } @@ -574,69 +593,11 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { case kWhatStart: { ALOGV("kWhatStart"); - - mVideoIsAVC = false; - mOffloadAudio = false; - mAudioEOS = false; - mVideoEOS = false; - mSkipRenderingAudioUntilMediaTimeUs = -1; - mSkipRenderingVideoUntilMediaTimeUs = -1; - mNumFramesTotal = 0; - mNumFramesDropped = 0; - mStarted = true; - - /* instantiate decoders now for secure playback */ - if (mSourceFlags & Source::FLAG_SECURE) { - if (mNativeWindow != NULL) { - instantiateDecoder(false, &mVideoDecoder); - } - - if (mAudioSink != NULL) { - instantiateDecoder(true, &mAudioDecoder); - } - } - - mSource->start(); - - uint32_t flags = 0; - - if (mSource->isRealTime()) { - flags |= Renderer::FLAG_REAL_TIME; - } - - sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */); - audio_stream_type_t streamType = AUDIO_STREAM_MUSIC; - if (mAudioSink != NULL) { - streamType = mAudioSink->getAudioStreamType(); - } - - sp<AMessage> videoFormat = mSource->getFormat(false /* audio */); - - mOffloadAudio = - canOffloadStream(audioMeta, (videoFormat != NULL), - true /* is_streaming */, streamType); - if (mOffloadAudio) { - flags |= Renderer::FLAG_OFFLOAD_AUDIO; - } - - sp<AMessage> notify = new AMessage(kWhatRendererNotify, id()); - ++mRendererGeneration; - notify->setInt32("generation", mRendererGeneration); - mRenderer = new Renderer(mAudioSink, notify, flags); - - mRendererLooper = new ALooper; - mRendererLooper->setName("NuPlayerRenderer"); - mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO); - mRendererLooper->registerHandler(mRenderer); - - sp<MetaData> meta = getFileMeta(); - int32_t rate; - if (meta != NULL - && meta->findInt32(kKeyFrameRate, &rate) && rate > 0) { - mRenderer->setVideoFrameRate(rate); + if (mStarted) { + onResume(); + } else { + onStart(); } - - postScanSources(); break; } @@ -663,11 +624,21 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { instantiateDecoder(false, &mVideoDecoder); } - if (mAudioSink != NULL) { - if (mOffloadAudio) { + // Don't try to re-open audio sink if there's an existing decoder. + if (mAudioSink != NULL && mAudioDecoder == NULL) { + sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */); + sp<AMessage> videoFormat = mSource->getFormat(false /* audio */); + audio_stream_type_t streamType = mAudioSink->getAudioStreamType(); + const bool hasVideo = (videoFormat != NULL); + const bool canOffload = canOffloadStream( + audioMeta, hasVideo, true /* is_streaming */, streamType); + if (canOffload) { + if (!mOffloadAudio) { + mRenderer->signalEnableOffloadAudio(); + } // open audio sink early under offload mode. sp<AMessage> format = mSource->getFormat(true /*audio*/); - openAudioSink(format, true /*offloadOnly*/); + tryOpenAudioSinkForOffload(format, hasVideo); } instantiateDecoder(true, &mAudioDecoder); } @@ -731,16 +702,26 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { int32_t what; CHECK(msg->findInt32("what", &what)); - if (what == Decoder::kWhatFillThisBuffer) { - status_t err = feedDecoderInputData( - audio, msg); + if (what == DecoderBase::kWhatInputDiscontinuity) { + int32_t formatChange; + CHECK(msg->findInt32("formatChange", &formatChange)); - if (err == -EWOULDBLOCK) { - if (mSource->feedMoreTSData() == OK) { - msg->post(10 * 1000ll); - } + ALOGV("%s discontinuity: formatChange %d", + audio ? "audio" : "video", formatChange); + + if (formatChange) { + mDeferredActions.push_back( + new FlushDecoderAction( + audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE, + audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN)); } - } else if (what == Decoder::kWhatEOS) { + + mDeferredActions.push_back( + new SimpleAction( + &NuPlayer::performScanSources)); + + processDeferredActions(); + } else if (what == DecoderBase::kWhatEOS) { int32_t err; CHECK(msg->findInt32("err", &err)); @@ -753,25 +734,20 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { } mRenderer->queueEOS(audio, err); - } else if (what == Decoder::kWhatFlushCompleted) { + } else if (what == DecoderBase::kWhatFlushCompleted) { ALOGV("decoder %s flush completed", audio ? "audio" : "video"); handleFlushComplete(audio, true /* isDecoder */); finishFlushIfPossible(); - } else if (what == Decoder::kWhatOutputFormatChanged) { + } else if (what == DecoderBase::kWhatVideoSizeChanged) { sp<AMessage> format; CHECK(msg->findMessage("format", &format)); - if (audio) { - openAudioSink(format, false /*offloadOnly*/); - } else { - // video - sp<AMessage> inputFormat = - mSource->getFormat(false /* audio */); + sp<AMessage> inputFormat = + mSource->getFormat(false /* audio */); - updateVideoSize(inputFormat, format); - } - } else if (what == Decoder::kWhatShutdownCompleted) { + updateVideoSize(inputFormat, format); + } else if (what == DecoderBase::kWhatShutdownCompleted) { ALOGV("%s shutdown completed", audio ? "audio" : "video"); if (audio) { mAudioDecoder.clear(); @@ -788,7 +764,9 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { } finishFlushIfPossible(); - } else if (what == Decoder::kWhatError) { + } else if (what == DecoderBase::kWhatResumeCompleted) { + finishResume(); + } else if (what == DecoderBase::kWhatError) { status_t err; if (!msg->findInt32("err", &err) || err == OK) { err = UNKNOWN_ERROR; @@ -797,6 +775,8 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { // Decoder errors can be due to Source (e.g. from streaming), // or from decoding corrupted bitstreams, or from other decoder // MediaCodec operations (e.g. from an ongoing reset or seek). + // They may also be due to openAudioSink failure at + // decoder start or after a format change. // // We try to gracefully shut down the affected decoder if possible, // rather than trying to force the shutdown with something @@ -811,7 +791,9 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { switch (*flushing) { case NONE: mDeferredActions.push_back( - new ShutdownDecoderAction(audio, !audio /* video */)); + new FlushDecoderAction( + audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE, + audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN)); processDeferredActions(); break; case FLUSHING_DECODER: @@ -834,8 +816,6 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { break; // Finish anyways. } notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err); - } else if (what == Decoder::kWhatDrainThisBuffer) { - renderBuffer(audio, msg); } else { ALOGV("Unhandled decoder notification %d '%c%c%c%c'.", what, @@ -901,7 +881,7 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { ALOGV("media rendering started"); notifyListener(MEDIA_STARTED, 0, 0); } else if (what == Renderer::kWhatAudioOffloadTearDown) { - ALOGV("Tear down audio offload, fall back to s/w path"); + ALOGV("Tear down audio offload, fall back to s/w path if due to error."); int64_t positionUs; CHECK(msg->findInt64("positionUs", &positionUs)); int32_t reason; @@ -909,15 +889,17 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { closeAudioSink(); mAudioDecoder.clear(); ++mAudioDecoderGeneration; - mRenderer->flush(true /* audio */); + mRenderer->flush( + true /* audio */, false /* notifyComplete */); if (mVideoDecoder != NULL) { - mRenderer->flush(false /* audio */); + mRenderer->flush( + false /* audio */, false /* notifyComplete */); } - mRenderer->signalDisableOffloadAudio(); - mOffloadAudio = false; performSeek(positionUs, false /* needNotify */); if (reason == Renderer::kDueToError) { + mRenderer->signalDisableOffloadAudio(); + mOffloadAudio = false; instantiateDecoder(true /* audio */, &mAudioDecoder); } } @@ -934,8 +916,9 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { ALOGV("kWhatReset"); mDeferredActions.push_back( - new ShutdownDecoderAction( - true /* audio */, true /* video */)); + new FlushDecoderAction( + FLUSH_CMD_SHUTDOWN /* audio */, + FLUSH_CMD_SHUTDOWN /* video */)); mDeferredActions.push_back( new SimpleAction(&NuPlayer::performReset)); @@ -955,11 +938,18 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { seekTimeUs, needNotify); mDeferredActions.push_back( - new SimpleAction(&NuPlayer::performDecoderFlush)); + new FlushDecoderAction(FLUSH_CMD_FLUSH /* audio */, + FLUSH_CMD_FLUSH /* video */)); mDeferredActions.push_back( new SeekAction(seekTimeUs, needNotify)); + // After a flush without shutdown, decoder is paused. + // Don't resume it until source seek is done, otherwise it could + // start pulling stale data too soon. + mDeferredActions.push_back( + new ResumeDecoderAction(needNotify)); + processDeferredActions(); break; } @@ -979,26 +969,6 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { break; } - case kWhatResume: - { - if (mSource != NULL) { - mSource->resume(); - } else { - ALOGW("resume called when source is gone or not set"); - } - // |mAudioDecoder| may have been released due to the pause timeout, so re-create it if - // needed. - if (audioDecoderStillNeeded() && mAudioDecoder == NULL) { - instantiateDecoder(true /* audio */, &mAudioDecoder); - } - if (mRenderer != NULL) { - mRenderer->resume(); - } else { - ALOGW("resume called when renderer is gone or not set"); - } - break; - } - case kWhatSourceNotify: { onSourceNotify(msg); @@ -1017,6 +987,91 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { } } +void NuPlayer::onResume() { + if (mSource != NULL) { + mSource->resume(); + } else { + ALOGW("resume called when source is gone or not set"); + } + // |mAudioDecoder| may have been released due to the pause timeout, so re-create it if + // needed. + if (audioDecoderStillNeeded() && mAudioDecoder == NULL) { + instantiateDecoder(true /* audio */, &mAudioDecoder); + } + if (mRenderer != NULL) { + mRenderer->resume(); + } else { + ALOGW("resume called when renderer is gone or not set"); + } +} + +void NuPlayer::onStart() { + mOffloadAudio = false; + mAudioEOS = false; + mVideoEOS = false; + mStarted = true; + + /* instantiate decoders now for secure playback */ + if (mSourceFlags & Source::FLAG_SECURE) { + if (mNativeWindow != NULL) { + instantiateDecoder(false, &mVideoDecoder); + } + + if (mAudioSink != NULL) { + instantiateDecoder(true, &mAudioDecoder); + } + } + + mSource->start(); + + uint32_t flags = 0; + + if (mSource->isRealTime()) { + flags |= Renderer::FLAG_REAL_TIME; + } + + sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */); + audio_stream_type_t streamType = AUDIO_STREAM_MUSIC; + if (mAudioSink != NULL) { + streamType = mAudioSink->getAudioStreamType(); + } + + sp<AMessage> videoFormat = mSource->getFormat(false /* audio */); + + mOffloadAudio = + canOffloadStream(audioMeta, (videoFormat != NULL), + true /* is_streaming */, streamType); + if (mOffloadAudio) { + flags |= Renderer::FLAG_OFFLOAD_AUDIO; + } + + sp<AMessage> notify = new AMessage(kWhatRendererNotify, id()); + ++mRendererGeneration; + notify->setInt32("generation", mRendererGeneration); + mRenderer = new Renderer(mAudioSink, notify, flags); + + mRendererLooper = new ALooper; + mRendererLooper->setName("NuPlayerRenderer"); + mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO); + mRendererLooper->registerHandler(mRenderer); + + sp<MetaData> meta = getFileMeta(); + int32_t rate; + if (meta != NULL + && meta->findInt32(kKeyFrameRate, &rate) && rate > 0) { + mRenderer->setVideoFrameRate(rate); + } + + if (mVideoDecoder != NULL) { + mVideoDecoder->setRenderer(mRenderer); + } + if (mAudioDecoder != NULL) { + mAudioDecoder->setRenderer(mRenderer); + } + + postScanSources(); +} + bool NuPlayer::audioDecoderStillNeeded() { // Audio decoder is no longer needed if it's in shut/shutting down status. return ((mFlushingAudio != SHUT_DOWN) && (mFlushingAudio != SHUTTING_DOWN_DECODER)); @@ -1074,22 +1129,6 @@ void NuPlayer::finishFlushIfPossible() { ALOGV("both audio and video are flushed now."); - mPendingAudioAccessUnit.clear(); - mAggregateBuffer.clear(); - - if (mTimeDiscontinuityPending) { - mRenderer->signalTimeDiscontinuity(); - mTimeDiscontinuityPending = false; - } - - if (mAudioDecoder != NULL && mFlushingAudio == FLUSHED) { - mAudioDecoder->signalResume(); - } - - if (mVideoDecoder != NULL && mFlushingVideo == FLUSHED) { - mVideoDecoder->signalResume(); - } - mFlushingAudio = NONE; mFlushingVideo = NONE; @@ -1110,28 +1149,16 @@ void NuPlayer::postScanSources() { mScanSourcesPending = true; } -void NuPlayer::openAudioSink(const sp<AMessage> &format, bool offloadOnly) { - uint32_t flags; - int64_t durationUs; - bool hasVideo = (mVideoDecoder != NULL); - // FIXME: we should handle the case where the video decoder - // is created after we receive the format change indication. - // Current code will just make that we select deep buffer - // with video which should not be a problem as it should - // not prevent from keeping A/V sync. - if (hasVideo && - mSource->getDuration(&durationUs) == OK && - durationUs - > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) { - flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER; - } else { - flags = AUDIO_OUTPUT_FLAG_NONE; - } +void NuPlayer::tryOpenAudioSinkForOffload(const sp<AMessage> &format, bool hasVideo) { + // Note: This is called early in NuPlayer to determine whether offloading + // is possible; otherwise the decoders call the renderer openAudioSink directly. - mOffloadAudio = mRenderer->openAudioSink( - format, offloadOnly, hasVideo, flags); - - if (mOffloadAudio) { + status_t err = mRenderer->openAudioSink( + format, true /* offloadOnly */, hasVideo, AUDIO_OUTPUT_FLAG_NONE, &mOffloadAudio); + if (err != OK) { + // Any failure we turn off mOffloadAudio. + mOffloadAudio = false; + } else if (mOffloadAudio) { sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */); sendMetaDataToHal(mAudioSink, audioMeta); @@ -1142,7 +1169,7 @@ void NuPlayer::closeAudioSink() { mRenderer->closeAudioSink(); } -status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) { +status_t NuPlayer::instantiateDecoder(bool audio, sp<DecoderBase> *decoder) { if (*decoder != NULL) { return OK; } @@ -1156,7 +1183,6 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) { if (!audio) { AString mime; CHECK(format->findString("mime", &mime)); - mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str()); sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id()); mCCDecoder = new CCDecoder(ccNotify); @@ -1172,16 +1198,28 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) { notify->setInt32("generation", mAudioDecoderGeneration); if (mOffloadAudio) { - *decoder = new DecoderPassThrough(notify); + *decoder = new DecoderPassThrough(notify, mSource, mRenderer); } else { - *decoder = new Decoder(notify); + *decoder = new Decoder(notify, mSource, mRenderer); } } else { sp<AMessage> notify = new AMessage(kWhatVideoNotify, id()); ++mVideoDecoderGeneration; notify->setInt32("generation", mVideoDecoderGeneration); - *decoder = new Decoder(notify, mNativeWindow); + *decoder = new Decoder( + notify, mSource, mRenderer, mNativeWindow, mCCDecoder); + + // enable FRC if high-quality AV sync is requested, even if not + // queuing to native window, as this will even improve textureview + // playback. + { + char value[PROPERTY_VALUE_MAX]; + if (property_get("persist.sys.media.avsync", value, NULL) && + (!strcmp("1", value) || !strcasecmp("true", value))) { + format->setInt32("auto-frc", 1); + } + } } (*decoder)->init(); (*decoder)->configure(format); @@ -1211,281 +1249,6 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) { return OK; } -status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) { - sp<AMessage> reply; - CHECK(msg->findMessage("reply", &reply)); - - if ((audio && mFlushingAudio != NONE) - || (!audio && mFlushingVideo != NONE) - || mSource == NULL) { - reply->setInt32("err", INFO_DISCONTINUITY); - reply->post(); - return OK; - } - - sp<ABuffer> accessUnit; - - // Aggregate smaller buffers into a larger buffer. - // The goal is to reduce power consumption. - // Note this will not work if the decoder requires one frame per buffer. - bool doBufferAggregation = (audio && mOffloadAudio); - bool needMoreData = false; - - bool dropAccessUnit; - do { - status_t err; - // Did we save an accessUnit earlier because of a discontinuity? - if (audio && (mPendingAudioAccessUnit != NULL)) { - accessUnit = mPendingAudioAccessUnit; - mPendingAudioAccessUnit.clear(); - err = mPendingAudioErr; - ALOGV("feedDecoderInputData() use mPendingAudioAccessUnit"); - } else { - err = mSource->dequeueAccessUnit(audio, &accessUnit); - } - - if (err == -EWOULDBLOCK) { - return err; - } else if (err != OK) { - if (err == INFO_DISCONTINUITY) { - if (doBufferAggregation && (mAggregateBuffer != NULL)) { - // We already have some data so save this for later. - mPendingAudioErr = err; - mPendingAudioAccessUnit = accessUnit; - accessUnit.clear(); - ALOGD("feedDecoderInputData() save discontinuity for later"); - break; - } - int32_t type; - CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); - - bool formatChange = - (audio && - (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT)) - || (!audio && - (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT)); - - bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0; - - ALOGI("%s discontinuity (formatChange=%d, time=%d)", - audio ? "audio" : "video", formatChange, timeChange); - - if (audio) { - mSkipRenderingAudioUntilMediaTimeUs = -1; - } else { - mSkipRenderingVideoUntilMediaTimeUs = -1; - } - - if (timeChange) { - sp<AMessage> extra; - if (accessUnit->meta()->findMessage("extra", &extra) - && extra != NULL) { - int64_t resumeAtMediaTimeUs; - if (extra->findInt64( - "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) { - ALOGI("suppressing rendering of %s until %lld us", - audio ? "audio" : "video", resumeAtMediaTimeUs); - - if (audio) { - mSkipRenderingAudioUntilMediaTimeUs = - resumeAtMediaTimeUs; - } else { - mSkipRenderingVideoUntilMediaTimeUs = - resumeAtMediaTimeUs; - } - } - } - } - - mTimeDiscontinuityPending = - mTimeDiscontinuityPending || timeChange; - - bool seamlessFormatChange = false; - sp<AMessage> newFormat = mSource->getFormat(audio); - if (formatChange) { - seamlessFormatChange = - getDecoder(audio)->supportsSeamlessFormatChange(newFormat); - // treat seamless format change separately - formatChange = !seamlessFormatChange; - } - bool shutdownOrFlush = formatChange || timeChange; - - // We want to queue up scan-sources only once per discontinuity. - // We control this by doing it only if neither audio nor video are - // flushing or shutting down. (After handling 1st discontinuity, one - // of the flushing states will not be NONE.) - // No need to scan sources if this discontinuity does not result - // in a flush or shutdown, as the flushing state will stay NONE. - if (mFlushingAudio == NONE && mFlushingVideo == NONE && - shutdownOrFlush) { - // And we'll resume scanning sources once we're done - // flushing. - mDeferredActions.push_front( - new SimpleAction( - &NuPlayer::performScanSources)); - } - - if (formatChange /* not seamless */) { - // must change decoder - flushDecoder(audio, /* needShutdown = */ true); - } else if (timeChange) { - // need to flush - flushDecoder(audio, /* needShutdown = */ false, newFormat); - err = OK; - } else if (seamlessFormatChange) { - // reuse existing decoder and don't flush - updateDecoderFormatWithoutFlush(audio, newFormat); - err = OK; - } else { - // This stream is unaffected by the discontinuity - return -EWOULDBLOCK; - } - } - - reply->setInt32("err", err); - reply->post(); - return OK; - } - - if (!audio) { - ++mNumFramesTotal; - } - - dropAccessUnit = false; - if (!audio - && !(mSourceFlags & Source::FLAG_SECURE) - && mRenderer->getVideoLateByUs() > 100000ll - && mVideoIsAVC - && !IsAVCReferenceFrame(accessUnit)) { - dropAccessUnit = true; - ++mNumFramesDropped; - } - - size_t smallSize = accessUnit->size(); - needMoreData = false; - if (doBufferAggregation && (mAggregateBuffer == NULL) - // Don't bother if only room for a few small buffers. - && (smallSize < (kAggregateBufferSizeBytes / 3))) { - // Create a larger buffer for combining smaller buffers from the extractor. - mAggregateBuffer = new ABuffer(kAggregateBufferSizeBytes); - mAggregateBuffer->setRange(0, 0); // start empty - } - - if (doBufferAggregation && (mAggregateBuffer != NULL)) { - int64_t timeUs; - int64_t dummy; - bool smallTimestampValid = accessUnit->meta()->findInt64("timeUs", &timeUs); - bool bigTimestampValid = mAggregateBuffer->meta()->findInt64("timeUs", &dummy); - // Will the smaller buffer fit? - size_t bigSize = mAggregateBuffer->size(); - size_t roomLeft = mAggregateBuffer->capacity() - bigSize; - // Should we save this small buffer for the next big buffer? - // If the first small buffer did not have a timestamp then save - // any buffer that does have a timestamp until the next big buffer. - if ((smallSize > roomLeft) - || (!bigTimestampValid && (bigSize > 0) && smallTimestampValid)) { - mPendingAudioErr = err; - mPendingAudioAccessUnit = accessUnit; - accessUnit.clear(); - } else { - // Grab time from first small buffer if available. - if ((bigSize == 0) && smallTimestampValid) { - mAggregateBuffer->meta()->setInt64("timeUs", timeUs); - } - // Append small buffer to the bigger buffer. - memcpy(mAggregateBuffer->base() + bigSize, accessUnit->data(), smallSize); - bigSize += smallSize; - mAggregateBuffer->setRange(0, bigSize); - - // Keep looping until we run out of room in the mAggregateBuffer. - needMoreData = true; - - ALOGV("feedDecoderInputData() smallSize = %zu, bigSize = %zu, capacity = %zu", - smallSize, bigSize, mAggregateBuffer->capacity()); - } - } - } while (dropAccessUnit || needMoreData); - - // ALOGV("returned a valid buffer of %s data", audio ? "audio" : "video"); - -#if 0 - int64_t mediaTimeUs; - CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs)); - ALOGV("feeding %s input buffer at media time %.2f secs", - audio ? "audio" : "video", - mediaTimeUs / 1E6); -#endif - - if (!audio) { - mCCDecoder->decode(accessUnit); - } - - if (doBufferAggregation && (mAggregateBuffer != NULL)) { - ALOGV("feedDecoderInputData() reply with aggregated buffer, %zu", - mAggregateBuffer->size()); - reply->setBuffer("buffer", mAggregateBuffer); - mAggregateBuffer.clear(); - } else { - reply->setBuffer("buffer", accessUnit); - } - - reply->post(); - - return OK; -} - -void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) { - // ALOGV("renderBuffer %s", audio ? "audio" : "video"); - - sp<AMessage> reply; - CHECK(msg->findMessage("reply", &reply)); - - if ((audio && mFlushingAudio != NONE) - || (!audio && mFlushingVideo != NONE)) { - // We're currently attempting to flush the decoder, in order - // to complete this, the decoder wants all its buffers back, - // so we don't want any output buffers it sent us (from before - // we initiated the flush) to be stuck in the renderer's queue. - - ALOGV("we're still flushing the %s decoder, sending its output buffer" - " right back.", audio ? "audio" : "video"); - - reply->post(); - return; - } - - sp<ABuffer> buffer; - CHECK(msg->findBuffer("buffer", &buffer)); - - int64_t mediaTimeUs; - CHECK(buffer->meta()->findInt64("timeUs", &mediaTimeUs)); - - int64_t &skipUntilMediaTimeUs = - audio - ? mSkipRenderingAudioUntilMediaTimeUs - : mSkipRenderingVideoUntilMediaTimeUs; - - if (skipUntilMediaTimeUs >= 0) { - - if (mediaTimeUs < skipUntilMediaTimeUs) { - ALOGV("dropping %s buffer at time %lld as requested.", - audio ? "audio" : "video", - mediaTimeUs); - - reply->post(); - return; - } - - skipUntilMediaTimeUs = -1; - } - - if (!audio && mCCDecoder->isSelected()) { - mCCDecoder->display(mediaTimeUs); - } - - mRenderer->queueBuffer(audio, buffer, reply); -} - void NuPlayer::updateVideoSize( const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) { @@ -1566,12 +1329,11 @@ void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) { driver->notifyListener(msg, ext1, ext2, in); } -void NuPlayer::flushDecoder( - bool audio, bool needShutdown, const sp<AMessage> &newFormat) { +void NuPlayer::flushDecoder(bool audio, bool needShutdown) { ALOGV("[%s] flushDecoder needShutdown=%d", audio ? "audio" : "video", needShutdown); - const sp<Decoder> &decoder = getDecoder(audio); + const sp<DecoderBase> &decoder = getDecoder(audio); if (decoder == NULL) { ALOGI("flushDecoder %s without decoder present", audio ? "audio" : "video"); @@ -1582,8 +1344,7 @@ void NuPlayer::flushDecoder( ++mScanSourcesGeneration; mScanSourcesPending = false; - decoder->signalFlush(newFormat); - mRenderer->flush(audio); + decoder->signalFlush(); FlushStatus newStatus = needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER; @@ -1598,33 +1359,17 @@ void NuPlayer::flushDecoder( ALOGE_IF(mFlushingVideo != NONE, "video flushDecoder() is called in state %d", mFlushingVideo); mFlushingVideo = newStatus; - - if (mCCDecoder != NULL) { - mCCDecoder->flush(); - } } } -void NuPlayer::updateDecoderFormatWithoutFlush( - bool audio, const sp<AMessage> &format) { - ALOGV("[%s] updateDecoderFormatWithoutFlush", audio ? "audio" : "video"); - - const sp<Decoder> &decoder = getDecoder(audio); - if (decoder == NULL) { - ALOGI("updateDecoderFormatWithoutFlush %s without decoder present", - audio ? "audio" : "video"); - return; - } - - decoder->signalUpdateFormat(format); -} - void NuPlayer::queueDecoderShutdown( bool audio, bool video, const sp<AMessage> &reply) { ALOGI("queueDecoderShutdown audio=%d, video=%d", audio, video); mDeferredActions.push_back( - new ShutdownDecoderAction(audio, video)); + new FlushDecoderAction( + audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE, + video ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE)); mDeferredActions.push_back( new SimpleAction(&NuPlayer::performScanSources)); @@ -1670,10 +1415,11 @@ status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const { return err; } -status_t NuPlayer::selectTrack(size_t trackIndex, bool select) { +status_t NuPlayer::selectTrack(size_t trackIndex, bool select, int64_t timeUs) { sp<AMessage> msg = new AMessage(kWhatSelectTrack, id()); msg->setSize("trackIndex", trackIndex); msg->setInt32("select", select); + msg->setInt64("timeUs", timeUs); sp<AMessage> response; status_t err = msg->postAndAwaitResponse(&response); @@ -1699,8 +1445,13 @@ status_t NuPlayer::getCurrentPosition(int64_t *mediaUs) { } void NuPlayer::getStats(int64_t *numFramesTotal, int64_t *numFramesDropped) { - *numFramesTotal = mNumFramesTotal; - *numFramesDropped = mNumFramesDropped; + sp<DecoderBase> decoder = getDecoder(false /* audio */); + if (decoder != NULL) { + decoder->getStats(numFramesTotal, numFramesDropped); + } else { + *numFramesTotal = 0; + *numFramesDropped = 0; + } } sp<MetaData> NuPlayer::getFileMeta() { @@ -1757,52 +1508,23 @@ void NuPlayer::performSeek(int64_t seekTimeUs, bool needNotify) { mSource->seekTo(seekTimeUs); ++mTimedTextGeneration; - if (mDriver != NULL) { - sp<NuPlayerDriver> driver = mDriver.promote(); - if (driver != NULL) { - if (needNotify) { - driver->notifySeekComplete(); - } - } - } - // everything's flushed, continue playback. } -void NuPlayer::performDecoderFlush() { - ALOGV("performDecoderFlush"); +void NuPlayer::performDecoderFlush(FlushCommand audio, FlushCommand video) { + ALOGV("performDecoderFlush audio=%d, video=%d", audio, video); - if (mAudioDecoder == NULL && mVideoDecoder == NULL) { + if ((audio == FLUSH_CMD_NONE || mAudioDecoder == NULL) + && (video == FLUSH_CMD_NONE || mVideoDecoder == NULL)) { return; } - mTimeDiscontinuityPending = true; - - if (mAudioDecoder != NULL) { - flushDecoder(true /* audio */, false /* needShutdown */); - } - - if (mVideoDecoder != NULL) { - flushDecoder(false /* audio */, false /* needShutdown */); - } -} - -void NuPlayer::performDecoderShutdown(bool audio, bool video) { - ALOGV("performDecoderShutdown audio=%d, video=%d", audio, video); - - if ((!audio || mAudioDecoder == NULL) - && (!video || mVideoDecoder == NULL)) { - return; - } - - mTimeDiscontinuityPending = true; - - if (audio && mAudioDecoder != NULL) { - flushDecoder(true /* audio */, true /* needShutdown */); + if (audio != FLUSH_CMD_NONE && mAudioDecoder != NULL) { + flushDecoder(true /* audio */, (audio == FLUSH_CMD_SHUTDOWN)); } - if (video && mVideoDecoder != NULL) { - flushDecoder(false /* audio */, true /* needShutdown */); + if (video != FLUSH_CMD_NONE && mVideoDecoder != NULL) { + flushDecoder(false /* audio */, (video == FLUSH_CMD_SHUTDOWN)); } } @@ -1871,6 +1593,42 @@ void NuPlayer::performSetSurface(const sp<NativeWindowWrapper> &wrapper) { } } +void NuPlayer::performResumeDecoders(bool needNotify) { + if (needNotify) { + mResumePending = true; + if (mVideoDecoder == NULL) { + // if audio-only, we can notify seek complete now, + // as the resume operation will be relatively fast. + finishResume(); + } + } + + if (mVideoDecoder != NULL) { + // When there is continuous seek, MediaPlayer will cache the seek + // position, and send down new seek request when previous seek is + // complete. Let's wait for at least one video output frame before + // notifying seek complete, so that the video thumbnail gets updated + // when seekbar is dragged. + mVideoDecoder->signalResume(needNotify); + } + + if (mAudioDecoder != NULL) { + mAudioDecoder->signalResume(false /* needNotify */); + } +} + +void NuPlayer::finishResume() { + if (mResumePending) { + mResumePending = false; + if (mDriver != NULL) { + sp<NuPlayerDriver> driver = mDriver.promote(); + if (driver != NULL) { + driver->notifySeekComplete(); + } + } + } +} + void NuPlayer::onSourceNotify(const sp<AMessage> &msg) { int32_t what; CHECK(msg->findInt32("what", &what)); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h index 24c06c9..bb32eac 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h @@ -54,7 +54,6 @@ struct NuPlayer : public AHandler { void start(); void pause(); - void resume(); // Will notify the driver through "notifyResetComplete" once finished. void resetAsync(); @@ -66,14 +65,12 @@ struct NuPlayer : public AHandler { status_t setVideoScalingMode(int32_t mode); status_t getTrackInfo(Parcel* reply) const; status_t getSelectedTrack(int32_t type, Parcel* reply) const; - status_t selectTrack(size_t trackIndex, bool select); + status_t selectTrack(size_t trackIndex, bool select, int64_t timeUs); status_t getCurrentPosition(int64_t *mediaUs); void getStats(int64_t *mNumFramesTotal, int64_t *mNumFramesDropped); sp<MetaData> getFileMeta(); - static const size_t kAggregateBufferSizeBytes; - protected: virtual ~NuPlayer(); @@ -85,6 +82,7 @@ public: private: struct Decoder; + struct DecoderBase; struct DecoderPassThrough; struct CCDecoder; struct GenericSource; @@ -95,7 +93,8 @@ private: struct Action; struct SeekAction; struct SetSurfaceAction; - struct ShutdownDecoderAction; + struct ResumeDecoderAction; + struct FlushDecoderAction; struct PostMessageAction; struct SimpleAction; @@ -129,10 +128,9 @@ private: uint32_t mSourceFlags; sp<NativeWindowWrapper> mNativeWindow; sp<MediaPlayerBase::AudioSink> mAudioSink; - sp<Decoder> mVideoDecoder; - bool mVideoIsAVC; + sp<DecoderBase> mVideoDecoder; bool mOffloadAudio; - sp<Decoder> mAudioDecoder; + sp<DecoderBase> mAudioDecoder; sp<CCDecoder> mCCDecoder; sp<Renderer> mRenderer; sp<ALooper> mRendererLooper; @@ -160,32 +158,26 @@ private: SHUT_DOWN, }; - // Once the current flush is complete this indicates whether the - // notion of time has changed. - bool mTimeDiscontinuityPending; + enum FlushCommand { + FLUSH_CMD_NONE, + FLUSH_CMD_FLUSH, + FLUSH_CMD_SHUTDOWN, + }; // Status of flush responses from the decoder and renderer. bool mFlushComplete[2][2]; - // Used by feedDecoderInputData to aggregate small buffers into - // one large buffer. - sp<ABuffer> mPendingAudioAccessUnit; - status_t mPendingAudioErr; - sp<ABuffer> mAggregateBuffer; - FlushStatus mFlushingAudio; FlushStatus mFlushingVideo; - int64_t mSkipRenderingAudioUntilMediaTimeUs; - int64_t mSkipRenderingVideoUntilMediaTimeUs; - - int64_t mNumFramesTotal, mNumFramesDropped; + // Status of flush responses from the decoder and renderer. + bool mResumePending; int32_t mVideoScalingMode; bool mStarted; - inline const sp<Decoder> &getDecoder(bool audio) { + inline const sp<DecoderBase> &getDecoder(bool audio) { return audio ? mAudioDecoder : mVideoDecoder; } @@ -196,28 +188,28 @@ private: mFlushComplete[1][1] = false; } - void openAudioSink(const sp<AMessage> &format, bool offloadOnly); + void tryOpenAudioSinkForOffload(const sp<AMessage> &format, bool hasVideo); void closeAudioSink(); - status_t instantiateDecoder(bool audio, sp<Decoder> *decoder); + status_t instantiateDecoder(bool audio, sp<DecoderBase> *decoder); void updateVideoSize( const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat = NULL); - status_t feedDecoderInputData(bool audio, const sp<AMessage> &msg); - void renderBuffer(bool audio, const sp<AMessage> &msg); - void notifyListener(int msg, int ext1, int ext2, const Parcel *in = NULL); void handleFlushComplete(bool audio, bool isDecoder); void finishFlushIfPossible(); + void onStart(); + void onResume(); + bool audioDecoderStillNeeded(); - void flushDecoder( - bool audio, bool needShutdown, const sp<AMessage> &newFormat = NULL); - void updateDecoderFormatWithoutFlush(bool audio, const sp<AMessage> &format); + void flushDecoder(bool audio, bool needShutdown); + + void finishResume(); void postScanSources(); @@ -227,11 +219,11 @@ private: void processDeferredActions(); void performSeek(int64_t seekTimeUs, bool needNotify); - void performDecoderFlush(); - void performDecoderShutdown(bool audio, bool video); + void performDecoderFlush(FlushCommand audio, FlushCommand video); void performReset(); void performScanSources(); void performSetSurface(const sp<NativeWindowWrapper> &wrapper); + void performResumeDecoders(bool needNotify); void onSourceNotify(const sp<AMessage> &msg); void onClosedCaptionNotify(const sp<AMessage> &msg); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.cpp new file mode 100644 index 0000000..9229704 --- /dev/null +++ b/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.cpp @@ -0,0 +1,361 @@ +/* + * Copyright 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "NuPlayerCCDecoder" +#include <utils/Log.h> +#include <inttypes.h> + +#include "NuPlayerCCDecoder.h" + +#include <media/stagefright/foundation/ABitReader.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/MediaDefs.h> + +namespace android { + +struct CCData { + CCData(uint8_t type, uint8_t data1, uint8_t data2) + : mType(type), mData1(data1), mData2(data2) { + } + bool getChannel(size_t *channel) const { + if (mData1 >= 0x10 && mData1 <= 0x1f) { + *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0); + return true; + } + return false; + } + + uint8_t mType; + uint8_t mData1; + uint8_t mData2; +}; + +static bool isNullPad(CCData *cc) { + return cc->mData1 < 0x10 && cc->mData2 < 0x10; +} + +static void dumpBytePair(const sp<ABuffer> &ccBuf) { + size_t offset = 0; + AString out; + + while (offset < ccBuf->size()) { + char tmp[128]; + + CCData *cc = (CCData *) (ccBuf->data() + offset); + + if (isNullPad(cc)) { + // 1 null pad or XDS metadata, ignore + offset += sizeof(CCData); + continue; + } + + if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) { + // 2 basic chars + sprintf(tmp, "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2); + } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19) + && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) { + // 1 special char + sprintf(tmp, "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2); + } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A) + && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){ + // 1 Spanish/French char + sprintf(tmp, "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2); + } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B) + && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){ + // 1 Portuguese/German/Danish char + sprintf(tmp, "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2); + } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19) + && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){ + // Mid-Row Codes (Table 69) + sprintf(tmp, "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2); + } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c) + && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f) + || + ((cc->mData1 == 0x17 || cc->mData1 == 0x1f) + && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){ + // Misc Control Codes (Table 70) + sprintf(tmp, "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2); + } else if ((cc->mData1 & 0x70) == 0x10 + && (cc->mData2 & 0x40) == 0x40 + && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) { + // Preamble Address Codes (Table 71) + sprintf(tmp, "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2); + } else { + sprintf(tmp, "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2); + } + + if (out.size() > 0) { + out.append(", "); + } + + out.append(tmp); + + offset += sizeof(CCData); + } + + ALOGI("%s", out.c_str()); +} + +NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> ¬ify) + : mNotify(notify), + mCurrentChannel(0), + mSelectedTrack(-1) { + for (size_t i = 0; i < sizeof(mTrackIndices)/sizeof(mTrackIndices[0]); ++i) { + mTrackIndices[i] = -1; + } +} + +size_t NuPlayer::CCDecoder::getTrackCount() const { + return mFoundChannels.size(); +} + +sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const { + if (!isTrackValid(index)) { + return NULL; + } + + sp<AMessage> format = new AMessage(); + + format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE); + format->setString("language", "und"); + format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608); + //CC1, field 0 channel 0 + bool isDefaultAuto = (mFoundChannels[index] == 0); + format->setInt32("auto", isDefaultAuto); + format->setInt32("default", isDefaultAuto); + format->setInt32("forced", 0); + + return format; +} + +status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) { + if (!isTrackValid(index)) { + return BAD_VALUE; + } + + if (select) { + if (mSelectedTrack == (ssize_t)index) { + ALOGE("track %zu already selected", index); + return BAD_VALUE; + } + ALOGV("selected track %zu", index); + mSelectedTrack = index; + } else { + if (mSelectedTrack != (ssize_t)index) { + ALOGE("track %zu is not selected", index); + return BAD_VALUE; + } + ALOGV("unselected track %zu", index); + mSelectedTrack = -1; + } + + return OK; +} + +bool NuPlayer::CCDecoder::isSelected() const { + return mSelectedTrack >= 0 && mSelectedTrack < (int32_t) getTrackCount(); +} + +bool NuPlayer::CCDecoder::isTrackValid(size_t index) const { + return index < getTrackCount(); +} + +int32_t NuPlayer::CCDecoder::getTrackIndex(size_t channel) const { + if (channel < sizeof(mTrackIndices)/sizeof(mTrackIndices[0])) { + return mTrackIndices[channel]; + } + return -1; +} + +// returns true if a new CC track is found +bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) { + int64_t timeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + + sp<ABuffer> sei; + if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) { + return false; + } + + bool trackAdded = false; + + NALBitReader br(sei->data() + 1, sei->size() - 1); + // sei_message() + while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message() + uint32_t payload_type = 0; + size_t payload_size = 0; + uint8_t last_byte; + + do { + last_byte = br.getBits(8); + payload_type += last_byte; + } while (last_byte == 0xFF); + + do { + last_byte = br.getBits(8); + payload_size += last_byte; + } while (last_byte == 0xFF); + + // sei_payload() + if (payload_type == 4) { + // user_data_registered_itu_t_t35() + + // ATSC A/72: 6.4.2 + uint8_t itu_t_t35_country_code = br.getBits(8); + uint16_t itu_t_t35_provider_code = br.getBits(16); + uint32_t user_identifier = br.getBits(32); + uint8_t user_data_type_code = br.getBits(8); + + payload_size -= 1 + 2 + 4 + 1; + + if (itu_t_t35_country_code == 0xB5 + && itu_t_t35_provider_code == 0x0031 + && user_identifier == 'GA94' + && user_data_type_code == 0x3) { + // MPEG_cc_data() + // ATSC A/53 Part 4: 6.2.3.1 + br.skipBits(1); //process_em_data_flag + bool process_cc_data_flag = br.getBits(1); + br.skipBits(1); //additional_data_flag + size_t cc_count = br.getBits(5); + br.skipBits(8); // em_data; + payload_size -= 2; + + if (process_cc_data_flag) { + AString out; + + sp<ABuffer> ccBuf = new ABuffer(cc_count * sizeof(CCData)); + ccBuf->setRange(0, 0); + + for (size_t i = 0; i < cc_count; i++) { + uint8_t marker = br.getBits(5); + CHECK_EQ(marker, 0x1f); + + bool cc_valid = br.getBits(1); + uint8_t cc_type = br.getBits(2); + // remove odd parity bit + uint8_t cc_data_1 = br.getBits(8) & 0x7f; + uint8_t cc_data_2 = br.getBits(8) & 0x7f; + + if (cc_valid + && (cc_type == 0 || cc_type == 1)) { + CCData cc(cc_type, cc_data_1, cc_data_2); + if (!isNullPad(&cc)) { + size_t channel; + if (cc.getChannel(&channel) && getTrackIndex(channel) < 0) { + mTrackIndices[channel] = mFoundChannels.size(); + mFoundChannels.push_back(channel); + trackAdded = true; + } + memcpy(ccBuf->data() + ccBuf->size(), + (void *)&cc, sizeof(cc)); + ccBuf->setRange(0, ccBuf->size() + sizeof(CCData)); + } + } + } + payload_size -= cc_count * 3; + + mCCMap.add(timeUs, ccBuf); + break; + } + } else { + ALOGV("Malformed SEI payload type 4"); + } + } else { + ALOGV("Unsupported SEI payload type %d", payload_type); + } + + // skipping remaining bits of this payload + br.skipBits(payload_size * 8); + } + + return trackAdded; +} + +sp<ABuffer> NuPlayer::CCDecoder::filterCCBuf( + const sp<ABuffer> &ccBuf, size_t index) { + sp<ABuffer> filteredCCBuf = new ABuffer(ccBuf->size()); + filteredCCBuf->setRange(0, 0); + + size_t cc_count = ccBuf->size() / sizeof(CCData); + const CCData* cc_data = (const CCData*)ccBuf->data(); + for (size_t i = 0; i < cc_count; ++i) { + size_t channel; + if (cc_data[i].getChannel(&channel)) { + mCurrentChannel = channel; + } + if (mCurrentChannel == mFoundChannels[index]) { + memcpy(filteredCCBuf->data() + filteredCCBuf->size(), + (void *)&cc_data[i], sizeof(CCData)); + filteredCCBuf->setRange(0, filteredCCBuf->size() + sizeof(CCData)); + } + } + + return filteredCCBuf; +} + +void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) { + if (extractFromSEI(accessUnit)) { + ALOGI("Found CEA-608 track"); + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("what", kWhatTrackAdded); + msg->post(); + } + // TODO: extract CC from other sources +} + +void NuPlayer::CCDecoder::display(int64_t timeUs) { + if (!isTrackValid(mSelectedTrack)) { + ALOGE("Could not find current track(index=%d)", mSelectedTrack); + return; + } + + ssize_t index = mCCMap.indexOfKey(timeUs); + if (index < 0) { + ALOGV("cc for timestamp %" PRId64 " not found", timeUs); + return; + } + + sp<ABuffer> ccBuf = filterCCBuf(mCCMap.valueAt(index), mSelectedTrack); + + if (ccBuf->size() > 0) { +#if 0 + dumpBytePair(ccBuf); +#endif + + ccBuf->meta()->setInt32("trackIndex", mSelectedTrack); + ccBuf->meta()->setInt64("timeUs", timeUs); + ccBuf->meta()->setInt64("durationUs", 0ll); + + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("what", kWhatClosedCaptionData); + msg->setBuffer("buffer", ccBuf); + msg->post(); + } + + // remove all entries before timeUs + mCCMap.removeItemsAt(0, index + 1); +} + +void NuPlayer::CCDecoder::flush() { + mCCMap.clear(); +} + +} // namespace android + diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.h b/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.h new file mode 100644 index 0000000..5e06f4e --- /dev/null +++ b/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.h @@ -0,0 +1,59 @@ +/* + * Copyright 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef NUPLAYER_CCDECODER_H_ + +#define NUPLAYER_CCDECODER_H_ + +#include "NuPlayer.h" + +namespace android { + +struct NuPlayer::CCDecoder : public RefBase { + enum { + kWhatClosedCaptionData, + kWhatTrackAdded, + }; + + CCDecoder(const sp<AMessage> ¬ify); + + size_t getTrackCount() const; + sp<AMessage> getTrackInfo(size_t index) const; + status_t selectTrack(size_t index, bool select); + bool isSelected() const; + void decode(const sp<ABuffer> &accessUnit); + void display(int64_t timeUs); + void flush(); + +private: + sp<AMessage> mNotify; + KeyedVector<int64_t, sp<ABuffer> > mCCMap; + size_t mCurrentChannel; + int32_t mSelectedTrack; + int32_t mTrackIndices[4]; + Vector<size_t> mFoundChannels; + + bool isTrackValid(size_t index) const; + int32_t getTrackIndex(size_t channel) const; + bool extractFromSEI(const sp<ABuffer> &accessUnit); + sp<ABuffer> filterCCBuf(const sp<ABuffer> &ccBuf, size_t index); + + DISALLOW_EVIL_CONSTRUCTORS(CCDecoder); +}; + +} // namespace android + +#endif // NUPLAYER_CCDECODER_H_ diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp index 27f6131..2abd9d6 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010 The Android Open Source Project + * Copyright 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,10 +19,12 @@ #include <utils/Log.h> #include <inttypes.h> +#include "NuPlayerCCDecoder.h" #include "NuPlayerDecoder.h" +#include "NuPlayerRenderer.h" +#include "NuPlayerSource.h" #include <media/ICrypto.h> -#include <media/stagefright/foundation/ABitReader.h> #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> @@ -31,71 +33,103 @@ #include <media/stagefright/MediaDefs.h> #include <media/stagefright/MediaErrors.h> +#include "avc_utils.h" +#include "ATSParser.h" + namespace android { NuPlayer::Decoder::Decoder( const sp<AMessage> ¬ify, - const sp<NativeWindowWrapper> &nativeWindow) - : mNotify(notify), + const sp<Source> &source, + const sp<Renderer> &renderer, + const sp<NativeWindowWrapper> &nativeWindow, + const sp<CCDecoder> &ccDecoder) + : DecoderBase(notify), mNativeWindow(nativeWindow), - mBufferGeneration(0), + mSource(source), + mRenderer(renderer), + mCCDecoder(ccDecoder), + mSkipRenderingUntilMediaTimeUs(-1ll), + mNumFramesTotal(0ll), + mNumFramesDropped(0ll), + mIsAudio(true), + mIsVideoAVC(false), + mIsSecure(false), + mFormatChangePending(false), mPaused(true), + mResumePending(false), mComponentName("decoder") { - // Every decoder has its own looper because MediaCodec operations - // are blocking, but NuPlayer needs asynchronous operations. - mDecoderLooper = new ALooper; - mDecoderLooper->setName("NPDecoder"); - mDecoderLooper->start(false, false, ANDROID_PRIORITY_AUDIO); - mCodecLooper = new ALooper; mCodecLooper->setName("NPDecoder-CL"); mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO); } NuPlayer::Decoder::~Decoder() { - mDecoderLooper->unregisterHandler(id()); - mDecoderLooper->stop(); - releaseAndResetMediaBuffers(); } -static -status_t PostAndAwaitResponse( - const sp<AMessage> &msg, sp<AMessage> *response) { - status_t err = msg->postAndAwaitResponse(response); +void NuPlayer::Decoder::getStats( + int64_t *numFramesTotal, + int64_t *numFramesDropped) const { + *numFramesTotal = mNumFramesTotal; + *numFramesDropped = mNumFramesDropped; +} - if (err != OK) { - return err; - } +void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) { + ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str()); - if (!(*response)->findInt32("err", &err)) { - err = OK; - } + switch (msg->what()) { + case kWhatCodecNotify: + { + if (!isStaleReply(msg)) { + int32_t numInput, numOutput; - return err; -} + if (!msg->findInt32("input-buffers", &numInput)) { + numInput = INT32_MAX; + } -void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) { - mCSDsForCurrentFormat.clear(); - for (int32_t i = 0; ; ++i) { - AString tag = "csd-"; - tag.append(i); - sp<ABuffer> buffer; - if (!format->findBuffer(tag.c_str(), &buffer)) { + if (!msg->findInt32("output-buffers", &numOutput)) { + numOutput = INT32_MAX; + } + + if (!mPaused) { + while (numInput-- > 0 && handleAnInputBuffer()) {} + } + + while (numOutput-- > 0 && handleAnOutputBuffer()) {} + } + + requestCodecNotification(); break; } - mCSDsForCurrentFormat.push(buffer); + + case kWhatRenderBuffer: + { + if (!isStaleReply(msg)) { + onRenderBuffer(msg); + } + break; + } + + default: + DecoderBase::onMessageReceived(msg); + break; } } void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) { CHECK(mCodec == NULL); + mFormatChangePending = false; + ++mBufferGeneration; AString mime; CHECK(format->findString("mime", &mime)); + mIsAudio = !strncasecmp("audio/", mime.c_str(), 6); + mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str()); + sp<Surface> surface = NULL; if (mNativeWindow != NULL) { surface = mNativeWindow->getSurfaceTextureClient(); @@ -123,6 +157,7 @@ void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) { handleError(UNKNOWN_ERROR); return; } + mIsSecure = secure; mCodec->getName(&mComponentName); @@ -169,83 +204,141 @@ void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) { mInputBuffers.size(), mOutputBuffers.size()); - requestCodecNotification(); + if (mRenderer != NULL) { + requestCodecNotification(); + } mPaused = false; + mResumePending = false; } -void NuPlayer::Decoder::releaseAndResetMediaBuffers() { - for (size_t i = 0; i < mMediaBuffers.size(); i++) { - if (mMediaBuffers[i] != NULL) { - mMediaBuffers[i]->release(); - mMediaBuffers.editItemAt(i) = NULL; - } - } - mMediaBuffers.resize(mInputBuffers.size()); - for (size_t i = 0; i < mMediaBuffers.size(); i++) { - mMediaBuffers.editItemAt(i) = NULL; +void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) { + bool hadNoRenderer = (mRenderer == NULL); + mRenderer = renderer; + if (hadNoRenderer && mRenderer != NULL) { + requestCodecNotification(); } - mInputBufferIsDequeued.clear(); - mInputBufferIsDequeued.resize(mInputBuffers.size()); - for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) { - mInputBufferIsDequeued.editItemAt(i) = false; +} + +void NuPlayer::Decoder::onGetInputBuffers( + Vector<sp<ABuffer> > *dstBuffers) { + dstBuffers->clear(); + for (size_t i = 0; i < mInputBuffers.size(); i++) { + dstBuffers->push(mInputBuffers[i]); } +} - mPendingInputMessages.clear(); +void NuPlayer::Decoder::onResume(bool notifyComplete) { + mPaused = false; + + if (notifyComplete) { + mResumePending = true; + } } -void NuPlayer::Decoder::requestCodecNotification() { +void NuPlayer::Decoder::onFlush(bool notifyComplete) { + if (mCCDecoder != NULL) { + mCCDecoder->flush(); + } + + if (mRenderer != NULL) { + mRenderer->flush(mIsAudio, notifyComplete); + mRenderer->signalTimeDiscontinuity(); + } + + status_t err = OK; if (mCodec != NULL) { - sp<AMessage> reply = new AMessage(kWhatCodecNotify, id()); - reply->setInt32("generation", mBufferGeneration); - mCodec->requestActivityNotification(reply); + err = mCodec->flush(); + mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator + ++mBufferGeneration; } -} -bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) { - int32_t generation; - CHECK(msg->findInt32("generation", &generation)); - return generation != mBufferGeneration; -} + if (err != OK) { + ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err); + handleError(err); + // finish with posting kWhatFlushCompleted. + // we attempt to release the buffers even if flush fails. + } + releaseAndResetMediaBuffers(); -void NuPlayer::Decoder::init() { - mDecoderLooper->registerHandler(this); + if (notifyComplete) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatFlushCompleted); + notify->post(); + mPaused = true; + } } -void NuPlayer::Decoder::configure(const sp<AMessage> &format) { - sp<AMessage> msg = new AMessage(kWhatConfigure, id()); - msg->setMessage("format", format); - msg->post(); -} +void NuPlayer::Decoder::onShutdown(bool notifyComplete) { + status_t err = OK; -void NuPlayer::Decoder::signalUpdateFormat(const sp<AMessage> &format) { - sp<AMessage> msg = new AMessage(kWhatUpdateFormat, id()); - msg->setMessage("format", format); - msg->post(); -} + // if there is a pending resume request, notify complete now + notifyResumeCompleteIfNecessary(); + + if (mCodec != NULL) { + err = mCodec->release(); + mCodec = NULL; + ++mBufferGeneration; + + if (mNativeWindow != NULL) { + // reconnect to surface as MediaCodec disconnected from it + status_t error = + native_window_api_connect( + mNativeWindow->getNativeWindow().get(), + NATIVE_WINDOW_API_MEDIA); + ALOGW_IF(error != NO_ERROR, + "[%s] failed to connect to native window, error=%d", + mComponentName.c_str(), error); + } + mComponentName = "decoder"; + } -status_t NuPlayer::Decoder::getInputBuffers(Vector<sp<ABuffer> > *buffers) const { - sp<AMessage> msg = new AMessage(kWhatGetInputBuffers, id()); - msg->setPointer("buffers", buffers); + releaseAndResetMediaBuffers(); - sp<AMessage> response; - return PostAndAwaitResponse(msg, &response); + if (err != OK) { + ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err); + handleError(err); + // finish with posting kWhatShutdownCompleted. + } + + if (notifyComplete) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatShutdownCompleted); + notify->post(); + mPaused = true; + } } -void NuPlayer::Decoder::handleError(int32_t err) -{ - // We cannot immediately release the codec due to buffers still outstanding - // in the renderer. We signal to the player the error so it can shutdown/release the - // decoder after flushing and increment the generation to discard unnecessary messages. +void NuPlayer::Decoder::doRequestBuffers() { + if (mFormatChangePending) { + return; + } + status_t err = OK; + while (!mDequeuedInputBuffers.empty()) { + size_t bufferIx = *mDequeuedInputBuffers.begin(); + sp<AMessage> msg = new AMessage(); + msg->setSize("buffer-ix", bufferIx); + err = fetchInputData(msg); + if (err != OK) { + break; + } + mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin()); - ++mBufferGeneration; + if (!mPendingInputMessages.empty() + || !onInputBufferFetched(msg)) { + mPendingInputMessages.push_back(msg); + } + } - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatError); - notify->setInt32("err", err); - notify->post(); + if (err == -EWOULDBLOCK + && mSource->feedMoreTSData() == OK) { + scheduleRequestBuffers(); + } } bool NuPlayer::Decoder::handleAnInputBuffer() { + if (mFormatChangePending) { + return false; + } size_t bufferIx = -1; status_t res = mCodec->dequeueInputBuffer(&bufferIx); ALOGV("[%s] dequeued input: %d", @@ -267,22 +360,21 @@ bool NuPlayer::Decoder::handleAnInputBuffer() { } mInputBufferIsDequeued.editItemAt(bufferIx) = true; - sp<AMessage> reply = new AMessage(kWhatInputBufferFilled, id()); - reply->setSize("buffer-ix", bufferIx); - reply->setInt32("generation", mBufferGeneration); - if (!mCSDsToSubmit.isEmpty()) { + sp<AMessage> msg = new AMessage(); + msg->setSize("buffer-ix", bufferIx); + sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0); ALOGI("[%s] resubmitting CSD", mComponentName.c_str()); - reply->setBuffer("buffer", buffer); + msg->setBuffer("buffer", buffer); mCSDsToSubmit.removeAt(0); - CHECK(onInputBufferFilled(reply)); + CHECK(onInputBufferFetched(msg)); return true; } while (!mPendingInputMessages.empty()) { sp<AMessage> msg = *mPendingInputMessages.begin(); - if (!onInputBufferFilled(msg)) { + if (!onInputBufferFetched(msg)) { break; } mPendingInputMessages.erase(mPendingInputMessages.begin()); @@ -292,15 +384,273 @@ bool NuPlayer::Decoder::handleAnInputBuffer() { return true; } - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatFillThisBuffer); - notify->setBuffer("buffer", mInputBuffers[bufferIx]); - notify->setMessage("reply", reply); - notify->post(); + mDequeuedInputBuffers.push_back(bufferIx); + + onRequestInputBuffers(); return true; } -bool android::NuPlayer::Decoder::onInputBufferFilled(const sp<AMessage> &msg) { +bool NuPlayer::Decoder::handleAnOutputBuffer() { + if (mFormatChangePending) { + return false; + } + size_t bufferIx = -1; + size_t offset; + size_t size; + int64_t timeUs; + uint32_t flags; + status_t res = mCodec->dequeueOutputBuffer( + &bufferIx, &offset, &size, &timeUs, &flags); + + if (res != OK) { + ALOGV("[%s] dequeued output: %d", mComponentName.c_str(), res); + } else { + ALOGV("[%s] dequeued output: %d (time=%lld flags=%" PRIu32 ")", + mComponentName.c_str(), (int)bufferIx, timeUs, flags); + } + + if (res == INFO_OUTPUT_BUFFERS_CHANGED) { + res = mCodec->getOutputBuffers(&mOutputBuffers); + if (res != OK) { + ALOGE("Failed to get output buffers for %s after INFO event (err=%d)", + mComponentName.c_str(), res); + handleError(res); + return false; + } + // NuPlayer ignores this + return true; + } else if (res == INFO_FORMAT_CHANGED) { + sp<AMessage> format = new AMessage(); + res = mCodec->getOutputFormat(&format); + if (res != OK) { + ALOGE("Failed to get output format for %s after INFO event (err=%d)", + mComponentName.c_str(), res); + handleError(res); + return false; + } + + if (!mIsAudio) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatVideoSizeChanged); + notify->setMessage("format", format); + notify->post(); + } else if (mRenderer != NULL) { + uint32_t flags; + int64_t durationUs; + bool hasVideo = (mSource->getFormat(false /* audio */) != NULL); + if (!hasVideo && + mSource->getDuration(&durationUs) == OK && + durationUs + > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) { + flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER; + } else { + flags = AUDIO_OUTPUT_FLAG_NONE; + } + + res = mRenderer->openAudioSink( + format, false /* offloadOnly */, hasVideo, flags, NULL /* isOffloaded */); + if (res != OK) { + ALOGE("Failed to open AudioSink on format change for %s (err=%d)", + mComponentName.c_str(), res); + handleError(res); + return false; + } + } + return true; + } else if (res == INFO_DISCONTINUITY) { + // nothing to do + return true; + } else if (res != OK) { + if (res != -EAGAIN) { + ALOGE("Failed to dequeue output buffer for %s (err=%d)", + mComponentName.c_str(), res); + handleError(res); + } + return false; + } + + CHECK_LT(bufferIx, mOutputBuffers.size()); + sp<ABuffer> buffer = mOutputBuffers[bufferIx]; + buffer->setRange(offset, size); + buffer->meta()->clear(); + buffer->meta()->setInt64("timeUs", timeUs); + if (flags & MediaCodec::BUFFER_FLAG_EOS) { + buffer->meta()->setInt32("eos", true); + notifyResumeCompleteIfNecessary(); + } + // we do not expect CODECCONFIG or SYNCFRAME for decoder + + sp<AMessage> reply = new AMessage(kWhatRenderBuffer, id()); + reply->setSize("buffer-ix", bufferIx); + reply->setInt32("generation", mBufferGeneration); + + if (mSkipRenderingUntilMediaTimeUs >= 0) { + if (timeUs < mSkipRenderingUntilMediaTimeUs) { + ALOGV("[%s] dropping buffer at time %lld as requested.", + mComponentName.c_str(), (long long)timeUs); + + reply->post(); + return true; + } + + mSkipRenderingUntilMediaTimeUs = -1; + } + + // wait until 1st frame comes out to signal resume complete + notifyResumeCompleteIfNecessary(); + + if (mRenderer != NULL) { + // send the buffer to renderer. + mRenderer->queueBuffer(mIsAudio, buffer, reply); + if (flags & MediaCodec::BUFFER_FLAG_EOS) { + mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM); + } + } + + return true; +} + +void NuPlayer::Decoder::releaseAndResetMediaBuffers() { + for (size_t i = 0; i < mMediaBuffers.size(); i++) { + if (mMediaBuffers[i] != NULL) { + mMediaBuffers[i]->release(); + mMediaBuffers.editItemAt(i) = NULL; + } + } + mMediaBuffers.resize(mInputBuffers.size()); + for (size_t i = 0; i < mMediaBuffers.size(); i++) { + mMediaBuffers.editItemAt(i) = NULL; + } + mInputBufferIsDequeued.clear(); + mInputBufferIsDequeued.resize(mInputBuffers.size()); + for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) { + mInputBufferIsDequeued.editItemAt(i) = false; + } + + mPendingInputMessages.clear(); + mDequeuedInputBuffers.clear(); + mSkipRenderingUntilMediaTimeUs = -1; +} + +void NuPlayer::Decoder::requestCodecNotification() { + if (mFormatChangePending) { + return; + } + if (mCodec != NULL) { + sp<AMessage> reply = new AMessage(kWhatCodecNotify, id()); + reply->setInt32("generation", mBufferGeneration); + mCodec->requestActivityNotification(reply); + } +} + +bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) { + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + return generation != mBufferGeneration; +} + +status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) { + sp<ABuffer> accessUnit; + bool dropAccessUnit; + do { + status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit); + + if (err == -EWOULDBLOCK) { + return err; + } else if (err != OK) { + if (err == INFO_DISCONTINUITY) { + int32_t type; + CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); + + bool formatChange = + (mIsAudio && + (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT)) + || (!mIsAudio && + (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT)); + + bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0; + + ALOGI("%s discontinuity (format=%d, time=%d)", + mIsAudio ? "audio" : "video", formatChange, timeChange); + + bool seamlessFormatChange = false; + sp<AMessage> newFormat = mSource->getFormat(mIsAudio); + if (formatChange) { + seamlessFormatChange = + supportsSeamlessFormatChange(newFormat); + // treat seamless format change separately + formatChange = !seamlessFormatChange; + } + + if (formatChange || timeChange) { + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("what", kWhatInputDiscontinuity); + msg->setInt32("formatChange", formatChange); + msg->post(); + } + + if (formatChange /* not seamless */) { + // must change decoder + // return EOS and wait to be killed + mFormatChangePending = true; + return ERROR_END_OF_STREAM; + } else if (timeChange) { + // need to flush + // TODO: Ideally we shouldn't need a flush upon time + // discontinuity, flushing will cause loss of frames. + // We probably should queue a time change marker to the + // output queue, and handles it in renderer instead. + rememberCodecSpecificData(newFormat); + onFlush(false /* notifyComplete */); + err = OK; + } else if (seamlessFormatChange) { + // reuse existing decoder and don't flush + rememberCodecSpecificData(newFormat); + err = OK; + } else { + // This stream is unaffected by the discontinuity + return -EWOULDBLOCK; + } + } + + reply->setInt32("err", err); + return OK; + } + + if (!mIsAudio) { + ++mNumFramesTotal; + } + + dropAccessUnit = false; + if (!mIsAudio + && !mIsSecure + && mRenderer->getVideoLateByUs() > 100000ll + && mIsVideoAVC + && !IsAVCReferenceFrame(accessUnit)) { + dropAccessUnit = true; + ++mNumFramesDropped; + } + } while (dropAccessUnit); + + // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video"); +#if 0 + int64_t mediaTimeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs)); + ALOGV("feeding %s input buffer at media time %.2f secs", + mIsAudio ? "audio" : "video", + mediaTimeUs / 1E6); +#endif + + if (mCCDecoder != NULL) { + mCCDecoder->decode(accessUnit); + } + + reply->setBuffer("buffer", accessUnit); + + return OK; +} + +bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) { size_t bufferIx; CHECK(msg->findSize("buffer-ix", &bufferIx)); CHECK_LT(bufferIx, mInputBuffers.size()); @@ -342,8 +692,6 @@ bool android::NuPlayer::Decoder::onInputBufferFilled(const sp<AMessage> &msg) { } } - - if (buffer == NULL /* includes !hasBuffer */) { int32_t streamErr = ERROR_END_OF_STREAM; CHECK(msg->findInt32("err", &streamErr) || !hasBuffer); @@ -375,6 +723,17 @@ bool android::NuPlayer::Decoder::onInputBufferFilled(const sp<AMessage> &msg) { handleError(streamErr); } } else { + sp<AMessage> extra; + if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) { + int64_t resumeAtMediaTimeUs; + if (extra->findInt64( + "resume-at-mediaTimeUs", &resumeAtMediaTimeUs)) { + ALOGI("[%s] suppressing rendering until %lld us", + mComponentName.c_str(), (long long)resumeAtMediaTimeUs); + mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs; + } + } + int64_t timeUs = 0; uint32_t flags = 0; CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); @@ -418,96 +777,22 @@ bool android::NuPlayer::Decoder::onInputBufferFilled(const sp<AMessage> &msg) { return true; } -bool NuPlayer::Decoder::handleAnOutputBuffer() { - size_t bufferIx = -1; - size_t offset; - size_t size; - int64_t timeUs; - uint32_t flags; - status_t res = mCodec->dequeueOutputBuffer( - &bufferIx, &offset, &size, &timeUs, &flags); - - if (res != OK) { - ALOGV("[%s] dequeued output: %d", mComponentName.c_str(), res); - } else { - ALOGV("[%s] dequeued output: %d (time=%lld flags=%" PRIu32 ")", - mComponentName.c_str(), (int)bufferIx, timeUs, flags); - } - - if (res == INFO_OUTPUT_BUFFERS_CHANGED) { - res = mCodec->getOutputBuffers(&mOutputBuffers); - if (res != OK) { - ALOGE("Failed to get output buffers for %s after INFO event (err=%d)", - mComponentName.c_str(), res); - handleError(res); - return false; - } - // NuPlayer ignores this - return true; - } else if (res == INFO_FORMAT_CHANGED) { - sp<AMessage> format = new AMessage(); - res = mCodec->getOutputFormat(&format); - if (res != OK) { - ALOGE("Failed to get output format for %s after INFO event (err=%d)", - mComponentName.c_str(), res); - handleError(res); - return false; - } - - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatOutputFormatChanged); - notify->setMessage("format", format); - notify->post(); - return true; - } else if (res == INFO_DISCONTINUITY) { - // nothing to do - return true; - } else if (res != OK) { - if (res != -EAGAIN) { - ALOGE("Failed to dequeue output buffer for %s (err=%d)", - mComponentName.c_str(), res); - handleError(res); - } - return false; - } - - CHECK_LT(bufferIx, mOutputBuffers.size()); - sp<ABuffer> buffer = mOutputBuffers[bufferIx]; - buffer->setRange(offset, size); - buffer->meta()->clear(); - buffer->meta()->setInt64("timeUs", timeUs); - if (flags & MediaCodec::BUFFER_FLAG_EOS) { - buffer->meta()->setInt32("eos", true); - } - // we do not expect CODECCONFIG or SYNCFRAME for decoder - - sp<AMessage> reply = new AMessage(kWhatRenderBuffer, id()); - reply->setSize("buffer-ix", bufferIx); - reply->setInt32("generation", mBufferGeneration); - - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatDrainThisBuffer); - notify->setBuffer("buffer", buffer); - notify->setMessage("reply", reply); - notify->post(); - - // FIXME: This should be handled after rendering is complete, - // but Renderer needs it now - if (flags & MediaCodec::BUFFER_FLAG_EOS) { - ALOGV("queueing eos [%s]", mComponentName.c_str()); - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatEOS); - notify->setInt32("err", ERROR_END_OF_STREAM); - notify->post(); - } - return true; -} - void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) { status_t err; int32_t render; size_t bufferIx; CHECK(msg->findSize("buffer-ix", &bufferIx)); + + if (!mIsAudio) { + int64_t timeUs; + sp<ABuffer> buffer = mOutputBuffers[bufferIx]; + buffer->meta()->findInt64("timeUs", &timeUs); + + if (mCCDecoder != NULL && mCCDecoder->isSelected()) { + mCCDecoder->display(timeUs); + } + } + if (msg->findInt32("render", &render) && render) { int64_t timestampNs; CHECK(msg->findInt64("timestampNs", ×tampNs)); @@ -522,192 +807,8 @@ void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) { } } -void NuPlayer::Decoder::onFlush() { - status_t err = OK; - if (mCodec != NULL) { - err = mCodec->flush(); - mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator - ++mBufferGeneration; - } - - if (err != OK) { - ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err); - handleError(err); - // finish with posting kWhatFlushCompleted. - // we attempt to release the buffers even if flush fails. - } - releaseAndResetMediaBuffers(); - - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatFlushCompleted); - notify->post(); - mPaused = true; -} - -void NuPlayer::Decoder::onResume() { - mPaused = false; -} - -void NuPlayer::Decoder::onShutdown() { - status_t err = OK; - if (mCodec != NULL) { - err = mCodec->release(); - mCodec = NULL; - ++mBufferGeneration; - - if (mNativeWindow != NULL) { - // reconnect to surface as MediaCodec disconnected from it - status_t error = - native_window_api_connect( - mNativeWindow->getNativeWindow().get(), - NATIVE_WINDOW_API_MEDIA); - ALOGW_IF(error != NO_ERROR, - "[%s] failed to connect to native window, error=%d", - mComponentName.c_str(), error); - } - mComponentName = "decoder"; - } - - releaseAndResetMediaBuffers(); - - if (err != OK) { - ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err); - handleError(err); - // finish with posting kWhatShutdownCompleted. - } - - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatShutdownCompleted); - notify->post(); - mPaused = true; -} - -void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) { - ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str()); - - switch (msg->what()) { - case kWhatConfigure: - { - sp<AMessage> format; - CHECK(msg->findMessage("format", &format)); - onConfigure(format); - break; - } - - case kWhatUpdateFormat: - { - sp<AMessage> format; - CHECK(msg->findMessage("format", &format)); - rememberCodecSpecificData(format); - break; - } - - case kWhatGetInputBuffers: - { - uint32_t replyID; - CHECK(msg->senderAwaitsResponse(&replyID)); - - Vector<sp<ABuffer> > *dstBuffers; - CHECK(msg->findPointer("buffers", (void **)&dstBuffers)); - - dstBuffers->clear(); - for (size_t i = 0; i < mInputBuffers.size(); i++) { - dstBuffers->push(mInputBuffers[i]); - } - - (new AMessage)->postReply(replyID); - break; - } - - case kWhatCodecNotify: - { - if (!isStaleReply(msg)) { - int32_t numInput, numOutput; - - if (!msg->findInt32("input-buffers", &numInput)) { - numInput = INT32_MAX; - } - - if (!msg->findInt32("output-buffers", &numOutput)) { - numOutput = INT32_MAX; - } - - if (!mPaused) { - while (numInput-- > 0 && handleAnInputBuffer()) {} - } - - while (numOutput-- > 0 && handleAnOutputBuffer()) {} - } - - requestCodecNotification(); - break; - } - - case kWhatInputBufferFilled: - { - if (!isStaleReply(msg)) { - if (!mPendingInputMessages.empty() - || !onInputBufferFilled(msg)) { - mPendingInputMessages.push_back(msg); - } - } - - break; - } - - case kWhatRenderBuffer: - { - if (!isStaleReply(msg)) { - onRenderBuffer(msg); - } - break; - } - - case kWhatFlush: - { - sp<AMessage> format; - if (msg->findMessage("new-format", &format)) { - rememberCodecSpecificData(format); - } - onFlush(); - break; - } - - case kWhatResume: - { - onResume(); - break; - } - - case kWhatShutdown: - { - onShutdown(); - break; - } - - default: - TRESPASS(); - break; - } -} - -void NuPlayer::Decoder::signalFlush(const sp<AMessage> &format) { - sp<AMessage> msg = new AMessage(kWhatFlush, id()); - if (format != NULL) { - msg->setMessage("new-format", format); - } - msg->post(); -} - -void NuPlayer::Decoder::signalResume() { - (new AMessage(kWhatResume, id()))->post(); -} - -void NuPlayer::Decoder::initiateShutdown() { - (new AMessage(kWhatShutdown, id()))->post(); -} - -bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(const sp<AMessage> &targetFormat) const { +bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange( + const sp<AMessage> &targetFormat) const { if (targetFormat == NULL) { return true; } @@ -772,332 +873,30 @@ bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetF return seamless; } -struct CCData { - CCData(uint8_t type, uint8_t data1, uint8_t data2) - : mType(type), mData1(data1), mData2(data2) { - } - bool getChannel(size_t *channel) const { - if (mData1 >= 0x10 && mData1 <= 0x1f) { - *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0); - return true; - } - return false; - } - - uint8_t mType; - uint8_t mData1; - uint8_t mData2; -}; - -static bool isNullPad(CCData *cc) { - return cc->mData1 < 0x10 && cc->mData2 < 0x10; -} - -static void dumpBytePair(const sp<ABuffer> &ccBuf) { - size_t offset = 0; - AString out; - - while (offset < ccBuf->size()) { - char tmp[128]; - - CCData *cc = (CCData *) (ccBuf->data() + offset); - - if (isNullPad(cc)) { - // 1 null pad or XDS metadata, ignore - offset += sizeof(CCData); - continue; - } - - if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) { - // 2 basic chars - sprintf(tmp, "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2); - } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19) - && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) { - // 1 special char - sprintf(tmp, "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A) - && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){ - // 1 Spanish/French char - sprintf(tmp, "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B) - && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){ - // 1 Portuguese/German/Danish char - sprintf(tmp, "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19) - && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){ - // Mid-Row Codes (Table 69) - sprintf(tmp, "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c) - && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f) - || - ((cc->mData1 == 0x17 || cc->mData1 == 0x1f) - && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){ - // Misc Control Codes (Table 70) - sprintf(tmp, "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else if ((cc->mData1 & 0x70) == 0x10 - && (cc->mData2 & 0x40) == 0x40 - && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) { - // Preamble Address Codes (Table 71) - sprintf(tmp, "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else { - sprintf(tmp, "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } - - if (out.size() > 0) { - out.append(", "); - } - - out.append(tmp); - - offset += sizeof(CCData); - } - - ALOGI("%s", out.c_str()); -} - -NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> ¬ify) - : mNotify(notify), - mCurrentChannel(0), - mSelectedTrack(-1) { - for (size_t i = 0; i < sizeof(mTrackIndices)/sizeof(mTrackIndices[0]); ++i) { - mTrackIndices[i] = -1; - } -} - -size_t NuPlayer::CCDecoder::getTrackCount() const { - return mFoundChannels.size(); -} - -sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const { - if (!isTrackValid(index)) { - return NULL; - } - - sp<AMessage> format = new AMessage(); - - format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE); - format->setString("language", "und"); - format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608); - //CC1, field 0 channel 0 - bool isDefaultAuto = (mFoundChannels[index] == 0); - format->setInt32("auto", isDefaultAuto); - format->setInt32("default", isDefaultAuto); - format->setInt32("forced", 0); - - return format; -} - -status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) { - if (!isTrackValid(index)) { - return BAD_VALUE; - } - - if (select) { - if (mSelectedTrack == (ssize_t)index) { - ALOGE("track %zu already selected", index); - return BAD_VALUE; - } - ALOGV("selected track %zu", index); - mSelectedTrack = index; - } else { - if (mSelectedTrack != (ssize_t)index) { - ALOGE("track %zu is not selected", index); - return BAD_VALUE; - } - ALOGV("unselected track %zu", index); - mSelectedTrack = -1; - } - - return OK; -} - -bool NuPlayer::CCDecoder::isSelected() const { - return mSelectedTrack >= 0 && mSelectedTrack < (int32_t) getTrackCount(); -} - -bool NuPlayer::CCDecoder::isTrackValid(size_t index) const { - return index < getTrackCount(); -} - -int32_t NuPlayer::CCDecoder::getTrackIndex(size_t channel) const { - if (channel < sizeof(mTrackIndices)/sizeof(mTrackIndices[0])) { - return mTrackIndices[channel]; - } - return -1; -} - -// returns true if a new CC track is found -bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) { - int64_t timeUs; - CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); - - sp<ABuffer> sei; - if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) { - return false; - } - - bool trackAdded = false; - - NALBitReader br(sei->data() + 1, sei->size() - 1); - // sei_message() - while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message() - uint32_t payload_type = 0; - size_t payload_size = 0; - uint8_t last_byte; - - do { - last_byte = br.getBits(8); - payload_type += last_byte; - } while (last_byte == 0xFF); - - do { - last_byte = br.getBits(8); - payload_size += last_byte; - } while (last_byte == 0xFF); - - // sei_payload() - if (payload_type == 4) { - // user_data_registered_itu_t_t35() - - // ATSC A/72: 6.4.2 - uint8_t itu_t_t35_country_code = br.getBits(8); - uint16_t itu_t_t35_provider_code = br.getBits(16); - uint32_t user_identifier = br.getBits(32); - uint8_t user_data_type_code = br.getBits(8); - - payload_size -= 1 + 2 + 4 + 1; - - if (itu_t_t35_country_code == 0xB5 - && itu_t_t35_provider_code == 0x0031 - && user_identifier == 'GA94' - && user_data_type_code == 0x3) { - // MPEG_cc_data() - // ATSC A/53 Part 4: 6.2.3.1 - br.skipBits(1); //process_em_data_flag - bool process_cc_data_flag = br.getBits(1); - br.skipBits(1); //additional_data_flag - size_t cc_count = br.getBits(5); - br.skipBits(8); // em_data; - payload_size -= 2; - - if (process_cc_data_flag) { - AString out; - - sp<ABuffer> ccBuf = new ABuffer(cc_count * sizeof(CCData)); - ccBuf->setRange(0, 0); - - for (size_t i = 0; i < cc_count; i++) { - uint8_t marker = br.getBits(5); - CHECK_EQ(marker, 0x1f); - - bool cc_valid = br.getBits(1); - uint8_t cc_type = br.getBits(2); - // remove odd parity bit - uint8_t cc_data_1 = br.getBits(8) & 0x7f; - uint8_t cc_data_2 = br.getBits(8) & 0x7f; - - if (cc_valid - && (cc_type == 0 || cc_type == 1)) { - CCData cc(cc_type, cc_data_1, cc_data_2); - if (!isNullPad(&cc)) { - size_t channel; - if (cc.getChannel(&channel) && getTrackIndex(channel) < 0) { - mTrackIndices[channel] = mFoundChannels.size(); - mFoundChannels.push_back(channel); - trackAdded = true; - } - memcpy(ccBuf->data() + ccBuf->size(), - (void *)&cc, sizeof(cc)); - ccBuf->setRange(0, ccBuf->size() + sizeof(CCData)); - } - } - } - payload_size -= cc_count * 3; - - mCCMap.add(timeUs, ccBuf); - break; - } - } else { - ALOGV("Malformed SEI payload type 4"); - } - } else { - ALOGV("Unsupported SEI payload type %d", payload_type); - } - - // skipping remaining bits of this payload - br.skipBits(payload_size * 8); +void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) { + if (format == NULL) { + return; } - - return trackAdded; -} - -sp<ABuffer> NuPlayer::CCDecoder::filterCCBuf( - const sp<ABuffer> &ccBuf, size_t index) { - sp<ABuffer> filteredCCBuf = new ABuffer(ccBuf->size()); - filteredCCBuf->setRange(0, 0); - - size_t cc_count = ccBuf->size() / sizeof(CCData); - const CCData* cc_data = (const CCData*)ccBuf->data(); - for (size_t i = 0; i < cc_count; ++i) { - size_t channel; - if (cc_data[i].getChannel(&channel)) { - mCurrentChannel = channel; - } - if (mCurrentChannel == mFoundChannels[index]) { - memcpy(filteredCCBuf->data() + filteredCCBuf->size(), - (void *)&cc_data[i], sizeof(CCData)); - filteredCCBuf->setRange(0, filteredCCBuf->size() + sizeof(CCData)); + mCSDsForCurrentFormat.clear(); + for (int32_t i = 0; ; ++i) { + AString tag = "csd-"; + tag.append(i); + sp<ABuffer> buffer; + if (!format->findBuffer(tag.c_str(), &buffer)) { + break; } + mCSDsForCurrentFormat.push(buffer); } - - return filteredCCBuf; -} - -void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) { - if (extractFromSEI(accessUnit)) { - ALOGI("Found CEA-608 track"); - sp<AMessage> msg = mNotify->dup(); - msg->setInt32("what", kWhatTrackAdded); - msg->post(); - } - // TODO: extract CC from other sources } -void NuPlayer::CCDecoder::display(int64_t timeUs) { - if (!isTrackValid(mSelectedTrack)) { - ALOGE("Could not find current track(index=%d)", mSelectedTrack); - return; - } - - ssize_t index = mCCMap.indexOfKey(timeUs); - if (index < 0) { - ALOGV("cc for timestamp %" PRId64 " not found", timeUs); - return; - } - - sp<ABuffer> ccBuf = filterCCBuf(mCCMap.valueAt(index), mSelectedTrack); +void NuPlayer::Decoder::notifyResumeCompleteIfNecessary() { + if (mResumePending) { + mResumePending = false; - if (ccBuf->size() > 0) { -#if 0 - dumpBytePair(ccBuf); -#endif - - ccBuf->meta()->setInt32("trackIndex", mSelectedTrack); - ccBuf->meta()->setInt64("timeUs", timeUs); - ccBuf->meta()->setInt64("durationUs", 0ll); - - sp<AMessage> msg = mNotify->dup(); - msg->setInt32("what", kWhatClosedCaptionData); - msg->setBuffer("buffer", ccBuf); - msg->post(); + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatResumeCompleted); + notify->post(); } - - // remove all entries before timeUs - mCCMap.removeItemsAt(0, index + 1); -} - -void NuPlayer::CCDecoder::flush() { - mCCMap.clear(); } } // namespace android diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h index dba3eee..1bfa94f 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010 The Android Open Source Project + * Copyright 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,70 +15,54 @@ */ #ifndef NUPLAYER_DECODER_H_ - #define NUPLAYER_DECODER_H_ #include "NuPlayer.h" -#include <media/stagefright/foundation/AHandler.h> +#include "NuPlayerDecoderBase.h" namespace android { -struct ABuffer; -struct MediaCodec; -struct MediaBuffer; - -struct NuPlayer::Decoder : public AHandler { +struct NuPlayer::Decoder : public DecoderBase { Decoder(const sp<AMessage> ¬ify, - const sp<NativeWindowWrapper> &nativeWindow = NULL); - - virtual void configure(const sp<AMessage> &format); - virtual void init(); - - status_t getInputBuffers(Vector<sp<ABuffer> > *dstBuffers) const; - virtual void signalFlush(const sp<AMessage> &format = NULL); - virtual void signalUpdateFormat(const sp<AMessage> &format); - virtual void signalResume(); - virtual void initiateShutdown(); + const sp<Source> &source, + const sp<Renderer> &renderer = NULL, + const sp<NativeWindowWrapper> &nativeWindow = NULL, + const sp<CCDecoder> &ccDecoder = NULL); - virtual bool supportsSeamlessFormatChange(const sp<AMessage> &to) const; - - enum { - kWhatFillThisBuffer = 'flTB', - kWhatDrainThisBuffer = 'drTB', - kWhatOutputFormatChanged = 'fmtC', - kWhatFlushCompleted = 'flsC', - kWhatShutdownCompleted = 'shDC', - kWhatEOS = 'eos ', - kWhatError = 'err ', - }; + virtual void getStats( + int64_t *mNumFramesTotal, + int64_t *mNumFramesDropped) const; protected: - virtual ~Decoder(); virtual void onMessageReceived(const sp<AMessage> &msg); + virtual void onConfigure(const sp<AMessage> &format); + virtual void onSetRenderer(const sp<Renderer> &renderer); + virtual void onGetInputBuffers(Vector<sp<ABuffer> > *dstBuffers); + virtual void onResume(bool notifyComplete); + virtual void onFlush(bool notifyComplete); + virtual void onShutdown(bool notifyComplete); + virtual void doRequestBuffers(); + private: enum { - kWhatCodecNotify = 'cdcN', - kWhatConfigure = 'conf', - kWhatGetInputBuffers = 'gInB', - kWhatInputBufferFilled = 'inpF', - kWhatRenderBuffer = 'rndr', - kWhatFlush = 'flus', - kWhatShutdown = 'shuD', - kWhatUpdateFormat = 'uFmt', + kWhatCodecNotify = 'cdcN', + kWhatRenderBuffer = 'rndr', }; - sp<AMessage> mNotify; sp<NativeWindowWrapper> mNativeWindow; + sp<Source> mSource; + sp<Renderer> mRenderer; + sp<CCDecoder> mCCDecoder; + sp<AMessage> mInputFormat; sp<AMessage> mOutputFormat; sp<MediaCodec> mCodec; sp<ALooper> mCodecLooper; - sp<ALooper> mDecoderLooper; List<sp<AMessage> > mPendingInputMessages; @@ -88,8 +72,20 @@ private: Vector<sp<ABuffer> > mCSDsToSubmit; Vector<bool> mInputBufferIsDequeued; Vector<MediaBuffer *> mMediaBuffers; + Vector<size_t> mDequeuedInputBuffers; + + int64_t mSkipRenderingUntilMediaTimeUs; + int64_t mNumFramesTotal; + int64_t mNumFramesDropped; + bool mIsAudio; + bool mIsVideoAVC; + bool mIsSecure; + bool mFormatChangePending; + + bool mPaused; + bool mResumePending; + AString mComponentName; - void handleError(int32_t err); bool handleAnInputBuffer(); bool handleAnOutputBuffer(); @@ -97,53 +93,17 @@ private: void requestCodecNotification(); bool isStaleReply(const sp<AMessage> &msg); - void onConfigure(const sp<AMessage> &format); - void onFlush(); - void onResume(); - bool onInputBufferFilled(const sp<AMessage> &msg); + status_t fetchInputData(sp<AMessage> &reply); + bool onInputBufferFetched(const sp<AMessage> &msg); void onRenderBuffer(const sp<AMessage> &msg); - void onShutdown(); - - int32_t mBufferGeneration; - bool mPaused; - AString mComponentName; + bool supportsSeamlessFormatChange(const sp<AMessage> &to) const; bool supportsSeamlessAudioFormatChange(const sp<AMessage> &targetFormat) const; void rememberCodecSpecificData(const sp<AMessage> &format); - DISALLOW_EVIL_CONSTRUCTORS(Decoder); -}; + void notifyResumeCompleteIfNecessary(); -struct NuPlayer::CCDecoder : public RefBase { - enum { - kWhatClosedCaptionData, - kWhatTrackAdded, - }; - - CCDecoder(const sp<AMessage> ¬ify); - - size_t getTrackCount() const; - sp<AMessage> getTrackInfo(size_t index) const; - status_t selectTrack(size_t index, bool select); - bool isSelected() const; - void decode(const sp<ABuffer> &accessUnit); - void display(int64_t timeUs); - void flush(); - -private: - sp<AMessage> mNotify; - KeyedVector<int64_t, sp<ABuffer> > mCCMap; - size_t mCurrentChannel; - int32_t mSelectedTrack; - int32_t mTrackIndices[4]; - Vector<size_t> mFoundChannels; - - bool isTrackValid(size_t index) const; - int32_t getTrackIndex(size_t channel) const; - bool extractFromSEI(const sp<ABuffer> &accessUnit); - sp<ABuffer> filterCCBuf(const sp<ABuffer> &ccBuf, size_t index); - - DISALLOW_EVIL_CONSTRUCTORS(CCDecoder); + DISALLOW_EVIL_CONSTRUCTORS(Decoder); }; } // namespace android diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.cpp new file mode 100644 index 0000000..d56fc4d --- /dev/null +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.cpp @@ -0,0 +1,200 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "NuPlayerDecoderBase" +#include <utils/Log.h> +#include <inttypes.h> + +#include "NuPlayerDecoderBase.h" + +#include "NuPlayerRenderer.h" + +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +namespace android { + +NuPlayer::DecoderBase::DecoderBase(const sp<AMessage> ¬ify) + : mNotify(notify), + mBufferGeneration(0), + mRequestInputBuffersPending(false) { + // Every decoder has its own looper because MediaCodec operations + // are blocking, but NuPlayer needs asynchronous operations. + mDecoderLooper = new ALooper; + mDecoderLooper->setName("NPDecoder"); + mDecoderLooper->start(false, false, ANDROID_PRIORITY_AUDIO); +} + +NuPlayer::DecoderBase::~DecoderBase() { + mDecoderLooper->unregisterHandler(id()); + mDecoderLooper->stop(); +} + +static +status_t PostAndAwaitResponse( + const sp<AMessage> &msg, sp<AMessage> *response) { + status_t err = msg->postAndAwaitResponse(response); + + if (err != OK) { + return err; + } + + if (!(*response)->findInt32("err", &err)) { + err = OK; + } + + return err; +} + +void NuPlayer::DecoderBase::configure(const sp<AMessage> &format) { + sp<AMessage> msg = new AMessage(kWhatConfigure, id()); + msg->setMessage("format", format); + msg->post(); +} + +void NuPlayer::DecoderBase::init() { + mDecoderLooper->registerHandler(this); +} + +void NuPlayer::DecoderBase::setRenderer(const sp<Renderer> &renderer) { + sp<AMessage> msg = new AMessage(kWhatSetRenderer, id()); + msg->setObject("renderer", renderer); + msg->post(); +} + +status_t NuPlayer::DecoderBase::getInputBuffers(Vector<sp<ABuffer> > *buffers) const { + sp<AMessage> msg = new AMessage(kWhatGetInputBuffers, id()); + msg->setPointer("buffers", buffers); + + sp<AMessage> response; + return PostAndAwaitResponse(msg, &response); +} + +void NuPlayer::DecoderBase::signalFlush() { + (new AMessage(kWhatFlush, id()))->post(); +} + +void NuPlayer::DecoderBase::signalResume(bool notifyComplete) { + sp<AMessage> msg = new AMessage(kWhatResume, id()); + msg->setInt32("notifyComplete", notifyComplete); + msg->post(); +} + +void NuPlayer::DecoderBase::initiateShutdown() { + (new AMessage(kWhatShutdown, id()))->post(); +} + +void NuPlayer::DecoderBase::onRequestInputBuffers() { + if (mRequestInputBuffersPending) { + return; + } + + doRequestBuffers(); +} + +void NuPlayer::DecoderBase::scheduleRequestBuffers() { + if (mRequestInputBuffersPending) { + return; + } + mRequestInputBuffersPending = true; + sp<AMessage> msg = new AMessage(kWhatRequestInputBuffers, id()); + msg->post(10 * 1000ll); +} + +void NuPlayer::DecoderBase::onMessageReceived(const sp<AMessage> &msg) { + + switch (msg->what()) { + case kWhatConfigure: + { + sp<AMessage> format; + CHECK(msg->findMessage("format", &format)); + onConfigure(format); + break; + } + + case kWhatSetRenderer: + { + sp<RefBase> obj; + CHECK(msg->findObject("renderer", &obj)); + onSetRenderer(static_cast<Renderer *>(obj.get())); + break; + } + + case kWhatGetInputBuffers: + { + uint32_t replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + + Vector<sp<ABuffer> > *dstBuffers; + CHECK(msg->findPointer("buffers", (void **)&dstBuffers)); + + onGetInputBuffers(dstBuffers); + + (new AMessage)->postReply(replyID); + break; + } + + case kWhatRequestInputBuffers: + { + mRequestInputBuffersPending = false; + onRequestInputBuffers(); + break; + } + + case kWhatFlush: + { + onFlush(true); + break; + } + + case kWhatResume: + { + int32_t notifyComplete; + CHECK(msg->findInt32("notifyComplete", ¬ifyComplete)); + + onResume(notifyComplete); + break; + } + + case kWhatShutdown: + { + onShutdown(true); + break; + } + + default: + TRESPASS(); + break; + } +} + +void NuPlayer::DecoderBase::handleError(int32_t err) +{ + // We cannot immediately release the codec due to buffers still outstanding + // in the renderer. We signal to the player the error so it can shutdown/release the + // decoder after flushing and increment the generation to discard unnecessary messages. + + ++mBufferGeneration; + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatError); + notify->setInt32("err", err); + notify->post(); +} + +} // namespace android + diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.h new file mode 100644 index 0000000..6732ff4 --- /dev/null +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.h @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef NUPLAYER_DECODER_BASE_H_ + +#define NUPLAYER_DECODER_BASE_H_ + +#include "NuPlayer.h" + +#include <media/stagefright/foundation/AHandler.h> + +namespace android { + +struct ABuffer; +struct MediaCodec; +struct MediaBuffer; + +struct NuPlayer::DecoderBase : public AHandler { + DecoderBase(const sp<AMessage> ¬ify); + + void configure(const sp<AMessage> &format); + void init(); + + void setRenderer(const sp<Renderer> &renderer); + + status_t getInputBuffers(Vector<sp<ABuffer> > *dstBuffers) const; + void signalFlush(); + void signalResume(bool notifyComplete); + void initiateShutdown(); + + virtual void getStats( + int64_t *mNumFramesTotal, + int64_t *mNumFramesDropped) const = 0; + + enum { + kWhatInputDiscontinuity = 'inDi', + kWhatVideoSizeChanged = 'viSC', + kWhatFlushCompleted = 'flsC', + kWhatShutdownCompleted = 'shDC', + kWhatResumeCompleted = 'resC', + kWhatEOS = 'eos ', + kWhatError = 'err ', + }; + +protected: + + virtual ~DecoderBase(); + + virtual void onMessageReceived(const sp<AMessage> &msg); + + virtual void onConfigure(const sp<AMessage> &format) = 0; + virtual void onSetRenderer(const sp<Renderer> &renderer) = 0; + virtual void onGetInputBuffers(Vector<sp<ABuffer> > *dstBuffers) = 0; + virtual void onResume(bool notifyComplete) = 0; + virtual void onFlush(bool notifyComplete) = 0; + virtual void onShutdown(bool notifyComplete) = 0; + + void onRequestInputBuffers(); + void scheduleRequestBuffers(); + virtual void doRequestBuffers() = 0; + virtual void handleError(int32_t err); + + sp<AMessage> mNotify; + int32_t mBufferGeneration; + +private: + enum { + kWhatConfigure = 'conf', + kWhatSetRenderer = 'setR', + kWhatGetInputBuffers = 'gInB', + kWhatRequestInputBuffers = 'reqB', + kWhatFlush = 'flus', + kWhatShutdown = 'shuD', + }; + + sp<ALooper> mDecoderLooper; + bool mRequestInputBuffersPending; + + DISALLOW_EVIL_CONSTRUCTORS(DecoderBase); +}; + +} // namespace android + +#endif // NUPLAYER_DECODER_BASE_H_ diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp index f7aacdd..9f7f09a 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp @@ -21,79 +21,80 @@ #include "NuPlayerDecoderPassThrough.h" +#include "NuPlayerRenderer.h" +#include "NuPlayerSource.h" + #include <media/ICrypto.h> #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> -#include <media/stagefright/MediaDefs.h> #include <media/stagefright/MediaErrors.h> +#include "ATSParser.h" + namespace android { +// TODO optimize buffer size for power consumption +// The offload read buffer size is 32 KB but 24 KB uses less power. +static const size_t kAggregateBufferSizeBytes = 24 * 1024; static const size_t kMaxCachedBytes = 200000; -// The buffers will contain a bit less than kAggregateBufferSizeBytes. -// So we can start off with just enough buffers to keep the cache full. -static const size_t kMaxPendingBuffers = 1 + (kMaxCachedBytes / NuPlayer::kAggregateBufferSizeBytes); NuPlayer::DecoderPassThrough::DecoderPassThrough( - const sp<AMessage> ¬ify) - : Decoder(notify), - mNotify(notify), - mBufferGeneration(0), + const sp<AMessage> ¬ify, + const sp<Source> &source, + const sp<Renderer> &renderer) + : DecoderBase(notify), + mSource(source), + mRenderer(renderer), + mSkipRenderingUntilMediaTimeUs(-1ll), + mPaused(false), mReachedEOS(true), - mPendingBuffersToFill(0), + mPendingAudioErr(OK), mPendingBuffersToDrain(0), mCachedBytes(0), mComponentName("pass through decoder") { - mDecoderLooper = new ALooper; - mDecoderLooper->setName("NuPlayerDecoderPassThrough"); - mDecoderLooper->start(false, false, ANDROID_PRIORITY_AUDIO); + ALOGW_IF(renderer == NULL, "expect a non-NULL renderer"); } NuPlayer::DecoderPassThrough::~DecoderPassThrough() { } -void NuPlayer::DecoderPassThrough::configure(const sp<AMessage> &format) { - sp<AMessage> msg = new AMessage(kWhatConfigure, id()); - msg->setMessage("format", format); - msg->post(); -} - -void NuPlayer::DecoderPassThrough::init() { - mDecoderLooper->registerHandler(this); -} - -void NuPlayer::DecoderPassThrough::signalFlush() { - (new AMessage(kWhatFlush, id()))->post(); -} - -void NuPlayer::DecoderPassThrough::signalResume() { - (new AMessage(kWhatResume, id()))->post(); -} - -void NuPlayer::DecoderPassThrough::initiateShutdown() { - (new AMessage(kWhatShutdown, id()))->post(); -} - -bool NuPlayer::DecoderPassThrough::supportsSeamlessFormatChange( - const sp<AMessage> & /* targetFormat */) const { - return true; +void NuPlayer::DecoderPassThrough::getStats( + int64_t *numFramesTotal, int64_t *numFramesDropped) const { + *numFramesTotal = 0; + *numFramesDropped = 0; } void NuPlayer::DecoderPassThrough::onConfigure(const sp<AMessage> &format) { ALOGV("[%s] onConfigure", mComponentName.c_str()); mCachedBytes = 0; - mPendingBuffersToFill = 0; mPendingBuffersToDrain = 0; mReachedEOS = false; ++mBufferGeneration; - requestMaxBuffers(); + onRequestInputBuffers(); - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatOutputFormatChanged); - notify->setMessage("format", format); - notify->post(); + // The audio sink is already opened before the PassThrough decoder is created. + // Opening again might be relevant if decoder is instantiated after shutdown and + // format is different. + status_t err = mRenderer->openAudioSink( + format, true /* offloadOnly */, false /* hasVideo */, + AUDIO_OUTPUT_FLAG_NONE /* flags */, NULL /* isOffloaded */); + if (err != OK) { + handleError(err); + } +} + +void NuPlayer::DecoderPassThrough::onSetRenderer( + const sp<Renderer> &renderer) { + // renderer can't be changed during offloading + ALOGW_IF(renderer != mRenderer, + "ignoring request to change renderer"); +} + +void NuPlayer::DecoderPassThrough::onGetInputBuffers( + Vector<sp<ABuffer> > * /* dstBuffers */) { + ALOGE("onGetInputBuffers() called unexpectedly"); } bool NuPlayer::DecoderPassThrough::isStaleReply(const sp<AMessage> &msg) { @@ -102,100 +103,301 @@ bool NuPlayer::DecoderPassThrough::isStaleReply(const sp<AMessage> &msg) { return generation != mBufferGeneration; } -bool NuPlayer::DecoderPassThrough::requestABuffer() { - if (mCachedBytes >= kMaxCachedBytes) { - ALOGV("[%s] mCachedBytes = %zu", - mComponentName.c_str(), mCachedBytes); - return false; +bool NuPlayer::DecoderPassThrough::isDoneFetching() const { + ALOGV("[%s] mCachedBytes = %zu, mReachedEOS = %d mPaused = %d", + mComponentName.c_str(), mCachedBytes, mReachedEOS, mPaused); + + return mCachedBytes >= kMaxCachedBytes || mReachedEOS || mPaused; +} + +void NuPlayer::DecoderPassThrough::doRequestBuffers() { + status_t err = OK; + while (!isDoneFetching()) { + sp<AMessage> msg = new AMessage(); + + err = fetchInputData(msg); + if (err != OK) { + break; + } + + onInputBufferFetched(msg); } - if (mReachedEOS) { - ALOGV("[%s] reached EOS", mComponentName.c_str()); - return false; + + if (err == -EWOULDBLOCK + && mSource->feedMoreTSData() == OK) { + scheduleRequestBuffers(); } +} - sp<AMessage> reply = new AMessage(kWhatInputBufferFilled, id()); - reply->setInt32("generation", mBufferGeneration); +status_t NuPlayer::DecoderPassThrough::dequeueAccessUnit(sp<ABuffer> *accessUnit) { + status_t err; + + // Did we save an accessUnit earlier because of a discontinuity? + if (mPendingAudioAccessUnit != NULL) { + *accessUnit = mPendingAudioAccessUnit; + mPendingAudioAccessUnit.clear(); + err = mPendingAudioErr; + ALOGV("feedDecoderInputData() use mPendingAudioAccessUnit"); + } else { + err = mSource->dequeueAccessUnit(true /* audio */, accessUnit); + } - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatFillThisBuffer); - notify->setMessage("reply", reply); - notify->post(); - mPendingBuffersToFill++; - ALOGV("requestABuffer: #ToFill = %zu, #ToDrain = %zu", mPendingBuffersToFill, - mPendingBuffersToDrain); + if (err == INFO_DISCONTINUITY || err == ERROR_END_OF_STREAM) { + if (mAggregateBuffer != NULL) { + // We already have some data so save this for later. + mPendingAudioErr = err; + mPendingAudioAccessUnit = *accessUnit; + (*accessUnit).clear(); + ALOGD("return aggregated buffer and save err(=%d) for later", err); + err = OK; + } + } - return true; + return err; } -void android::NuPlayer::DecoderPassThrough::onInputBufferFilled( +sp<ABuffer> NuPlayer::DecoderPassThrough::aggregateBuffer( + const sp<ABuffer> &accessUnit) { + sp<ABuffer> aggregate; + + if (accessUnit == NULL) { + // accessUnit is saved to mPendingAudioAccessUnit + // return current mAggregateBuffer + aggregate = mAggregateBuffer; + mAggregateBuffer.clear(); + return aggregate; + } + + size_t smallSize = accessUnit->size(); + if ((mAggregateBuffer == NULL) + // Don't bother if only room for a few small buffers. + && (smallSize < (kAggregateBufferSizeBytes / 3))) { + // Create a larger buffer for combining smaller buffers from the extractor. + mAggregateBuffer = new ABuffer(kAggregateBufferSizeBytes); + mAggregateBuffer->setRange(0, 0); // start empty + } + + if (mAggregateBuffer != NULL) { + int64_t timeUs; + int64_t dummy; + bool smallTimestampValid = accessUnit->meta()->findInt64("timeUs", &timeUs); + bool bigTimestampValid = mAggregateBuffer->meta()->findInt64("timeUs", &dummy); + // Will the smaller buffer fit? + size_t bigSize = mAggregateBuffer->size(); + size_t roomLeft = mAggregateBuffer->capacity() - bigSize; + // Should we save this small buffer for the next big buffer? + // If the first small buffer did not have a timestamp then save + // any buffer that does have a timestamp until the next big buffer. + if ((smallSize > roomLeft) + || (!bigTimestampValid && (bigSize > 0) && smallTimestampValid)) { + mPendingAudioErr = OK; + mPendingAudioAccessUnit = accessUnit; + aggregate = mAggregateBuffer; + mAggregateBuffer.clear(); + } else { + // Grab time from first small buffer if available. + if ((bigSize == 0) && smallTimestampValid) { + mAggregateBuffer->meta()->setInt64("timeUs", timeUs); + } + // Append small buffer to the bigger buffer. + memcpy(mAggregateBuffer->base() + bigSize, accessUnit->data(), smallSize); + bigSize += smallSize; + mAggregateBuffer->setRange(0, bigSize); + + ALOGV("feedDecoderInputData() smallSize = %zu, bigSize = %zu, capacity = %zu", + smallSize, bigSize, mAggregateBuffer->capacity()); + } + } else { + // decided not to aggregate + aggregate = accessUnit; + } + + return aggregate; +} + +status_t NuPlayer::DecoderPassThrough::fetchInputData(sp<AMessage> &reply) { + sp<ABuffer> accessUnit; + + do { + status_t err = dequeueAccessUnit(&accessUnit); + + if (err == -EWOULDBLOCK) { + return err; + } else if (err != OK) { + if (err == INFO_DISCONTINUITY) { + int32_t type; + CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); + + bool formatChange = + (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT) != 0; + + bool timeChange = + (type & ATSParser::DISCONTINUITY_TIME) != 0; + + ALOGI("audio discontinuity (formatChange=%d, time=%d)", + formatChange, timeChange); + + if (formatChange || timeChange) { + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("what", kWhatInputDiscontinuity); + // will perform seamless format change, + // only notify NuPlayer to scan sources + msg->setInt32("formatChange", false); + msg->post(); + } + + if (timeChange) { + onFlush(false /* notifyComplete */); + err = OK; + } else if (formatChange) { + // do seamless format change + err = OK; + } else { + // This stream is unaffected by the discontinuity + return -EWOULDBLOCK; + } + } + + reply->setInt32("err", err); + return OK; + } + + accessUnit = aggregateBuffer(accessUnit); + } while (accessUnit == NULL); + +#if 0 + int64_t mediaTimeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs)); + ALOGV("feeding audio input buffer at media time %.2f secs", + mediaTimeUs / 1E6); +#endif + + reply->setBuffer("buffer", accessUnit); + + return OK; +} + +void NuPlayer::DecoderPassThrough::onInputBufferFetched( const sp<AMessage> &msg) { - --mPendingBuffersToFill; if (mReachedEOS) { return; } sp<ABuffer> buffer; - msg->findBuffer("buffer", &buffer); + bool hasBuffer = msg->findBuffer("buffer", &buffer); if (buffer == NULL) { - mReachedEOS = true; + int32_t streamErr = ERROR_END_OF_STREAM; + CHECK(msg->findInt32("err", &streamErr) || !hasBuffer); + if (streamErr == OK) { + return; + } - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatEOS); - notify->setInt32("err", ERROR_END_OF_STREAM); - notify->post(); + mReachedEOS = true; + if (mRenderer != NULL) { + mRenderer->queueEOS(true /* audio */, ERROR_END_OF_STREAM); + } return; } - mCachedBytes += buffer->size(); + sp<AMessage> extra; + if (buffer->meta()->findMessage("extra", &extra) && extra != NULL) { + int64_t resumeAtMediaTimeUs; + if (extra->findInt64( + "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) { + ALOGI("[%s] suppressing rendering until %lld us", + mComponentName.c_str(), (long long)resumeAtMediaTimeUs); + mSkipRenderingUntilMediaTimeUs = resumeAtMediaTimeUs; + } + } + + int32_t bufferSize = buffer->size(); + mCachedBytes += bufferSize; + + if (mSkipRenderingUntilMediaTimeUs >= 0) { + int64_t timeUs = 0; + CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); + + if (timeUs < mSkipRenderingUntilMediaTimeUs) { + ALOGV("[%s] dropping buffer at time %lld as requested.", + mComponentName.c_str(), (long long)timeUs); + + onBufferConsumed(bufferSize); + return; + } + + mSkipRenderingUntilMediaTimeUs = -1; + } + + if (mRenderer == NULL) { + onBufferConsumed(bufferSize); + return; + } sp<AMessage> reply = new AMessage(kWhatBufferConsumed, id()); reply->setInt32("generation", mBufferGeneration); - reply->setInt32("size", buffer->size()); + reply->setInt32("size", bufferSize); + + mRenderer->queueBuffer(true /* audio */, buffer, reply); - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatDrainThisBuffer); - notify->setBuffer("buffer", buffer); - notify->setMessage("reply", reply); - notify->post(); ++mPendingBuffersToDrain; - ALOGV("onInputBufferFilled: #ToFill = %zu, #ToDrain = %zu, cachedBytes = %zu", - mPendingBuffersToFill, mPendingBuffersToDrain, mCachedBytes); + ALOGV("onInputBufferFilled: #ToDrain = %zu, cachedBytes = %zu", + mPendingBuffersToDrain, mCachedBytes); } void NuPlayer::DecoderPassThrough::onBufferConsumed(int32_t size) { --mPendingBuffersToDrain; mCachedBytes -= size; - ALOGV("onBufferConsumed: #ToFill = %zu, #ToDrain = %zu, cachedBytes = %zu", - mPendingBuffersToFill, mPendingBuffersToDrain, mCachedBytes); - requestABuffer(); + ALOGV("onBufferConsumed: #ToDrain = %zu, cachedBytes = %zu", + mPendingBuffersToDrain, mCachedBytes); + onRequestInputBuffers(); +} + +void NuPlayer::DecoderPassThrough::onResume(bool notifyComplete) { + mPaused = false; + + onRequestInputBuffers(); + + if (notifyComplete) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatResumeCompleted); + notify->post(); + } } -void NuPlayer::DecoderPassThrough::onFlush() { +void NuPlayer::DecoderPassThrough::onFlush(bool notifyComplete) { ++mBufferGeneration; + mSkipRenderingUntilMediaTimeUs = -1; + mPendingAudioAccessUnit.clear(); + mPendingAudioErr = OK; + mAggregateBuffer.clear(); + + if (mRenderer != NULL) { + mRenderer->flush(true /* audio */, notifyComplete); + mRenderer->signalTimeDiscontinuity(); + } + + if (notifyComplete) { + mPaused = true; + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatFlushCompleted); + notify->post(); + } - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatFlushCompleted); - notify->post(); - mPendingBuffersToFill = 0; mPendingBuffersToDrain = 0; mCachedBytes = 0; mReachedEOS = false; } -void NuPlayer::DecoderPassThrough::requestMaxBuffers() { - for (size_t i = 0; i < kMaxPendingBuffers; i++) { - if (!requestABuffer()) { - break; - } - } -} - -void NuPlayer::DecoderPassThrough::onShutdown() { +void NuPlayer::DecoderPassThrough::onShutdown(bool notifyComplete) { ++mBufferGeneration; + mSkipRenderingUntilMediaTimeUs = -1; + + if (notifyComplete) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatShutdownCompleted); + notify->post(); + } - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatShutdownCompleted); - notify->post(); mReachedEOS = true; } @@ -204,31 +406,6 @@ void NuPlayer::DecoderPassThrough::onMessageReceived(const sp<AMessage> &msg) { msg->debugString().c_str()); switch (msg->what()) { - case kWhatConfigure: - { - sp<AMessage> format; - CHECK(msg->findMessage("format", &format)); - onConfigure(format); - break; - } - - case kWhatRequestABuffer: - { - if (!isStaleReply(msg)) { - requestABuffer(); - } - - break; - } - - case kWhatInputBufferFilled: - { - if (!isStaleReply(msg)) { - onInputBufferFilled(msg); - } - break; - } - case kWhatBufferConsumed: { if (!isStaleReply(msg)) { @@ -239,26 +416,8 @@ void NuPlayer::DecoderPassThrough::onMessageReceived(const sp<AMessage> &msg) { break; } - case kWhatFlush: - { - onFlush(); - break; - } - - case kWhatResume: - { - requestMaxBuffers(); - break; - } - - case kWhatShutdown: - { - onShutdown(); - break; - } - default: - TRESPASS(); + DecoderBase::onMessageReceived(msg); break; } } diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h index fb20257..a6e1faf 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h @@ -20,21 +20,18 @@ #include "NuPlayer.h" -#include "NuPlayerDecoder.h" +#include "NuPlayerDecoderBase.h" namespace android { -struct NuPlayer::DecoderPassThrough : public Decoder { - DecoderPassThrough(const sp<AMessage> ¬ify); +struct NuPlayer::DecoderPassThrough : public DecoderBase { + DecoderPassThrough(const sp<AMessage> ¬ify, + const sp<Source> &source, + const sp<Renderer> &renderer); - virtual void configure(const sp<AMessage> &format); - virtual void init(); - - virtual void signalFlush(); - virtual void signalResume(); - virtual void initiateShutdown(); - - bool supportsSeamlessFormatChange(const sp<AMessage> &to) const; + virtual void getStats( + int64_t *mNumFramesTotal, + int64_t *mNumFramesDropped) const; protected: @@ -42,41 +39,48 @@ protected: virtual void onMessageReceived(const sp<AMessage> &msg); + virtual void onConfigure(const sp<AMessage> &format); + virtual void onSetRenderer(const sp<Renderer> &renderer); + virtual void onGetInputBuffers(Vector<sp<ABuffer> > *dstBuffers); + virtual void onResume(bool notifyComplete); + virtual void onFlush(bool notifyComplete); + virtual void onShutdown(bool notifyComplete); + virtual void doRequestBuffers(); + private: enum { - kWhatRequestABuffer = 'reqB', - kWhatConfigure = 'conf', - kWhatInputBufferFilled = 'inpF', kWhatBufferConsumed = 'bufC', - kWhatFlush = 'flus', - kWhatShutdown = 'shuD', }; - sp<AMessage> mNotify; - sp<ALooper> mDecoderLooper; + sp<Source> mSource; + sp<Renderer> mRenderer; + int64_t mSkipRenderingUntilMediaTimeUs; + bool mPaused; - /** Returns true if a buffer was requested. - * Returns false if at EOS or cache already full. - */ - bool requestABuffer(); - bool isStaleReply(const sp<AMessage> &msg); + bool mReachedEOS; - void onConfigure(const sp<AMessage> &format); - void onFlush(); - void onInputBufferFilled(const sp<AMessage> &msg); - void onBufferConsumed(int32_t size); - void requestMaxBuffers(); - void onShutdown(); + // Used by feedDecoderInputData to aggregate small buffers into + // one large buffer. + sp<ABuffer> mPendingAudioAccessUnit; + status_t mPendingAudioErr; + sp<ABuffer> mAggregateBuffer; - int32_t mBufferGeneration; - bool mReachedEOS; - // TODO mPendingBuffersToFill and mPendingBuffersToDrain are only for - // debugging. They can be removed when the power investigation is done. - size_t mPendingBuffersToFill; + // mPendingBuffersToDrain are only for debugging. It can be removed + // when the power investigation is done. size_t mPendingBuffersToDrain; size_t mCachedBytes; AString mComponentName; + bool isStaleReply(const sp<AMessage> &msg); + bool isDoneFetching() const; + + status_t dequeueAccessUnit(sp<ABuffer> *accessUnit); + sp<ABuffer> aggregateBuffer(const sp<ABuffer> &accessUnit); + status_t fetchInputData(sp<AMessage> &reply); + + void onInputBufferFetched(const sp<AMessage> &msg); + void onBufferConsumed(int32_t size); + DISALLOW_EVIL_CONSTRUCTORS(DecoderPassThrough); }; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp index ab46074..d050c78 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp @@ -239,16 +239,24 @@ status_t NuPlayerDriver::start() { // fall through } + case STATE_PAUSED: + case STATE_STOPPED_AND_PREPARED: + { + if (mAtEOS && mStartupSeekTimeUs < 0) { + mStartupSeekTimeUs = 0; + mPositionUs = -1; + } + + // fall through + } + case STATE_PREPARED: { mAtEOS = false; mPlayer->start(); if (mStartupSeekTimeUs >= 0) { - if (mStartupSeekTimeUs > 0) { - mPlayer->seekToAsync(mStartupSeekTimeUs); - } - + mPlayer->seekToAsync(mStartupSeekTimeUs); mStartupSeekTimeUs = -1; } break; @@ -264,20 +272,6 @@ status_t NuPlayerDriver::start() { break; } - case STATE_PAUSED: - case STATE_STOPPED_AND_PREPARED: - { - if (mAtEOS) { - mPlayer->seekToAsync(0); - mAtEOS = false; - mPlayer->resume(); - mPositionUs = -1; - } else { - mPlayer->resume(); - } - break; - } - default: return INVALID_OPERATION; } @@ -316,6 +310,13 @@ status_t NuPlayerDriver::stop() { } status_t NuPlayerDriver::pause() { + // The NuPlayerRenderer may get flushed if pause for long enough, e.g. the pause timeout tear + // down for audio offload mode. If that happens, the NuPlayerRenderer will no longer know the + // current position. So similar to seekTo, update |mPositionUs| to the pause position by calling + // getCurrentPosition here. + int msec; + getCurrentPosition(&msec); + Mutex::Autolock autoLock(mLock); switch (mState) { @@ -348,6 +349,7 @@ status_t NuPlayerDriver::seekTo(int msec) { switch (mState) { case STATE_PREPARED: + case STATE_STOPPED_AND_PREPARED: { mStartupSeekTimeUs = seekTimeUs; // pretend that the seek completed. It will actually happen when starting playback. @@ -485,13 +487,16 @@ status_t NuPlayerDriver::invoke(const Parcel &request, Parcel *reply) { case INVOKE_ID_SELECT_TRACK: { int trackIndex = request.readInt32(); - return mPlayer->selectTrack(trackIndex, true /* select */); + int msec = 0; + // getCurrentPosition should always return OK + getCurrentPosition(&msec); + return mPlayer->selectTrack(trackIndex, true /* select */, msec * 1000ll); } case INVOKE_ID_UNSELECT_TRACK: { int trackIndex = request.readInt32(); - return mPlayer->selectTrack(trackIndex, false /* select */); + return mPlayer->selectTrack(trackIndex, false /* select */, 0xdeadbeef /* not used */); } case INVOKE_ID_GET_SELECTED_TRACK: @@ -630,9 +635,24 @@ void NuPlayerDriver::notifyListener_l( case MEDIA_PLAYBACK_COMPLETE: { if (mState != STATE_RESET_IN_PROGRESS) { + if (mAutoLoop) { + audio_stream_type_t streamType = AUDIO_STREAM_MUSIC; + if (mAudioSink != NULL) { + streamType = mAudioSink->getAudioStreamType(); + } + if (streamType == AUDIO_STREAM_NOTIFICATION) { + ALOGW("disabling auto-loop for notification"); + mAutoLoop = false; + } + } if (mLooping || (mAutoLoop && (mAudioSink == NULL || mAudioSink->realtime()))) { mPlayer->seekToAsync(0); + if (mAudioSink != NULL) { + // The renderer has stopped the sink at the end in order to play out + // the last little bit of audio. If we're looping, we need to restart it. + mAudioSink->start(); + } break; } diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp index 638d9bc..21b74ee 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp @@ -20,8 +20,6 @@ #include "NuPlayerRenderer.h" -#include <cutils/properties.h> - #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> @@ -38,21 +36,11 @@ namespace android { // Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink // is closed to allow the audio DSP to power down. -static const int64_t kOffloadPauseMaxUs = 60000000ll; +static const int64_t kOffloadPauseMaxUs = 10000000ll; // static const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll; -static bool sFrameAccurateAVsync = false; - -static void readProperties() { - char value[PROPERTY_VALUE_MAX]; - if (property_get("persist.sys.media.avsync", value, NULL)) { - sFrameAccurateAVsync = - !strcmp("1", value) || !strcasecmp("true", value); - } -} - NuPlayer::Renderer::Renderer( const sp<MediaPlayerBase::AudioSink> &sink, const sp<AMessage> ¬ify, @@ -68,12 +56,16 @@ NuPlayer::Renderer::Renderer( mAudioFirstAnchorTimeMediaUs(-1), mAnchorTimeMediaUs(-1), mAnchorTimeRealUs(-1), + mAnchorNumFramesWritten(-1), + mAnchorMaxMediaUs(-1), mVideoLateByUs(0ll), mHasAudio(false), mHasVideo(false), mPauseStartedTimeRealUs(-1), mFlushingAudio(false), mFlushingVideo(false), + mNotifyCompleteAudio(false), + mNotifyCompleteVideo(false), mSyncQueues(false), mPaused(false), mVideoSampleReceived(false), @@ -85,7 +77,6 @@ NuPlayer::Renderer::Renderer( mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER), mTotalBuffersQueued(0), mLastAudioBufferDrained(0) { - readProperties(); } NuPlayer::Renderer::~Renderer() { @@ -116,15 +107,17 @@ void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) { msg->post(); } -void NuPlayer::Renderer::flush(bool audio) { +void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) { { Mutex::Autolock autoLock(mFlushLock); if (audio) { + mNotifyCompleteAudio |= notifyComplete; if (mFlushingAudio) { return; } mFlushingAudio = true; } else { + mNotifyCompleteVideo |= notifyComplete; if (mFlushingVideo) { return; } @@ -155,6 +148,10 @@ void NuPlayer::Renderer::signalDisableOffloadAudio() { (new AMessage(kWhatDisableOffloadAudio, id()))->post(); } +void NuPlayer::Renderer::signalEnableOffloadAudio() { + (new AMessage(kWhatEnableOffloadAudio, id()))->post(); +} + void NuPlayer::Renderer::pause() { (new AMessage(kWhatPause, id()))->post(); } @@ -173,7 +170,8 @@ status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) { return getCurrentPosition(mediaUs, ALooper::GetNowUs()); } -status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs, int64_t nowUs) { +status_t NuPlayer::Renderer::getCurrentPosition( + int64_t *mediaUs, int64_t nowUs, bool allowPastQueuedVideo) { Mutex::Autolock autoLock(mTimeLock); if (!mHasAudio && !mHasVideo) { return NO_INIT; @@ -182,12 +180,21 @@ status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs, int64_t nowUs) if (mAnchorTimeMediaUs < 0) { return NO_INIT; } + int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs; if (mPauseStartedTimeRealUs != -1) { positionUs -= (nowUs - mPauseStartedTimeRealUs); } + // limit position to the last queued media time (for video only stream + // position will be discrete as we don't know how long each frame lasts) + if (mAnchorMaxMediaUs >= 0 && !allowPastQueuedVideo) { + if (positionUs > mAnchorMaxMediaUs) { + positionUs = mAnchorMaxMediaUs; + } + } + if (positionUs < mAudioFirstAnchorTimeMediaUs) { positionUs = mAudioFirstAnchorTimeMediaUs; } @@ -217,10 +224,12 @@ void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded(int64_t mediaUs) { } } -void NuPlayer::Renderer::setAnchorTime(int64_t mediaUs, int64_t realUs, bool resume) { +void NuPlayer::Renderer::setAnchorTime( + int64_t mediaUs, int64_t realUs, int64_t numFramesWritten, bool resume) { Mutex::Autolock autoLock(mTimeLock); mAnchorTimeMediaUs = mediaUs; mAnchorTimeRealUs = realUs; + mAnchorNumFramesWritten = numFramesWritten; if (resume) { mPauseStartedTimeRealUs = -1; } @@ -241,11 +250,12 @@ void NuPlayer::Renderer::setPauseStartedTimeRealUs(int64_t realUs) { mPauseStartedTimeRealUs = realUs; } -bool NuPlayer::Renderer::openAudioSink( +status_t NuPlayer::Renderer::openAudioSink( const sp<AMessage> &format, bool offloadOnly, bool hasVideo, - uint32_t flags) { + uint32_t flags, + bool *isOffloaded) { sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, id()); msg->setMessage("format", format); msg->setInt32("offload-only", offloadOnly); @@ -255,9 +265,15 @@ bool NuPlayer::Renderer::openAudioSink( sp<AMessage> response; msg->postAndAwaitResponse(&response); - int32_t offload; - CHECK(response->findInt32("offload", &offload)); - return (offload != 0); + int32_t err; + if (!response->findInt32("err", &err)) { + err = INVALID_OPERATION; + } else if (err == OK && isOffloaded != NULL) { + int32_t offload; + CHECK(response->findInt32("offload", &offload)); + *isOffloaded = (offload != 0); + } + return err; } void NuPlayer::Renderer::closeAudioSink() { @@ -283,10 +299,11 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { uint32_t flags; CHECK(msg->findInt32("flags", (int32_t *)&flags)); - bool offload = onOpenAudioSink(format, offloadOnly, hasVideo, flags); + status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags); sp<AMessage> response = new AMessage; - response->setInt32("offload", offload); + response->setInt32("err", err); + response->setInt32("offload", offloadingAudio()); uint32_t replyID; CHECK(msg->senderAwaitsResponse(&replyID)); @@ -406,6 +423,12 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatEnableOffloadAudio: + { + onEnableOffloadAudio(); + break; + } + case kWhatPause: { onPause(); @@ -541,7 +564,7 @@ size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) { int64_t mediaTimeUs; CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6); - onNewAudioMediaTime(mediaTimeUs); + setAudioFirstAnchorTimeIfNeeded(mediaTimeUs); } size_t copy = entry->mBuffer->size() - entry->mOffset; @@ -564,6 +587,14 @@ size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) { notifyIfMediaRenderingStarted(); } + if (mAudioFirstAnchorTimeMediaUs >= 0) { + int64_t nowUs = ALooper::GetNowUs(); + setAnchorTime(mAudioFirstAnchorTimeMediaUs, nowUs - getPlayedOutAudioDurationUs(nowUs)); + } + + // we don't know how much data we are queueing for offloaded tracks + mAnchorMaxMediaUs = -1; + if (hasEOS) { (new AMessage(kWhatStopAudioSink, id()))->post(); } @@ -607,6 +638,10 @@ bool NuPlayer::Renderer::onDrainAudioQueue() { mAudioQueue.erase(mAudioQueue.begin()); entry = NULL; + // Need to stop the track here, because that will play out the last + // little bit at the end of the file. Otherwise short files won't play. + mAudioSink->stop(); + mNumFramesWritten = 0; return false; } @@ -624,8 +659,9 @@ bool NuPlayer::Renderer::onDrainAudioQueue() { ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset, copy); if (written < 0) { - // An error in AudioSink write is fatal here. - LOG_ALWAYS_FATAL("AudioSink write error(%zd) when writing %zu bytes", written, copy); + // An error in AudioSink write. Perhaps the AudioSink was not properly opened. + ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy); + break; } entry->mOffset += written; @@ -663,6 +699,11 @@ bool NuPlayer::Renderer::onDrainAudioQueue() { break; } } + mAnchorMaxMediaUs = + mAnchorTimeMediaUs + + (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL) + * 1000LL * mAudioSink->msecsPerFrame()); + return !mAudioQueue.empty(); } @@ -674,7 +715,7 @@ int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) { int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) { int64_t currentPositionUs; - if (getCurrentPosition(¤tPositionUs, nowUs) != OK) { + if (getCurrentPosition(¤tPositionUs, nowUs, true /* allowPastQueuedVideo */) != OK) { // If failed to get current position, e.g. due to audio clock is not ready, then just // play out video immediately without delay. return nowUs; @@ -690,7 +731,8 @@ void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) { } setAudioFirstAnchorTimeIfNeeded(mediaTimeUs); int64_t nowUs = ALooper::GetNowUs(); - setAnchorTime(mediaTimeUs, nowUs + getPendingAudioPlayoutDurationUs(nowUs)); + setAnchorTime( + mediaTimeUs, nowUs + getPendingAudioPlayoutDurationUs(nowUs), mNumFramesWritten); } void NuPlayer::Renderer::postDrainVideoQueue() { @@ -729,10 +771,14 @@ void NuPlayer::Renderer::postDrainVideoQueue() { if (mAnchorTimeMediaUs < 0) { setAnchorTime(mediaTimeUs, nowUs); + mAnchorMaxMediaUs = mediaTimeUs; realTimeUs = nowUs; } else { realTimeUs = getRealTimeUs(mediaTimeUs, nowUs); } + if (!mHasAudio) { + mAnchorMaxMediaUs = mediaTimeUs + 100000; // smooth out videos >= 10fps + } // Heuristics to handle situation when media time changed without a // discontinuity. If we have not drained an audio buffer that was @@ -761,11 +807,6 @@ void NuPlayer::Renderer::postDrainVideoQueue() { ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs); // post 2 display refreshes before rendering is due - // FIXME currently this increases power consumption, so unless frame-accurate - // AV sync is requested, post closer to required render time (at 0.63 vsyncs) - if (!sFrameAccurateAVsync) { - twoVsyncsUs >>= 4; - } msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0); mDrainVideoQueuePending = true; @@ -988,15 +1029,19 @@ void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) { } void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { - int32_t audio; + int32_t audio, notifyComplete; CHECK(msg->findInt32("audio", &audio)); { Mutex::Autolock autoLock(mFlushLock); if (audio) { mFlushingAudio = false; + notifyComplete = mNotifyCompleteAudio; + mNotifyCompleteAudio = false; } else { mFlushingVideo = false; + notifyComplete = mNotifyCompleteVideo; + mNotifyCompleteVideo = false; } } @@ -1012,6 +1057,7 @@ void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { Mutex::Autolock autoLock(mLock); syncQueuesDone_l(); setPauseStartedTimeRealUs(-1); + setAnchorTime(-1, -1); } ALOGV("flushing %s", audio ? "audio" : "video"); @@ -1049,7 +1095,10 @@ void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { } mVideoSampleReceived = false; - notifyFlushComplete(audio); + + if (notifyComplete) { + notifyFlushComplete(audio); + } } void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) { @@ -1103,6 +1152,7 @@ void NuPlayer::Renderer::onAudioSinkChanged() { } CHECK(!mDrainAudioQueuePending); mNumFramesWritten = 0; + mAnchorNumFramesWritten = -1; uint32_t written; if (mAudioSink->getFramesWritten(&written) == OK) { mNumFramesWritten = written; @@ -1115,6 +1165,12 @@ void NuPlayer::Renderer::onDisableOffloadAudio() { ++mAudioQueueGeneration; } +void NuPlayer::Renderer::onEnableOffloadAudio() { + Mutex::Autolock autoLock(mLock); + mFlags |= FLAG_OFFLOAD_AUDIO; + ++mAudioQueueGeneration; +} + void NuPlayer::Renderer::onPause() { if (mPaused) { ALOGW("Renderer::onPause() called while already paused!"); @@ -1142,8 +1198,6 @@ void NuPlayer::Renderer::onPause() { } void NuPlayer::Renderer::onResume() { - readProperties(); - if (!mPaused) { return; } @@ -1158,7 +1212,8 @@ void NuPlayer::Renderer::onResume() { if (mPauseStartedTimeRealUs != -1) { int64_t newAnchorRealUs = mAnchorTimeRealUs + ALooper::GetNowUs() - mPauseStartedTimeRealUs; - setAnchorTime(mAnchorTimeMediaUs, newAnchorRealUs, true /* resume */); + setAnchorTime( + mAnchorTimeMediaUs, newAnchorRealUs, mAnchorNumFramesWritten, true /* resume */); } if (!mAudioQueue.empty()) { @@ -1276,7 +1331,7 @@ void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() { } } -bool NuPlayer::Renderer::onOpenAudioSink( +status_t NuPlayer::Renderer::onOpenAudioSink( const sp<AMessage> &format, bool offloadOnly, bool hasVideo, @@ -1338,11 +1393,12 @@ bool NuPlayer::Renderer::onOpenAudioSink( if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) { ALOGV("openAudioSink: no change in offload mode"); // no change from previous configuration, everything ok. - return offloadingAudio(); + return OK; } ALOGV("openAudioSink: try to open AudioSink in offload mode"); - flags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD; - flags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER; + uint32_t offloadFlags = flags; + offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD; + offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER; audioSinkChanged = true; mAudioSink->close(); err = mAudioSink->open( @@ -1353,7 +1409,7 @@ bool NuPlayer::Renderer::onOpenAudioSink( 8 /* bufferCount */, &NuPlayer::Renderer::AudioSinkCallback, this, - (audio_output_flags_t)flags, + (audio_output_flags_t)offloadFlags, &offloadInfo); if (err == OK) { @@ -1377,13 +1433,13 @@ bool NuPlayer::Renderer::onOpenAudioSink( } } if (!offloadOnly && !offloadingAudio()) { - flags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD; ALOGV("openAudioSink: open AudioSink in NON-offload mode"); - + uint32_t pcmFlags = flags; + pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD; audioSinkChanged = true; mAudioSink->close(); mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER; - CHECK_EQ(mAudioSink->open( + status_t err = mAudioSink->open( sampleRate, numChannels, (audio_channel_mask_t)channelMask, @@ -1391,15 +1447,20 @@ bool NuPlayer::Renderer::onOpenAudioSink( 8 /* bufferCount */, NULL, NULL, - (audio_output_flags_t)flags), - (status_t)OK); + (audio_output_flags_t)pcmFlags); + if (err != OK) { + ALOGW("openAudioSink: non offloaded open failed status: %d", err); + return err; + } mAudioSink->start(); } if (audioSinkChanged) { onAudioSinkChanged(); } - - return offloadingAudio(); + if (offloadingAudio()) { + mAudioOffloadTornDown = false; + } + return OK; } void NuPlayer::Renderer::onCloseAudioSink() { diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h index b15a266..406c64c 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h @@ -46,13 +46,14 @@ struct NuPlayer::Renderer : public AHandler { void queueEOS(bool audio, status_t finalResult); - void flush(bool audio); + void flush(bool audio, bool notifyComplete); void signalTimeDiscontinuity(); void signalAudioSinkChanged(); void signalDisableOffloadAudio(); + void signalEnableOffloadAudio(); void pause(); void resume(); @@ -61,20 +62,23 @@ struct NuPlayer::Renderer : public AHandler { // Following setters and getters are protected by mTimeLock. status_t getCurrentPosition(int64_t *mediaUs); - status_t getCurrentPosition(int64_t *mediaUs, int64_t nowUs); + status_t getCurrentPosition( + int64_t *mediaUs, int64_t nowUs, bool allowPastQueuedVideo = false); void setHasMedia(bool audio); void setAudioFirstAnchorTime(int64_t mediaUs); void setAudioFirstAnchorTimeIfNeeded(int64_t mediaUs); - void setAnchorTime(int64_t mediaUs, int64_t realUs, bool resume = false); + void setAnchorTime( + int64_t mediaUs, int64_t realUs, int64_t numFramesWritten = -1, bool resume = false); void setVideoLateByUs(int64_t lateUs); int64_t getVideoLateByUs(); void setPauseStartedTimeRealUs(int64_t realUs); - bool openAudioSink( + status_t openAudioSink( const sp<AMessage> &format, bool offloadOnly, bool hasVideo, - uint32_t flags); + uint32_t flags, + bool *isOffloaded); void closeAudioSink(); enum { @@ -112,6 +116,7 @@ private: kWhatCloseAudioSink = 'clsA', kWhatStopAudioSink = 'stpA', kWhatDisableOffloadAudio = 'noOA', + kWhatEnableOffloadAudio = 'enOA', kWhatSetVideoFrameRate = 'sVFR', }; @@ -148,6 +153,8 @@ private: int64_t mAudioFirstAnchorTimeMediaUs; int64_t mAnchorTimeMediaUs; int64_t mAnchorTimeRealUs; + int64_t mAnchorNumFramesWritten; + int64_t mAnchorMaxMediaUs; int64_t mVideoLateByUs; bool mHasAudio; bool mHasVideo; @@ -156,6 +163,8 @@ private: Mutex mFlushLock; // protects the following 2 member vars. bool mFlushingAudio; bool mFlushingVideo; + bool mNotifyCompleteAudio; + bool mNotifyCompleteVideo; bool mSyncQueues; @@ -174,6 +183,7 @@ private: int32_t mTotalBuffersQueued; int32_t mLastAudioBufferDrained; + size_t fillAudioBuffer(void *buffer, size_t size); bool onDrainAudioQueue(); @@ -195,11 +205,12 @@ private: void onFlush(const sp<AMessage> &msg); void onAudioSinkChanged(); void onDisableOffloadAudio(); + void onEnableOffloadAudio(); void onPause(); void onResume(); void onSetVideoFrameRate(float fps); void onAudioOffloadTearDown(AudioOffloadTearDownReason reason); - bool onOpenAudioSink( + status_t onOpenAudioSink( const sp<AMessage> &format, bool offloadOnly, bool hasVideo, diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerSource.h b/media/libmediaplayerservice/nuplayer/NuPlayerSource.h index 9924f18..e92cd36 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerSource.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerSource.h @@ -96,7 +96,7 @@ struct NuPlayer::Source : public AHandler { return INVALID_OPERATION; } - virtual status_t selectTrack(size_t /* trackIndex */, bool /* select */) { + virtual status_t selectTrack(size_t /* trackIndex */, bool /* select */, int64_t /* timeUs*/) { return INVALID_OPERATION; } diff --git a/media/libmediaplayerservice/nuplayer/RTSPSource.cpp b/media/libmediaplayerservice/nuplayer/RTSPSource.cpp index ffacb8f..0282a9f 100644 --- a/media/libmediaplayerservice/nuplayer/RTSPSource.cpp +++ b/media/libmediaplayerservice/nuplayer/RTSPSource.cpp @@ -50,7 +50,7 @@ NuPlayer::RTSPSource::RTSPSource( mState(DISCONNECTED), mFinalResult(OK), mDisconnectReplyID(0), - mBuffering(true), + mBuffering(false), mSeekGeneration(0), mEOSTimeoutAudio(0), mEOSTimeoutVideo(0) { @@ -106,9 +106,7 @@ void NuPlayer::RTSPSource::prepareAsync() { mHandler->connect(); } - sp<AMessage> notifyStart = dupNotify(); - notifyStart->setInt32("what", kWhatBufferingStart); - notifyStart->post(); + startBufferingIfNecessary(); } void NuPlayer::RTSPSource::start() { @@ -144,6 +142,7 @@ void NuPlayer::RTSPSource::resume() { } status_t NuPlayer::RTSPSource::feedMoreTSData() { + Mutex::Autolock _l(mBufferingLock); return mFinalResult; } @@ -195,16 +194,8 @@ bool NuPlayer::RTSPSource::haveSufficientDataOnAllTracks() { status_t NuPlayer::RTSPSource::dequeueAccessUnit( bool audio, sp<ABuffer> *accessUnit) { - if (mBuffering) { - if (!haveSufficientDataOnAllTracks()) { - return -EWOULDBLOCK; - } - - mBuffering = false; - - sp<AMessage> notify = dupNotify(); - notify->setInt32("what", kWhatBufferingEnd); - notify->post(); + if (!stopBufferingIfNecessary()) { + return -EWOULDBLOCK; } sp<AnotherPacketSource> source = getSource(audio); @@ -246,11 +237,7 @@ status_t NuPlayer::RTSPSource::dequeueAccessUnit( if (!(otherSource != NULL && otherSource->isFinished(mediaDurationUs))) { // We should not enter buffering mode // if any of the sources already have detected EOS. - mBuffering = true; - - sp<AMessage> notify = dupNotify(); - notify->setInt32("what", kWhatBufferingStart); - notify->post(); + startBufferingIfNecessary(); } return -EWOULDBLOCK; @@ -506,7 +493,7 @@ void NuPlayer::RTSPSource::onMessageReceived(const sp<AMessage> &msg) { sp<AnotherPacketSource> source = info->mSource; if (source != NULL) { source->queueDiscontinuity( - ATSParser::DISCONTINUITY_SEEK, + ATSParser::DISCONTINUITY_TIME, NULL, true /* discard */); } @@ -630,7 +617,7 @@ void NuPlayer::RTSPSource::onSDPLoaded(const sp<AMessage> &msg) { } mState = DISCONNECTED; - mFinalResult = err; + setError(err); if (mDisconnectReplyID != 0) { finishDisconnectIfPossible(); @@ -657,7 +644,7 @@ void NuPlayer::RTSPSource::onDisconnected(const sp<AMessage> &msg) { } mState = DISCONNECTED; - mFinalResult = err; + setError(err); if (mDisconnectReplyID != 0) { finishDisconnectIfPossible(); @@ -678,4 +665,40 @@ void NuPlayer::RTSPSource::finishDisconnectIfPossible() { mDisconnectReplyID = 0; } +void NuPlayer::RTSPSource::setError(status_t err) { + Mutex::Autolock _l(mBufferingLock); + mFinalResult = err; +} + +void NuPlayer::RTSPSource::startBufferingIfNecessary() { + Mutex::Autolock _l(mBufferingLock); + + if (!mBuffering) { + mBuffering = true; + + sp<AMessage> notify = dupNotify(); + notify->setInt32("what", kWhatBufferingStart); + notify->post(); + } +} + +bool NuPlayer::RTSPSource::stopBufferingIfNecessary() { + Mutex::Autolock _l(mBufferingLock); + + if (mBuffering) { + if (!haveSufficientDataOnAllTracks()) { + return false; + } + + mBuffering = false; + + sp<AMessage> notify = dupNotify(); + notify->setInt32("what", kWhatBufferingEnd); + notify->post(); + } + + return true; +} + + } // namespace android diff --git a/media/libmediaplayerservice/nuplayer/RTSPSource.h b/media/libmediaplayerservice/nuplayer/RTSPSource.h index f1cae53..ac3299a 100644 --- a/media/libmediaplayerservice/nuplayer/RTSPSource.h +++ b/media/libmediaplayerservice/nuplayer/RTSPSource.h @@ -97,6 +97,7 @@ private: State mState; status_t mFinalResult; uint32_t mDisconnectReplyID; + Mutex mBufferingLock; bool mBuffering; sp<ALooper> mLooper; @@ -126,6 +127,9 @@ private: bool haveSufficientDataOnAllTracks(); void setEOSTimeout(bool audio, int64_t timeout); + void setError(status_t err); + void startBufferingIfNecessary(); + bool stopBufferingIfNecessary(); DISALLOW_EVIL_CONSTRUCTORS(RTSPSource); }; diff --git a/media/libmediaplayerservice/nuplayer/StreamingSource.cpp b/media/libmediaplayerservice/nuplayer/StreamingSource.cpp index 2e9a29a..b3f224d 100644 --- a/media/libmediaplayerservice/nuplayer/StreamingSource.cpp +++ b/media/libmediaplayerservice/nuplayer/StreamingSource.cpp @@ -37,13 +37,26 @@ NuPlayer::StreamingSource::StreamingSource( const sp<IStreamSource> &source) : Source(notify), mSource(source), - mFinalResult(OK) { + mFinalResult(OK), + mBuffering(false) { } NuPlayer::StreamingSource::~StreamingSource() { + if (mLooper != NULL) { + mLooper->unregisterHandler(id()); + mLooper->stop(); + } } void NuPlayer::StreamingSource::prepareAsync() { + if (mLooper == NULL) { + mLooper = new ALooper; + mLooper->setName("streaming"); + mLooper->start(); + + mLooper->registerHandler(this); + } + notifyVideoSizeChanged(); notifyFlagsChanged(0); notifyPrepared(); @@ -62,13 +75,15 @@ void NuPlayer::StreamingSource::start() { mTSParser = new ATSParser(parserFlags); mStreamListener->start(); + + postReadBuffer(); } status_t NuPlayer::StreamingSource::feedMoreTSData() { - if (mFinalResult != OK) { - return mFinalResult; - } + return postReadBuffer(); +} +void NuPlayer::StreamingSource::onReadBuffer() { for (int32_t i = 0; i < 50; ++i) { char buffer[188]; sp<AMessage> extra; @@ -77,10 +92,10 @@ status_t NuPlayer::StreamingSource::feedMoreTSData() { if (n == 0) { ALOGI("input data EOS reached."); mTSParser->signalEOS(ERROR_END_OF_STREAM); - mFinalResult = ERROR_END_OF_STREAM; + setError(ERROR_END_OF_STREAM); break; } else if (n == INFO_DISCONTINUITY) { - int32_t type = ATSParser::DISCONTINUITY_SEEK; + int32_t type = ATSParser::DISCONTINUITY_TIME; int32_t mask; if (extra != NULL @@ -88,7 +103,8 @@ status_t NuPlayer::StreamingSource::feedMoreTSData() { IStreamListener::kKeyDiscontinuityMask, &mask)) { if (mask == 0) { ALOGE("Client specified an illegal discontinuity type."); - return ERROR_UNSUPPORTED; + setError(ERROR_UNSUPPORTED); + break; } type = mask; @@ -97,7 +113,6 @@ status_t NuPlayer::StreamingSource::feedMoreTSData() { mTSParser->signalDiscontinuity( (ATSParser::DiscontinuityType)type, extra); } else if (n < 0) { - CHECK_EQ(n, -EWOULDBLOCK); break; } else { if (buffer[0] == 0x00) { @@ -118,7 +133,7 @@ status_t NuPlayer::StreamingSource::feedMoreTSData() { mTSParser->signalDiscontinuity( ((type & 1) == 0) - ? ATSParser::DISCONTINUITY_SEEK + ? ATSParser::DISCONTINUITY_TIME : ATSParser::DISCONTINUITY_FORMATCHANGE, extra); } else { @@ -128,22 +143,80 @@ status_t NuPlayer::StreamingSource::feedMoreTSData() { ALOGE("TS Parser returned error %d", err); mTSParser->signalEOS(err); - mFinalResult = err; + setError(err); break; } } } } +} + +status_t NuPlayer::StreamingSource::postReadBuffer() { + { + Mutex::Autolock _l(mBufferingLock); + if (mFinalResult != OK) { + return mFinalResult; + } + if (mBuffering) { + return OK; + } + mBuffering = true; + } + (new AMessage(kWhatReadBuffer, id()))->post(); return OK; } -sp<MetaData> NuPlayer::StreamingSource::getFormatMeta(bool audio) { - ATSParser::SourceType type = - audio ? ATSParser::AUDIO : ATSParser::VIDEO; +bool NuPlayer::StreamingSource::haveSufficientDataOnAllTracks() { + // We're going to buffer at least 2 secs worth data on all tracks before + // starting playback (both at startup and after a seek). - sp<AnotherPacketSource> source = - static_cast<AnotherPacketSource *>(mTSParser->getSource(type).get()); + static const int64_t kMinDurationUs = 2000000ll; + + sp<AnotherPacketSource> audioTrack = getSource(true /*audio*/); + sp<AnotherPacketSource> videoTrack = getSource(false /*audio*/); + + status_t err; + int64_t durationUs; + if (audioTrack != NULL + && (durationUs = audioTrack->getBufferedDurationUs(&err)) + < kMinDurationUs + && err == OK) { + ALOGV("audio track doesn't have enough data yet. (%.2f secs buffered)", + durationUs / 1E6); + return false; + } + + if (videoTrack != NULL + && (durationUs = videoTrack->getBufferedDurationUs(&err)) + < kMinDurationUs + && err == OK) { + ALOGV("video track doesn't have enough data yet. (%.2f secs buffered)", + durationUs / 1E6); + return false; + } + + return true; +} + +void NuPlayer::StreamingSource::setError(status_t err) { + Mutex::Autolock _l(mBufferingLock); + mFinalResult = err; +} + +sp<AnotherPacketSource> NuPlayer::StreamingSource::getSource(bool audio) { + if (mTSParser == NULL) { + return NULL; + } + + sp<MediaSource> source = mTSParser->getSource( + audio ? ATSParser::AUDIO : ATSParser::VIDEO); + + return static_cast<AnotherPacketSource *>(source.get()); +} + +sp<MetaData> NuPlayer::StreamingSource::getFormatMeta(bool audio) { + sp<AnotherPacketSource> source = getSource(audio); if (source == NULL) { return NULL; @@ -154,16 +227,16 @@ sp<MetaData> NuPlayer::StreamingSource::getFormatMeta(bool audio) { status_t NuPlayer::StreamingSource::dequeueAccessUnit( bool audio, sp<ABuffer> *accessUnit) { - ATSParser::SourceType type = - audio ? ATSParser::AUDIO : ATSParser::VIDEO; - - sp<AnotherPacketSource> source = - static_cast<AnotherPacketSource *>(mTSParser->getSource(type).get()); + sp<AnotherPacketSource> source = getSource(audio); if (source == NULL) { return -EWOULDBLOCK; } + if (!haveSufficientDataOnAllTracks()) { + postReadBuffer(); + } + status_t finalResult; if (!source->hasBufferAvailable(&finalResult)) { return finalResult == OK ? -EWOULDBLOCK : finalResult; @@ -186,5 +259,26 @@ bool NuPlayer::StreamingSource::isRealTime() const { return mSource->flags() & IStreamSource::kFlagIsRealTimeData; } +void NuPlayer::StreamingSource::onMessageReceived( + const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatReadBuffer: + { + onReadBuffer(); + + { + Mutex::Autolock _l(mBufferingLock); + mBuffering = false; + } + break; + } + default: + { + TRESPASS(); + } + } +} + + } // namespace android diff --git a/media/libmediaplayerservice/nuplayer/StreamingSource.h b/media/libmediaplayerservice/nuplayer/StreamingSource.h index 412b6c4..1f95f3c 100644 --- a/media/libmediaplayerservice/nuplayer/StreamingSource.h +++ b/media/libmediaplayerservice/nuplayer/StreamingSource.h @@ -25,6 +25,7 @@ namespace android { struct ABuffer; struct ATSParser; +struct AnotherPacketSource; struct NuPlayer::StreamingSource : public NuPlayer::Source { StreamingSource( @@ -43,14 +44,29 @@ struct NuPlayer::StreamingSource : public NuPlayer::Source { protected: virtual ~StreamingSource(); + virtual void onMessageReceived(const sp<AMessage> &msg); + virtual sp<MetaData> getFormatMeta(bool audio); private: + enum { + kWhatReadBuffer, + }; sp<IStreamSource> mSource; status_t mFinalResult; sp<NuPlayerStreamListener> mStreamListener; sp<ATSParser> mTSParser; + bool mBuffering; + Mutex mBufferingLock; + sp<ALooper> mLooper; + + void setError(status_t err); + sp<AnotherPacketSource> getSource(bool audio); + bool haveSufficientDataOnAllTracks(); + status_t postReadBuffer(); + void onReadBuffer(); + DISALLOW_EVIL_CONSTRUCTORS(StreamingSource); }; |