From 7137ec7e005a5a6e3c0edb91cfacf16a31f4bf6a Mon Sep 17 00:00:00 2001 From: Chong Zhang Date: Wed, 12 Nov 2014 16:41:05 -0800 Subject: tunnel NuPlayer source and decoder input Bug: 18342383 Change-Id: Ieff1cd3bad2b39d46f127ddd5d5139b919992461 --- media/libmediaplayerservice/nuplayer/Android.mk | 2 + media/libmediaplayerservice/nuplayer/NuPlayer.cpp | 361 +----- media/libmediaplayerservice/nuplayer/NuPlayer.h | 32 +- .../nuplayer/NuPlayerCCDecoder.cpp | 361 ++++++ .../nuplayer/NuPlayerCCDecoder.h | 59 + .../nuplayer/NuPlayerDecoder.cpp | 1253 ++++++++------------ .../nuplayer/NuPlayerDecoder.h | 119 +- .../nuplayer/NuPlayerDecoderBase.cpp | 179 +++ .../nuplayer/NuPlayerDecoderBase.h | 92 ++ .../nuplayer/NuPlayerDecoderPassThrough.cpp | 350 ++++-- .../nuplayer/NuPlayerDecoderPassThrough.h | 62 +- .../nuplayer/NuPlayerRenderer.cpp | 17 +- .../nuplayer/NuPlayerRenderer.h | 4 +- 13 files changed, 1550 insertions(+), 1341 deletions(-) create mode 100644 media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.cpp create mode 100644 media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.h create mode 100644 media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.cpp create mode 100644 media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.h (limited to 'media') diff --git a/media/libmediaplayerservice/nuplayer/Android.mk b/media/libmediaplayerservice/nuplayer/Android.mk index 676c0a6..6609874 100644 --- a/media/libmediaplayerservice/nuplayer/Android.mk +++ b/media/libmediaplayerservice/nuplayer/Android.mk @@ -5,7 +5,9 @@ LOCAL_SRC_FILES:= \ GenericSource.cpp \ HTTPLiveSource.cpp \ NuPlayer.cpp \ + NuPlayerCCDecoder.cpp \ NuPlayerDecoder.cpp \ + NuPlayerDecoderBase.cpp \ NuPlayerDecoderPassThrough.cpp \ NuPlayerDriver.cpp \ NuPlayerRenderer.cpp \ diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp index c01f16a..405278c 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp @@ -21,7 +21,9 @@ #include "NuPlayer.h" #include "HTTPLiveSource.h" +#include "NuPlayerCCDecoder.h" #include "NuPlayerDecoder.h" +#include "NuPlayerDecoderBase.h" #include "NuPlayerDecoderPassThrough.h" #include "NuPlayerDriver.h" #include "NuPlayerRenderer.h" @@ -52,10 +54,6 @@ namespace android { -// TODO optimize buffer size for power consumption -// The offload read buffer size is 32 KB but 24 KB uses less power. -const size_t NuPlayer::kAggregateBufferSizeBytes = 24 * 1024; - struct NuPlayer::Action : public RefBase { Action() {} @@ -153,7 +151,6 @@ private: NuPlayer::NuPlayer() : mUIDValid(false), mSourceFlags(0), - mVideoIsAVC(false), mOffloadAudio(false), mAudioDecoderGeneration(0), mVideoDecoderGeneration(0), @@ -164,11 +161,8 @@ NuPlayer::NuPlayer() mScanSourcesGeneration(0), mPollDurationGeneration(0), mTimedTextGeneration(0), - mTimeDiscontinuityPending(false), mFlushingAudio(NONE), mFlushingVideo(NONE), - mNumFramesTotal(0ll), - mNumFramesDropped(0ll), mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW), mStarted(false) { clearFlushComplete(); @@ -559,6 +553,12 @@ void NuPlayer::onMessageReceived(const sp &msg) { new SimpleAction(&NuPlayer::performScanSources)); } + // After a flush wihtout shutdown, decoder is paused. + // Don't resume it until source is seeked, otherwise it could + // start pulling stale data too soon. + mDeferredActions.push_back( + new SimpleAction(&NuPlayer::performResumeDecoders)); + processDeferredActions(); break; } @@ -685,16 +685,26 @@ void NuPlayer::onMessageReceived(const sp &msg) { int32_t what; CHECK(msg->findInt32("what", &what)); - if (what == Decoder::kWhatFillThisBuffer) { - status_t err = feedDecoderInputData( - audio, msg); + if (what == DecoderBase::kWhatInputDiscontinuity) { + int32_t formatChange; + CHECK(msg->findInt32("formatChange", &formatChange)); - if (err == -EWOULDBLOCK) { - if (mSource->feedMoreTSData() == OK) { - msg->post(10 * 1000ll); - } + ALOGV("%s discontinuity: formatChange %d", + audio ? "audio" : "video", formatChange); + + if (formatChange) { + mDeferredActions.push_back( + new FlushDecoderAction( + audio ? FLUSH_CMD_SHUTDOWN : FLUSH_CMD_NONE, + audio ? FLUSH_CMD_NONE : FLUSH_CMD_SHUTDOWN)); } - } else if (what == Decoder::kWhatEOS) { + + mDeferredActions.push_back( + new SimpleAction( + &NuPlayer::performScanSources)); + + processDeferredActions(); + } else if (what == DecoderBase::kWhatEOS) { int32_t err; CHECK(msg->findInt32("err", &err)); @@ -707,12 +717,12 @@ void NuPlayer::onMessageReceived(const sp &msg) { } mRenderer->queueEOS(audio, err); - } else if (what == Decoder::kWhatFlushCompleted) { + } else if (what == DecoderBase::kWhatFlushCompleted) { ALOGV("decoder %s flush completed", audio ? "audio" : "video"); handleFlushComplete(audio, true /* isDecoder */); finishFlushIfPossible(); - } else if (what == Decoder::kWhatVideoSizeChanged) { + } else if (what == DecoderBase::kWhatVideoSizeChanged) { sp format; CHECK(msg->findMessage("format", &format)); @@ -720,7 +730,7 @@ void NuPlayer::onMessageReceived(const sp &msg) { mSource->getFormat(false /* audio */); updateVideoSize(inputFormat, format); - } else if (what == Decoder::kWhatShutdownCompleted) { + } else if (what == DecoderBase::kWhatShutdownCompleted) { ALOGV("%s shutdown completed", audio ? "audio" : "video"); if (audio) { mAudioDecoder.clear(); @@ -737,7 +747,7 @@ void NuPlayer::onMessageReceived(const sp &msg) { } finishFlushIfPossible(); - } else if (what == Decoder::kWhatError) { + } else if (what == DecoderBase::kWhatError) { status_t err; if (!msg->findInt32("err", &err) || err == OK) { err = UNKNOWN_ERROR; @@ -785,8 +795,6 @@ void NuPlayer::onMessageReceived(const sp &msg) { break; // Finish anyways. } notifyListener(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err); - } else if (what == Decoder::kWhatRenderBufferTime) { - renderBuffer(audio, msg); } else { ALOGV("Unhandled decoder notification %d '%c%c%c%c'.", what, @@ -860,9 +868,11 @@ void NuPlayer::onMessageReceived(const sp &msg) { closeAudioSink(); mAudioDecoder.clear(); ++mAudioDecoderGeneration; - mRenderer->flush(true /* audio */); + mRenderer->flush( + true /* audio */, false /* notifyComplete */); if (mVideoDecoder != NULL) { - mRenderer->flush(false /* audio */); + mRenderer->flush( + false /* audio */, false /* notifyComplete */); } performSeek(positionUs, false /* needNotify */); @@ -913,6 +923,12 @@ void NuPlayer::onMessageReceived(const sp &msg) { mDeferredActions.push_back( new SeekAction(seekTimeUs, needNotify)); + // After a flush wihtout shutdown, decoder is paused. + // Don't resume it until source is seeked, otherwise it could + // start pulling stale data too soon. + mDeferredActions.push_back( + new SimpleAction(&NuPlayer::performResumeDecoders)); + processDeferredActions(); break; } @@ -969,12 +985,9 @@ void NuPlayer::onResume() { } void NuPlayer::onStart() { - mVideoIsAVC = false; mOffloadAudio = false; mAudioEOS = false; mVideoEOS = false; - mNumFramesTotal = 0; - mNumFramesDropped = 0; mStarted = true; /* instantiate decoders now for secure playback */ @@ -1095,22 +1108,6 @@ void NuPlayer::finishFlushIfPossible() { ALOGV("both audio and video are flushed now."); - mPendingAudioAccessUnit.clear(); - mAggregateBuffer.clear(); - - if (mTimeDiscontinuityPending) { - mRenderer->signalTimeDiscontinuity(); - mTimeDiscontinuityPending = false; - } - - if (mAudioDecoder != NULL && mFlushingAudio == FLUSHED) { - mAudioDecoder->signalResume(); - } - - if (mVideoDecoder != NULL && mFlushingVideo == FLUSHED) { - mVideoDecoder->signalResume(); - } - mFlushingAudio = NONE; mFlushingVideo = NONE; @@ -1163,7 +1160,7 @@ void NuPlayer::closeAudioSink() { mRenderer->closeAudioSink(); } -status_t NuPlayer::instantiateDecoder(bool audio, sp *decoder) { +status_t NuPlayer::instantiateDecoder(bool audio, sp *decoder) { if (*decoder != NULL) { return OK; } @@ -1177,7 +1174,6 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp *decoder) { if (!audio) { AString mime; CHECK(format->findString("mime", &mime)); - mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str()); sp ccNotify = new AMessage(kWhatClosedCaptionNotify, id()); mCCDecoder = new CCDecoder(ccNotify); @@ -1202,7 +1198,8 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp *decoder) { ++mVideoDecoderGeneration; notify->setInt32("generation", mVideoDecoderGeneration); - *decoder = new Decoder(notify, mSource, mRenderer, mNativeWindow); + *decoder = new Decoder( + notify, mSource, mRenderer, mNativeWindow, mCCDecoder); // enable FRC if high-quality AV sync is requested, even if not // queuing to native window, as this will even improve textureview @@ -1243,232 +1240,6 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp *decoder) { return OK; } -status_t NuPlayer::feedDecoderInputData(bool audio, const sp &msg) { - sp reply; - CHECK(msg->findMessage("reply", &reply)); - - if ((audio && mFlushingAudio != NONE) - || (!audio && mFlushingVideo != NONE) - || mSource == NULL) { - reply->setInt32("err", INFO_DISCONTINUITY); - reply->post(); - return OK; - } - - sp accessUnit; - - // Aggregate smaller buffers into a larger buffer. - // The goal is to reduce power consumption. - // Note this will not work if the decoder requires one frame per buffer. - bool doBufferAggregation = (audio && mOffloadAudio); - bool needMoreData = false; - - bool dropAccessUnit; - do { - status_t err; - // Did we save an accessUnit earlier because of a discontinuity? - if (audio && (mPendingAudioAccessUnit != NULL)) { - accessUnit = mPendingAudioAccessUnit; - mPendingAudioAccessUnit.clear(); - err = mPendingAudioErr; - ALOGV("feedDecoderInputData() use mPendingAudioAccessUnit"); - } else { - err = mSource->dequeueAccessUnit(audio, &accessUnit); - } - - if (err == -EWOULDBLOCK) { - return err; - } else if (err != OK) { - if (err == INFO_DISCONTINUITY) { - if (doBufferAggregation && (mAggregateBuffer != NULL)) { - // We already have some data so save this for later. - mPendingAudioErr = err; - mPendingAudioAccessUnit = accessUnit; - accessUnit.clear(); - ALOGD("feedDecoderInputData() save discontinuity for later"); - break; - } - int32_t type; - CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); - - bool formatChange = - (audio && - (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT)) - || (!audio && - (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT)); - - bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0; - - ALOGI("%s discontinuity (formatChange=%d, time=%d)", - audio ? "audio" : "video", formatChange, timeChange); - - mTimeDiscontinuityPending = - mTimeDiscontinuityPending || timeChange; - - bool seamlessFormatChange = false; - sp newFormat = mSource->getFormat(audio); - if (formatChange) { - seamlessFormatChange = - getDecoder(audio)->supportsSeamlessFormatChange(newFormat); - // treat seamless format change separately - formatChange = !seamlessFormatChange; - } - bool shutdownOrFlush = formatChange || timeChange; - - // We want to queue up scan-sources only once per discontinuity. - // We control this by doing it only if neither audio nor video are - // flushing or shutting down. (After handling 1st discontinuity, one - // of the flushing states will not be NONE.) - // No need to scan sources if this discontinuity does not result - // in a flush or shutdown, as the flushing state will stay NONE. - if (mFlushingAudio == NONE && mFlushingVideo == NONE && - shutdownOrFlush) { - // And we'll resume scanning sources once we're done - // flushing. - mDeferredActions.push_front( - new SimpleAction( - &NuPlayer::performScanSources)); - } - - if (formatChange /* not seamless */) { - // must change decoder - flushDecoder(audio, /* needShutdown = */ true); - } else if (timeChange) { - // need to flush - flushDecoder(audio, /* needShutdown = */ false, newFormat); - err = OK; - } else if (seamlessFormatChange) { - // reuse existing decoder and don't flush - updateDecoderFormatWithoutFlush(audio, newFormat); - err = OK; - } else { - // This stream is unaffected by the discontinuity - return -EWOULDBLOCK; - } - } else if (err == ERROR_END_OF_STREAM - && doBufferAggregation && (mAggregateBuffer != NULL)) { - // send out the last bit of aggregated data - reply->setBuffer("buffer", mAggregateBuffer); - mAggregateBuffer.clear(); - err = OK; - } - - reply->setInt32("err", err); - reply->post(); - return OK; - } - - if (!audio) { - ++mNumFramesTotal; - } - - dropAccessUnit = false; - if (!audio - && !(mSourceFlags & Source::FLAG_SECURE) - && mRenderer->getVideoLateByUs() > 100000ll - && mVideoIsAVC - && !IsAVCReferenceFrame(accessUnit)) { - dropAccessUnit = true; - ++mNumFramesDropped; - } - - size_t smallSize = accessUnit->size(); - needMoreData = false; - if (doBufferAggregation && (mAggregateBuffer == NULL) - // Don't bother if only room for a few small buffers. - && (smallSize < (kAggregateBufferSizeBytes / 3))) { - // Create a larger buffer for combining smaller buffers from the extractor. - mAggregateBuffer = new ABuffer(kAggregateBufferSizeBytes); - mAggregateBuffer->setRange(0, 0); // start empty - } - - if (doBufferAggregation && (mAggregateBuffer != NULL)) { - int64_t timeUs; - int64_t dummy; - bool smallTimestampValid = accessUnit->meta()->findInt64("timeUs", &timeUs); - bool bigTimestampValid = mAggregateBuffer->meta()->findInt64("timeUs", &dummy); - // Will the smaller buffer fit? - size_t bigSize = mAggregateBuffer->size(); - size_t roomLeft = mAggregateBuffer->capacity() - bigSize; - // Should we save this small buffer for the next big buffer? - // If the first small buffer did not have a timestamp then save - // any buffer that does have a timestamp until the next big buffer. - if ((smallSize > roomLeft) - || (!bigTimestampValid && (bigSize > 0) && smallTimestampValid)) { - mPendingAudioErr = err; - mPendingAudioAccessUnit = accessUnit; - accessUnit.clear(); - } else { - // Grab time from first small buffer if available. - if ((bigSize == 0) && smallTimestampValid) { - mAggregateBuffer->meta()->setInt64("timeUs", timeUs); - } - // Append small buffer to the bigger buffer. - memcpy(mAggregateBuffer->base() + bigSize, accessUnit->data(), smallSize); - bigSize += smallSize; - mAggregateBuffer->setRange(0, bigSize); - - // Keep looping until we run out of room in the mAggregateBuffer. - needMoreData = true; - - ALOGV("feedDecoderInputData() smallSize = %zu, bigSize = %zu, capacity = %zu", - smallSize, bigSize, mAggregateBuffer->capacity()); - } - } - } while (dropAccessUnit || needMoreData); - - // ALOGV("returned a valid buffer of %s data", audio ? "audio" : "video"); - -#if 0 - int64_t mediaTimeUs; - CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs)); - ALOGV("feeding %s input buffer at media time %.2f secs", - audio ? "audio" : "video", - mediaTimeUs / 1E6); -#endif - - if (!audio) { - mCCDecoder->decode(accessUnit); - } - - if (doBufferAggregation && (mAggregateBuffer != NULL)) { - ALOGV("feedDecoderInputData() reply with aggregated buffer, %zu", - mAggregateBuffer->size()); - reply->setBuffer("buffer", mAggregateBuffer); - mAggregateBuffer.clear(); - } else { - reply->setBuffer("buffer", accessUnit); - } - - reply->post(); - - return OK; -} - -void NuPlayer::renderBuffer(bool audio, const sp &msg) { - // ALOGV("renderBuffer %s", audio ? "audio" : "video"); - - if ((audio && mFlushingAudio != NONE) - || (!audio && mFlushingVideo != NONE)) { - // We're currently attempting to flush the decoder, in order - // to complete this, the decoder wants all its buffers back, - // so we don't want any output buffers it sent us (from before - // we initiated the flush) to be stuck in the renderer's queue. - - ALOGV("we're still flushing the %s decoder, sending its output buffer" - " right back.", audio ? "audio" : "video"); - - return; - } - - int64_t mediaTimeUs; - CHECK(msg->findInt64("timeUs", &mediaTimeUs)); - - if (!audio && mCCDecoder->isSelected()) { - mCCDecoder->display(mediaTimeUs); - } -} - void NuPlayer::updateVideoSize( const sp &inputFormat, const sp &outputFormat) { @@ -1549,12 +1320,11 @@ void NuPlayer::notifyListener(int msg, int ext1, int ext2, const Parcel *in) { driver->notifyListener(msg, ext1, ext2, in); } -void NuPlayer::flushDecoder( - bool audio, bool needShutdown, const sp &newFormat) { +void NuPlayer::flushDecoder(bool audio, bool needShutdown) { ALOGV("[%s] flushDecoder needShutdown=%d", audio ? "audio" : "video", needShutdown); - const sp &decoder = getDecoder(audio); + const sp &decoder = getDecoder(audio); if (decoder == NULL) { ALOGI("flushDecoder %s without decoder present", audio ? "audio" : "video"); @@ -1565,7 +1335,7 @@ void NuPlayer::flushDecoder( ++mScanSourcesGeneration; mScanSourcesPending = false; - decoder->signalFlush(newFormat); + decoder->signalFlush(); FlushStatus newStatus = needShutdown ? FLUSHING_DECODER_SHUTDOWN : FLUSHING_DECODER; @@ -1580,25 +1350,7 @@ void NuPlayer::flushDecoder( ALOGE_IF(mFlushingVideo != NONE, "video flushDecoder() is called in state %d", mFlushingVideo); mFlushingVideo = newStatus; - - if (mCCDecoder != NULL) { - mCCDecoder->flush(); - } - } -} - -void NuPlayer::updateDecoderFormatWithoutFlush( - bool audio, const sp &format) { - ALOGV("[%s] updateDecoderFormatWithoutFlush", audio ? "audio" : "video"); - - const sp &decoder = getDecoder(audio); - if (decoder == NULL) { - ALOGI("updateDecoderFormatWithoutFlush %s without decoder present", - audio ? "audio" : "video"); - return; } - - decoder->signalUpdateFormat(format); } void NuPlayer::queueDecoderShutdown( @@ -1684,8 +1436,13 @@ status_t NuPlayer::getCurrentPosition(int64_t *mediaUs) { } void NuPlayer::getStats(int64_t *numFramesTotal, int64_t *numFramesDropped) { - *numFramesTotal = mNumFramesTotal; - *numFramesDropped = mNumFramesDropped; + sp decoder = getDecoder(false /* audio */); + if (decoder != NULL) { + decoder->getStats(numFramesTotal, numFramesDropped); + } else { + *numFramesTotal = 0; + *numFramesDropped = 0; + } } sp NuPlayer::getFileMeta() { @@ -1762,8 +1519,6 @@ void NuPlayer::performDecoderFlush(FlushCommand audio, FlushCommand video) { return; } - mTimeDiscontinuityPending = true; - if (audio != FLUSH_CMD_NONE && mAudioDecoder != NULL) { flushDecoder(true /* audio */, (audio == FLUSH_CMD_SHUTDOWN)); } @@ -1838,6 +1593,16 @@ void NuPlayer::performSetSurface(const sp &wrapper) { } } +void NuPlayer::performResumeDecoders() { + if (mVideoDecoder != NULL) { + mVideoDecoder->signalResume(); + } + + if (mAudioDecoder != NULL) { + mAudioDecoder->signalResume(); + } +} + void NuPlayer::onSourceNotify(const sp &msg) { int32_t what; CHECK(msg->findInt32("what", &what)); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h index 901cfbd..6856af1 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h @@ -71,8 +71,6 @@ struct NuPlayer : public AHandler { sp getFileMeta(); - static const size_t kAggregateBufferSizeBytes; - protected: virtual ~NuPlayer(); @@ -84,6 +82,7 @@ public: private: struct Decoder; + struct DecoderBase; struct DecoderPassThrough; struct CCDecoder; struct GenericSource; @@ -128,10 +127,9 @@ private: uint32_t mSourceFlags; sp mNativeWindow; sp mAudioSink; - sp mVideoDecoder; - bool mVideoIsAVC; + sp mVideoDecoder; bool mOffloadAudio; - sp mAudioDecoder; + sp mAudioDecoder; sp mCCDecoder; sp mRenderer; sp mRendererLooper; @@ -165,29 +163,17 @@ private: FLUSH_CMD_SHUTDOWN, }; - // Once the current flush is complete this indicates whether the - // notion of time has changed. - bool mTimeDiscontinuityPending; - // Status of flush responses from the decoder and renderer. bool mFlushComplete[2][2]; - // Used by feedDecoderInputData to aggregate small buffers into - // one large buffer. - sp mPendingAudioAccessUnit; - status_t mPendingAudioErr; - sp mAggregateBuffer; - FlushStatus mFlushingAudio; FlushStatus mFlushingVideo; - int64_t mNumFramesTotal, mNumFramesDropped; - int32_t mVideoScalingMode; bool mStarted; - inline const sp &getDecoder(bool audio) { + inline const sp &getDecoder(bool audio) { return audio ? mAudioDecoder : mVideoDecoder; } @@ -201,15 +187,12 @@ private: void openAudioSink(const sp &format, bool offloadOnly); void closeAudioSink(); - status_t instantiateDecoder(bool audio, sp *decoder); + status_t instantiateDecoder(bool audio, sp *decoder); void updateVideoSize( const sp &inputFormat, const sp &outputFormat = NULL); - status_t feedDecoderInputData(bool audio, const sp &msg); - void renderBuffer(bool audio, const sp &msg); - void notifyListener(int msg, int ext1, int ext2, const Parcel *in = NULL); void handleFlushComplete(bool audio, bool isDecoder); @@ -220,9 +203,7 @@ private: bool audioDecoderStillNeeded(); - void flushDecoder( - bool audio, bool needShutdown, const sp &newFormat = NULL); - void updateDecoderFormatWithoutFlush(bool audio, const sp &format); + void flushDecoder(bool audio, bool needShutdown); void postScanSources(); @@ -236,6 +217,7 @@ private: void performReset(); void performScanSources(); void performSetSurface(const sp &wrapper); + void performResumeDecoders(); void onSourceNotify(const sp &msg); void onClosedCaptionNotify(const sp &msg); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.cpp new file mode 100644 index 0000000..9229704 --- /dev/null +++ b/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.cpp @@ -0,0 +1,361 @@ +/* + * Copyright 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "NuPlayerCCDecoder" +#include +#include + +#include "NuPlayerCCDecoder.h" + +#include +#include +#include +#include +#include + +namespace android { + +struct CCData { + CCData(uint8_t type, uint8_t data1, uint8_t data2) + : mType(type), mData1(data1), mData2(data2) { + } + bool getChannel(size_t *channel) const { + if (mData1 >= 0x10 && mData1 <= 0x1f) { + *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0); + return true; + } + return false; + } + + uint8_t mType; + uint8_t mData1; + uint8_t mData2; +}; + +static bool isNullPad(CCData *cc) { + return cc->mData1 < 0x10 && cc->mData2 < 0x10; +} + +static void dumpBytePair(const sp &ccBuf) { + size_t offset = 0; + AString out; + + while (offset < ccBuf->size()) { + char tmp[128]; + + CCData *cc = (CCData *) (ccBuf->data() + offset); + + if (isNullPad(cc)) { + // 1 null pad or XDS metadata, ignore + offset += sizeof(CCData); + continue; + } + + if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) { + // 2 basic chars + sprintf(tmp, "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2); + } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19) + && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) { + // 1 special char + sprintf(tmp, "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2); + } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A) + && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){ + // 1 Spanish/French char + sprintf(tmp, "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2); + } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B) + && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){ + // 1 Portuguese/German/Danish char + sprintf(tmp, "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2); + } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19) + && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){ + // Mid-Row Codes (Table 69) + sprintf(tmp, "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2); + } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c) + && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f) + || + ((cc->mData1 == 0x17 || cc->mData1 == 0x1f) + && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){ + // Misc Control Codes (Table 70) + sprintf(tmp, "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2); + } else if ((cc->mData1 & 0x70) == 0x10 + && (cc->mData2 & 0x40) == 0x40 + && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) { + // Preamble Address Codes (Table 71) + sprintf(tmp, "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2); + } else { + sprintf(tmp, "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2); + } + + if (out.size() > 0) { + out.append(", "); + } + + out.append(tmp); + + offset += sizeof(CCData); + } + + ALOGI("%s", out.c_str()); +} + +NuPlayer::CCDecoder::CCDecoder(const sp ¬ify) + : mNotify(notify), + mCurrentChannel(0), + mSelectedTrack(-1) { + for (size_t i = 0; i < sizeof(mTrackIndices)/sizeof(mTrackIndices[0]); ++i) { + mTrackIndices[i] = -1; + } +} + +size_t NuPlayer::CCDecoder::getTrackCount() const { + return mFoundChannels.size(); +} + +sp NuPlayer::CCDecoder::getTrackInfo(size_t index) const { + if (!isTrackValid(index)) { + return NULL; + } + + sp format = new AMessage(); + + format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE); + format->setString("language", "und"); + format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608); + //CC1, field 0 channel 0 + bool isDefaultAuto = (mFoundChannels[index] == 0); + format->setInt32("auto", isDefaultAuto); + format->setInt32("default", isDefaultAuto); + format->setInt32("forced", 0); + + return format; +} + +status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) { + if (!isTrackValid(index)) { + return BAD_VALUE; + } + + if (select) { + if (mSelectedTrack == (ssize_t)index) { + ALOGE("track %zu already selected", index); + return BAD_VALUE; + } + ALOGV("selected track %zu", index); + mSelectedTrack = index; + } else { + if (mSelectedTrack != (ssize_t)index) { + ALOGE("track %zu is not selected", index); + return BAD_VALUE; + } + ALOGV("unselected track %zu", index); + mSelectedTrack = -1; + } + + return OK; +} + +bool NuPlayer::CCDecoder::isSelected() const { + return mSelectedTrack >= 0 && mSelectedTrack < (int32_t) getTrackCount(); +} + +bool NuPlayer::CCDecoder::isTrackValid(size_t index) const { + return index < getTrackCount(); +} + +int32_t NuPlayer::CCDecoder::getTrackIndex(size_t channel) const { + if (channel < sizeof(mTrackIndices)/sizeof(mTrackIndices[0])) { + return mTrackIndices[channel]; + } + return -1; +} + +// returns true if a new CC track is found +bool NuPlayer::CCDecoder::extractFromSEI(const sp &accessUnit) { + int64_t timeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + + sp sei; + if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) { + return false; + } + + bool trackAdded = false; + + NALBitReader br(sei->data() + 1, sei->size() - 1); + // sei_message() + while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message() + uint32_t payload_type = 0; + size_t payload_size = 0; + uint8_t last_byte; + + do { + last_byte = br.getBits(8); + payload_type += last_byte; + } while (last_byte == 0xFF); + + do { + last_byte = br.getBits(8); + payload_size += last_byte; + } while (last_byte == 0xFF); + + // sei_payload() + if (payload_type == 4) { + // user_data_registered_itu_t_t35() + + // ATSC A/72: 6.4.2 + uint8_t itu_t_t35_country_code = br.getBits(8); + uint16_t itu_t_t35_provider_code = br.getBits(16); + uint32_t user_identifier = br.getBits(32); + uint8_t user_data_type_code = br.getBits(8); + + payload_size -= 1 + 2 + 4 + 1; + + if (itu_t_t35_country_code == 0xB5 + && itu_t_t35_provider_code == 0x0031 + && user_identifier == 'GA94' + && user_data_type_code == 0x3) { + // MPEG_cc_data() + // ATSC A/53 Part 4: 6.2.3.1 + br.skipBits(1); //process_em_data_flag + bool process_cc_data_flag = br.getBits(1); + br.skipBits(1); //additional_data_flag + size_t cc_count = br.getBits(5); + br.skipBits(8); // em_data; + payload_size -= 2; + + if (process_cc_data_flag) { + AString out; + + sp ccBuf = new ABuffer(cc_count * sizeof(CCData)); + ccBuf->setRange(0, 0); + + for (size_t i = 0; i < cc_count; i++) { + uint8_t marker = br.getBits(5); + CHECK_EQ(marker, 0x1f); + + bool cc_valid = br.getBits(1); + uint8_t cc_type = br.getBits(2); + // remove odd parity bit + uint8_t cc_data_1 = br.getBits(8) & 0x7f; + uint8_t cc_data_2 = br.getBits(8) & 0x7f; + + if (cc_valid + && (cc_type == 0 || cc_type == 1)) { + CCData cc(cc_type, cc_data_1, cc_data_2); + if (!isNullPad(&cc)) { + size_t channel; + if (cc.getChannel(&channel) && getTrackIndex(channel) < 0) { + mTrackIndices[channel] = mFoundChannels.size(); + mFoundChannels.push_back(channel); + trackAdded = true; + } + memcpy(ccBuf->data() + ccBuf->size(), + (void *)&cc, sizeof(cc)); + ccBuf->setRange(0, ccBuf->size() + sizeof(CCData)); + } + } + } + payload_size -= cc_count * 3; + + mCCMap.add(timeUs, ccBuf); + break; + } + } else { + ALOGV("Malformed SEI payload type 4"); + } + } else { + ALOGV("Unsupported SEI payload type %d", payload_type); + } + + // skipping remaining bits of this payload + br.skipBits(payload_size * 8); + } + + return trackAdded; +} + +sp NuPlayer::CCDecoder::filterCCBuf( + const sp &ccBuf, size_t index) { + sp filteredCCBuf = new ABuffer(ccBuf->size()); + filteredCCBuf->setRange(0, 0); + + size_t cc_count = ccBuf->size() / sizeof(CCData); + const CCData* cc_data = (const CCData*)ccBuf->data(); + for (size_t i = 0; i < cc_count; ++i) { + size_t channel; + if (cc_data[i].getChannel(&channel)) { + mCurrentChannel = channel; + } + if (mCurrentChannel == mFoundChannels[index]) { + memcpy(filteredCCBuf->data() + filteredCCBuf->size(), + (void *)&cc_data[i], sizeof(CCData)); + filteredCCBuf->setRange(0, filteredCCBuf->size() + sizeof(CCData)); + } + } + + return filteredCCBuf; +} + +void NuPlayer::CCDecoder::decode(const sp &accessUnit) { + if (extractFromSEI(accessUnit)) { + ALOGI("Found CEA-608 track"); + sp msg = mNotify->dup(); + msg->setInt32("what", kWhatTrackAdded); + msg->post(); + } + // TODO: extract CC from other sources +} + +void NuPlayer::CCDecoder::display(int64_t timeUs) { + if (!isTrackValid(mSelectedTrack)) { + ALOGE("Could not find current track(index=%d)", mSelectedTrack); + return; + } + + ssize_t index = mCCMap.indexOfKey(timeUs); + if (index < 0) { + ALOGV("cc for timestamp %" PRId64 " not found", timeUs); + return; + } + + sp ccBuf = filterCCBuf(mCCMap.valueAt(index), mSelectedTrack); + + if (ccBuf->size() > 0) { +#if 0 + dumpBytePair(ccBuf); +#endif + + ccBuf->meta()->setInt32("trackIndex", mSelectedTrack); + ccBuf->meta()->setInt64("timeUs", timeUs); + ccBuf->meta()->setInt64("durationUs", 0ll); + + sp msg = mNotify->dup(); + msg->setInt32("what", kWhatClosedCaptionData); + msg->setBuffer("buffer", ccBuf); + msg->post(); + } + + // remove all entries before timeUs + mCCMap.removeItemsAt(0, index + 1); +} + +void NuPlayer::CCDecoder::flush() { + mCCMap.clear(); +} + +} // namespace android + diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.h b/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.h new file mode 100644 index 0000000..5e06f4e --- /dev/null +++ b/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.h @@ -0,0 +1,59 @@ +/* + * Copyright 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef NUPLAYER_CCDECODER_H_ + +#define NUPLAYER_CCDECODER_H_ + +#include "NuPlayer.h" + +namespace android { + +struct NuPlayer::CCDecoder : public RefBase { + enum { + kWhatClosedCaptionData, + kWhatTrackAdded, + }; + + CCDecoder(const sp ¬ify); + + size_t getTrackCount() const; + sp getTrackInfo(size_t index) const; + status_t selectTrack(size_t index, bool select); + bool isSelected() const; + void decode(const sp &accessUnit); + void display(int64_t timeUs); + void flush(); + +private: + sp mNotify; + KeyedVector > mCCMap; + size_t mCurrentChannel; + int32_t mSelectedTrack; + int32_t mTrackIndices[4]; + Vector mFoundChannels; + + bool isTrackValid(size_t index) const; + int32_t getTrackIndex(size_t channel) const; + bool extractFromSEI(const sp &accessUnit); + sp filterCCBuf(const sp &ccBuf, size_t index); + + DISALLOW_EVIL_CONSTRUCTORS(CCDecoder); +}; + +} // namespace android + +#endif // NUPLAYER_CCDECODER_H_ diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp index e695c43..0439a9a 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010 The Android Open Source Project + * Copyright 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,13 +19,12 @@ #include #include +#include "NuPlayerCCDecoder.h" #include "NuPlayerDecoder.h" - #include "NuPlayerRenderer.h" #include "NuPlayerSource.h" #include -#include #include #include #include @@ -34,76 +33,103 @@ #include #include +#include "avc_utils.h" +#include "ATSParser.h" + namespace android { NuPlayer::Decoder::Decoder( const sp ¬ify, const sp &source, const sp &renderer, - const sp &nativeWindow) + const sp &nativeWindow, + const sp &ccDecoder) : mNotify(notify), mNativeWindow(nativeWindow), mSource(source), mRenderer(renderer), + mCCDecoder(ccDecoder), mSkipRenderingUntilMediaTimeUs(-1ll), + mNumFramesTotal(0ll), + mNumFramesDropped(0ll), + mIsAudio(true), + mIsVideoAVC(false), + mIsSecure(false), + mFormatChangePending(false), mBufferGeneration(0), mPaused(true), mComponentName("decoder") { - // Every decoder has its own looper because MediaCodec operations - // are blocking, but NuPlayer needs asynchronous operations. - mDecoderLooper = new ALooper; - mDecoderLooper->setName("NPDecoder"); - mDecoderLooper->start(false, false, ANDROID_PRIORITY_AUDIO); - mCodecLooper = new ALooper; mCodecLooper->setName("NPDecoder-CL"); mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO); } NuPlayer::Decoder::~Decoder() { - mDecoderLooper->unregisterHandler(id()); - mDecoderLooper->stop(); - releaseAndResetMediaBuffers(); } -static -status_t PostAndAwaitResponse( - const sp &msg, sp *response) { - status_t err = msg->postAndAwaitResponse(response); +void NuPlayer::Decoder::getStats( + int64_t *numFramesTotal, + int64_t *numFramesDropped) const { + *numFramesTotal = mNumFramesTotal; + *numFramesDropped = mNumFramesDropped; +} - if (err != OK) { - return err; - } +void NuPlayer::Decoder::onMessageReceived(const sp &msg) { + ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str()); - if (!(*response)->findInt32("err", &err)) { - err = OK; - } + switch (msg->what()) { + case kWhatCodecNotify: + { + if (!isStaleReply(msg)) { + int32_t numInput, numOutput; - return err; -} + if (!msg->findInt32("input-buffers", &numInput)) { + numInput = INT32_MAX; + } -void NuPlayer::Decoder::rememberCodecSpecificData(const sp &format) { - mCSDsForCurrentFormat.clear(); - for (int32_t i = 0; ; ++i) { - AString tag = "csd-"; - tag.append(i); - sp buffer; - if (!format->findBuffer(tag.c_str(), &buffer)) { + if (!msg->findInt32("output-buffers", &numOutput)) { + numOutput = INT32_MAX; + } + + if (!mPaused) { + while (numInput-- > 0 && handleAnInputBuffer()) {} + } + + while (numOutput-- > 0 && handleAnOutputBuffer()) {} + } + + requestCodecNotification(); break; } - mCSDsForCurrentFormat.push(buffer); + + case kWhatRenderBuffer: + { + if (!isStaleReply(msg)) { + onRenderBuffer(msg); + } + break; + } + + default: + DecoderBase::onMessageReceived(msg); + break; } } void NuPlayer::Decoder::onConfigure(const sp &format) { CHECK(mCodec == NULL); + mFormatChangePending = false; + ++mBufferGeneration; AString mime; CHECK(format->findString("mime", &mime)); + mIsAudio = !strncasecmp("audio/", mime.c_str(), 6); + mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str()); + sp surface = NULL; if (mNativeWindow != NULL) { surface = mNativeWindow->getSurfaceTextureClient(); @@ -131,6 +157,7 @@ void NuPlayer::Decoder::onConfigure(const sp &format) { handleError(UNKNOWN_ERROR); return; } + mIsSecure = secure; mCodec->getName(&mComponentName); @@ -183,69 +210,120 @@ void NuPlayer::Decoder::onConfigure(const sp &format) { mPaused = false; } -void NuPlayer::Decoder::releaseAndResetMediaBuffers() { - for (size_t i = 0; i < mMediaBuffers.size(); i++) { - if (mMediaBuffers[i] != NULL) { - mMediaBuffers[i]->release(); - mMediaBuffers.editItemAt(i) = NULL; - } - } - mMediaBuffers.resize(mInputBuffers.size()); - for (size_t i = 0; i < mMediaBuffers.size(); i++) { - mMediaBuffers.editItemAt(i) = NULL; +void NuPlayer::Decoder::onSetRenderer(const sp &renderer) { + bool hadNoRenderer = (mRenderer == NULL); + mRenderer = renderer; + if (hadNoRenderer && mRenderer != NULL) { + requestCodecNotification(); } - mInputBufferIsDequeued.clear(); - mInputBufferIsDequeued.resize(mInputBuffers.size()); - for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) { - mInputBufferIsDequeued.editItemAt(i) = false; +} + +void NuPlayer::Decoder::onGetInputBuffers( + Vector > *dstBuffers) { + dstBuffers->clear(); + for (size_t i = 0; i < mInputBuffers.size(); i++) { + dstBuffers->push(mInputBuffers[i]); } +} - mPendingInputMessages.clear(); - mSkipRenderingUntilMediaTimeUs = -1; +void NuPlayer::Decoder::onResume() { + mPaused = false; } -void NuPlayer::Decoder::requestCodecNotification() { +void NuPlayer::Decoder::onFlush(bool notifyComplete) { + if (mCCDecoder != NULL) { + mCCDecoder->flush(); + } + + if (mRenderer != NULL) { + mRenderer->flush(mIsAudio, notifyComplete); + mRenderer->signalTimeDiscontinuity(); + } + + status_t err = OK; if (mCodec != NULL) { - sp reply = new AMessage(kWhatCodecNotify, id()); - reply->setInt32("generation", mBufferGeneration); - mCodec->requestActivityNotification(reply); + err = mCodec->flush(); + mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator + ++mBufferGeneration; } -} -bool NuPlayer::Decoder::isStaleReply(const sp &msg) { - int32_t generation; - CHECK(msg->findInt32("generation", &generation)); - return generation != mBufferGeneration; -} + if (err != OK) { + ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err); + handleError(err); + // finish with posting kWhatFlushCompleted. + // we attempt to release the buffers even if flush fails. + } + releaseAndResetMediaBuffers(); -void NuPlayer::Decoder::init() { - mDecoderLooper->registerHandler(this); + if (notifyComplete) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatFlushCompleted); + notify->post(); + mPaused = true; + } } -void NuPlayer::Decoder::configure(const sp &format) { - sp msg = new AMessage(kWhatConfigure, id()); - msg->setMessage("format", format); - msg->post(); -} +void NuPlayer::Decoder::onShutdown(bool notifyComplete) { + status_t err = OK; + if (mCodec != NULL) { + err = mCodec->release(); + mCodec = NULL; + ++mBufferGeneration; -void NuPlayer::Decoder::setRenderer(const sp &renderer) { - sp msg = new AMessage(kWhatSetRenderer, id()); - msg->setObject("renderer", renderer); - msg->post(); -} + if (mNativeWindow != NULL) { + // reconnect to surface as MediaCodec disconnected from it + status_t error = + native_window_api_connect( + mNativeWindow->getNativeWindow().get(), + NATIVE_WINDOW_API_MEDIA); + ALOGW_IF(error != NO_ERROR, + "[%s] failed to connect to native window, error=%d", + mComponentName.c_str(), error); + } + mComponentName = "decoder"; + } + + releaseAndResetMediaBuffers(); + + if (err != OK) { + ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err); + handleError(err); + // finish with posting kWhatShutdownCompleted. + } -void NuPlayer::Decoder::signalUpdateFormat(const sp &format) { - sp msg = new AMessage(kWhatUpdateFormat, id()); - msg->setMessage("format", format); - msg->post(); + if (notifyComplete) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatShutdownCompleted); + notify->post(); + mPaused = true; + } } -status_t NuPlayer::Decoder::getInputBuffers(Vector > *buffers) const { - sp msg = new AMessage(kWhatGetInputBuffers, id()); - msg->setPointer("buffers", buffers); +void NuPlayer::Decoder::doRequestBuffers() { + if (mFormatChangePending) { + return; + } + status_t err = OK; + while (!mDequeuedInputBuffers.empty()) { + size_t bufferIx = *mDequeuedInputBuffers.begin(); + sp msg = new AMessage(); + msg->setSize("buffer-ix", bufferIx); + err = fetchInputData(msg); + if (err != OK) { + break; + } + mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin()); + + if (!mPendingInputMessages.empty() + || !onInputBufferFetched(msg)) { + mPendingInputMessages.push_back(msg); + } + } - sp response; - return PostAndAwaitResponse(msg, &response); + if (err == -EWOULDBLOCK + && mSource->feedMoreTSData() == OK) { + scheduleRequestBuffers(); + } } void NuPlayer::Decoder::handleError(int32_t err) @@ -263,6 +341,9 @@ void NuPlayer::Decoder::handleError(int32_t err) } bool NuPlayer::Decoder::handleAnInputBuffer() { + if (mFormatChangePending) { + return false; + } size_t bufferIx = -1; status_t res = mCodec->dequeueInputBuffer(&bufferIx); ALOGV("[%s] dequeued input: %d", @@ -284,22 +365,21 @@ bool NuPlayer::Decoder::handleAnInputBuffer() { } mInputBufferIsDequeued.editItemAt(bufferIx) = true; - sp reply = new AMessage(kWhatInputBufferFilled, id()); - reply->setSize("buffer-ix", bufferIx); - reply->setInt32("generation", mBufferGeneration); - if (!mCSDsToSubmit.isEmpty()) { + sp msg = new AMessage(); + msg->setSize("buffer-ix", bufferIx); + sp buffer = mCSDsToSubmit.itemAt(0); ALOGI("[%s] resubmitting CSD", mComponentName.c_str()); - reply->setBuffer("buffer", buffer); + msg->setBuffer("buffer", buffer); mCSDsToSubmit.removeAt(0); - CHECK(onInputBufferFilled(reply)); + CHECK(onInputBufferFetched(msg)); return true; } while (!mPendingInputMessages.empty()) { sp msg = *mPendingInputMessages.begin(); - if (!onInputBufferFilled(msg)) { + if (!onInputBufferFetched(msg)) { break; } mPendingInputMessages.erase(mPendingInputMessages.begin()); @@ -309,62 +389,310 @@ bool NuPlayer::Decoder::handleAnInputBuffer() { return true; } - sp notify = mNotify->dup(); - notify->setInt32("what", kWhatFillThisBuffer); - notify->setBuffer("buffer", mInputBuffers[bufferIx]); - notify->setMessage("reply", reply); - notify->post(); + mDequeuedInputBuffers.push_back(bufferIx); + + onRequestInputBuffers(); return true; } -bool android::NuPlayer::Decoder::onInputBufferFilled(const sp &msg) { - size_t bufferIx; - CHECK(msg->findSize("buffer-ix", &bufferIx)); - CHECK_LT(bufferIx, mInputBuffers.size()); - sp codecBuffer = mInputBuffers[bufferIx]; +bool NuPlayer::Decoder::handleAnOutputBuffer() { + if (mFormatChangePending) { + return false; + } + size_t bufferIx = -1; + size_t offset; + size_t size; + int64_t timeUs; + uint32_t flags; + status_t res = mCodec->dequeueOutputBuffer( + &bufferIx, &offset, &size, &timeUs, &flags); - sp buffer; - bool hasBuffer = msg->findBuffer("buffer", &buffer); + if (res != OK) { + ALOGV("[%s] dequeued output: %d", mComponentName.c_str(), res); + } else { + ALOGV("[%s] dequeued output: %d (time=%lld flags=%" PRIu32 ")", + mComponentName.c_str(), (int)bufferIx, timeUs, flags); + } - // handle widevine classic source - that fills an arbitrary input buffer - MediaBuffer *mediaBuffer = NULL; - if (hasBuffer) { - mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase()); - if (mediaBuffer != NULL) { - // likely filled another buffer than we requested: adjust buffer index - size_t ix; - for (ix = 0; ix < mInputBuffers.size(); ix++) { - const sp &buf = mInputBuffers[ix]; - if (buf->data() == mediaBuffer->data()) { - // all input buffers are dequeued on start, hence the check - if (!mInputBufferIsDequeued[ix]) { - ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu", - mComponentName.c_str(), ix, bufferIx); - mediaBuffer->release(); - return false; - } + if (res == INFO_OUTPUT_BUFFERS_CHANGED) { + res = mCodec->getOutputBuffers(&mOutputBuffers); + if (res != OK) { + ALOGE("Failed to get output buffers for %s after INFO event (err=%d)", + mComponentName.c_str(), res); + handleError(res); + return false; + } + // NuPlayer ignores this + return true; + } else if (res == INFO_FORMAT_CHANGED) { + sp format = new AMessage(); + res = mCodec->getOutputFormat(&format); + if (res != OK) { + ALOGE("Failed to get output format for %s after INFO event (err=%d)", + mComponentName.c_str(), res); + handleError(res); + return false; + } - // TRICKY: need buffer for the metadata, so instead, set - // codecBuffer to the same (though incorrect) buffer to - // avoid a memcpy into the codecBuffer - codecBuffer = buffer; - codecBuffer->setRange( - mediaBuffer->range_offset(), - mediaBuffer->range_length()); - bufferIx = ix; - break; - } + if (!mIsAudio) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatVideoSizeChanged); + notify->setMessage("format", format); + notify->post(); + } else if (mRenderer != NULL) { + uint32_t flags; + int64_t durationUs; + bool hasVideo = (mSource->getFormat(false /* audio */) != NULL); + if (!hasVideo && + mSource->getDuration(&durationUs) == OK && + durationUs + > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) { + flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER; + } else { + flags = AUDIO_OUTPUT_FLAG_NONE; } - CHECK(ix < mInputBuffers.size()); + + mRenderer->openAudioSink( + format, false /* offloadOnly */, hasVideo, flags); + } + return true; + } else if (res == INFO_DISCONTINUITY) { + // nothing to do + return true; + } else if (res != OK) { + if (res != -EAGAIN) { + ALOGE("Failed to dequeue output buffer for %s (err=%d)", + mComponentName.c_str(), res); + handleError(res); } + return false; } - if (buffer == NULL /* includes !hasBuffer */) { - int32_t streamErr = ERROR_END_OF_STREAM; - CHECK(msg->findInt32("err", &streamErr) || !hasBuffer); - - if (streamErr == OK) { - /* buffers are returned to hold on to */ + CHECK_LT(bufferIx, mOutputBuffers.size()); + sp buffer = mOutputBuffers[bufferIx]; + buffer->setRange(offset, size); + buffer->meta()->clear(); + buffer->meta()->setInt64("timeUs", timeUs); + if (flags & MediaCodec::BUFFER_FLAG_EOS) { + buffer->meta()->setInt32("eos", true); + } + // we do not expect CODECCONFIG or SYNCFRAME for decoder + + sp reply = new AMessage(kWhatRenderBuffer, id()); + reply->setSize("buffer-ix", bufferIx); + reply->setInt32("generation", mBufferGeneration); + + if (mSkipRenderingUntilMediaTimeUs >= 0) { + if (timeUs < mSkipRenderingUntilMediaTimeUs) { + ALOGV("[%s] dropping buffer at time %lld as requested.", + mComponentName.c_str(), (long long)timeUs); + + reply->post(); + return true; + } + + mSkipRenderingUntilMediaTimeUs = -1; + } + + if (mRenderer != NULL) { + // send the buffer to renderer. + mRenderer->queueBuffer(mIsAudio, buffer, reply); + if (flags & MediaCodec::BUFFER_FLAG_EOS) { + mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM); + } + } + + return true; +} + +void NuPlayer::Decoder::releaseAndResetMediaBuffers() { + for (size_t i = 0; i < mMediaBuffers.size(); i++) { + if (mMediaBuffers[i] != NULL) { + mMediaBuffers[i]->release(); + mMediaBuffers.editItemAt(i) = NULL; + } + } + mMediaBuffers.resize(mInputBuffers.size()); + for (size_t i = 0; i < mMediaBuffers.size(); i++) { + mMediaBuffers.editItemAt(i) = NULL; + } + mInputBufferIsDequeued.clear(); + mInputBufferIsDequeued.resize(mInputBuffers.size()); + for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) { + mInputBufferIsDequeued.editItemAt(i) = false; + } + + mPendingInputMessages.clear(); + mDequeuedInputBuffers.clear(); + mSkipRenderingUntilMediaTimeUs = -1; +} + +void NuPlayer::Decoder::requestCodecNotification() { + if (mFormatChangePending) { + return; + } + if (mCodec != NULL) { + sp reply = new AMessage(kWhatCodecNotify, id()); + reply->setInt32("generation", mBufferGeneration); + mCodec->requestActivityNotification(reply); + } +} + +bool NuPlayer::Decoder::isStaleReply(const sp &msg) { + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + return generation != mBufferGeneration; +} + +status_t NuPlayer::Decoder::fetchInputData(sp &reply) { + sp accessUnit; + bool dropAccessUnit; + do { + status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit); + + if (err == -EWOULDBLOCK) { + return err; + } else if (err != OK) { + if (err == INFO_DISCONTINUITY) { + int32_t type; + CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); + + bool formatChange = + (mIsAudio && + (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT)) + || (!mIsAudio && + (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT)); + + bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0; + + ALOGI("%s discontinuity (format=%d, time=%d)", + mIsAudio ? "audio" : "video", formatChange, timeChange); + + bool seamlessFormatChange = false; + sp newFormat = mSource->getFormat(mIsAudio); + if (formatChange) { + seamlessFormatChange = + supportsSeamlessFormatChange(newFormat); + // treat seamless format change separately + formatChange = !seamlessFormatChange; + } + + if (formatChange || timeChange) { + sp msg = mNotify->dup(); + msg->setInt32("what", kWhatInputDiscontinuity); + msg->setInt32("formatChange", formatChange); + msg->post(); + } + + if (formatChange /* not seamless */) { + // must change decoder + // return EOS and wait to be killed + mFormatChangePending = true; + return ERROR_END_OF_STREAM; + } else if (timeChange) { + // need to flush + // TODO: Ideally we shouldn't need a flush upon time + // discontinuity, flushing will cause loss of frames. + // We probably should queue a time change marker to the + // output queue, and handles it in renderer instead. + rememberCodecSpecificData(newFormat); + onFlush(false /* notifyComplete */); + err = OK; + } else if (seamlessFormatChange) { + // reuse existing decoder and don't flush + rememberCodecSpecificData(newFormat); + err = OK; + } else { + // This stream is unaffected by the discontinuity + return -EWOULDBLOCK; + } + } + + reply->setInt32("err", err); + return OK; + } + + if (!mIsAudio) { + ++mNumFramesTotal; + } + + dropAccessUnit = false; + if (!mIsAudio + && !mIsSecure + && mRenderer->getVideoLateByUs() > 100000ll + && mIsVideoAVC + && !IsAVCReferenceFrame(accessUnit)) { + dropAccessUnit = true; + ++mNumFramesDropped; + } + } while (dropAccessUnit); + + // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video"); +#if 0 + int64_t mediaTimeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs)); + ALOGV("feeding %s input buffer at media time %.2f secs", + mIsAudio ? "audio" : "video", + mediaTimeUs / 1E6); +#endif + + if (mCCDecoder != NULL) { + mCCDecoder->decode(accessUnit); + } + + reply->setBuffer("buffer", accessUnit); + + return OK; +} + +bool NuPlayer::Decoder::onInputBufferFetched(const sp &msg) { + size_t bufferIx; + CHECK(msg->findSize("buffer-ix", &bufferIx)); + CHECK_LT(bufferIx, mInputBuffers.size()); + sp codecBuffer = mInputBuffers[bufferIx]; + + sp buffer; + bool hasBuffer = msg->findBuffer("buffer", &buffer); + + // handle widevine classic source - that fills an arbitrary input buffer + MediaBuffer *mediaBuffer = NULL; + if (hasBuffer) { + mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase()); + if (mediaBuffer != NULL) { + // likely filled another buffer than we requested: adjust buffer index + size_t ix; + for (ix = 0; ix < mInputBuffers.size(); ix++) { + const sp &buf = mInputBuffers[ix]; + if (buf->data() == mediaBuffer->data()) { + // all input buffers are dequeued on start, hence the check + if (!mInputBufferIsDequeued[ix]) { + ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu", + mComponentName.c_str(), ix, bufferIx); + mediaBuffer->release(); + return false; + } + + // TRICKY: need buffer for the metadata, so instead, set + // codecBuffer to the same (though incorrect) buffer to + // avoid a memcpy into the codecBuffer + codecBuffer = buffer; + codecBuffer->setRange( + mediaBuffer->range_offset(), + mediaBuffer->range_length()); + bufferIx = ix; + break; + } + } + CHECK(ix < mInputBuffers.size()); + } + } + + if (buffer == NULL /* includes !hasBuffer */) { + int32_t streamErr = ERROR_END_OF_STREAM; + CHECK(msg->findInt32("err", &streamErr) || !hasBuffer); + + if (streamErr == OK) { + /* buffers are returned to hold on to */ return true; } @@ -444,127 +772,20 @@ bool android::NuPlayer::Decoder::onInputBufferFilled(const sp &msg) { return true; } -bool NuPlayer::Decoder::handleAnOutputBuffer() { - size_t bufferIx = -1; - size_t offset; - size_t size; - int64_t timeUs; - uint32_t flags; - status_t res = mCodec->dequeueOutputBuffer( - &bufferIx, &offset, &size, &timeUs, &flags); - - if (res != OK) { - ALOGV("[%s] dequeued output: %d", mComponentName.c_str(), res); - } else { - ALOGV("[%s] dequeued output: %d (time=%lld flags=%" PRIu32 ")", - mComponentName.c_str(), (int)bufferIx, timeUs, flags); - } - - if (res == INFO_OUTPUT_BUFFERS_CHANGED) { - res = mCodec->getOutputBuffers(&mOutputBuffers); - if (res != OK) { - ALOGE("Failed to get output buffers for %s after INFO event (err=%d)", - mComponentName.c_str(), res); - handleError(res); - return false; - } - // NuPlayer ignores this - return true; - } else if (res == INFO_FORMAT_CHANGED) { - sp format = new AMessage(); - res = mCodec->getOutputFormat(&format); - if (res != OK) { - ALOGE("Failed to get output format for %s after INFO event (err=%d)", - mComponentName.c_str(), res); - handleError(res); - return false; - } - - if (isVideo()) { - sp notify = mNotify->dup(); - notify->setInt32("what", kWhatVideoSizeChanged); - notify->setMessage("format", format); - notify->post(); - } else if (mRenderer != NULL) { - uint32_t flags; - int64_t durationUs; - bool hasVideo = (mSource->getFormat(false /* audio */) != NULL); - if (!hasVideo && - mSource->getDuration(&durationUs) == OK && - durationUs - > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) { - flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER; - } else { - flags = AUDIO_OUTPUT_FLAG_NONE; - } - - mRenderer->openAudioSink( - format, false /* offloadOnly */, hasVideo, flags); - } - return true; - } else if (res == INFO_DISCONTINUITY) { - // nothing to do - return true; - } else if (res != OK) { - if (res != -EAGAIN) { - ALOGE("Failed to dequeue output buffer for %s (err=%d)", - mComponentName.c_str(), res); - handleError(res); - } - return false; - } - - CHECK_LT(bufferIx, mOutputBuffers.size()); - sp buffer = mOutputBuffers[bufferIx]; - buffer->setRange(offset, size); - buffer->meta()->clear(); - buffer->meta()->setInt64("timeUs", timeUs); - if (flags & MediaCodec::BUFFER_FLAG_EOS) { - buffer->meta()->setInt32("eos", true); - } - // we do not expect CODECCONFIG or SYNCFRAME for decoder - - sp reply = new AMessage(kWhatRenderBuffer, id()); - reply->setSize("buffer-ix", bufferIx); - reply->setInt32("generation", mBufferGeneration); - - if (mSkipRenderingUntilMediaTimeUs >= 0) { - if (timeUs < mSkipRenderingUntilMediaTimeUs) { - ALOGV("[%s] dropping buffer at time %lld as requested.", - mComponentName.c_str(), (long long)timeUs); - - reply->post(); - return true; - } - - mSkipRenderingUntilMediaTimeUs = -1; - } - - if (mRenderer != NULL) { - // send the buffer to renderer. - mRenderer->queueBuffer(!isVideo(), buffer, reply); - if (flags & MediaCodec::BUFFER_FLAG_EOS) { - mRenderer->queueEOS(!isVideo(), ERROR_END_OF_STREAM); - } - } - - return true; -} - void NuPlayer::Decoder::onRenderBuffer(const sp &msg) { status_t err; int32_t render; size_t bufferIx; CHECK(msg->findSize("buffer-ix", &bufferIx)); - if (isVideo()) { + if (!mIsAudio) { int64_t timeUs; sp buffer = mOutputBuffers[bufferIx]; buffer->meta()->findInt64("timeUs", &timeUs); - sp notify = mNotify->dup(); - notify->setInt32("what", kWhatRenderBufferTime); - notify->setInt64("timeUs", timeUs); - notify->post(); + + if (mCCDecoder != NULL && mCCDecoder->isSelected()) { + mCCDecoder->display(timeUs); + } } if (msg->findInt32("render", &render) && render) { @@ -581,208 +802,8 @@ void NuPlayer::Decoder::onRenderBuffer(const sp &msg) { } } -void NuPlayer::Decoder::onFlush() { - if (mRenderer != NULL) { - mRenderer->flush(!isVideo()); - } - - status_t err = OK; - if (mCodec != NULL) { - err = mCodec->flush(); - mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator - ++mBufferGeneration; - } - - if (err != OK) { - ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err); - handleError(err); - // finish with posting kWhatFlushCompleted. - // we attempt to release the buffers even if flush fails. - } - releaseAndResetMediaBuffers(); - - sp notify = mNotify->dup(); - notify->setInt32("what", kWhatFlushCompleted); - notify->post(); - mPaused = true; -} - -void NuPlayer::Decoder::onResume() { - mPaused = false; -} - -void NuPlayer::Decoder::onShutdown() { - status_t err = OK; - if (mCodec != NULL) { - err = mCodec->release(); - mCodec = NULL; - ++mBufferGeneration; - - if (mNativeWindow != NULL) { - // reconnect to surface as MediaCodec disconnected from it - status_t error = - native_window_api_connect( - mNativeWindow->getNativeWindow().get(), - NATIVE_WINDOW_API_MEDIA); - ALOGW_IF(error != NO_ERROR, - "[%s] failed to connect to native window, error=%d", - mComponentName.c_str(), error); - } - mComponentName = "decoder"; - } - - releaseAndResetMediaBuffers(); - - if (err != OK) { - ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err); - handleError(err); - // finish with posting kWhatShutdownCompleted. - } - - sp notify = mNotify->dup(); - notify->setInt32("what", kWhatShutdownCompleted); - notify->post(); - mPaused = true; -} - -void NuPlayer::Decoder::onMessageReceived(const sp &msg) { - ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str()); - - switch (msg->what()) { - case kWhatConfigure: - { - sp format; - CHECK(msg->findMessage("format", &format)); - onConfigure(format); - break; - } - - case kWhatSetRenderer: - { - bool hadNoRenderer = (mRenderer == NULL); - sp obj; - CHECK(msg->findObject("renderer", &obj)); - mRenderer = static_cast(obj.get()); - if (hadNoRenderer && mRenderer != NULL) { - requestCodecNotification(); - } - break; - } - - case kWhatUpdateFormat: - { - sp format; - CHECK(msg->findMessage("format", &format)); - rememberCodecSpecificData(format); - break; - } - - case kWhatGetInputBuffers: - { - uint32_t replyID; - CHECK(msg->senderAwaitsResponse(&replyID)); - - Vector > *dstBuffers; - CHECK(msg->findPointer("buffers", (void **)&dstBuffers)); - - dstBuffers->clear(); - for (size_t i = 0; i < mInputBuffers.size(); i++) { - dstBuffers->push(mInputBuffers[i]); - } - - (new AMessage)->postReply(replyID); - break; - } - - case kWhatCodecNotify: - { - if (!isStaleReply(msg)) { - int32_t numInput, numOutput; - - if (!msg->findInt32("input-buffers", &numInput)) { - numInput = INT32_MAX; - } - - if (!msg->findInt32("output-buffers", &numOutput)) { - numOutput = INT32_MAX; - } - - if (!mPaused) { - while (numInput-- > 0 && handleAnInputBuffer()) {} - } - - while (numOutput-- > 0 && handleAnOutputBuffer()) {} - } - - requestCodecNotification(); - break; - } - - case kWhatInputBufferFilled: - { - if (!isStaleReply(msg)) { - if (!mPendingInputMessages.empty() - || !onInputBufferFilled(msg)) { - mPendingInputMessages.push_back(msg); - } - } - - break; - } - - case kWhatRenderBuffer: - { - if (!isStaleReply(msg)) { - onRenderBuffer(msg); - } - break; - } - - case kWhatFlush: - { - sp format; - if (msg->findMessage("new-format", &format)) { - rememberCodecSpecificData(format); - } - onFlush(); - break; - } - - case kWhatResume: - { - onResume(); - break; - } - - case kWhatShutdown: - { - onShutdown(); - break; - } - - default: - TRESPASS(); - break; - } -} - -void NuPlayer::Decoder::signalFlush(const sp &format) { - sp msg = new AMessage(kWhatFlush, id()); - if (format != NULL) { - msg->setMessage("new-format", format); - } - msg->post(); -} - -void NuPlayer::Decoder::signalResume() { - (new AMessage(kWhatResume, id()))->post(); -} - -void NuPlayer::Decoder::initiateShutdown() { - (new AMessage(kWhatShutdown, id()))->post(); -} - -bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(const sp &targetFormat) const { +bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange( + const sp &targetFormat) const { if (targetFormat == NULL) { return true; } @@ -847,336 +868,20 @@ bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp &targetF return seamless; } -bool NuPlayer::Decoder::isVideo() { - return mNativeWindow != NULL; -} - -struct CCData { - CCData(uint8_t type, uint8_t data1, uint8_t data2) - : mType(type), mData1(data1), mData2(data2) { - } - bool getChannel(size_t *channel) const { - if (mData1 >= 0x10 && mData1 <= 0x1f) { - *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0); - return true; - } - return false; - } - - uint8_t mType; - uint8_t mData1; - uint8_t mData2; -}; - -static bool isNullPad(CCData *cc) { - return cc->mData1 < 0x10 && cc->mData2 < 0x10; -} - -static void dumpBytePair(const sp &ccBuf) { - size_t offset = 0; - AString out; - - while (offset < ccBuf->size()) { - char tmp[128]; - - CCData *cc = (CCData *) (ccBuf->data() + offset); - - if (isNullPad(cc)) { - // 1 null pad or XDS metadata, ignore - offset += sizeof(CCData); - continue; - } - - if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) { - // 2 basic chars - sprintf(tmp, "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2); - } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19) - && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) { - // 1 special char - sprintf(tmp, "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A) - && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){ - // 1 Spanish/French char - sprintf(tmp, "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B) - && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){ - // 1 Portuguese/German/Danish char - sprintf(tmp, "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19) - && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){ - // Mid-Row Codes (Table 69) - sprintf(tmp, "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c) - && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f) - || - ((cc->mData1 == 0x17 || cc->mData1 == 0x1f) - && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){ - // Misc Control Codes (Table 70) - sprintf(tmp, "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else if ((cc->mData1 & 0x70) == 0x10 - && (cc->mData2 & 0x40) == 0x40 - && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) { - // Preamble Address Codes (Table 71) - sprintf(tmp, "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else { - sprintf(tmp, "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } - - if (out.size() > 0) { - out.append(", "); - } - - out.append(tmp); - - offset += sizeof(CCData); - } - - ALOGI("%s", out.c_str()); -} - -NuPlayer::CCDecoder::CCDecoder(const sp ¬ify) - : mNotify(notify), - mCurrentChannel(0), - mSelectedTrack(-1) { - for (size_t i = 0; i < sizeof(mTrackIndices)/sizeof(mTrackIndices[0]); ++i) { - mTrackIndices[i] = -1; - } -} - -size_t NuPlayer::CCDecoder::getTrackCount() const { - return mFoundChannels.size(); -} - -sp NuPlayer::CCDecoder::getTrackInfo(size_t index) const { - if (!isTrackValid(index)) { - return NULL; - } - - sp format = new AMessage(); - - format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE); - format->setString("language", "und"); - format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608); - //CC1, field 0 channel 0 - bool isDefaultAuto = (mFoundChannels[index] == 0); - format->setInt32("auto", isDefaultAuto); - format->setInt32("default", isDefaultAuto); - format->setInt32("forced", 0); - - return format; -} - -status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) { - if (!isTrackValid(index)) { - return BAD_VALUE; - } - - if (select) { - if (mSelectedTrack == (ssize_t)index) { - ALOGE("track %zu already selected", index); - return BAD_VALUE; - } - ALOGV("selected track %zu", index); - mSelectedTrack = index; - } else { - if (mSelectedTrack != (ssize_t)index) { - ALOGE("track %zu is not selected", index); - return BAD_VALUE; - } - ALOGV("unselected track %zu", index); - mSelectedTrack = -1; - } - - return OK; -} - -bool NuPlayer::CCDecoder::isSelected() const { - return mSelectedTrack >= 0 && mSelectedTrack < (int32_t) getTrackCount(); -} - -bool NuPlayer::CCDecoder::isTrackValid(size_t index) const { - return index < getTrackCount(); -} - -int32_t NuPlayer::CCDecoder::getTrackIndex(size_t channel) const { - if (channel < sizeof(mTrackIndices)/sizeof(mTrackIndices[0])) { - return mTrackIndices[channel]; - } - return -1; -} - -// returns true if a new CC track is found -bool NuPlayer::CCDecoder::extractFromSEI(const sp &accessUnit) { - int64_t timeUs; - CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); - - sp sei; - if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) { - return false; - } - - bool trackAdded = false; - - NALBitReader br(sei->data() + 1, sei->size() - 1); - // sei_message() - while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message() - uint32_t payload_type = 0; - size_t payload_size = 0; - uint8_t last_byte; - - do { - last_byte = br.getBits(8); - payload_type += last_byte; - } while (last_byte == 0xFF); - - do { - last_byte = br.getBits(8); - payload_size += last_byte; - } while (last_byte == 0xFF); - - // sei_payload() - if (payload_type == 4) { - // user_data_registered_itu_t_t35() - - // ATSC A/72: 6.4.2 - uint8_t itu_t_t35_country_code = br.getBits(8); - uint16_t itu_t_t35_provider_code = br.getBits(16); - uint32_t user_identifier = br.getBits(32); - uint8_t user_data_type_code = br.getBits(8); - - payload_size -= 1 + 2 + 4 + 1; - - if (itu_t_t35_country_code == 0xB5 - && itu_t_t35_provider_code == 0x0031 - && user_identifier == 'GA94' - && user_data_type_code == 0x3) { - // MPEG_cc_data() - // ATSC A/53 Part 4: 6.2.3.1 - br.skipBits(1); //process_em_data_flag - bool process_cc_data_flag = br.getBits(1); - br.skipBits(1); //additional_data_flag - size_t cc_count = br.getBits(5); - br.skipBits(8); // em_data; - payload_size -= 2; - - if (process_cc_data_flag) { - AString out; - - sp ccBuf = new ABuffer(cc_count * sizeof(CCData)); - ccBuf->setRange(0, 0); - - for (size_t i = 0; i < cc_count; i++) { - uint8_t marker = br.getBits(5); - CHECK_EQ(marker, 0x1f); - - bool cc_valid = br.getBits(1); - uint8_t cc_type = br.getBits(2); - // remove odd parity bit - uint8_t cc_data_1 = br.getBits(8) & 0x7f; - uint8_t cc_data_2 = br.getBits(8) & 0x7f; - - if (cc_valid - && (cc_type == 0 || cc_type == 1)) { - CCData cc(cc_type, cc_data_1, cc_data_2); - if (!isNullPad(&cc)) { - size_t channel; - if (cc.getChannel(&channel) && getTrackIndex(channel) < 0) { - mTrackIndices[channel] = mFoundChannels.size(); - mFoundChannels.push_back(channel); - trackAdded = true; - } - memcpy(ccBuf->data() + ccBuf->size(), - (void *)&cc, sizeof(cc)); - ccBuf->setRange(0, ccBuf->size() + sizeof(CCData)); - } - } - } - payload_size -= cc_count * 3; - - mCCMap.add(timeUs, ccBuf); - break; - } - } else { - ALOGV("Malformed SEI payload type 4"); - } - } else { - ALOGV("Unsupported SEI payload type %d", payload_type); - } - - // skipping remaining bits of this payload - br.skipBits(payload_size * 8); - } - - return trackAdded; -} - -sp NuPlayer::CCDecoder::filterCCBuf( - const sp &ccBuf, size_t index) { - sp filteredCCBuf = new ABuffer(ccBuf->size()); - filteredCCBuf->setRange(0, 0); - - size_t cc_count = ccBuf->size() / sizeof(CCData); - const CCData* cc_data = (const CCData*)ccBuf->data(); - for (size_t i = 0; i < cc_count; ++i) { - size_t channel; - if (cc_data[i].getChannel(&channel)) { - mCurrentChannel = channel; - } - if (mCurrentChannel == mFoundChannels[index]) { - memcpy(filteredCCBuf->data() + filteredCCBuf->size(), - (void *)&cc_data[i], sizeof(CCData)); - filteredCCBuf->setRange(0, filteredCCBuf->size() + sizeof(CCData)); - } - } - - return filteredCCBuf; -} - -void NuPlayer::CCDecoder::decode(const sp &accessUnit) { - if (extractFromSEI(accessUnit)) { - ALOGI("Found CEA-608 track"); - sp msg = mNotify->dup(); - msg->setInt32("what", kWhatTrackAdded); - msg->post(); - } - // TODO: extract CC from other sources -} - -void NuPlayer::CCDecoder::display(int64_t timeUs) { - if (!isTrackValid(mSelectedTrack)) { - ALOGE("Could not find current track(index=%d)", mSelectedTrack); - return; - } - - ssize_t index = mCCMap.indexOfKey(timeUs); - if (index < 0) { - ALOGV("cc for timestamp %" PRId64 " not found", timeUs); +void NuPlayer::Decoder::rememberCodecSpecificData(const sp &format) { + if (format == NULL) { return; } - - sp ccBuf = filterCCBuf(mCCMap.valueAt(index), mSelectedTrack); - - if (ccBuf->size() > 0) { -#if 0 - dumpBytePair(ccBuf); -#endif - - ccBuf->meta()->setInt32("trackIndex", mSelectedTrack); - ccBuf->meta()->setInt64("timeUs", timeUs); - ccBuf->meta()->setInt64("durationUs", 0ll); - - sp msg = mNotify->dup(); - msg->setInt32("what", kWhatClosedCaptionData); - msg->setBuffer("buffer", ccBuf); - msg->post(); + mCSDsForCurrentFormat.clear(); + for (int32_t i = 0; ; ++i) { + AString tag = "csd-"; + tag.append(i); + sp buffer; + if (!format->findBuffer(tag.c_str(), &buffer)) { + break; + } + mCSDsForCurrentFormat.push(buffer); } - - // remove all entries before timeUs - mCCMap.removeItemsAt(0, index + 1); -} - -void NuPlayer::CCDecoder::flush() { - mCCMap.clear(); } } // namespace android diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h index c6ceb4e..07401b0 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010 The Android Open Source Project + * Copyright 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,78 +15,55 @@ */ #ifndef NUPLAYER_DECODER_H_ - #define NUPLAYER_DECODER_H_ #include "NuPlayer.h" -#include +#include "NuPlayerDecoderBase.h" namespace android { -struct ABuffer; -struct MediaCodec; -struct MediaBuffer; - -struct NuPlayer::Decoder : public AHandler { +struct NuPlayer::Decoder : public DecoderBase { Decoder(const sp ¬ify, const sp &source, const sp &renderer = NULL, - const sp &nativeWindow = NULL); - - virtual void configure(const sp &format); - virtual void init(); - - virtual void setRenderer(const sp &renderer); + const sp &nativeWindow = NULL, + const sp &ccDecoder = NULL); - status_t getInputBuffers(Vector > *dstBuffers) const; - virtual void signalFlush(const sp &format = NULL); - virtual void signalUpdateFormat(const sp &format); - virtual void signalResume(); - virtual void initiateShutdown(); - - virtual bool supportsSeamlessFormatChange(const sp &to) const; - - enum { - kWhatFillThisBuffer = 'flTB', - kWhatRenderBufferTime = 'rnBT', - kWhatVideoSizeChanged = 'viSC', - kWhatFlushCompleted = 'flsC', - kWhatShutdownCompleted = 'shDC', - kWhatEOS = 'eos ', - kWhatError = 'err ', - }; + virtual void getStats( + int64_t *mNumFramesTotal, + int64_t *mNumFramesDropped) const; protected: - virtual ~Decoder(); virtual void onMessageReceived(const sp &msg); + virtual void onConfigure(const sp &format); + virtual void onSetRenderer(const sp &renderer); + virtual void onGetInputBuffers(Vector > *dstBuffers); + virtual void onResume(); + virtual void onFlush(bool notifyComplete); + virtual void onShutdown(bool notifyComplete); + virtual void doRequestBuffers(); + +private: enum { - kWhatCodecNotify = 'cdcN', - kWhatConfigure = 'conf', - kWhatSetRenderer = 'setR', - kWhatGetInputBuffers = 'gInB', - kWhatInputBufferFilled = 'inpF', - kWhatRenderBuffer = 'rndr', - kWhatFlush = 'flus', - kWhatShutdown = 'shuD', - kWhatUpdateFormat = 'uFmt', + kWhatCodecNotify = 'cdcN', + kWhatRenderBuffer = 'rndr', }; -private: sp mNotify; sp mNativeWindow; sp mSource; sp mRenderer; + sp mCCDecoder; sp mInputFormat; sp mOutputFormat; sp mCodec; sp mCodecLooper; - sp mDecoderLooper; List > mPendingInputMessages; @@ -96,8 +73,19 @@ private: Vector > mCSDsToSubmit; Vector mInputBufferIsDequeued; Vector mMediaBuffers; + Vector mDequeuedInputBuffers; int64_t mSkipRenderingUntilMediaTimeUs; + int64_t mNumFramesTotal; + int64_t mNumFramesDropped; + bool mIsAudio; + bool mIsVideoAVC; + bool mIsSecure; + bool mFormatChangePending; + + int32_t mBufferGeneration; + bool mPaused; + AString mComponentName; void handleError(int32_t err); bool handleAnInputBuffer(); @@ -107,56 +95,17 @@ private: void requestCodecNotification(); bool isStaleReply(const sp &msg); - void onConfigure(const sp &format); - void onFlush(); - void onResume(); - bool onInputBufferFilled(const sp &msg); + status_t fetchInputData(sp &reply); + bool onInputBufferFetched(const sp &msg); void onRenderBuffer(const sp &msg); - void onShutdown(); - - int32_t mBufferGeneration; - bool mPaused; - AString mComponentName; + bool supportsSeamlessFormatChange(const sp &to) const; bool supportsSeamlessAudioFormatChange(const sp &targetFormat) const; void rememberCodecSpecificData(const sp &format); - bool isVideo(); DISALLOW_EVIL_CONSTRUCTORS(Decoder); }; -struct NuPlayer::CCDecoder : public RefBase { - enum { - kWhatClosedCaptionData, - kWhatTrackAdded, - }; - - CCDecoder(const sp ¬ify); - - size_t getTrackCount() const; - sp getTrackInfo(size_t index) const; - status_t selectTrack(size_t index, bool select); - bool isSelected() const; - void decode(const sp &accessUnit); - void display(int64_t timeUs); - void flush(); - -private: - sp mNotify; - KeyedVector > mCCMap; - size_t mCurrentChannel; - int32_t mSelectedTrack; - int32_t mTrackIndices[4]; - Vector mFoundChannels; - - bool isTrackValid(size_t index) const; - int32_t getTrackIndex(size_t channel) const; - bool extractFromSEI(const sp &accessUnit); - sp filterCCBuf(const sp &ccBuf, size_t index); - - DISALLOW_EVIL_CONSTRUCTORS(CCDecoder); -}; - } // namespace android #endif // NUPLAYER_DECODER_H_ diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.cpp new file mode 100644 index 0000000..6941f77 --- /dev/null +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.cpp @@ -0,0 +1,179 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "NuPlayerDecoderBase" +#include +#include + +#include "NuPlayerDecoderBase.h" + +#include "NuPlayerRenderer.h" + +#include +#include + +namespace android { + +NuPlayer::DecoderBase::DecoderBase() + : mRequestInputBuffersPending(false) { + // Every decoder has its own looper because MediaCodec operations + // are blocking, but NuPlayer needs asynchronous operations. + mDecoderLooper = new ALooper; + mDecoderLooper->setName("NPDecoder"); + mDecoderLooper->start(false, false, ANDROID_PRIORITY_AUDIO); +} + +NuPlayer::DecoderBase::~DecoderBase() { + mDecoderLooper->unregisterHandler(id()); + mDecoderLooper->stop(); +} + +static +status_t PostAndAwaitResponse( + const sp &msg, sp *response) { + status_t err = msg->postAndAwaitResponse(response); + + if (err != OK) { + return err; + } + + if (!(*response)->findInt32("err", &err)) { + err = OK; + } + + return err; +} + +void NuPlayer::DecoderBase::configure(const sp &format) { + sp msg = new AMessage(kWhatConfigure, id()); + msg->setMessage("format", format); + msg->post(); +} + +void NuPlayer::DecoderBase::init() { + mDecoderLooper->registerHandler(this); +} + +void NuPlayer::DecoderBase::setRenderer(const sp &renderer) { + sp msg = new AMessage(kWhatSetRenderer, id()); + msg->setObject("renderer", renderer); + msg->post(); +} + +status_t NuPlayer::DecoderBase::getInputBuffers(Vector > *buffers) const { + sp msg = new AMessage(kWhatGetInputBuffers, id()); + msg->setPointer("buffers", buffers); + + sp response; + return PostAndAwaitResponse(msg, &response); +} + +void NuPlayer::DecoderBase::signalFlush() { + (new AMessage(kWhatFlush, id()))->post(); +} + +void NuPlayer::DecoderBase::signalResume() { + (new AMessage(kWhatResume, id()))->post(); +} + +void NuPlayer::DecoderBase::initiateShutdown() { + (new AMessage(kWhatShutdown, id()))->post(); +} + +void NuPlayer::DecoderBase::onRequestInputBuffers() { + if (mRequestInputBuffersPending) { + return; + } + + doRequestBuffers(); +} + +void NuPlayer::DecoderBase::scheduleRequestBuffers() { + if (mRequestInputBuffersPending) { + return; + } + mRequestInputBuffersPending = true; + sp msg = new AMessage(kWhatRequestInputBuffers, id()); + msg->post(10 * 1000ll); +} + +void NuPlayer::DecoderBase::onMessageReceived(const sp &msg) { + + switch (msg->what()) { + case kWhatConfigure: + { + sp format; + CHECK(msg->findMessage("format", &format)); + onConfigure(format); + break; + } + + case kWhatSetRenderer: + { + sp obj; + CHECK(msg->findObject("renderer", &obj)); + onSetRenderer(static_cast(obj.get())); + break; + } + + case kWhatGetInputBuffers: + { + uint32_t replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + + Vector > *dstBuffers; + CHECK(msg->findPointer("buffers", (void **)&dstBuffers)); + + onGetInputBuffers(dstBuffers); + + (new AMessage)->postReply(replyID); + break; + } + + case kWhatRequestInputBuffers: + { + mRequestInputBuffersPending = false; + onRequestInputBuffers(); + break; + } + + case kWhatFlush: + { + onFlush(true); + break; + } + + case kWhatResume: + { + onResume(); + break; + } + + case kWhatShutdown: + { + onShutdown(true); + break; + } + + default: + TRESPASS(); + break; + } +} + +} // namespace android + diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.h new file mode 100644 index 0000000..1b24c4f --- /dev/null +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.h @@ -0,0 +1,92 @@ +/* + * Copyright (C) 2010 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef NUPLAYER_DECODER_BASE_H_ + +#define NUPLAYER_DECODER_BASE_H_ + +#include "NuPlayer.h" + +#include + +namespace android { + +struct ABuffer; +struct MediaCodec; +struct MediaBuffer; + +struct NuPlayer::DecoderBase : public AHandler { + DecoderBase(); + + void configure(const sp &format); + void init(); + + void setRenderer(const sp &renderer); + + status_t getInputBuffers(Vector > *dstBuffers) const; + void signalFlush(); + void signalResume(); + void initiateShutdown(); + + virtual void getStats( + int64_t *mNumFramesTotal, + int64_t *mNumFramesDropped) const = 0; + + enum { + kWhatInputDiscontinuity = 'inDi', + kWhatVideoSizeChanged = 'viSC', + kWhatFlushCompleted = 'flsC', + kWhatShutdownCompleted = 'shDC', + kWhatEOS = 'eos ', + kWhatError = 'err ', + }; + +protected: + + virtual ~DecoderBase(); + + virtual void onMessageReceived(const sp &msg); + + virtual void onConfigure(const sp &format) = 0; + virtual void onSetRenderer(const sp &renderer) = 0; + virtual void onGetInputBuffers(Vector > *dstBuffers) = 0; + virtual void onResume() = 0; + virtual void onFlush(bool notifyComplete) = 0; + virtual void onShutdown(bool notifyComplete) = 0; + + void onRequestInputBuffers(); + void scheduleRequestBuffers(); + virtual void doRequestBuffers() = 0; + +private: + enum { + kWhatConfigure = 'conf', + kWhatSetRenderer = 'setR', + kWhatGetInputBuffers = 'gInB', + kWhatRequestInputBuffers = 'reqB', + kWhatFlush = 'flus', + kWhatShutdown = 'shuD', + }; + + sp mDecoderLooper; + bool mRequestInputBuffersPending; + + DISALLOW_EVIL_CONSTRUCTORS(DecoderBase); +}; + +} // namespace android + +#endif // NUPLAYER_DECODER_BASE_H_ diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp index d2721ed..3b4c0a7 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp @@ -28,76 +28,51 @@ #include #include #include -#include #include +#include "ATSParser.h" + namespace android { +// TODO optimize buffer size for power consumption +// The offload read buffer size is 32 KB but 24 KB uses less power. +static const size_t kAggregateBufferSizeBytes = 24 * 1024; static const size_t kMaxCachedBytes = 200000; -// The buffers will contain a bit less than kAggregateBufferSizeBytes. -// So we can start off with just enough buffers to keep the cache full. -static const size_t kMaxPendingBuffers = 1 + (kMaxCachedBytes / NuPlayer::kAggregateBufferSizeBytes); NuPlayer::DecoderPassThrough::DecoderPassThrough( const sp ¬ify, const sp &source, const sp &renderer) - : Decoder(notify, source), - mNotify(notify), + : mNotify(notify), mSource(source), mRenderer(renderer), mSkipRenderingUntilMediaTimeUs(-1ll), mBufferGeneration(0), mReachedEOS(true), - mPendingBuffersToFill(0), + mPendingAudioErr(OK), mPendingBuffersToDrain(0), mCachedBytes(0), mComponentName("pass through decoder") { ALOGW_IF(renderer == NULL, "expect a non-NULL renderer"); - mDecoderLooper = new ALooper; - mDecoderLooper->setName("NuPlayerDecoderPassThrough"); - mDecoderLooper->start(false, false, ANDROID_PRIORITY_AUDIO); } NuPlayer::DecoderPassThrough::~DecoderPassThrough() { } -void NuPlayer::DecoderPassThrough::configure(const sp &format) { - sp msg = new AMessage(kWhatConfigure, id()); - msg->setMessage("format", format); - msg->post(); -} - -void NuPlayer::DecoderPassThrough::init() { - mDecoderLooper->registerHandler(this); -} - -void NuPlayer::DecoderPassThrough::signalFlush() { - (new AMessage(kWhatFlush, id()))->post(); -} - -void NuPlayer::DecoderPassThrough::signalResume() { - (new AMessage(kWhatResume, id()))->post(); -} - -void NuPlayer::DecoderPassThrough::initiateShutdown() { - (new AMessage(kWhatShutdown, id()))->post(); -} - -bool NuPlayer::DecoderPassThrough::supportsSeamlessFormatChange( - const sp & /* targetFormat */) const { - return true; +void NuPlayer::DecoderPassThrough::getStats( + int64_t *numFramesTotal, int64_t *numFramesDropped) const { + *numFramesTotal = 0; + *numFramesDropped = 0; } void NuPlayer::DecoderPassThrough::onConfigure(const sp &format) { ALOGV("[%s] onConfigure", mComponentName.c_str()); mCachedBytes = 0; - mPendingBuffersToFill = 0; mPendingBuffersToDrain = 0; mReachedEOS = false; ++mBufferGeneration; - requestMaxBuffers(); + onRequestInputBuffers(); uint32_t flags; int64_t durationUs; @@ -112,47 +87,213 @@ void NuPlayer::DecoderPassThrough::onConfigure(const sp &format) { format, true /* offloadOnly */, false /* hasVideo */, flags); } +void NuPlayer::DecoderPassThrough::onSetRenderer( + const sp &renderer) { + // renderer can't be changed during offloading + ALOGW_IF(renderer != mRenderer, + "ignoring request to change renderer"); +} + +void NuPlayer::DecoderPassThrough::onGetInputBuffers( + Vector > * /* dstBuffers */) { + ALOGE("onGetInputBuffers() called unexpectedly"); +} + bool NuPlayer::DecoderPassThrough::isStaleReply(const sp &msg) { int32_t generation; CHECK(msg->findInt32("generation", &generation)); return generation != mBufferGeneration; } -bool NuPlayer::DecoderPassThrough::requestABuffer() { - if (mCachedBytes >= kMaxCachedBytes) { - ALOGV("[%s] mCachedBytes = %zu", - mComponentName.c_str(), mCachedBytes); - return false; +bool NuPlayer::DecoderPassThrough::isCacheFullOrEOS() const { + ALOGV("[%s] mCachedBytes = %zu, mReachedEOS = %d", + mComponentName.c_str(), mCachedBytes, mReachedEOS); + + return mCachedBytes >= kMaxCachedBytes || mReachedEOS; +} + +void NuPlayer::DecoderPassThrough::doRequestBuffers() { + status_t err = OK; + while (!isCacheFullOrEOS()) { + sp msg = new AMessage(); + + err = fetchInputData(msg); + if (err != OK) { + break; + } + + onInputBufferFetched(msg); } - if (mReachedEOS) { - ALOGV("[%s] reached EOS", mComponentName.c_str()); - return false; + + if (err == -EWOULDBLOCK + && mSource->feedMoreTSData() == OK) { + scheduleRequestBuffers(); } +} - sp reply = new AMessage(kWhatInputBufferFilled, id()); - reply->setInt32("generation", mBufferGeneration); +status_t NuPlayer::DecoderPassThrough::dequeueAccessUnit(sp *accessUnit) { + status_t err; + + // Did we save an accessUnit earlier because of a discontinuity? + if (mPendingAudioAccessUnit != NULL) { + *accessUnit = mPendingAudioAccessUnit; + mPendingAudioAccessUnit.clear(); + err = mPendingAudioErr; + ALOGV("feedDecoderInputData() use mPendingAudioAccessUnit"); + } else { + err = mSource->dequeueAccessUnit(true /* audio */, accessUnit); + } + + if (err == INFO_DISCONTINUITY || err == ERROR_END_OF_STREAM) { + if (mAggregateBuffer != NULL) { + // We already have some data so save this for later. + mPendingAudioErr = err; + mPendingAudioAccessUnit = *accessUnit; + (*accessUnit).clear(); + ALOGD("return aggregated buffer and save err(=%d) for later", err); + err = OK; + } + } + + return err; +} + +sp NuPlayer::DecoderPassThrough::aggregateBuffer( + const sp &accessUnit) { + sp aggregate; + + if (accessUnit == NULL) { + // accessUnit is saved to mPendingAudioAccessUnit + // return current mAggregateBuffer + aggregate = mAggregateBuffer; + mAggregateBuffer.clear(); + return aggregate; + } + + size_t smallSize = accessUnit->size(); + if ((mAggregateBuffer == NULL) + // Don't bother if only room for a few small buffers. + && (smallSize < (kAggregateBufferSizeBytes / 3))) { + // Create a larger buffer for combining smaller buffers from the extractor. + mAggregateBuffer = new ABuffer(kAggregateBufferSizeBytes); + mAggregateBuffer->setRange(0, 0); // start empty + } + + if (mAggregateBuffer != NULL) { + int64_t timeUs; + int64_t dummy; + bool smallTimestampValid = accessUnit->meta()->findInt64("timeUs", &timeUs); + bool bigTimestampValid = mAggregateBuffer->meta()->findInt64("timeUs", &dummy); + // Will the smaller buffer fit? + size_t bigSize = mAggregateBuffer->size(); + size_t roomLeft = mAggregateBuffer->capacity() - bigSize; + // Should we save this small buffer for the next big buffer? + // If the first small buffer did not have a timestamp then save + // any buffer that does have a timestamp until the next big buffer. + if ((smallSize > roomLeft) + || (!bigTimestampValid && (bigSize > 0) && smallTimestampValid)) { + mPendingAudioErr = OK; + mPendingAudioAccessUnit = accessUnit; + aggregate = mAggregateBuffer; + mAggregateBuffer.clear(); + } else { + // Grab time from first small buffer if available. + if ((bigSize == 0) && smallTimestampValid) { + mAggregateBuffer->meta()->setInt64("timeUs", timeUs); + } + // Append small buffer to the bigger buffer. + memcpy(mAggregateBuffer->base() + bigSize, accessUnit->data(), smallSize); + bigSize += smallSize; + mAggregateBuffer->setRange(0, bigSize); + + ALOGV("feedDecoderInputData() smallSize = %zu, bigSize = %zu, capacity = %zu", + smallSize, bigSize, mAggregateBuffer->capacity()); + } + } else { + // decided not to aggregate + aggregate = accessUnit; + } + + return aggregate; +} + +status_t NuPlayer::DecoderPassThrough::fetchInputData(sp &reply) { + sp accessUnit; + + do { + status_t err = dequeueAccessUnit(&accessUnit); + + if (err == -EWOULDBLOCK) { + return err; + } else if (err != OK) { + if (err == INFO_DISCONTINUITY) { + int32_t type; + CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); + + bool formatChange = + (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT) != 0; + + bool timeChange = + (type & ATSParser::DISCONTINUITY_TIME) != 0; + + ALOGI("audio discontinuity (formatChange=%d, time=%d)", + formatChange, timeChange); + + if (formatChange || timeChange) { + sp msg = mNotify->dup(); + msg->setInt32("what", kWhatInputDiscontinuity); + // will perform seamless format change, + // only notify NuPlayer to scan sources + msg->setInt32("formatChange", false); + msg->post(); + } + + if (timeChange) { + onFlush(false /* notifyComplete */); + err = OK; + } else if (formatChange) { + // do seamless format change + err = OK; + } else { + // This stream is unaffected by the discontinuity + return -EWOULDBLOCK; + } + } + + reply->setInt32("err", err); + return OK; + } - sp notify = mNotify->dup(); - notify->setInt32("what", kWhatFillThisBuffer); - notify->setMessage("reply", reply); - notify->post(); - mPendingBuffersToFill++; - ALOGV("requestABuffer: #ToFill = %zu, #ToDrain = %zu", mPendingBuffersToFill, - mPendingBuffersToDrain); + accessUnit = aggregateBuffer(accessUnit); + } while (accessUnit == NULL); - return true; +#if 0 + int64_t mediaTimeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs)); + ALOGV("feeding audio input buffer at media time %.2f secs", + mediaTimeUs / 1E6); +#endif + + reply->setBuffer("buffer", accessUnit); + + return OK; } -void android::NuPlayer::DecoderPassThrough::onInputBufferFilled( +void NuPlayer::DecoderPassThrough::onInputBufferFetched( const sp &msg) { - --mPendingBuffersToFill; if (mReachedEOS) { return; } sp buffer; - msg->findBuffer("buffer", &buffer); + bool hasBuffer = msg->findBuffer("buffer", &buffer); if (buffer == NULL) { + int32_t streamErr = ERROR_END_OF_STREAM; + CHECK(msg->findInt32("err", &streamErr) || !hasBuffer); + if (streamErr == OK) { + return; + } + mReachedEOS = true; if (mRenderer != NULL) { mRenderer->queueEOS(true /* audio */, ERROR_END_OF_STREAM); @@ -201,50 +342,52 @@ void android::NuPlayer::DecoderPassThrough::onInputBufferFilled( mRenderer->queueBuffer(true /* audio */, buffer, reply); ++mPendingBuffersToDrain; - ALOGV("onInputBufferFilled: #ToFill = %zu, #ToDrain = %zu, cachedBytes = %zu", - mPendingBuffersToFill, mPendingBuffersToDrain, mCachedBytes); + ALOGV("onInputBufferFilled: #ToDrain = %zu, cachedBytes = %zu", + mPendingBuffersToDrain, mCachedBytes); } void NuPlayer::DecoderPassThrough::onBufferConsumed(int32_t size) { --mPendingBuffersToDrain; mCachedBytes -= size; - ALOGV("onBufferConsumed: #ToFill = %zu, #ToDrain = %zu, cachedBytes = %zu", - mPendingBuffersToFill, mPendingBuffersToDrain, mCachedBytes); - requestABuffer(); + ALOGV("onBufferConsumed: #ToDrain = %zu, cachedBytes = %zu", + mPendingBuffersToDrain, mCachedBytes); + onRequestInputBuffers(); +} + +void NuPlayer::DecoderPassThrough::onResume() { + onRequestInputBuffers(); } -void NuPlayer::DecoderPassThrough::onFlush() { +void NuPlayer::DecoderPassThrough::onFlush(bool notifyComplete) { ++mBufferGeneration; mSkipRenderingUntilMediaTimeUs = -1; if (mRenderer != NULL) { - mRenderer->flush(true /* audio */); + mRenderer->flush(true /* audio */, notifyComplete); + mRenderer->signalTimeDiscontinuity(); + } + + if (notifyComplete) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatFlushCompleted); + notify->post(); } - sp notify = mNotify->dup(); - notify->setInt32("what", kWhatFlushCompleted); - notify->post(); - mPendingBuffersToFill = 0; mPendingBuffersToDrain = 0; mCachedBytes = 0; mReachedEOS = false; } -void NuPlayer::DecoderPassThrough::requestMaxBuffers() { - for (size_t i = 0; i < kMaxPendingBuffers; i++) { - if (!requestABuffer()) { - break; - } - } -} - -void NuPlayer::DecoderPassThrough::onShutdown() { +void NuPlayer::DecoderPassThrough::onShutdown(bool notifyComplete) { ++mBufferGeneration; mSkipRenderingUntilMediaTimeUs = -1; - sp notify = mNotify->dup(); - notify->setInt32("what", kWhatShutdownCompleted); - notify->post(); + if (notifyComplete) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatShutdownCompleted); + notify->post(); + } + mReachedEOS = true; } @@ -253,31 +396,6 @@ void NuPlayer::DecoderPassThrough::onMessageReceived(const sp &msg) { msg->debugString().c_str()); switch (msg->what()) { - case kWhatConfigure: - { - sp format; - CHECK(msg->findMessage("format", &format)); - onConfigure(format); - break; - } - - case kWhatRequestABuffer: - { - if (!isStaleReply(msg)) { - requestABuffer(); - } - - break; - } - - case kWhatInputBufferFilled: - { - if (!isStaleReply(msg)) { - onInputBufferFilled(msg); - } - break; - } - case kWhatBufferConsumed: { if (!isStaleReply(msg)) { @@ -288,26 +406,8 @@ void NuPlayer::DecoderPassThrough::onMessageReceived(const sp &msg) { break; } - case kWhatFlush: - { - onFlush(); - break; - } - - case kWhatResume: - { - requestMaxBuffers(); - break; - } - - case kWhatShutdown: - { - onShutdown(); - break; - } - default: - TRESPASS(); + DecoderBase::onMessageReceived(msg); break; } } diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h index 7742d30..3fe32b6 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h @@ -20,23 +20,18 @@ #include "NuPlayer.h" -#include "NuPlayerDecoder.h" +#include "NuPlayerDecoderBase.h" namespace android { -struct NuPlayer::DecoderPassThrough : public Decoder { +struct NuPlayer::DecoderPassThrough : public DecoderBase { DecoderPassThrough(const sp ¬ify, const sp &source, const sp &renderer); - virtual void configure(const sp &format); - virtual void init(); - - virtual void signalFlush(); - virtual void signalResume(); - virtual void initiateShutdown(); - - bool supportsSeamlessFormatChange(const sp &to) const; + virtual void getStats( + int64_t *mNumFramesTotal, + int64_t *mNumFramesDropped) const; protected: @@ -44,42 +39,49 @@ protected: virtual void onMessageReceived(const sp &msg); + virtual void onConfigure(const sp &format); + virtual void onSetRenderer(const sp &renderer); + virtual void onGetInputBuffers(Vector > *dstBuffers); + virtual void onResume(); + virtual void onFlush(bool notifyComplete); + virtual void onShutdown(bool notifyComplete); + virtual void doRequestBuffers(); + private: enum { - kWhatRequestABuffer = 'reqB', kWhatBufferConsumed = 'bufC', }; sp mNotify; - sp mDecoderLooper; - sp mSource; sp mRenderer; - - /** Returns true if a buffer was requested. - * Returns false if at EOS or cache already full. - */ - bool requestABuffer(); - bool isStaleReply(const sp &msg); - - void onConfigure(const sp &format); - void onFlush(); - void onInputBufferFilled(const sp &msg); - void onBufferConsumed(int32_t size); - void requestMaxBuffers(); - void onShutdown(); - int64_t mSkipRenderingUntilMediaTimeUs; int32_t mBufferGeneration; bool mReachedEOS; - // TODO mPendingBuffersToFill and mPendingBuffersToDrain are only for - // debugging. They can be removed when the power investigation is done. - size_t mPendingBuffersToFill; + + // Used by feedDecoderInputData to aggregate small buffers into + // one large buffer. + sp mPendingAudioAccessUnit; + status_t mPendingAudioErr; + sp mAggregateBuffer; + + // mPendingBuffersToDrain are only for debugging. It can be removed + // when the power investigation is done. size_t mPendingBuffersToDrain; size_t mCachedBytes; AString mComponentName; + bool isStaleReply(const sp &msg); + bool isCacheFullOrEOS() const; + + status_t dequeueAccessUnit(sp *accessUnit); + sp aggregateBuffer(const sp &accessUnit); + status_t fetchInputData(sp &reply); + + void onInputBufferFetched(const sp &msg); + void onBufferConsumed(int32_t size); + DISALLOW_EVIL_CONSTRUCTORS(DecoderPassThrough); }; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp index 42288a3..920554b 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp @@ -64,6 +64,8 @@ NuPlayer::Renderer::Renderer( mPauseStartedTimeRealUs(-1), mFlushingAudio(false), mFlushingVideo(false), + mNotifyCompleteAudio(false), + mNotifyCompleteVideo(false), mSyncQueues(false), mPaused(false), mVideoSampleReceived(false), @@ -105,15 +107,17 @@ void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) { msg->post(); } -void NuPlayer::Renderer::flush(bool audio) { +void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) { { Mutex::Autolock autoLock(mFlushLock); if (audio) { + mNotifyCompleteAudio |= notifyComplete; if (mFlushingAudio) { return; } mFlushingAudio = true; } else { + mNotifyCompleteVideo |= notifyComplete; if (mFlushingVideo) { return; } @@ -1015,15 +1019,19 @@ void NuPlayer::Renderer::onQueueEOS(const sp &msg) { } void NuPlayer::Renderer::onFlush(const sp &msg) { - int32_t audio; + int32_t audio, notifyComplete; CHECK(msg->findInt32("audio", &audio)); { Mutex::Autolock autoLock(mFlushLock); if (audio) { mFlushingAudio = false; + notifyComplete = mNotifyCompleteAudio; + mNotifyCompleteAudio = false; } else { mFlushingVideo = false; + notifyComplete = mNotifyCompleteVideo; + mNotifyCompleteVideo = false; } } @@ -1076,7 +1084,10 @@ void NuPlayer::Renderer::onFlush(const sp &msg) { } mVideoSampleReceived = false; - notifyFlushComplete(audio); + + if (notifyComplete) { + notifyFlushComplete(audio); + } } void NuPlayer::Renderer::flushQueue(List *queue) { diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h index 985ec49..14ae924 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h @@ -46,7 +46,7 @@ struct NuPlayer::Renderer : public AHandler { void queueEOS(bool audio, status_t finalResult); - void flush(bool audio); + void flush(bool audio, bool notifyComplete); void signalTimeDiscontinuity(); @@ -162,6 +162,8 @@ private: Mutex mFlushLock; // protects the following 2 member vars. bool mFlushingAudio; bool mFlushingVideo; + bool mNotifyCompleteAudio; + bool mNotifyCompleteVideo; bool mSyncQueues; -- cgit v1.1