diff options
Diffstat (limited to 'media')
27 files changed, 1950 insertions, 377 deletions
diff --git a/media/libmedia/AudioRecord.cpp b/media/libmedia/AudioRecord.cpp index 666fafa..ccbc5a3 100644 --- a/media/libmedia/AudioRecord.cpp +++ b/media/libmedia/AudioRecord.cpp @@ -545,13 +545,13 @@ status_t AudioRecord::obtainBuffer(Buffer* audioBuffer, int32_t waitCount) } const struct timespec *requested; + struct timespec timeout; if (waitCount == -1) { requested = &ClientProxy::kForever; } else if (waitCount == 0) { requested = &ClientProxy::kNonBlocking; } else if (waitCount > 0) { long long ms = WAIT_PERIOD_MS * (long long) waitCount; - struct timespec timeout; timeout.tv_sec = ms / 1000; timeout.tv_nsec = (int) (ms % 1000) * 1000000; requested = &timeout; diff --git a/media/libmedia/AudioTrack.cpp b/media/libmedia/AudioTrack.cpp index a9d6993..3f3a88c 100644 --- a/media/libmedia/AudioTrack.cpp +++ b/media/libmedia/AudioTrack.cpp @@ -85,7 +85,8 @@ AudioTrack::AudioTrack() : mStatus(NO_INIT), mIsTimed(false), mPreviousPriority(ANDROID_PRIORITY_NORMAL), - mPreviousSchedulingGroup(SP_DEFAULT) + mPreviousSchedulingGroup(SP_DEFAULT), + mPausedPosition(0) { } @@ -106,7 +107,8 @@ AudioTrack::AudioTrack( : mStatus(NO_INIT), mIsTimed(false), mPreviousPriority(ANDROID_PRIORITY_NORMAL), - mPreviousSchedulingGroup(SP_DEFAULT) + mPreviousSchedulingGroup(SP_DEFAULT), + mPausedPosition(0) { mStatus = set(streamType, sampleRate, format, channelMask, frameCount, flags, cbf, user, notificationFrames, @@ -131,7 +133,8 @@ AudioTrack::AudioTrack( : mStatus(NO_INIT), mIsTimed(false), mPreviousPriority(ANDROID_PRIORITY_NORMAL), - mPreviousSchedulingGroup(SP_DEFAULT) + mPreviousSchedulingGroup(SP_DEFAULT), + mPausedPosition(0) { mStatus = set(streamType, sampleRate, format, channelMask, 0 /*frameCount*/, flags, cbf, user, notificationFrames, @@ -529,6 +532,16 @@ void AudioTrack::pause() } mProxy->interrupt(); mAudioTrack->pause(); + + if (isOffloaded()) { + if (mOutput != 0) { + uint32_t halFrames; + // OffloadThread sends HAL pause in its threadLoop.. time saved + // here can be slightly off + AudioSystem::getRenderPosition(mOutput, &halFrames, &mPausedPosition); + ALOGV("AudioTrack::pause for offload, cache current position %u", mPausedPosition); + } + } } status_t AudioTrack::setVolume(float left, float right) @@ -747,6 +760,12 @@ status_t AudioTrack::getPosition(uint32_t *position) const if (isOffloaded()) { uint32_t dspFrames = 0; + if ((mState == STATE_PAUSED) || (mState == STATE_PAUSED_STOPPING)) { + ALOGV("getPosition called in paused state, return cached position %u", mPausedPosition); + *position = mPausedPosition; + return NO_ERROR; + } + if (mOutput != 0) { uint32_t halFrames; AudioSystem::getRenderPosition(mOutput, &halFrames, &dspFrames); @@ -1113,13 +1132,13 @@ status_t AudioTrack::obtainBuffer(Buffer* audioBuffer, int32_t waitCount) } const struct timespec *requested; + struct timespec timeout; if (waitCount == -1) { requested = &ClientProxy::kForever; } else if (waitCount == 0) { requested = &ClientProxy::kNonBlocking; } else if (waitCount > 0) { long long ms = WAIT_PERIOD_MS * (long long) waitCount; - struct timespec timeout; timeout.tv_sec = ms / 1000; timeout.tv_nsec = (int) (ms % 1000) * 1000000; requested = &timeout; @@ -1451,6 +1470,7 @@ nsecs_t AudioTrack::processAudioBuffer(const sp<AudioTrackThread>& thread) } size_t misalignment = mProxy->getMisalignment(); uint32_t sequence = mSequence; + sp<AudioTrackClientProxy> proxy = mProxy; // These fields don't need to be cached, because they are assigned only by set(): // mTransfer, mCbf, mUserData, mFormat, mFrameSize, mFrameSizeAF, mFlags @@ -1459,35 +1479,32 @@ nsecs_t AudioTrack::processAudioBuffer(const sp<AudioTrackThread>& thread) mLock.unlock(); if (waitStreamEnd) { - AutoMutex lock(mLock); - - sp<AudioTrackClientProxy> proxy = mProxy; - sp<IMemory> iMem = mCblkMemory; - struct timespec timeout; timeout.tv_sec = WAIT_STREAM_END_TIMEOUT_SEC; timeout.tv_nsec = 0; - mLock.unlock(); - status_t status = mProxy->waitStreamEndDone(&timeout); - mLock.lock(); + status_t status = proxy->waitStreamEndDone(&timeout); switch (status) { case NO_ERROR: case DEAD_OBJECT: case TIMED_OUT: - mLock.unlock(); mCbf(EVENT_STREAM_END, mUserData, NULL); - mLock.lock(); - if (mState == STATE_STOPPING) { - mState = STATE_STOPPED; - if (status != DEAD_OBJECT) { - return NS_INACTIVE; + { + AutoMutex lock(mLock); + // The previously assigned value of waitStreamEnd is no longer valid, + // since the mutex has been unlocked and either the callback handler + // or another thread could have re-started the AudioTrack during that time. + waitStreamEnd = mState == STATE_STOPPING; + if (waitStreamEnd) { + mState = STATE_STOPPED; } } - return 0; - default: - return 0; + if (waitStreamEnd && status != DEAD_OBJECT) { + return NS_INACTIVE; + } + break; } + return 0; } // perform callbacks while unlocked diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp index 3669a5b..25d55a3 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp @@ -1011,7 +1011,14 @@ status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) { &NuPlayer::performScanSources)); } - flushDecoder(audio, formatChange); + sp<AMessage> newFormat = mSource->getFormat(audio); + sp<Decoder> &decoder = audio ? mAudioDecoder : mVideoDecoder; + if (formatChange && !decoder->supportsSeamlessFormatChange(newFormat)) { + flushDecoder(audio, /* needShutdown = */ true); + } else { + flushDecoder(audio, /* needShutdown = */ false); + err = OK; + } } else { // This stream is unaffected by the discontinuity diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp index 22f699e..2423fd5 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp @@ -67,6 +67,7 @@ void NuPlayer::Decoder::configure(const sp<AMessage> &format) { // queue. bool needDedicatedLooper = !strncasecmp(mime.c_str(), "video/", 6); + mFormat = format; mCodec = new ACodec; if (needDedicatedLooper && mCodecLooper == NULL) { @@ -147,5 +148,65 @@ void NuPlayer::Decoder::initiateShutdown() { } } +bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(const sp<AMessage> &targetFormat) const { + if (targetFormat == NULL) { + return true; + } + + AString mime; + if (!targetFormat->findString("mime", &mime)) { + return false; + } + + if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) { + // field-by-field comparison + const char * keys[] = { "channel-count", "sample-rate", "is-adts" }; + for (unsigned int i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) { + int32_t oldVal, newVal; + if (!mFormat->findInt32(keys[i], &oldVal) || !targetFormat->findInt32(keys[i], &newVal) + || oldVal != newVal) { + return false; + } + } + + sp<ABuffer> oldBuf, newBuf; + if (mFormat->findBuffer("csd-0", &oldBuf) && targetFormat->findBuffer("csd-0", &newBuf)) { + if (oldBuf->size() != newBuf->size()) { + return false; + } + return !memcmp(oldBuf->data(), newBuf->data(), oldBuf->size()); + } + } + return false; +} + +bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetFormat) const { + if (mFormat == NULL) { + return false; + } + + if (targetFormat == NULL) { + return true; + } + + AString oldMime, newMime; + if (!mFormat->findString("mime", &oldMime) + || !targetFormat->findString("mime", &newMime) + || !(oldMime == newMime)) { + return false; + } + + bool audio = !strncasecmp(oldMime.c_str(), "audio/", strlen("audio/")); + bool seamless; + if (audio) { + seamless = supportsSeamlessAudioFormatChange(targetFormat); + } else { + seamless = mCodec != NULL && mCodec->isConfiguredForAdaptivePlayback(); + } + + ALOGV("%s seamless support for %s", seamless ? "yes" : "no", oldMime.c_str()); + return seamless; +} + } // namespace android diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h index a876148..78ea74a 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h @@ -36,6 +36,8 @@ struct NuPlayer::Decoder : public AHandler { void signalResume(); void initiateShutdown(); + bool supportsSeamlessFormatChange(const sp<AMessage> &to) const; + protected: virtual ~Decoder(); @@ -49,6 +51,7 @@ private: sp<AMessage> mNotify; sp<NativeWindowWrapper> mNativeWindow; + sp<AMessage> mFormat; sp<ACodec> mCodec; sp<ALooper> mCodecLooper; @@ -59,6 +62,8 @@ private: void onFillThisBuffer(const sp<AMessage> &msg); + bool supportsSeamlessAudioFormatChange(const sp<AMessage> &targetFormat) const; + DISALLOW_EVIL_CONSTRUCTORS(Decoder); }; diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp index 76a3358..6dcedca 100644 --- a/media/libstagefright/ACodec.cpp +++ b/media/libstagefright/ACodec.cpp @@ -35,7 +35,9 @@ #include <media/hardware/HardwareAPI.h> +#include <OMX_AudioExt.h> #include <OMX_Component.h> +#include <OMX_IndexExt.h> #include "include/avc_utils.h" @@ -363,6 +365,7 @@ ACodec::ACodec() mIsEncoder(false), mUseMetadataOnEncoderOutput(false), mShutdownInProgress(false), + mIsConfiguredForAdaptivePlayback(false), mEncoderDelay(0), mEncoderPadding(0), mChannelMaskPresent(false), @@ -370,7 +373,8 @@ ACodec::ACodec() mDequeueCounter(0), mStoreMetaDataInOutputBuffers(false), mMetaDataBuffersToSubmit(0), - mRepeatFrameDelayUs(-1ll) { + mRepeatFrameDelayUs(-1ll), + mMaxPtsGapUs(-1l) { mUninitializedState = new UninitializedState(this); mLoadedState = new LoadedState(this); mLoadedToIdleState = new LoadedToIdleState(this); @@ -452,6 +456,18 @@ void ACodec::signalRequestIDRFrame() { (new AMessage(kWhatRequestIDRFrame, id()))->post(); } +// *** NOTE: THE FOLLOWING WORKAROUND WILL BE REMOVED *** +// Some codecs may return input buffers before having them processed. +// This causes a halt if we already signaled an EOS on the input +// port. For now keep submitting an output buffer if there was an +// EOS on the input port, but not yet on the output port. +void ACodec::signalSubmitOutputMetaDataBufferIfEOS_workaround() { + if (mPortEOS[kPortIndexInput] && !mPortEOS[kPortIndexOutput] && + mMetaDataBuffersToSubmit > 0) { + (new AMessage(kWhatSubmitOutputMetaDataBufferIfEOS, id()))->post(); + } +} + status_t ACodec::allocateBuffersOnPort(OMX_U32 portIndex) { CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); @@ -965,6 +981,10 @@ status_t ACodec::setComponentRole( "audio_decoder.flac", "audio_encoder.flac" }, { MEDIA_MIMETYPE_AUDIO_MSGSM, "audio_decoder.gsm", "audio_encoder.gsm" }, + { MEDIA_MIMETYPE_VIDEO_MPEG2, + "video_decoder.mpeg2", "video_encoder.mpeg2" }, + { MEDIA_MIMETYPE_AUDIO_AC3, + "audio_decoder.ac3", "audio_encoder.ac3" }, }; static const size_t kNumMimeToRole = @@ -1096,6 +1116,10 @@ status_t ACodec::configureCodec( &mRepeatFrameDelayUs)) { mRepeatFrameDelayUs = -1ll; } + + if (!msg->findInt64("max-pts-gap-to-encoder", &mMaxPtsGapUs)) { + mMaxPtsGapUs = -1l; + } } // Always try to enable dynamic output buffers on native surface @@ -1103,6 +1127,7 @@ status_t ACodec::configureCodec( int32_t haveNativeWindow = msg->findObject("native-window", &obj) && obj != NULL; mStoreMetaDataInOutputBuffers = false; + mIsConfiguredForAdaptivePlayback = false; if (!encoder && video && haveNativeWindow) { err = mOMX->storeMetaDataInBuffers(mNode, kPortIndexOutput, OMX_TRUE); if (err != OK) { @@ -1147,12 +1172,14 @@ status_t ACodec::configureCodec( ALOGW_IF(err != OK, "[%s] prepareForAdaptivePlayback failed w/ err %d", mComponentName.c_str(), err); + mIsConfiguredForAdaptivePlayback = (err == OK); } // allow failure err = OK; } else { ALOGV("[%s] storeMetaDataInBuffers succeeded", mComponentName.c_str()); mStoreMetaDataInOutputBuffers = true; + mIsConfiguredForAdaptivePlayback = true; } int32_t push; @@ -1256,6 +1283,15 @@ status_t ACodec::configureCodec( } else { err = setupRawAudioFormat(kPortIndexInput, sampleRate, numChannels); } + } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AC3)) { + int32_t numChannels; + int32_t sampleRate; + if (!msg->findInt32("channel-count", &numChannels) + || !msg->findInt32("sample-rate", &sampleRate)) { + err = INVALID_OPERATION; + } else { + err = setupAC3Codec(encoder, numChannels, sampleRate); + } } if (err != OK) { @@ -1452,6 +1488,44 @@ status_t ACodec::setupAACCodec( mNode, OMX_IndexParamAudioAac, &profile, sizeof(profile)); } +status_t ACodec::setupAC3Codec( + bool encoder, int32_t numChannels, int32_t sampleRate) { + status_t err = setupRawAudioFormat( + encoder ? kPortIndexInput : kPortIndexOutput, sampleRate, numChannels); + + if (err != OK) { + return err; + } + + if (encoder) { + ALOGW("AC3 encoding is not supported."); + return INVALID_OPERATION; + } + + OMX_AUDIO_PARAM_ANDROID_AC3TYPE def; + InitOMXParams(&def); + def.nPortIndex = kPortIndexInput; + + err = mOMX->getParameter( + mNode, + (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAc3, + &def, + sizeof(def)); + + if (err != OK) { + return err; + } + + def.nChannels = numChannels; + def.nSampleRate = sampleRate; + + return mOMX->setParameter( + mNode, + (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAc3, + &def, + sizeof(def)); +} + static OMX_AUDIO_AMRBANDMODETYPE pickModeFromBitRate( bool isAMRWB, int32_t bps) { if (isAMRWB) { @@ -2546,7 +2620,7 @@ void ACodec::sendFormatChange(const sp<AMessage> &reply) { { OMX_AUDIO_PORTDEFINITIONTYPE *audioDef = &def.format.audio; - switch (audioDef->eEncoding) { + switch ((int)audioDef->eEncoding) { case OMX_AUDIO_CodingPCM: { OMX_AUDIO_PARAM_PCMMODETYPE params; @@ -2652,6 +2726,24 @@ void ACodec::sendFormatChange(const sp<AMessage> &reply) { break; } + case OMX_AUDIO_CodingAndroidAC3: + { + OMX_AUDIO_PARAM_ANDROID_AC3TYPE params; + InitOMXParams(¶ms); + params.nPortIndex = kPortIndexOutput; + + CHECK_EQ((status_t)OK, mOMX->getParameter( + mNode, + (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAc3, + ¶ms, + sizeof(params))); + + notify->setString("mime", MEDIA_MIMETYPE_AUDIO_AC3); + notify->setInt32("channel-count", params.nChannels); + notify->setInt32("sample-rate", params.nSampleRate); + break; + } + default: TRESPASS(); } @@ -3235,11 +3327,11 @@ void ACodec::BaseState::onInputBufferFilled(const sp<AMessage> &msg) { mCodec->mInputEOSResult = err; } break; - - default: - CHECK_EQ((int)mode, (int)FREE_BUFFERS); - break; } + + default: + CHECK_EQ((int)mode, (int)FREE_BUFFERS); + break; } } @@ -3690,6 +3782,7 @@ void ACodec::LoadedState::stateEntered() { mCodec->mDequeueCounter = 0; mCodec->mMetaDataBuffersToSubmit = 0; mCodec->mRepeatFrameDelayUs = -1ll; + mCodec->mIsConfiguredForAdaptivePlayback = false; if (mCodec->mShutdownInProgress) { bool keepComponentAllocated = mCodec->mKeepComponentAllocated; @@ -3838,6 +3931,21 @@ void ACodec::LoadedState::onCreateInputSurface( } } + if (err == OK && mCodec->mMaxPtsGapUs > 0l) { + err = mCodec->mOMX->setInternalOption( + mCodec->mNode, + kPortIndexInput, + IOMX::INTERNAL_OPTION_MAX_TIMESTAMP_GAP, + &mCodec->mMaxPtsGapUs, + sizeof(mCodec->mMaxPtsGapUs)); + + if (err != OK) { + ALOGE("[%s] Unable to configure max timestamp gap (err %d)", + mCodec->mComponentName.c_str(), + err); + } + } + if (err == OK) { notify->setObject("input-surface", new BufferProducerWrapper(bufferProducer)); @@ -4036,6 +4144,9 @@ void ACodec::ExecutingState::submitOutputMetaBuffers() { break; } } + + // *** NOTE: THE FOLLOWING WORKAROUND WILL BE REMOVED *** + mCodec->signalSubmitOutputMetaDataBufferIfEOS_workaround(); } void ACodec::ExecutingState::submitRegularOutputBuffers() { @@ -4184,6 +4295,19 @@ bool ACodec::ExecutingState::onMessageReceived(const sp<AMessage> &msg) { break; } + // *** NOTE: THE FOLLOWING WORKAROUND WILL BE REMOVED *** + case kWhatSubmitOutputMetaDataBufferIfEOS: + { + if (mCodec->mPortEOS[kPortIndexInput] && + !mCodec->mPortEOS[kPortIndexOutput]) { + status_t err = mCodec->submitOutputMetaDataBuffer(); + if (err == OK) { + mCodec->signalSubmitOutputMetaDataBufferIfEOS_workaround(); + } + } + return true; + } + default: handled = BaseState::onMessageReceived(msg); break; diff --git a/media/libstagefright/MediaDefs.cpp b/media/libstagefright/MediaDefs.cpp index b5d4e44..340cba7 100644 --- a/media/libstagefright/MediaDefs.cpp +++ b/media/libstagefright/MediaDefs.cpp @@ -42,6 +42,7 @@ const char *MEDIA_MIMETYPE_AUDIO_RAW = "audio/raw"; const char *MEDIA_MIMETYPE_AUDIO_FLAC = "audio/flac"; const char *MEDIA_MIMETYPE_AUDIO_AAC_ADTS = "audio/aac-adts"; const char *MEDIA_MIMETYPE_AUDIO_MSGSM = "audio/gsm"; +const char *MEDIA_MIMETYPE_AUDIO_AC3 = "audio/ac3"; const char *MEDIA_MIMETYPE_CONTAINER_MPEG4 = "video/mp4"; const char *MEDIA_MIMETYPE_CONTAINER_WAV = "audio/x-wav"; diff --git a/media/libstagefright/OMXCodec.cpp b/media/libstagefright/OMXCodec.cpp index 43736ad..625922f 100644 --- a/media/libstagefright/OMXCodec.cpp +++ b/media/libstagefright/OMXCodec.cpp @@ -40,7 +40,9 @@ #include <utils/Vector.h> #include <OMX_Audio.h> +#include <OMX_AudioExt.h> #include <OMX_Component.h> +#include <OMX_IndexExt.h> #include "include/avc_utils.h" @@ -528,6 +530,17 @@ status_t OMXCodec::configureCodec(const sp<MetaData> &meta) { sampleRate, numChannels); } + } else if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_AC3, mMIME)) { + int32_t numChannels; + int32_t sampleRate; + CHECK(meta->findInt32(kKeyChannelCount, &numChannels)); + CHECK(meta->findInt32(kKeySampleRate, &sampleRate)); + + status_t err = setAC3Format(numChannels, sampleRate); + if (err != OK) { + CODEC_LOGE("setAC3Format() failed (err = %d)", err); + return err; + } } else if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_G711_ALAW, mMIME) || !strcasecmp(MEDIA_MIMETYPE_AUDIO_G711_MLAW, mMIME)) { // These are PCM-like formats with a fixed sample rate but @@ -1394,6 +1407,10 @@ void OMXCodec::setComponentRole( "audio_decoder.flac", "audio_encoder.flac" }, { MEDIA_MIMETYPE_AUDIO_MSGSM, "audio_decoder.gsm", "audio_encoder.gsm" }, + { MEDIA_MIMETYPE_VIDEO_MPEG2, + "video_decoder.mpeg2", "video_encoder.mpeg2" }, + { MEDIA_MIMETYPE_AUDIO_AC3, + "audio_decoder.ac3", "audio_encoder.ac3" }, }; static const size_t kNumMimeToRole = @@ -3489,6 +3506,31 @@ status_t OMXCodec::setAACFormat( return OK; } +status_t OMXCodec::setAC3Format(int32_t numChannels, int32_t sampleRate) { + OMX_AUDIO_PARAM_ANDROID_AC3TYPE def; + InitOMXParams(&def); + def.nPortIndex = kPortIndexInput; + + status_t err = mOMX->getParameter( + mNode, + (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAc3, + &def, + sizeof(def)); + + if (err != OK) { + return err; + } + + def.nChannels = numChannels; + def.nSampleRate = sampleRate; + + return mOMX->setParameter( + mNode, + (OMX_INDEXTYPE)OMX_IndexParamAudioAndroidAc3, + &def, + sizeof(def)); +} + void OMXCodec::setG711Format(int32_t numChannels) { CHECK(!mIsEncoder); setRawAudioFormat(kPortIndexInput, 8000, numChannels); @@ -4422,6 +4464,17 @@ void OMXCodec::initOutputFormat(const sp<MetaData> &inputFormat) { mOutputFormat->setInt32(kKeyChannelCount, numChannels); mOutputFormat->setInt32(kKeySampleRate, sampleRate); mOutputFormat->setInt32(kKeyBitRate, bitRate); + } else if (audio_def->eEncoding == + (OMX_AUDIO_CODINGTYPE)OMX_AUDIO_CodingAndroidAC3) { + mOutputFormat->setCString( + kKeyMIMEType, MEDIA_MIMETYPE_AUDIO_AC3); + int32_t numChannels, sampleRate, bitRate; + inputFormat->findInt32(kKeyChannelCount, &numChannels); + inputFormat->findInt32(kKeySampleRate, &sampleRate); + inputFormat->findInt32(kKeyBitRate, &bitRate); + mOutputFormat->setInt32(kKeyChannelCount, numChannels); + mOutputFormat->setInt32(kKeySampleRate, sampleRate); + mOutputFormat->setInt32(kKeyBitRate, bitRate); } else { CHECK(!"Should not be here. Unknown audio encoding."); } diff --git a/media/libstagefright/chromium_http/Android.mk b/media/libstagefright/chromium_http/Android.mk index 25a25ab..f191a4b 100644 --- a/media/libstagefright/chromium_http/Android.mk +++ b/media/libstagefright/chromium_http/Android.mk @@ -21,6 +21,7 @@ LOCAL_SHARED_LIBRARIES += \ libstlport \ libchromium_net \ libutils \ + libbinder \ libcutils \ liblog \ libstagefright_foundation \ diff --git a/media/libstagefright/chromium_http/support.cpp b/media/libstagefright/chromium_http/support.cpp index 3b33212..d4e82ee 100644 --- a/media/libstagefright/chromium_http/support.cpp +++ b/media/libstagefright/chromium_http/support.cpp @@ -40,8 +40,90 @@ #include <media/stagefright/Utils.h> #include <string> +#include <utils/Errors.h> +#include <binder/IInterface.h> +#include <binder/IServiceManager.h> + namespace android { +// must be kept in sync with interface defined in IAudioService.aidl +class IAudioService : public IInterface +{ +public: + DECLARE_META_INTERFACE(AudioService); + + virtual int verifyX509CertChain( + const std::vector<std::string>& cert_chain, + const std::string& hostname, + const std::string& auth_type) = 0; +}; + +class BpAudioService : public BpInterface<IAudioService> +{ +public: + BpAudioService(const sp<IBinder>& impl) + : BpInterface<IAudioService>(impl) + { + } + + virtual int verifyX509CertChain( + const std::vector<std::string>& cert_chain, + const std::string& hostname, + const std::string& auth_type) + { + Parcel data, reply; + data.writeInterfaceToken(IAudioService::getInterfaceDescriptor()); + + // The vector of std::string we get isn't really a vector of strings, + // but rather a vector of binary certificate data. If we try to pass + // it to Java language code as a string, it ends up mangled on the other + // side, so send them as bytes instead. + // Since we can't send an array of byte arrays, send a single array, + // which will be split out by the recipient. + + int numcerts = cert_chain.size(); + data.writeInt32(numcerts); + size_t total = 0; + for (int i = 0; i < numcerts; i++) { + total += cert_chain[i].size(); + } + size_t bytesize = total + numcerts * 4; + uint8_t *bytes = (uint8_t*) malloc(bytesize); + if (!bytes) { + return 5; // SSL_INVALID + } + ALOGV("%d certs: %d -> %d", numcerts, total, bytesize); + + int offset = 0; + for (int i = 0; i < numcerts; i++) { + int32_t certsize = cert_chain[i].size(); + // store this in a known order, which just happens to match the default + // byte order of a java ByteBuffer + int32_t bigsize = htonl(certsize); + ALOGV("cert %d, size %d", i, certsize); + memcpy(bytes + offset, &bigsize, sizeof(bigsize)); + offset += sizeof(bigsize); + memcpy(bytes + offset, cert_chain[i].data(), certsize); + offset += certsize; + } + data.writeByteArray(bytesize, bytes); + free(bytes); + data.writeString16(String16(hostname.c_str())); + data.writeString16(String16(auth_type.c_str())); + + int32_t result; + if (remote()->transact(IBinder::FIRST_CALL_TRANSACTION, data, &reply) != NO_ERROR + || reply.readExceptionCode() < 0 || reply.readInt32(&result) != NO_ERROR) { + return 5; // SSL_INVALID; + } + return result; + } + +}; + +IMPLEMENT_META_INTERFACE(AudioService, "android.media.IAudioService"); + + static Mutex gNetworkThreadLock; static base::Thread *gNetworkThread = NULL; static scoped_refptr<SfRequestContext> gReqContext; @@ -226,7 +308,24 @@ SfNetworkLibrary::VerifyResult SfNetworkLibrary::VerifyX509CertChain( const std::vector<std::string>& cert_chain, const std::string& hostname, const std::string& auth_type) { - return VERIFY_OK; + + sp<IBinder> binder = + defaultServiceManager()->checkService(String16("audio")); + if (binder == 0) { + ALOGW("Thread cannot connect to the audio service"); + } else { + sp<IAudioService> service = interface_cast<IAudioService>(binder); + int code = service->verifyX509CertChain(cert_chain, hostname, auth_type); + ALOGV("verified: %d", code); + if (code == -1) { + return VERIFY_OK; + } else if (code == 2) { // SSL_IDMISMATCH + return VERIFY_BAD_HOSTNAME; + } else if (code == 3) { // SSL_UNTRUSTED + return VERIFY_NO_TRUSTED_ROOT; + } + } + return VERIFY_INVOCATION_ERROR; } //////////////////////////////////////////////////////////////////////////////// diff --git a/media/libstagefright/httplive/LiveSession.cpp b/media/libstagefright/httplive/LiveSession.cpp index bdf5787..90ee3b0 100644 --- a/media/libstagefright/httplive/LiveSession.cpp +++ b/media/libstagefright/httplive/LiveSession.cpp @@ -37,6 +37,8 @@ #include <media/stagefright/MetaData.h> #include <media/stagefright/Utils.h> +#include <utils/Mutex.h> + #include <ctype.h> #include <openssl/aes.h> #include <openssl/md5.h> @@ -57,32 +59,57 @@ LiveSession::LiveSession( : 0)), mPrevBandwidthIndex(-1), mStreamMask(0), + mNewStreamMask(0), + mSwapMask(0), mCheckBandwidthGeneration(0), + mSwitchGeneration(0), mLastDequeuedTimeUs(0ll), mRealTimeBaseUs(0ll), mReconfigurationInProgress(false), - mDisconnectReplyID(0) { + mSwitchInProgress(false), + mDisconnectReplyID(0), + mSeekReplyID(0) { if (mUIDValid) { mHTTPDataSource->setUID(mUID); } - mPacketSources.add( - STREAMTYPE_AUDIO, new AnotherPacketSource(NULL /* meta */)); - - mPacketSources.add( - STREAMTYPE_VIDEO, new AnotherPacketSource(NULL /* meta */)); + mStreams[kAudioIndex] = StreamItem("audio"); + mStreams[kVideoIndex] = StreamItem("video"); + mStreams[kSubtitleIndex] = StreamItem("subtitles"); - mPacketSources.add( - STREAMTYPE_SUBTITLES, new AnotherPacketSource(NULL /* meta */)); + for (size_t i = 0; i < kMaxStreams; ++i) { + mPacketSources.add(indexToType(i), new AnotherPacketSource(NULL /* meta */)); + mPacketSources2.add(indexToType(i), new AnotherPacketSource(NULL /* meta */)); + } } LiveSession::~LiveSession() { } +sp<ABuffer> LiveSession::createFormatChangeBuffer(bool swap) { + ABuffer *discontinuity = new ABuffer(0); + discontinuity->meta()->setInt32("discontinuity", ATSParser::DISCONTINUITY_FORMATCHANGE); + discontinuity->meta()->setInt32("swapPacketSource", swap); + discontinuity->meta()->setInt32("switchGeneration", mSwitchGeneration); + discontinuity->meta()->setInt64("timeUs", -1); + return discontinuity; +} + +void LiveSession::swapPacketSource(StreamType stream) { + sp<AnotherPacketSource> &aps = mPacketSources.editValueFor(stream); + sp<AnotherPacketSource> &aps2 = mPacketSources2.editValueFor(stream); + sp<AnotherPacketSource> tmp = aps; + aps = aps2; + aps2 = tmp; + aps2->clear(); +} + status_t LiveSession::dequeueAccessUnit( StreamType stream, sp<ABuffer> *accessUnit) { if (!(mStreamMask & stream)) { - return UNKNOWN_ERROR; + // return -EWOULDBLOCK to avoid halting the decoder + // when switching between audio/video and audio only. + return -EWOULDBLOCK; } sp<AnotherPacketSource> packetSource = mPacketSources.valueFor(stream); @@ -122,6 +149,25 @@ status_t LiveSession::dequeueAccessUnit( streamStr, type, extra == NULL ? "NULL" : extra->debugString().c_str()); + + int32_t swap; + if (type == ATSParser::DISCONTINUITY_FORMATCHANGE + && (*accessUnit)->meta()->findInt32("swapPacketSource", &swap) + && swap) { + + int32_t switchGeneration; + CHECK((*accessUnit)->meta()->findInt32("switchGeneration", &switchGeneration)); + { + Mutex::Autolock lock(mSwapMutex); + if (switchGeneration == mSwitchGeneration) { + swapPacketSource(stream); + sp<AMessage> msg = new AMessage(kWhatSwapped, id()); + msg->setInt32("stream", stream); + msg->setInt32("switchGeneration", switchGeneration); + msg->post(); + } + } + } } else if (err == OK) { if (stream == STREAMTYPE_AUDIO || stream == STREAMTYPE_VIDEO) { int64_t timeUs; @@ -143,6 +189,7 @@ status_t LiveSession::dequeueAccessUnit( } status_t LiveSession::getStreamFormat(StreamType stream, sp<AMessage> *format) { + // No swapPacketSource race condition; called from the same thread as dequeueAccessUnit. if (!(mStreamMask & stream)) { return UNKNOWN_ERROR; } @@ -188,6 +235,10 @@ status_t LiveSession::seekTo(int64_t timeUs) { sp<AMessage> response; status_t err = msg->postAndAwaitResponse(&response); + uint32_t replyID; + CHECK(response == mSeekReply && 0 != mSeekReplyID); + mSeekReply.clear(); + mSeekReplyID = 0; return err; } @@ -213,15 +264,12 @@ void LiveSession::onMessageReceived(const sp<AMessage> &msg) { case kWhatSeek: { - uint32_t replyID; - CHECK(msg->senderAwaitsResponse(&replyID)); + CHECK(msg->senderAwaitsResponse(&mSeekReplyID)); status_t err = onSeek(msg); - sp<AMessage> response = new AMessage; - response->setInt32("err", err); - - response->postReply(replyID); + mSeekReply = new AMessage; + mSeekReply->setInt32("err", err); break; } @@ -239,13 +287,23 @@ void LiveSession::onMessageReceived(const sp<AMessage> &msg) { if (what == PlaylistFetcher::kWhatStopped) { AString uri; CHECK(msg->findString("uri", &uri)); - mFetcherInfos.removeItem(uri); + if (mFetcherInfos.removeItem(uri) < 0) { + // ignore duplicated kWhatStopped messages. + break; + } + + tryToFinishBandwidthSwitch(); } if (mContinuation != NULL) { CHECK_GT(mContinuationCounter, 0); if (--mContinuationCounter == 0) { mContinuation->post(); + + if (mSeekReplyID != 0) { + CHECK(mSeekReply != NULL); + mSeekReply->postReply(mSeekReplyID); + } } } break; @@ -275,6 +333,8 @@ void LiveSession::onMessageReceived(const sp<AMessage> &msg) { postPrepared(err); } + cancelBandwidthSwitch(); + mPacketSources.valueFor(STREAMTYPE_AUDIO)->signalEOS(err); mPacketSources.valueFor(STREAMTYPE_VIDEO)->signalEOS(err); @@ -313,6 +373,27 @@ void LiveSession::onMessageReceived(const sp<AMessage> &msg) { break; } + case PlaylistFetcher::kWhatStartedAt: + { + int32_t switchGeneration; + CHECK(msg->findInt32("switchGeneration", &switchGeneration)); + + if (switchGeneration != mSwitchGeneration) { + break; + } + + // Resume fetcher for the original variant; the resumed fetcher should + // continue until the timestamps found in msg, which is stored by the + // new fetcher to indicate where the new variant has started buffering. + for (size_t i = 0; i < mFetcherInfos.size(); i++) { + const FetcherInfo info = mFetcherInfos.valueAt(i); + if (info.mToBeRemoved) { + info.mFetcher->resumeUntilAsync(msg); + } + } + break; + } + default: TRESPASS(); } @@ -357,6 +438,11 @@ void LiveSession::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatSwapped: + { + onSwapped(msg); + break; + } default: TRESPASS(); break; @@ -374,6 +460,12 @@ int LiveSession::SortByBandwidth(const BandwidthItem *a, const BandwidthItem *b) return 1; } +// static +LiveSession::StreamType LiveSession::indexToType(int idx) { + CHECK(idx >= 0 && idx < kMaxStreams); + return (StreamType)(1 << idx); +} + void LiveSession::onConnect(const sp<AMessage> &msg) { AString url; CHECK(msg->findString("url", &url)); @@ -461,6 +553,10 @@ void LiveSession::finishDisconnect() { // during disconnection either. cancelCheckBandwidthEvent(); + // Protect mPacketSources from a swapPacketSource race condition through disconnect. + // (finishDisconnect, onFinishDisconnect2) + cancelBandwidthSwitch(); + for (size_t i = 0; i < mFetcherInfos.size(); ++i) { mFetcherInfos.valueAt(i).mFetcher->stopAsync(); } @@ -500,11 +596,13 @@ sp<PlaylistFetcher> LiveSession::addFetcher(const char *uri) { sp<AMessage> notify = new AMessage(kWhatFetcherNotify, id()); notify->setString("uri", uri); + notify->setInt32("switchGeneration", mSwitchGeneration); FetcherInfo info; info.mFetcher = new PlaylistFetcher(notify, this, uri); info.mDurationUs = -1ll; info.mIsPrepared = false; + info.mToBeRemoved = false; looper()->registerHandler(info.mFetcher); mFetcherInfos.add(uri, info); @@ -512,54 +610,81 @@ sp<PlaylistFetcher> LiveSession::addFetcher(const char *uri) { return info.mFetcher; } +/* + * Illustration of parameters: + * + * 0 `range_offset` + * +------------+-------------------------------------------------------+--+--+ + * | | | next block to fetch | | | + * | | `source` handle => `out` buffer | | | | + * | `url` file |<--------- buffer size --------->|<--- `block_size` -->| | | + * | |<----------- `range_length` / buffer capacity ----------->| | + * |<------------------------------ file_size ------------------------------->| + * + * Special parameter values: + * - range_length == -1 means entire file + * - block_size == 0 means entire range + * + */ status_t LiveSession::fetchFile( const char *url, sp<ABuffer> *out, int64_t range_offset, int64_t range_length, + uint32_t block_size, /* download block size */ + sp<DataSource> *source, /* to return and reuse source */ String8 *actualUrl) { - *out = NULL; + off64_t size; + sp<DataSource> temp_source; + if (source == NULL) { + source = &temp_source; + } - sp<DataSource> source; + if (*source == NULL) { + if (!strncasecmp(url, "file://", 7)) { + *source = new FileSource(url + 7); + } else if (strncasecmp(url, "http://", 7) + && strncasecmp(url, "https://", 8)) { + return ERROR_UNSUPPORTED; + } else { + KeyedVector<String8, String8> headers = mExtraHeaders; + if (range_offset > 0 || range_length >= 0) { + headers.add( + String8("Range"), + String8( + StringPrintf( + "bytes=%lld-%s", + range_offset, + range_length < 0 + ? "" : StringPrintf("%lld", + range_offset + range_length - 1).c_str()).c_str())); + } + status_t err = mHTTPDataSource->connect(url, &headers); - if (!strncasecmp(url, "file://", 7)) { - source = new FileSource(url + 7); - } else if (strncasecmp(url, "http://", 7) - && strncasecmp(url, "https://", 8)) { - return ERROR_UNSUPPORTED; - } else { - KeyedVector<String8, String8> headers = mExtraHeaders; - if (range_offset > 0 || range_length >= 0) { - headers.add( - String8("Range"), - String8( - StringPrintf( - "bytes=%lld-%s", - range_offset, - range_length < 0 - ? "" : StringPrintf("%lld", range_offset + range_length - 1).c_str()).c_str())); - } - status_t err = mHTTPDataSource->connect(url, &headers); + if (err != OK) { + return err; + } - if (err != OK) { - return err; + *source = mHTTPDataSource; } - - source = mHTTPDataSource; } - off64_t size; - status_t err = source->getSize(&size); - - if (err != OK) { + status_t getSizeErr = (*source)->getSize(&size); + if (getSizeErr != OK) { size = 65536; } - sp<ABuffer> buffer = new ABuffer(size); - buffer->setRange(0, 0); + sp<ABuffer> buffer = *out != NULL ? *out : new ABuffer(size); + if (*out == NULL) { + buffer->setRange(0, 0); + } + // adjust range_length if only reading partial block + if (block_size > 0 && (range_length == -1 || buffer->size() + block_size < range_length)) { + range_length = buffer->size() + block_size; + } for (;;) { + // Only resize when we don't know the size. size_t bufferRemaining = buffer->capacity() - buffer->size(); - - if (bufferRemaining == 0) { + if (bufferRemaining == 0 && getSizeErr != OK) { bufferRemaining = 32768; ALOGV("increasing download buffer to %d bytes", @@ -584,7 +709,9 @@ status_t LiveSession::fetchFile( } } - ssize_t n = source->readAt( + // The DataSource is responsible for informing us of error (n < 0) or eof (n == 0) + // to help us break out of the loop. + ssize_t n = (*source)->readAt( buffer->size(), buffer->data() + buffer->size(), maxBytesToRead); @@ -601,7 +728,7 @@ status_t LiveSession::fetchFile( *out = buffer; if (actualUrl != NULL) { - *actualUrl = source->getUri(); + *actualUrl = (*source)->getUri(); if (actualUrl->isEmpty()) { *actualUrl = url; } @@ -618,7 +745,7 @@ sp<M3UParser> LiveSession::fetchPlaylist( sp<ABuffer> buffer; String8 actualUrl; - status_t err = fetchFile(url, &buffer, 0, -1, &actualUrl); + status_t err = fetchFile(url, &buffer, 0, -1, 0, NULL, &actualUrl); if (err != OK) { return NULL; @@ -815,8 +942,25 @@ status_t LiveSession::selectTrack(size_t index, bool select) { return err; } +bool LiveSession::canSwitchUp() { + // Allow upwards bandwidth switch when a stream has buffered at least 10 seconds. + status_t err = OK; + for (size_t i = 0; i < mPacketSources.size(); ++i) { + sp<AnotherPacketSource> source = mPacketSources.valueAt(i); + int64_t dur = source->getBufferedDurationUs(&err); + if (err == OK && dur > 10000000) { + return true; + } + } + return false; +} + void LiveSession::changeConfiguration( int64_t timeUs, size_t bandwidthIndex, bool pickTrack) { + // Protect mPacketSources from a swapPacketSource race condition through reconfiguration. + // (changeConfiguration, onChangeConfiguration2, onChangeConfiguration3). + cancelBandwidthSwitch(); + CHECK(!mReconfigurationInProgress); mReconfigurationInProgress = true; @@ -832,21 +976,14 @@ void LiveSession::changeConfiguration( CHECK_LT(bandwidthIndex, mBandwidthItems.size()); const BandwidthItem &item = mBandwidthItems.itemAt(bandwidthIndex); - uint32_t streamMask = 0; - - AString audioURI; - if (mPlaylist->getAudioURI(item.mPlaylistIndex, &audioURI)) { - streamMask |= STREAMTYPE_AUDIO; - } - - AString videoURI; - if (mPlaylist->getVideoURI(item.mPlaylistIndex, &videoURI)) { - streamMask |= STREAMTYPE_VIDEO; - } + uint32_t streamMask = 0; // streams that should be fetched by the new fetcher + uint32_t resumeMask = 0; // streams that should be fetched by the original fetcher - AString subtitleURI; - if (mPlaylist->getSubtitleURI(item.mPlaylistIndex, &subtitleURI)) { - streamMask |= STREAMTYPE_SUBTITLES; + AString URIs[kMaxStreams]; + for (size_t i = 0; i < kMaxStreams; ++i) { + if (mPlaylist->getTypeURI(item.mPlaylistIndex, mStreams[i].mType, &URIs[i])) { + streamMask |= indexToType(i); + } } // Step 1, stop and discard fetchers that are no longer needed. @@ -858,10 +995,15 @@ void LiveSession::changeConfiguration( // If we're seeking all current fetchers are discarded. if (timeUs < 0ll) { - if (((streamMask & STREAMTYPE_AUDIO) && uri == audioURI) - || ((streamMask & STREAMTYPE_VIDEO) && uri == videoURI) - || ((streamMask & STREAMTYPE_SUBTITLES) && uri == subtitleURI)) { - discardFetcher = false; + // delay fetcher removal + discardFetcher = false; + + for (size_t j = 0; j < kMaxStreams; ++j) { + StreamType type = indexToType(j); + if ((streamMask & type) && uri == URIs[j]) { + resumeMask |= type; + streamMask &= ~type; + } } } @@ -872,17 +1014,20 @@ void LiveSession::changeConfiguration( } } - sp<AMessage> msg = new AMessage(kWhatChangeConfiguration2, id()); + sp<AMessage> msg; + if (timeUs < 0ll) { + // skip onChangeConfiguration2 (decoder destruction) if switching. + msg = new AMessage(kWhatChangeConfiguration3, id()); + } else { + msg = new AMessage(kWhatChangeConfiguration2, id()); + } msg->setInt32("streamMask", streamMask); + msg->setInt32("resumeMask", resumeMask); msg->setInt64("timeUs", timeUs); - if (streamMask & STREAMTYPE_AUDIO) { - msg->setString("audioURI", audioURI.c_str()); - } - if (streamMask & STREAMTYPE_VIDEO) { - msg->setString("videoURI", videoURI.c_str()); - } - if (streamMask & STREAMTYPE_SUBTITLES) { - msg->setString("subtitleURI", subtitleURI.c_str()); + for (size_t i = 0; i < kMaxStreams; ++i) { + if (streamMask & indexToType(i)) { + msg->setString(mStreams[i].uriKey().c_str(), URIs[i].c_str()); + } } // Every time a fetcher acknowledges the stopAsync or pauseAsync request @@ -894,6 +1039,11 @@ void LiveSession::changeConfiguration( if (mContinuationCounter == 0) { msg->post(); + + if (mSeekReplyID != 0) { + CHECK(mSeekReply != NULL); + mSeekReply->postReply(mSeekReplyID); + } } } @@ -913,18 +1063,13 @@ void LiveSession::onChangeConfiguration2(const sp<AMessage> &msg) { uint32_t streamMask; CHECK(msg->findInt32("streamMask", (int32_t *)&streamMask)); - AString audioURI, videoURI, subtitleURI; - if (streamMask & STREAMTYPE_AUDIO) { - CHECK(msg->findString("audioURI", &audioURI)); - ALOGV("audioURI = '%s'", audioURI.c_str()); - } - if (streamMask & STREAMTYPE_VIDEO) { - CHECK(msg->findString("videoURI", &videoURI)); - ALOGV("videoURI = '%s'", videoURI.c_str()); - } - if (streamMask & STREAMTYPE_SUBTITLES) { - CHECK(msg->findString("subtitleURI", &subtitleURI)); - ALOGV("subtitleURI = '%s'", subtitleURI.c_str()); + AString URIs[kMaxStreams]; + for (size_t i = 0; i < kMaxStreams; ++i) { + if (streamMask & indexToType(i)) { + const AString &uriKey = mStreams[i].uriKey(); + CHECK(msg->findString(uriKey.c_str(), &URIs[i])); + ALOGV("%s = '%s'", uriKey.c_str(), URIs[i].c_str()); + } } // Determine which decoders to shutdown on the player side, @@ -934,15 +1079,12 @@ void LiveSession::onChangeConfiguration2(const sp<AMessage> &msg) { // 2) its streamtype was already active and still is but the URI // has changed. uint32_t changedMask = 0; - if (((mStreamMask & streamMask & STREAMTYPE_AUDIO) - && !(audioURI == mAudioURI)) - || (mStreamMask & ~streamMask & STREAMTYPE_AUDIO)) { - changedMask |= STREAMTYPE_AUDIO; - } - if (((mStreamMask & streamMask & STREAMTYPE_VIDEO) - && !(videoURI == mVideoURI)) - || (mStreamMask & ~streamMask & STREAMTYPE_VIDEO)) { - changedMask |= STREAMTYPE_VIDEO; + for (size_t i = 0; i < kMaxStreams && i != kSubtitleIndex; ++i) { + if (((mStreamMask & streamMask & indexToType(i)) + && !(URIs[i] == mStreams[i].mUri)) + || (mStreamMask & ~streamMask & indexToType(i))) { + changedMask |= indexToType(i); + } } if (changedMask == 0) { @@ -968,68 +1110,54 @@ void LiveSession::onChangeConfiguration2(const sp<AMessage> &msg) { } void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { + mContinuation.clear(); // All remaining fetchers are still suspended, the player has shutdown // any decoders that needed it. - uint32_t streamMask; + uint32_t streamMask, resumeMask; CHECK(msg->findInt32("streamMask", (int32_t *)&streamMask)); + CHECK(msg->findInt32("resumeMask", (int32_t *)&resumeMask)); - AString audioURI, videoURI, subtitleURI; - if (streamMask & STREAMTYPE_AUDIO) { - CHECK(msg->findString("audioURI", &audioURI)); - } - if (streamMask & STREAMTYPE_VIDEO) { - CHECK(msg->findString("videoURI", &videoURI)); - } - if (streamMask & STREAMTYPE_SUBTITLES) { - CHECK(msg->findString("subtitleURI", &subtitleURI)); + for (size_t i = 0; i < kMaxStreams; ++i) { + if (streamMask & indexToType(i)) { + CHECK(msg->findString(mStreams[i].uriKey().c_str(), &mStreams[i].mUri)); + } } int64_t timeUs; + bool switching = false; CHECK(msg->findInt64("timeUs", &timeUs)); if (timeUs < 0ll) { timeUs = mLastDequeuedTimeUs; + switching = true; } mRealTimeBaseUs = ALooper::GetNowUs() - timeUs; - mStreamMask = streamMask; - mAudioURI = audioURI; - mVideoURI = videoURI; - mSubtitleURI = subtitleURI; + mNewStreamMask = streamMask; - // Resume all existing fetchers and assign them packet sources. + // Of all existing fetchers: + // * Resume fetchers that are still needed and assign them original packet sources. + // * Mark otherwise unneeded fetchers for removal. + ALOGV("resuming fetchers for mask 0x%08x", resumeMask); for (size_t i = 0; i < mFetcherInfos.size(); ++i) { const AString &uri = mFetcherInfos.keyAt(i); - uint32_t resumeMask = 0; - - sp<AnotherPacketSource> audioSource; - if ((streamMask & STREAMTYPE_AUDIO) && uri == audioURI) { - audioSource = mPacketSources.valueFor(STREAMTYPE_AUDIO); - resumeMask |= STREAMTYPE_AUDIO; - } - - sp<AnotherPacketSource> videoSource; - if ((streamMask & STREAMTYPE_VIDEO) && uri == videoURI) { - videoSource = mPacketSources.valueFor(STREAMTYPE_VIDEO); - resumeMask |= STREAMTYPE_VIDEO; + sp<AnotherPacketSource> sources[kMaxStreams]; + for (size_t j = 0; j < kMaxStreams; ++j) { + if ((resumeMask & indexToType(j)) && uri == mStreams[j].mUri) { + sources[j] = mPacketSources.valueFor(indexToType(j)); + } } - sp<AnotherPacketSource> subtitleSource; - if ((streamMask & STREAMTYPE_SUBTITLES) && uri == subtitleURI) { - subtitleSource = mPacketSources.valueFor(STREAMTYPE_SUBTITLES); - resumeMask |= STREAMTYPE_SUBTITLES; + FetcherInfo &info = mFetcherInfos.editValueAt(i); + if (sources[kAudioIndex] != NULL || sources[kVideoIndex] != NULL + || sources[kSubtitleIndex] != NULL) { + info.mFetcher->startAsync( + sources[kAudioIndex], sources[kVideoIndex], sources[kSubtitleIndex]); + } else { + info.mToBeRemoved = true; } - - CHECK_NE(resumeMask, 0u); - - ALOGV("resuming fetchers for mask 0x%08x", resumeMask); - - streamMask &= ~resumeMask; - - mFetcherInfos.valueAt(i).mFetcher->startAsync( - audioSource, videoSource, subtitleSource); } // streamMask now only contains the types that need a new fetcher created. @@ -1038,52 +1166,65 @@ void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { ALOGV("creating new fetchers for mask 0x%08x", streamMask); } - while (streamMask != 0) { - StreamType streamType = (StreamType)(streamMask & ~(streamMask - 1)); + // Find out when the original fetchers have buffered up to and start the new fetchers + // at a later timestamp. + for (size_t i = 0; i < kMaxStreams; i++) { + if (!(indexToType(i) & streamMask)) { + continue; + } AString uri; - switch (streamType) { - case STREAMTYPE_AUDIO: - uri = audioURI; - break; - case STREAMTYPE_VIDEO: - uri = videoURI; - break; - case STREAMTYPE_SUBTITLES: - uri = subtitleURI; - break; - default: - TRESPASS(); - } + uri = mStreams[i].mUri; sp<PlaylistFetcher> fetcher = addFetcher(uri.c_str()); CHECK(fetcher != NULL); - sp<AnotherPacketSource> audioSource; - if ((streamMask & STREAMTYPE_AUDIO) && uri == audioURI) { - audioSource = mPacketSources.valueFor(STREAMTYPE_AUDIO); - audioSource->clear(); - - streamMask &= ~STREAMTYPE_AUDIO; - } - - sp<AnotherPacketSource> videoSource; - if ((streamMask & STREAMTYPE_VIDEO) && uri == videoURI) { - videoSource = mPacketSources.valueFor(STREAMTYPE_VIDEO); - videoSource->clear(); - - streamMask &= ~STREAMTYPE_VIDEO; - } + int32_t latestSeq = -1; + int64_t latestTimeUs = 0ll; + sp<AnotherPacketSource> sources[kMaxStreams]; + + // TRICKY: looping from i as earlier streams are already removed from streamMask + for (size_t j = i; j < kMaxStreams; ++j) { + if ((streamMask & indexToType(j)) && uri == mStreams[j].mUri) { + sources[j] = mPacketSources.valueFor(indexToType(j)); + + if (!switching) { + sources[j]->clear(); + } else { + int32_t type, seq; + int64_t srcTimeUs; + sp<AMessage> meta = sources[j]->getLatestMeta(); + + if (meta != NULL && !meta->findInt32("discontinuity", &type)) { + CHECK(meta->findInt32("seq", &seq)); + if (seq > latestSeq) { + latestSeq = seq; + } + CHECK(meta->findInt64("timeUs", &srcTimeUs)); + if (srcTimeUs > latestTimeUs) { + latestTimeUs = srcTimeUs; + } + } - sp<AnotherPacketSource> subtitleSource; - if ((streamMask & STREAMTYPE_SUBTITLES) && uri == subtitleURI) { - subtitleSource = mPacketSources.valueFor(STREAMTYPE_SUBTITLES); - subtitleSource->clear(); + sources[j] = mPacketSources2.valueFor(indexToType(j)); + sources[j]->clear(); + uint32_t extraStreams = mNewStreamMask & (~mStreamMask); + if (extraStreams & indexToType(j)) { + sources[j]->queueAccessUnit(createFormatChangeBuffer(/* swap = */ false)); + } + } - streamMask &= ~STREAMTYPE_SUBTITLES; + streamMask &= ~indexToType(j); + } } - fetcher->startAsync(audioSource, videoSource, subtitleSource, timeUs); + fetcher->startAsync( + sources[kAudioIndex], + sources[kVideoIndex], + sources[kSubtitleIndex], + timeUs, + latestTimeUs /* min start time(us) */, + latestSeq >= 0 ? latestSeq + 1 : -1 /* starting sequence number hint */ ); } // All fetchers have now been started, the configuration change @@ -1092,14 +1233,61 @@ void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { scheduleCheckBandwidthEvent(); ALOGV("XXX configuration change completed."); - mReconfigurationInProgress = false; + if (switching) { + mSwitchInProgress = true; + } else { + mStreamMask = mNewStreamMask; + } if (mDisconnectReplyID != 0) { finishDisconnect(); } } +void LiveSession::onSwapped(const sp<AMessage> &msg) { + int32_t switchGeneration; + CHECK(msg->findInt32("switchGeneration", &switchGeneration)); + if (switchGeneration != mSwitchGeneration) { + return; + } + + int32_t stream; + CHECK(msg->findInt32("stream", &stream)); + mSwapMask |= stream; + if (mSwapMask != mStreamMask) { + return; + } + + // Check if new variant contains extra streams. + uint32_t extraStreams = mNewStreamMask & (~mStreamMask); + while (extraStreams) { + StreamType extraStream = (StreamType) (extraStreams & ~(extraStreams - 1)); + swapPacketSource(extraStream); + extraStreams &= ~extraStream; + } + + tryToFinishBandwidthSwitch(); +} + +// Mark switch done when: +// 1. all old buffers are swapped out, AND +// 2. all old fetchers are removed. +void LiveSession::tryToFinishBandwidthSwitch() { + bool needToRemoveFetchers = false; + for (size_t i = 0; i < mFetcherInfos.size(); ++i) { + if (mFetcherInfos.valueAt(i).mToBeRemoved) { + needToRemoveFetchers = true; + break; + } + } + if (!needToRemoveFetchers && mSwapMask == mStreamMask) { + mStreamMask = mNewStreamMask; + mSwitchInProgress = false; + mSwapMask = 0; + } +} + void LiveSession::scheduleCheckBandwidthEvent() { sp<AMessage> msg = new AMessage(kWhatCheckBandwidth, id()); msg->setInt32("generation", mCheckBandwidthGeneration); @@ -1110,16 +1298,37 @@ void LiveSession::cancelCheckBandwidthEvent() { ++mCheckBandwidthGeneration; } -void LiveSession::onCheckBandwidth() { - if (mReconfigurationInProgress) { - scheduleCheckBandwidthEvent(); - return; +void LiveSession::cancelBandwidthSwitch() { + Mutex::Autolock lock(mSwapMutex); + mSwitchGeneration++; + mSwitchInProgress = false; + mSwapMask = 0; +} + +bool LiveSession::canSwitchBandwidthTo(size_t bandwidthIndex) { + if (mReconfigurationInProgress || mSwitchInProgress) { + return false; } + if (mPrevBandwidthIndex < 0) { + return true; + } + + if (bandwidthIndex == (size_t)mPrevBandwidthIndex) { + return false; + } else if (bandwidthIndex > (size_t)mPrevBandwidthIndex) { + return canSwitchUp(); + } else { + return true; + } +} + +void LiveSession::onCheckBandwidth() { size_t bandwidthIndex = getBandwidthIndex(); - if (mPrevBandwidthIndex < 0 - || bandwidthIndex != (size_t)mPrevBandwidthIndex) { + if (canSwitchBandwidthTo(bandwidthIndex)) { changeConfiguration(-1ll /* timeUs */, bandwidthIndex); + } else { + scheduleCheckBandwidthEvent(); } // Handling the kWhatCheckBandwidth even here does _not_ automatically diff --git a/media/libstagefright/httplive/LiveSession.h b/media/libstagefright/httplive/LiveSession.h index 8f6a4ea..680792d 100644 --- a/media/libstagefright/httplive/LiveSession.h +++ b/media/libstagefright/httplive/LiveSession.h @@ -42,10 +42,17 @@ struct LiveSession : public AHandler { const sp<AMessage> ¬ify, uint32_t flags = 0, bool uidValid = false, uid_t uid = 0); + enum StreamIndex { + kAudioIndex = 0, + kVideoIndex = 1, + kSubtitleIndex = 2, + kMaxStreams = 3, + }; + enum StreamType { - STREAMTYPE_AUDIO = 1, - STREAMTYPE_VIDEO = 2, - STREAMTYPE_SUBTITLES = 4, + STREAMTYPE_AUDIO = 1 << kAudioIndex, + STREAMTYPE_VIDEO = 1 << kVideoIndex, + STREAMTYPE_SUBTITLES = 1 << kSubtitleIndex, }; status_t dequeueAccessUnit(StreamType stream, sp<ABuffer> *accessUnit); @@ -74,6 +81,11 @@ struct LiveSession : public AHandler { kWhatPreparationFailed, }; + // create a format-change discontinuity + // + // swap: + // whether is format-change discontinuity should trigger a buffer swap + sp<ABuffer> createFormatChangeBuffer(bool swap = true); protected: virtual ~LiveSession(); @@ -92,6 +104,7 @@ private: kWhatChangeConfiguration2 = 'chC2', kWhatChangeConfiguration3 = 'chC3', kWhatFinishDisconnect2 = 'fin2', + kWhatSwapped = 'swap', }; struct BandwidthItem { @@ -103,8 +116,22 @@ private: sp<PlaylistFetcher> mFetcher; int64_t mDurationUs; bool mIsPrepared; + bool mToBeRemoved; }; + struct StreamItem { + const char *mType; + AString mUri; + StreamItem() : mType("") {} + StreamItem(const char *type) : mType(type) {} + AString uriKey() { + AString key(mType); + key.append("URI"); + return key; + } + }; + StreamItem mStreams[kMaxStreams]; + sp<AMessage> mNotify; uint32_t mFlags; bool mUIDValid; @@ -123,21 +150,40 @@ private: sp<M3UParser> mPlaylist; KeyedVector<AString, FetcherInfo> mFetcherInfos; - AString mAudioURI, mVideoURI, mSubtitleURI; uint32_t mStreamMask; + // Masks used during reconfiguration: + // mNewStreamMask: streams in the variant playlist we're switching to; + // we don't want to immediately overwrite the original value. + uint32_t mNewStreamMask; + + // mSwapMask: streams that have started to playback content in the new variant playlist; + // we use this to track reconfiguration progress. + uint32_t mSwapMask; + KeyedVector<StreamType, sp<AnotherPacketSource> > mPacketSources; + // A second set of packet sources that buffer content for the variant we're switching to. + KeyedVector<StreamType, sp<AnotherPacketSource> > mPacketSources2; + + // A mutex used to serialize two sets of events: + // * the swapping of packet sources in dequeueAccessUnit on the player thread, AND + // * a forced bandwidth switch termination in cancelSwitch on the live looper. + Mutex mSwapMutex; int32_t mCheckBandwidthGeneration; + int32_t mSwitchGeneration; size_t mContinuationCounter; sp<AMessage> mContinuation; + sp<AMessage> mSeekReply; int64_t mLastDequeuedTimeUs; int64_t mRealTimeBaseUs; bool mReconfigurationInProgress; + bool mSwitchInProgress; uint32_t mDisconnectReplyID; + uint32_t mSeekReplyID; sp<PlaylistFetcher> addFetcher(const char *uri); @@ -145,9 +191,25 @@ private: status_t onSeek(const sp<AMessage> &msg); void onFinishDisconnect2(); + // If given a non-zero block_size (default 0), it is used to cap the number of + // bytes read in from the DataSource. If given a non-NULL buffer, new content + // is read into the end. + // + // The DataSource we read from is responsible for signaling error or EOF to help us + // break out of the read loop. The DataSource can be returned to the caller, so + // that the caller can reuse it for subsequent fetches (within the initially + // requested range). + // + // For reused HTTP sources, the caller must download a file sequentially without + // any overlaps or gaps to prevent reconnection. status_t fetchFile( const char *url, sp<ABuffer> *out, + /* request/open a file starting at range_offset for range_length bytes */ int64_t range_offset = 0, int64_t range_length = -1, + /* download block size */ + uint32_t block_size = 0, + /* reuse DataSource if doing partial fetch */ + sp<DataSource> *source = NULL, String8 *actualUrl = NULL); sp<M3UParser> fetchPlaylist( @@ -156,22 +218,34 @@ private: size_t getBandwidthIndex(); static int SortByBandwidth(const BandwidthItem *, const BandwidthItem *); + static StreamType indexToType(int idx); void changeConfiguration( int64_t timeUs, size_t bandwidthIndex, bool pickTrack = false); void onChangeConfiguration(const sp<AMessage> &msg); void onChangeConfiguration2(const sp<AMessage> &msg); void onChangeConfiguration3(const sp<AMessage> &msg); + void onSwapped(const sp<AMessage> &msg); + void tryToFinishBandwidthSwitch(); void scheduleCheckBandwidthEvent(); void cancelCheckBandwidthEvent(); + // cancelBandwidthSwitch is atomic wrt swapPacketSource; call it to prevent packet sources + // from being swapped out on stale discontinuities while manipulating + // mPacketSources/mPacketSources2. + void cancelBandwidthSwitch(); + + bool canSwitchBandwidthTo(size_t bandwidthIndex); void onCheckBandwidth(); void finishDisconnect(); void postPrepared(status_t err); + void swapPacketSource(StreamType stream); + bool canSwitchUp(); + DISALLOW_EVIL_CONSTRUCTORS(LiveSession); }; diff --git a/media/libstagefright/httplive/M3UParser.cpp b/media/libstagefright/httplive/M3UParser.cpp index 0f390c3..7b06a2f 100644 --- a/media/libstagefright/httplive/M3UParser.cpp +++ b/media/libstagefright/httplive/M3UParser.cpp @@ -24,6 +24,7 @@ #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> #include <media/stagefright/MediaErrors.h> +#include <media/stagefright/Utils.h> #include <media/mediaplayer.h> namespace android { @@ -352,9 +353,28 @@ bool M3UParser::getTypeURI(size_t index, const char *key, AString *uri) const { if (!meta->findString(key, &groupID)) { *uri = mItems.itemAt(index).mURI; - // Assume media without any more specific attribute contains - // audio and video, but no subtitles. - return !strcmp("audio", key) || !strcmp("video", key); + AString codecs; + if (!meta->findString("codecs", &codecs)) { + // Assume media without any more specific attribute contains + // audio and video, but no subtitles. + return !strcmp("audio", key) || !strcmp("video", key); + } else { + // Split the comma separated list of codecs. + size_t offset = 0; + ssize_t commaPos = -1; + codecs.append(','); + while ((commaPos = codecs.find(",", offset)) >= 0) { + AString codec(codecs, offset, commaPos - offset); + codec.trim(); + // return true only if a codec of type `key` ("audio"/"video") + // is found. + if (codecIsType(codec, key)) { + return true; + } + offset = commaPos + 1; + } + return false; + } } sp<MediaGroup> group = mMediaGroups.valueFor(groupID); @@ -369,18 +389,6 @@ bool M3UParser::getTypeURI(size_t index, const char *key, AString *uri) const { return true; } -bool M3UParser::getAudioURI(size_t index, AString *uri) const { - return getTypeURI(index, "audio", uri); -} - -bool M3UParser::getVideoURI(size_t index, AString *uri) const { - return getTypeURI(index, "video", uri); -} - -bool M3UParser::getSubtitleURI(size_t index, AString *uri) const { - return getTypeURI(index, "subtitles", uri); -} - static bool MakeURL(const char *baseURL, const char *url, AString *out) { out->clear(); @@ -694,12 +702,22 @@ status_t M3UParser::parseStreamInf( *meta = new AMessage; } (*meta)->setInt32("bandwidth", x); + } else if (!strcasecmp("codecs", key.c_str())) { + if (!isQuotedString(val)) { + ALOGE("Expected quoted string for %s attribute, " + "got '%s' instead.", + key.c_str(), val.c_str());; + + return ERROR_MALFORMED; + } + + key.tolower(); + const AString &codecs = unquoteString(val); + (*meta)->setString(key.c_str(), codecs.c_str()); } else if (!strcasecmp("audio", key.c_str()) || !strcasecmp("video", key.c_str()) || !strcasecmp("subtitles", key.c_str())) { - if (val.size() < 2 - || val.c_str()[0] != '"' - || val.c_str()[val.size() - 1] != '"') { + if (!isQuotedString(val)) { ALOGE("Expected quoted string for %s attribute, " "got '%s' instead.", key.c_str(), val.c_str()); @@ -707,7 +725,7 @@ status_t M3UParser::parseStreamInf( return ERROR_MALFORMED; } - AString groupID(val, 1, val.size() - 2); + const AString &groupID = unquoteString(val); ssize_t groupIndex = mMediaGroups.indexOfKey(groupID); if (groupIndex < 0) { @@ -1095,4 +1113,121 @@ status_t M3UParser::ParseDouble(const char *s, double *x) { return OK; } +// static +bool M3UParser::isQuotedString(const AString &str) { + if (str.size() < 2 + || str.c_str()[0] != '"' + || str.c_str()[str.size() - 1] != '"') { + return false; + } + return true; +} + +// static +AString M3UParser::unquoteString(const AString &str) { + if (!isQuotedString(str)) { + return str; + } + return AString(str, 1, str.size() - 2); +} + +// static +bool M3UParser::codecIsType(const AString &codec, const char *type) { + if (codec.size() < 4) { + return false; + } + const char *c = codec.c_str(); + switch (FOURCC(c[0], c[1], c[2], c[3])) { + // List extracted from http://www.mp4ra.org/codecs.html + case 'ac-3': + case 'alac': + case 'dra1': + case 'dtsc': + case 'dtse': + case 'dtsh': + case 'dtsl': + case 'ec-3': + case 'enca': + case 'g719': + case 'g726': + case 'm4ae': + case 'mlpa': + case 'mp4a': + case 'raw ': + case 'samr': + case 'sawb': + case 'sawp': + case 'sevc': + case 'sqcp': + case 'ssmv': + case 'twos': + case 'agsm': + case 'alaw': + case 'dvi ': + case 'fl32': + case 'fl64': + case 'ima4': + case 'in24': + case 'in32': + case 'lpcm': + case 'Qclp': + case 'QDM2': + case 'QDMC': + case 'ulaw': + case 'vdva': + return !strcmp("audio", type); + + case 'avc1': + case 'avc2': + case 'avcp': + case 'drac': + case 'encv': + case 'mjp2': + case 'mp4v': + case 'mvc1': + case 'mvc2': + case 'resv': + case 's263': + case 'svc1': + case 'vc-1': + case 'CFHD': + case 'civd': + case 'DV10': + case 'dvh5': + case 'dvh6': + case 'dvhp': + case 'DVOO': + case 'DVOR': + case 'DVTV': + case 'DVVT': + case 'flic': + case 'gif ': + case 'h261': + case 'h263': + case 'HD10': + case 'jpeg': + case 'M105': + case 'mjpa': + case 'mjpb': + case 'png ': + case 'PNTG': + case 'rle ': + case 'rpza': + case 'Shr0': + case 'Shr1': + case 'Shr2': + case 'Shr3': + case 'Shr4': + case 'SVQ1': + case 'SVQ3': + case 'tga ': + case 'tiff': + case 'WRLE': + return !strcmp("video", type); + + default: + return false; + } +} + } // namespace android diff --git a/media/libstagefright/httplive/M3UParser.h b/media/libstagefright/httplive/M3UParser.h index 5248004..ccd6556 100644 --- a/media/libstagefright/httplive/M3UParser.h +++ b/media/libstagefright/httplive/M3UParser.h @@ -45,9 +45,7 @@ struct M3UParser : public RefBase { status_t getTrackInfo(Parcel* reply) const; ssize_t getSelectedIndex() const; - bool getAudioURI(size_t index, AString *uri) const; - bool getVideoURI(size_t index, AString *uri) const; - bool getSubtitleURI(size_t index, AString *uri) const; + bool getTypeURI(size_t index, const char *key, AString *uri) const; protected: virtual ~M3UParser(); @@ -95,11 +93,13 @@ private: status_t parseMedia(const AString &line); - bool getTypeURI(size_t index, const char *key, AString *uri) const; - static status_t ParseInt32(const char *s, int32_t *x); static status_t ParseDouble(const char *s, double *x); + static bool isQuotedString(const AString &str); + static AString unquoteString(const AString &str); + static bool codecIsType(const AString &codec, const char *type); + DISALLOW_EVIL_CONSTRUCTORS(M3UParser); }; diff --git a/media/libstagefright/httplive/PlaylistFetcher.cpp b/media/libstagefright/httplive/PlaylistFetcher.cpp index 973b779..012a68b 100644 --- a/media/libstagefright/httplive/PlaylistFetcher.cpp +++ b/media/libstagefright/httplive/PlaylistFetcher.cpp @@ -47,26 +47,34 @@ namespace android { // static const int64_t PlaylistFetcher::kMinBufferedDurationUs = 10000000ll; +const int64_t PlaylistFetcher::kMaxMonitorDelayUs = 3000000ll; +const int32_t PlaylistFetcher::kNumSkipFrames = 10; PlaylistFetcher::PlaylistFetcher( const sp<AMessage> ¬ify, const sp<LiveSession> &session, const char *uri) : mNotify(notify), + mStartTimeUsNotify(notify->dup()), mSession(session), mURI(uri), mStreamTypeMask(0), mStartTimeUs(-1ll), + mMinStartTimeUs(0ll), + mStopParams(NULL), mLastPlaylistFetchTimeUs(-1ll), mSeqNumber(-1), mNumRetries(0), mStartup(true), + mPrepared(false), mNextPTSTimeUs(-1ll), mMonitorQueueGeneration(0), mRefreshState(INITIAL_MINIMUM_RELOAD_DELAY), mFirstPTSValid(false), mAbsoluteTimeAnchorUs(0ll) { memset(mPlaylistHash, 0, sizeof(mPlaylistHash)); + mStartTimeUsNotify->setInt32("what", kWhatStartedAt); + mStartTimeUsNotify->setInt32("streamMask", 0); } PlaylistFetcher::~PlaylistFetcher() { @@ -103,10 +111,16 @@ int64_t PlaylistFetcher::getSegmentStartTimeUs(int32_t seqNumber) const { return segmentStartUs; } -bool PlaylistFetcher::timeToRefreshPlaylist(int64_t nowUs) const { - if (mPlaylist == NULL) { +int64_t PlaylistFetcher::delayUsToRefreshPlaylist() const { + int64_t nowUs = ALooper::GetNowUs(); + + if (mPlaylist == NULL || mLastPlaylistFetchTimeUs < 0ll) { CHECK_EQ((int)mRefreshState, (int)INITIAL_MINIMUM_RELOAD_DELAY); - return true; + return 0ll; + } + + if (mPlaylist->isComplete()) { + return (~0llu >> 1); } int32_t targetDurationSecs; @@ -157,11 +171,13 @@ bool PlaylistFetcher::timeToRefreshPlaylist(int64_t nowUs) const { break; } - return mLastPlaylistFetchTimeUs + minPlaylistAgeUs <= nowUs; + int64_t delayUs = mLastPlaylistFetchTimeUs + minPlaylistAgeUs - nowUs; + return delayUs > 0ll ? delayUs : 0ll; } status_t PlaylistFetcher::decryptBuffer( - size_t playlistIndex, const sp<ABuffer> &buffer) { + size_t playlistIndex, const sp<ABuffer> &buffer, + bool first) { sp<AMessage> itemMeta; bool found = false; AString method; @@ -179,6 +195,7 @@ status_t PlaylistFetcher::decryptBuffer( if (!found) { method = "NONE"; } + buffer->meta()->setString("cipher-method", method.c_str()); if (method == "NONE") { return OK; @@ -218,63 +235,89 @@ status_t PlaylistFetcher::decryptBuffer( return UNKNOWN_ERROR; } - unsigned char aes_ivec[16]; + size_t n = buffer->size(); + if (!n) { + return OK; + } + CHECK(n % 16 == 0); - AString iv; - if (itemMeta->findString("cipher-iv", &iv)) { - if ((!iv.startsWith("0x") && !iv.startsWith("0X")) - || iv.size() != 16 * 2 + 2) { - ALOGE("malformed cipher IV '%s'.", iv.c_str()); - return ERROR_MALFORMED; - } + if (first) { + // If decrypting the first block in a file, read the iv from the manifest + // or derive the iv from the file's sequence number. - memset(aes_ivec, 0, sizeof(aes_ivec)); - for (size_t i = 0; i < 16; ++i) { - char c1 = tolower(iv.c_str()[2 + 2 * i]); - char c2 = tolower(iv.c_str()[3 + 2 * i]); - if (!isxdigit(c1) || !isxdigit(c2)) { + AString iv; + if (itemMeta->findString("cipher-iv", &iv)) { + if ((!iv.startsWith("0x") && !iv.startsWith("0X")) + || iv.size() != 16 * 2 + 2) { ALOGE("malformed cipher IV '%s'.", iv.c_str()); return ERROR_MALFORMED; } - uint8_t nibble1 = isdigit(c1) ? c1 - '0' : c1 - 'a' + 10; - uint8_t nibble2 = isdigit(c2) ? c2 - '0' : c2 - 'a' + 10; - aes_ivec[i] = nibble1 << 4 | nibble2; + memset(mAESInitVec, 0, sizeof(mAESInitVec)); + for (size_t i = 0; i < 16; ++i) { + char c1 = tolower(iv.c_str()[2 + 2 * i]); + char c2 = tolower(iv.c_str()[3 + 2 * i]); + if (!isxdigit(c1) || !isxdigit(c2)) { + ALOGE("malformed cipher IV '%s'.", iv.c_str()); + return ERROR_MALFORMED; + } + uint8_t nibble1 = isdigit(c1) ? c1 - '0' : c1 - 'a' + 10; + uint8_t nibble2 = isdigit(c2) ? c2 - '0' : c2 - 'a' + 10; + + mAESInitVec[i] = nibble1 << 4 | nibble2; + } + } else { + memset(mAESInitVec, 0, sizeof(mAESInitVec)); + mAESInitVec[15] = mSeqNumber & 0xff; + mAESInitVec[14] = (mSeqNumber >> 8) & 0xff; + mAESInitVec[13] = (mSeqNumber >> 16) & 0xff; + mAESInitVec[12] = (mSeqNumber >> 24) & 0xff; } - } else { - memset(aes_ivec, 0, sizeof(aes_ivec)); - aes_ivec[15] = mSeqNumber & 0xff; - aes_ivec[14] = (mSeqNumber >> 8) & 0xff; - aes_ivec[13] = (mSeqNumber >> 16) & 0xff; - aes_ivec[12] = (mSeqNumber >> 24) & 0xff; } AES_cbc_encrypt( buffer->data(), buffer->data(), buffer->size(), - &aes_key, aes_ivec, AES_DECRYPT); + &aes_key, mAESInitVec, AES_DECRYPT); - // hexdump(buffer->data(), buffer->size()); - - size_t n = buffer->size(); - CHECK_GT(n, 0u); + return OK; +} - size_t pad = buffer->data()[n - 1]; +status_t PlaylistFetcher::checkDecryptPadding(const sp<ABuffer> &buffer) { + status_t err; + AString method; + CHECK(buffer->meta()->findString("cipher-method", &method)); + if (method == "NONE") { + return OK; + } - CHECK_GT(pad, 0u); - CHECK_LE(pad, 16u); - CHECK_GE((size_t)n, pad); - for (size_t i = 0; i < pad; ++i) { - CHECK_EQ((unsigned)buffer->data()[n - 1 - i], pad); + uint8_t padding = 0; + if (buffer->size() > 0) { + padding = buffer->data()[buffer->size() - 1]; } - n -= pad; + if (padding > 16) { + return ERROR_MALFORMED; + } - buffer->setRange(buffer->offset(), n); + for (size_t i = buffer->size() - padding; i < padding; i++) { + if (buffer->data()[i] != padding) { + return ERROR_MALFORMED; + } + } + buffer->setRange(buffer->offset(), buffer->size() - padding); return OK; } -void PlaylistFetcher::postMonitorQueue(int64_t delayUs) { +void PlaylistFetcher::postMonitorQueue(int64_t delayUs, int64_t minDelayUs) { + int64_t maxDelayUs = delayUsToRefreshPlaylist(); + if (maxDelayUs < minDelayUs) { + maxDelayUs = minDelayUs; + } + if (delayUs > maxDelayUs) { + ALOGV("Need to refresh playlist in %lld", maxDelayUs); + delayUs = maxDelayUs; + } sp<AMessage> msg = new AMessage(kWhatMonitorQueue, id()); msg->setInt32("generation", mMonitorQueueGeneration); msg->post(delayUs); @@ -288,7 +331,9 @@ void PlaylistFetcher::startAsync( const sp<AnotherPacketSource> &audioSource, const sp<AnotherPacketSource> &videoSource, const sp<AnotherPacketSource> &subtitleSource, - int64_t startTimeUs) { + int64_t startTimeUs, + int64_t minStartTimeUs, + int32_t startSeqNumberHint) { sp<AMessage> msg = new AMessage(kWhatStart, id()); uint32_t streamTypeMask = 0ul; @@ -310,6 +355,8 @@ void PlaylistFetcher::startAsync( msg->setInt32("streamTypeMask", streamTypeMask); msg->setInt64("startTimeUs", startTimeUs); + msg->setInt64("minStartTimeUs", minStartTimeUs); + msg->setInt32("startSeqNumberHint", startSeqNumberHint); msg->post(); } @@ -317,8 +364,16 @@ void PlaylistFetcher::pauseAsync() { (new AMessage(kWhatPause, id()))->post(); } -void PlaylistFetcher::stopAsync() { - (new AMessage(kWhatStop, id()))->post(); +void PlaylistFetcher::stopAsync(bool selfTriggered) { + sp<AMessage> msg = new AMessage(kWhatStop, id()); + msg->setInt32("selfTriggered", selfTriggered); + msg->post(); +} + +void PlaylistFetcher::resumeUntilAsync(const sp<AMessage> ¶ms) { + AMessage* msg = new AMessage(kWhatResumeUntil, id()); + msg->setMessage("params", params); + msg->post(); } void PlaylistFetcher::onMessageReceived(const sp<AMessage> &msg) { @@ -346,7 +401,7 @@ void PlaylistFetcher::onMessageReceived(const sp<AMessage> &msg) { case kWhatStop: { - onStop(); + onStop(msg); sp<AMessage> notify = mNotify->dup(); notify->setInt32("what", kWhatStopped); @@ -355,6 +410,7 @@ void PlaylistFetcher::onMessageReceived(const sp<AMessage> &msg) { } case kWhatMonitorQueue: + case kWhatDownloadNext: { int32_t generation; CHECK(msg->findInt32("generation", &generation)); @@ -364,7 +420,17 @@ void PlaylistFetcher::onMessageReceived(const sp<AMessage> &msg) { break; } - onMonitorQueue(); + if (msg->what() == kWhatMonitorQueue) { + onMonitorQueue(); + } else { + onDownloadNext(); + } + break; + } + + case kWhatResumeUntil: + { + onResumeUntil(msg); break; } @@ -380,7 +446,10 @@ status_t PlaylistFetcher::onStart(const sp<AMessage> &msg) { CHECK(msg->findInt32("streamTypeMask", (int32_t *)&streamTypeMask)); int64_t startTimeUs; + int32_t startSeqNumberHint; CHECK(msg->findInt64("startTimeUs", &startTimeUs)); + CHECK(msg->findInt64("minStartTimeUs", (int64_t *) &mMinStartTimeUs)); + CHECK(msg->findInt32("startSeqNumberHint", &startSeqNumberHint)); if (streamTypeMask & LiveSession::STREAMTYPE_AUDIO) { void *ptr; @@ -415,6 +484,11 @@ status_t PlaylistFetcher::onStart(const sp<AMessage> &msg) { if (mStartTimeUs >= 0ll) { mSeqNumber = -1; mStartup = true; + mPrepared = false; + } + + if (startSeqNumberHint >= 0) { + mSeqNumber = startSeqNumberHint; } postMonitorQueue(); @@ -424,22 +498,83 @@ status_t PlaylistFetcher::onStart(const sp<AMessage> &msg) { void PlaylistFetcher::onPause() { cancelMonitorQueue(); - - mPacketSources.clear(); - mStreamTypeMask = 0; } -void PlaylistFetcher::onStop() { +void PlaylistFetcher::onStop(const sp<AMessage> &msg) { cancelMonitorQueue(); - for (size_t i = 0; i < mPacketSources.size(); ++i) { - mPacketSources.valueAt(i)->clear(); + int32_t selfTriggered; + CHECK(msg->findInt32("selfTriggered", &selfTriggered)); + if (!selfTriggered) { + // Self triggered stops only happen during switching, in which case we do not want + // to clear the discontinuities queued at the end of packet sources. + for (size_t i = 0; i < mPacketSources.size(); i++) { + sp<AnotherPacketSource> packetSource = mPacketSources.valueAt(i); + packetSource->clear(); + } } mPacketSources.clear(); mStreamTypeMask = 0; } +// Resume until we have reached the boundary timestamps listed in `msg`; when +// the remaining time is too short (within a resume threshold) stop immediately +// instead. +status_t PlaylistFetcher::onResumeUntil(const sp<AMessage> &msg) { + sp<AMessage> params; + CHECK(msg->findMessage("params", ¶ms)); + + bool stop = false; + for (size_t i = 0; i < mPacketSources.size(); i++) { + sp<AnotherPacketSource> packetSource = mPacketSources.valueAt(i); + + const char *stopKey; + int streamType = mPacketSources.keyAt(i); + switch (streamType) { + case LiveSession::STREAMTYPE_VIDEO: + stopKey = "timeUsVideo"; + break; + + case LiveSession::STREAMTYPE_AUDIO: + stopKey = "timeUsAudio"; + break; + + case LiveSession::STREAMTYPE_SUBTITLES: + stopKey = "timeUsSubtitle"; + break; + + default: + TRESPASS(); + } + + // Don't resume if we would stop within a resume threshold. + int64_t latestTimeUs = 0, stopTimeUs = 0; + sp<AMessage> latestMeta = packetSource->getLatestMeta(); + if (latestMeta != NULL + && (latestMeta->findInt64("timeUs", &latestTimeUs) + && params->findInt64(stopKey, &stopTimeUs))) { + int64_t diffUs = stopTimeUs - latestTimeUs; + if (diffUs < resumeThreshold(latestMeta)) { + stop = true; + } + } + } + + if (stop) { + for (size_t i = 0; i < mPacketSources.size(); i++) { + mPacketSources.valueAt(i)->queueAccessUnit(mSession->createFormatChangeBuffer()); + } + stopAsync(/* selfTriggered = */ true); + return OK; + } + + mStopParams = params; + postMonitorQueue(); + + return OK; +} + void PlaylistFetcher::notifyError(status_t err) { sp<AMessage> notify = mNotify->dup(); notify->setInt32("what", kWhatError); @@ -456,41 +591,70 @@ void PlaylistFetcher::queueDiscontinuity( void PlaylistFetcher::onMonitorQueue() { bool downloadMore = false; + refreshPlaylist(); - status_t finalResult; + int32_t targetDurationSecs; + int64_t targetDurationUs = kMinBufferedDurationUs; + if (mPlaylist != NULL) { + CHECK(mPlaylist->meta()->findInt32("target-duration", &targetDurationSecs)); + targetDurationUs = targetDurationSecs * 1000000ll; + } + + // buffer at least 3 times the target duration, or up to 10 seconds + int64_t durationToBufferUs = targetDurationUs * 3; + if (durationToBufferUs > kMinBufferedDurationUs) { + durationToBufferUs = kMinBufferedDurationUs; + } + + int64_t bufferedDurationUs = 0ll; + status_t finalResult = NOT_ENOUGH_DATA; if (mStreamTypeMask == LiveSession::STREAMTYPE_SUBTITLES) { sp<AnotherPacketSource> packetSource = mPacketSources.valueFor(LiveSession::STREAMTYPE_SUBTITLES); - int64_t bufferedDurationUs = + bufferedDurationUs = packetSource->getBufferedDurationUs(&finalResult); - - downloadMore = (bufferedDurationUs < kMinBufferedDurationUs); finalResult = OK; } else { - bool first = true; - int64_t minBufferedDurationUs = 0ll; - + // Use max stream duration to prevent us from waiting on a non-existent stream; + // when we cannot make out from the manifest what streams are included in a playlist + // we might assume extra streams. for (size_t i = 0; i < mPacketSources.size(); ++i) { if ((mStreamTypeMask & mPacketSources.keyAt(i)) == 0) { continue; } - int64_t bufferedDurationUs = + int64_t bufferedStreamDurationUs = mPacketSources.valueAt(i)->getBufferedDurationUs(&finalResult); - - if (first || bufferedDurationUs < minBufferedDurationUs) { - minBufferedDurationUs = bufferedDurationUs; - first = false; + ALOGV("buffered %lld for stream %d", + bufferedStreamDurationUs, mPacketSources.keyAt(i)); + if (bufferedStreamDurationUs > bufferedDurationUs) { + bufferedDurationUs = bufferedStreamDurationUs; } } + } + downloadMore = (bufferedDurationUs < durationToBufferUs); + + // signal start if buffered up at least the target size + if (!mPrepared && bufferedDurationUs > targetDurationUs && downloadMore) { + mPrepared = true; - downloadMore = - !first && (minBufferedDurationUs < kMinBufferedDurationUs); + ALOGV("prepared, buffered=%lld > %lld", + bufferedDurationUs, targetDurationUs); + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("what", kWhatTemporarilyDoneFetching); + msg->post(); } if (finalResult == OK && downloadMore) { - onDownloadNext(); + ALOGV("monitoring, buffered=%lld < %lld", + bufferedDurationUs, durationToBufferUs); + // delay the next download slightly; hopefully this gives other concurrent fetchers + // a better chance to run. + // onDownloadNext(); + sp<AMessage> msg = new AMessage(kWhatDownloadNext, id()); + msg->setInt32("generation", mMonitorQueueGeneration); + msg->post(1000l); } else { // Nothing to do yet, try again in a second. @@ -498,15 +662,17 @@ void PlaylistFetcher::onMonitorQueue() { msg->setInt32("what", kWhatTemporarilyDoneFetching); msg->post(); - postMonitorQueue(1000000ll); + int64_t delayUs = mPrepared ? kMaxMonitorDelayUs : targetDurationUs / 2; + ALOGV("pausing for %lld, buffered=%lld > %lld", + delayUs, bufferedDurationUs, durationToBufferUs); + // :TRICKY: need to enforce minimum delay because the delay to + // refresh the playlist will become 0 + postMonitorQueue(delayUs, mPrepared ? targetDurationUs * 2 : 0); } } -void PlaylistFetcher::onDownloadNext() { - int64_t nowUs = ALooper::GetNowUs(); - - if (mLastPlaylistFetchTimeUs < 0ll - || (!mPlaylist->isComplete() && timeToRefreshPlaylist(nowUs))) { +status_t PlaylistFetcher::refreshPlaylist() { + if (delayUsToRefreshPlaylist() <= 0) { bool unchanged; sp<M3UParser> playlist = mSession->fetchPlaylist( mURI.c_str(), mPlaylistHash, &unchanged); @@ -522,7 +688,7 @@ void PlaylistFetcher::onDownloadNext() { } else { ALOGE("failed to load playlist at url '%s'", mURI.c_str()); notifyError(ERROR_IO); - return; + return ERROR_IO; } } else { mRefreshState = INITIAL_MINIMUM_RELOAD_DELAY; @@ -535,6 +701,13 @@ void PlaylistFetcher::onDownloadNext() { mLastPlaylistFetchTimeUs = ALooper::GetNowUs(); } + return OK; +} + +void PlaylistFetcher::onDownloadNext() { + if (refreshPlaylist() != OK) { + return; + } int32_t firstSeqNumberInPlaylist; if (mPlaylist->meta() == NULL || !mPlaylist->meta()->findInt32( @@ -548,17 +721,29 @@ void PlaylistFetcher::onDownloadNext() { const int32_t lastSeqNumberInPlaylist = firstSeqNumberInPlaylist + (int32_t)mPlaylist->size() - 1; + if (mStartup && mSeqNumber >= 0 + && (mSeqNumber < firstSeqNumberInPlaylist || mSeqNumber > lastSeqNumberInPlaylist)) { + // in case we guessed wrong during reconfiguration, try fetching the latest content. + mSeqNumber = lastSeqNumberInPlaylist; + } + if (mSeqNumber < 0) { CHECK_GE(mStartTimeUs, 0ll); if (mPlaylist->isComplete() || mPlaylist->isEvent()) { mSeqNumber = getSeqNumberForTime(mStartTimeUs); + ALOGV("Initial sequence number for time %lld is %ld from (%ld .. %ld)", + mStartTimeUs, mSeqNumber, firstSeqNumberInPlaylist, + lastSeqNumberInPlaylist); } else { // If this is a live session, start 3 segments from the end. mSeqNumber = lastSeqNumberInPlaylist - 3; if (mSeqNumber < firstSeqNumberInPlaylist) { mSeqNumber = firstSeqNumberInPlaylist; } + ALOGV("Initial sequence number for live event %ld from (%ld .. %ld)", + mSeqNumber, firstSeqNumberInPlaylist, + lastSeqNumberInPlaylist); } mStartTimeUs = -1ll; @@ -570,16 +755,34 @@ void PlaylistFetcher::onDownloadNext() { ++mNumRetries; if (mSeqNumber > lastSeqNumberInPlaylist) { - mLastPlaylistFetchTimeUs = -1; - postMonitorQueue(3000000ll); + // refresh in increasing fraction (1/2, 1/3, ...) of the + // playlist's target duration or 3 seconds, whichever is less + int32_t targetDurationSecs; + CHECK(mPlaylist->meta()->findInt32( + "target-duration", &targetDurationSecs)); + int64_t delayUs = mPlaylist->size() * targetDurationSecs * + 1000000ll / (1 + mNumRetries); + if (delayUs > kMaxMonitorDelayUs) { + delayUs = kMaxMonitorDelayUs; + } + ALOGV("sequence number high: %ld from (%ld .. %ld), monitor in %lld (retry=%d)", + mSeqNumber, firstSeqNumberInPlaylist, + lastSeqNumberInPlaylist, delayUs, mNumRetries); + postMonitorQueue(delayUs); return; } // we've missed the boat, let's start from the lowest sequence // number available and signal a discontinuity. - ALOGI("We've missed the boat, restarting playback."); - mSeqNumber = lastSeqNumberInPlaylist; + ALOGI("We've missed the boat, restarting playback." + " mStartup=%d, was looking for %d in %d-%d", + mStartup, mSeqNumber, firstSeqNumberInPlaylist, + lastSeqNumberInPlaylist); + mSeqNumber = lastSeqNumberInPlaylist - 3; + if (mSeqNumber < firstSeqNumberInPlaylist) { + mSeqNumber = firstSeqNumberInPlaylist; + } explicitDiscontinuity = true; // fall through @@ -633,6 +836,9 @@ void PlaylistFetcher::onDownloadNext() { CHECK(buffer != NULL); err = decryptBuffer(mSeqNumber - firstSeqNumberInPlaylist, buffer); + if (err == OK) { + err = checkDecryptPadding(buffer); + } if (err != OK) { ALOGE("decryptBuffer failed w/ error %d", err); @@ -665,6 +871,18 @@ void PlaylistFetcher::onDownloadNext() { err = extractAndQueueAccessUnits(buffer, itemMeta); + if (err == -EAGAIN) { + // bad starting sequence number hint + postMonitorQueue(); + return; + } + + if (err == ERROR_OUT_OF_RANGE) { + // reached stopping point + stopAsync(/* selfTriggered = */ true); + return; + } + if (err != OK) { notifyError(err); return; @@ -721,12 +939,15 @@ status_t PlaylistFetcher::extractAndQueueAccessUnits( } if (mTSParser == NULL) { - mTSParser = new ATSParser; + // Use TS_TIMESTAMPS_ARE_ABSOLUTE so pts carry over between fetchers. + mTSParser = new ATSParser(ATSParser::TS_TIMESTAMPS_ARE_ABSOLUTE); } if (mNextPTSTimeUs >= 0ll) { sp<AMessage> extra = new AMessage; - extra->setInt64(IStreamListener::kKeyMediaTimeUs, mNextPTSTimeUs); + // Since we are using absolute timestamps, signal an offset of 0 to prevent + // ATSParser from skewing the timestamps of access units. + extra->setInt64(IStreamListener::kKeyMediaTimeUs, 0); mTSParser->signalDiscontinuity( ATSParser::DISCONTINUITY_SEEK, extra); @@ -745,17 +966,23 @@ status_t PlaylistFetcher::extractAndQueueAccessUnits( offset += 188; } + status_t err = OK; for (size_t i = mPacketSources.size(); i-- > 0;) { sp<AnotherPacketSource> packetSource = mPacketSources.valueAt(i); + const char *key; ATSParser::SourceType type; - switch (mPacketSources.keyAt(i)) { + const LiveSession::StreamType stream = mPacketSources.keyAt(i); + switch (stream) { + case LiveSession::STREAMTYPE_VIDEO: type = ATSParser::VIDEO; + key = "timeUsVideo"; break; case LiveSession::STREAMTYPE_AUDIO: type = ATSParser::AUDIO; + key = "timeUsAudio"; break; case LiveSession::STREAMTYPE_SUBTITLES: @@ -782,18 +1009,87 @@ status_t PlaylistFetcher::extractAndQueueAccessUnits( continue; } + int64_t timeUs; sp<ABuffer> accessUnit; status_t finalResult; while (source->hasBufferAvailable(&finalResult) && source->dequeueAccessUnit(&accessUnit) == OK) { - // Note that we do NOT dequeue any discontinuities. + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + if (mMinStartTimeUs > 0) { + if (timeUs < mMinStartTimeUs) { + // TODO untested path + // try a later ts + int32_t targetDuration; + mPlaylist->meta()->findInt32("target-duration", &targetDuration); + int32_t incr = (mMinStartTimeUs - timeUs) / 1000000 / targetDuration; + if (incr == 0) { + // increment mSeqNumber by at least one + incr = 1; + } + mSeqNumber += incr; + err = -EAGAIN; + break; + } else { + int64_t startTimeUs; + if (mStartTimeUsNotify != NULL + && !mStartTimeUsNotify->findInt64(key, &startTimeUs)) { + mStartTimeUsNotify->setInt64(key, timeUs); + + uint32_t streamMask = 0; + mStartTimeUsNotify->findInt32("streamMask", (int32_t *) &streamMask); + streamMask |= mPacketSources.keyAt(i); + mStartTimeUsNotify->setInt32("streamMask", streamMask); + + if (streamMask == mStreamTypeMask) { + mStartTimeUsNotify->post(); + mStartTimeUsNotify.clear(); + } + } + } + } + + if (mStopParams != NULL) { + // Queue discontinuity in original stream. + int64_t stopTimeUs; + if (!mStopParams->findInt64(key, &stopTimeUs) || timeUs >= stopTimeUs) { + packetSource->queueAccessUnit(mSession->createFormatChangeBuffer()); + mStreamTypeMask &= ~stream; + mPacketSources.removeItemsAt(i); + break; + } + } + + // Note that we do NOT dequeue any discontinuities except for format change. + + // for simplicity, store a reference to the format in each unit + sp<MetaData> format = source->getFormat(); + if (format != NULL) { + accessUnit->meta()->setObject("format", format); + } + + // Stash the sequence number so we can hint future fetchers where to start at. + accessUnit->meta()->setInt32("seq", mSeqNumber); packetSource->queueAccessUnit(accessUnit); } - if (packetSource->getFormat() == NULL) { - packetSource->setFormat(source->getFormat()); + if (err != OK) { + break; + } + } + + if (err != OK) { + for (size_t i = mPacketSources.size(); i-- > 0;) { + sp<AnotherPacketSource> packetSource = mPacketSources.valueAt(i); + packetSource->clear(); } + return err; + } + + if (!mStreamTypeMask) { + // Signal gap is filled between original and new stream. + ALOGV("ERROR OUT OF RANGE"); + return ERROR_OUT_OF_RANGE; } return OK; @@ -810,6 +1106,7 @@ status_t PlaylistFetcher::extractAndQueueAccessUnits( CHECK(itemMeta->findInt64("durationUs", &durationUs)); buffer->meta()->setInt64("timeUs", getSegmentStartTimeUs(mSeqNumber)); buffer->meta()->setInt64("durationUs", durationUs); + buffer->meta()->setInt32("seq", mSeqNumber); packetSource->queueAccessUnit(buffer); return OK; @@ -946,6 +1243,7 @@ status_t PlaylistFetcher::extractAndQueueAccessUnits( // Each AAC frame encodes 1024 samples. numSamples += 1024; + unit->meta()->setInt32("seq", mSeqNumber); packetSource->queueAccessUnit(unit); offset += aac_frame_length; @@ -973,4 +1271,33 @@ void PlaylistFetcher::updateDuration() { msg->post(); } +int64_t PlaylistFetcher::resumeThreshold(const sp<AMessage> &msg) { + int64_t durationUs, threshold; + if (msg->findInt64("durationUs", &durationUs)) { + return kNumSkipFrames * durationUs; + } + + sp<RefBase> obj; + msg->findObject("format", &obj); + MetaData *format = static_cast<MetaData *>(obj.get()); + + const char *mime; + CHECK(format->findCString(kKeyMIMEType, &mime)); + bool audio = !strncasecmp(mime, "audio/", 6); + if (audio) { + // Assumes 1000 samples per frame. + int32_t sampleRate; + CHECK(format->findInt32(kKeySampleRate, &sampleRate)); + return kNumSkipFrames /* frames */ * 1000 /* samples */ + * (1000000 / sampleRate) /* sample duration (us) */; + } else { + int32_t frameRate; + if (format->findInt32(kKeyFrameRate, &frameRate) && frameRate > 0) { + return kNumSkipFrames * (1000000 / frameRate); + } + } + + return 500000ll; +} + } // namespace android diff --git a/media/libstagefright/httplive/PlaylistFetcher.h b/media/libstagefright/httplive/PlaylistFetcher.h index 1648e02..8404b8d 100644 --- a/media/libstagefright/httplive/PlaylistFetcher.h +++ b/media/libstagefright/httplive/PlaylistFetcher.h @@ -43,6 +43,7 @@ struct PlaylistFetcher : public AHandler { kWhatTemporarilyDoneFetching, kWhatPrepared, kWhatPreparationFailed, + kWhatStartedAt, }; PlaylistFetcher( @@ -56,11 +57,15 @@ struct PlaylistFetcher : public AHandler { const sp<AnotherPacketSource> &audioSource, const sp<AnotherPacketSource> &videoSource, const sp<AnotherPacketSource> &subtitleSource, - int64_t startTimeUs = -1ll); + int64_t startTimeUs = -1ll, + int64_t minStartTimeUs = 0ll /* start after this timestamp */, + int32_t startSeqNumberHint = -1 /* try starting at this sequence number */); void pauseAsync(); - void stopAsync(); + void stopAsync(bool selfTriggered = false); + + void resumeUntilAsync(const sp<AMessage> ¶ms); protected: virtual ~PlaylistFetcher(); @@ -76,16 +81,25 @@ private: kWhatPause = 'paus', kWhatStop = 'stop', kWhatMonitorQueue = 'moni', + kWhatResumeUntil = 'rsme', + kWhatDownloadNext = 'dlnx', }; static const int64_t kMinBufferedDurationUs; + static const int64_t kMaxMonitorDelayUs; + static const int32_t kNumSkipFrames; + // notifications to mSession sp<AMessage> mNotify; + sp<AMessage> mStartTimeUsNotify; + sp<LiveSession> mSession; AString mURI; uint32_t mStreamTypeMask; int64_t mStartTimeUs; + int64_t mMinStartTimeUs; // start fetching no earlier than this value + sp<AMessage> mStopParams; // message containing the latest timestamps we should fetch. KeyedVector<LiveSession::StreamType, sp<AnotherPacketSource> > mPacketSources; @@ -97,6 +111,7 @@ private: int32_t mSeqNumber; int32_t mNumRetries; bool mStartup; + bool mPrepared; int64_t mNextPTSTimeUs; int32_t mMonitorQueueGeneration; @@ -117,13 +132,29 @@ private: uint64_t mFirstPTS; int64_t mAbsoluteTimeAnchorUs; + // Stores the initialization vector to decrypt the next block of cipher text, which can + // either be derived from the sequence number, read from the manifest, or copied from + // the last block of cipher text (cipher-block chaining). + unsigned char mAESInitVec[16]; + + // Set first to true if decrypting the first segment of a playlist segment. When + // first is true, reset the initialization vector based on the available + // information in the manifest; otherwise, use the initialization vector as + // updated by the last call to AES_cbc_encrypt. + // + // For the input to decrypt correctly, decryptBuffer must be called on + // consecutive byte ranges on block boundaries, e.g. 0..15, 16..47, 48..63, + // and so on. status_t decryptBuffer( - size_t playlistIndex, const sp<ABuffer> &buffer); + size_t playlistIndex, const sp<ABuffer> &buffer, + bool first = true); + status_t checkDecryptPadding(const sp<ABuffer> &buffer); - void postMonitorQueue(int64_t delayUs = 0); + void postMonitorQueue(int64_t delayUs = 0, int64_t minDelayUs = 0); void cancelMonitorQueue(); - bool timeToRefreshPlaylist(int64_t nowUs) const; + int64_t delayUsToRefreshPlaylist() const; + status_t refreshPlaylist(); // Returns the media time in us of the segment specified by seqNumber. // This is computed by summing the durations of all segments before it. @@ -131,10 +162,13 @@ private: status_t onStart(const sp<AMessage> &msg); void onPause(); - void onStop(); + void onStop(const sp<AMessage> &msg); void onMonitorQueue(); void onDownloadNext(); + // Resume a fetcher to continue until the stopping point stored in msg. + status_t onResumeUntil(const sp<AMessage> &msg); + status_t extractAndQueueAccessUnits( const sp<ABuffer> &buffer, const sp<AMessage> &itemMeta); @@ -147,6 +181,10 @@ private: void updateDuration(); + // Before resuming a fetcher in onResume, check the remaining duration is longer than that + // returned by resumeThreshold. + int64_t resumeThreshold(const sp<AMessage> &msg); + DISALLOW_EVIL_CONSTRUCTORS(PlaylistFetcher); }; diff --git a/media/libstagefright/matroska/MatroskaExtractor.cpp b/media/libstagefright/matroska/MatroskaExtractor.cpp index d260d0f..dcb1cda 100644 --- a/media/libstagefright/matroska/MatroskaExtractor.cpp +++ b/media/libstagefright/matroska/MatroskaExtractor.cpp @@ -716,41 +716,61 @@ bool MatroskaExtractor::isLiveStreaming() const { return mIsLiveStreaming; } +static int bytesForSize(size_t size) { + // use at most 28 bits (4 times 7) + CHECK(size <= 0xfffffff); + + if (size > 0x1fffff) { + return 4; + } else if (size > 0x3fff) { + return 3; + } else if (size > 0x7f) { + return 2; + } + return 1; +} + +static void storeSize(uint8_t *data, size_t &idx, size_t size) { + int numBytes = bytesForSize(size); + idx += numBytes; + + data += idx; + size_t next = 0; + while (numBytes--) { + *--data = (size & 0x7f) | next; + size >>= 7; + next = 0x80; + } +} + static void addESDSFromCodecPrivate( const sp<MetaData> &meta, bool isAudio, const void *priv, size_t privSize) { - static const uint8_t kStaticESDS[] = { - 0x03, 22, - 0x00, 0x00, // ES_ID - 0x00, // streamDependenceFlag, URL_Flag, OCRstreamFlag - - 0x04, 17, - 0x40, // ObjectTypeIndication - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - - 0x05, - // CodecSpecificInfo (with size prefix) follows - }; - // Make sure all sizes can be coded in a single byte. - CHECK(privSize + 22 - 2 < 128); - size_t esdsSize = sizeof(kStaticESDS) + privSize + 1; + int privSizeBytesRequired = bytesForSize(privSize); + int esdsSize2 = 14 + privSizeBytesRequired + privSize; + int esdsSize2BytesRequired = bytesForSize(esdsSize2); + int esdsSize1 = 4 + esdsSize2BytesRequired + esdsSize2; + int esdsSize1BytesRequired = bytesForSize(esdsSize1); + size_t esdsSize = 1 + esdsSize1BytesRequired + esdsSize1; uint8_t *esds = new uint8_t[esdsSize]; - memcpy(esds, kStaticESDS, sizeof(kStaticESDS)); - uint8_t *ptr = esds + sizeof(kStaticESDS); - *ptr++ = privSize; - memcpy(ptr, priv, privSize); - - // Increment by codecPrivateSize less 2 bytes that are accounted for - // already in lengths of 22/17 - esds[1] += privSize - 2; - esds[6] += privSize - 2; - - // Set ObjectTypeIndication. - esds[7] = isAudio ? 0x40 // Audio ISO/IEC 14496-3 - : 0x20; // Visual ISO/IEC 14496-2 + + size_t idx = 0; + esds[idx++] = 0x03; + storeSize(esds, idx, esdsSize1); + esds[idx++] = 0x00; // ES_ID + esds[idx++] = 0x00; // ES_ID + esds[idx++] = 0x00; // streamDependenceFlag, URL_Flag, OCRstreamFlag + esds[idx++] = 0x04; + storeSize(esds, idx, esdsSize2); + esds[idx++] = isAudio ? 0x40 // Audio ISO/IEC 14496-3 + : 0x20; // Visual ISO/IEC 14496-2 + for (int i = 0; i < 12; i++) { + esds[idx++] = 0x00; + } + esds[idx++] = 0x05; + storeSize(esds, idx, privSize); + memcpy(esds + idx, priv, privSize); meta->setData(kKeyESDS, 0, esds, esdsSize); diff --git a/media/libstagefright/mpeg2ts/ATSParser.cpp b/media/libstagefright/mpeg2ts/ATSParser.cpp index 175a263..cb57a2f 100644 --- a/media/libstagefright/mpeg2ts/ATSParser.cpp +++ b/media/libstagefright/mpeg2ts/ATSParser.cpp @@ -506,6 +506,11 @@ ATSParser::Stream::Stream( ElementaryStreamQueue::PCM_AUDIO); break; + case STREAMTYPE_AC3: + mQueue = new ElementaryStreamQueue( + ElementaryStreamQueue::AC3); + break; + default: break; } @@ -614,6 +619,7 @@ bool ATSParser::Stream::isAudio() const { case STREAMTYPE_MPEG2_AUDIO: case STREAMTYPE_MPEG2_AUDIO_ADTS: case STREAMTYPE_PCM_AUDIO: + case STREAMTYPE_AC3: return true; default: diff --git a/media/libstagefright/mpeg2ts/ATSParser.h b/media/libstagefright/mpeg2ts/ATSParser.h index a10edc9..d4e30b4 100644 --- a/media/libstagefright/mpeg2ts/ATSParser.h +++ b/media/libstagefright/mpeg2ts/ATSParser.h @@ -88,6 +88,10 @@ struct ATSParser : public RefBase { STREAMTYPE_MPEG2_AUDIO_ADTS = 0x0f, STREAMTYPE_MPEG4_VIDEO = 0x10, STREAMTYPE_H264 = 0x1b, + + // From ATSC A/53 Part 3:2009, 6.7.1 + STREAMTYPE_AC3 = 0x81, + STREAMTYPE_PCM_AUDIO = 0x83, }; diff --git a/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp b/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp index 3153c8b..2b0bf30 100644 --- a/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp +++ b/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp @@ -34,7 +34,8 @@ AnotherPacketSource::AnotherPacketSource(const sp<MetaData> &meta) : mIsAudio(false), mFormat(NULL), mLastQueuedTimeUs(0), - mEOSResult(OK) { + mEOSResult(OK), + mLatestEnqueuedMeta(NULL) { setFormat(meta); } @@ -70,7 +71,27 @@ status_t AnotherPacketSource::stop() { } sp<MetaData> AnotherPacketSource::getFormat() { - return mFormat; + Mutex::Autolock autoLock(mLock); + if (mFormat != NULL) { + return mFormat; + } + + List<sp<ABuffer> >::iterator it = mBuffers.begin(); + while (it != mBuffers.end()) { + sp<ABuffer> buffer = *it; + int32_t discontinuity; + if (buffer->meta()->findInt32("discontinuity", &discontinuity)) { + break; + } + + sp<RefBase> object; + if (buffer->meta()->findObject("format", &object)) { + return static_cast<MetaData*>(object.get()); + } + + ++it; + } + return NULL; } status_t AnotherPacketSource::dequeueAccessUnit(sp<ABuffer> *buffer) { @@ -94,6 +115,11 @@ status_t AnotherPacketSource::dequeueAccessUnit(sp<ABuffer> *buffer) { return INFO_DISCONTINUITY; } + sp<RefBase> object; + if ((*buffer)->meta()->findObject("format", &object)) { + mFormat = static_cast<MetaData*>(object.get()); + } + return OK; } @@ -120,17 +146,22 @@ status_t AnotherPacketSource::read( } return INFO_DISCONTINUITY; - } else { - int64_t timeUs; - CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); + } + + sp<RefBase> object; + if (buffer->meta()->findObject("format", &object)) { + mFormat = static_cast<MetaData*>(object.get()); + } - MediaBuffer *mediaBuffer = new MediaBuffer(buffer); + int64_t timeUs; + CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); - mediaBuffer->meta_data()->setInt64(kKeyTime, timeUs); + MediaBuffer *mediaBuffer = new MediaBuffer(buffer); - *out = mediaBuffer; - return OK; - } + mediaBuffer->meta_data()->setInt64(kKeyTime, timeUs); + + *out = mediaBuffer; + return OK; } return mEOSResult; @@ -152,12 +183,24 @@ void AnotherPacketSource::queueAccessUnit(const sp<ABuffer> &buffer) { return; } - CHECK(buffer->meta()->findInt64("timeUs", &mLastQueuedTimeUs)); + int64_t lastQueuedTimeUs; + CHECK(buffer->meta()->findInt64("timeUs", &lastQueuedTimeUs)); + mLastQueuedTimeUs = lastQueuedTimeUs; ALOGV("queueAccessUnit timeUs=%lld us (%.2f secs)", mLastQueuedTimeUs, mLastQueuedTimeUs / 1E6); Mutex::Autolock autoLock(mLock); mBuffers.push_back(buffer); mCondition.signal(); + + if (!mLatestEnqueuedMeta.get()) { + mLatestEnqueuedMeta = buffer->meta(); + } else { + int64_t latestTimeUs = 0; + CHECK(mLatestEnqueuedMeta->findInt64("timeUs", &latestTimeUs)); + if (lastQueuedTimeUs > latestTimeUs) { + mLatestEnqueuedMeta = buffer->meta(); + } + } } void AnotherPacketSource::clear() { @@ -167,6 +210,7 @@ void AnotherPacketSource::clear() { mEOSResult = OK; mFormat = NULL; + mLatestEnqueuedMeta = NULL; } void AnotherPacketSource::queueDiscontinuity( @@ -191,6 +235,7 @@ void AnotherPacketSource::queueDiscontinuity( mEOSResult = OK; mLastQueuedTimeUs = 0; + mLatestEnqueuedMeta = NULL; sp<ABuffer> buffer = new ABuffer(0); buffer->meta()->setInt32("discontinuity", static_cast<int32_t>(type)); @@ -278,4 +323,9 @@ bool AnotherPacketSource::isFinished(int64_t duration) const { return (mEOSResult != OK); } +sp<AMessage> AnotherPacketSource::getLatestMeta() { + Mutex::Autolock autoLock(mLock); + return mLatestEnqueuedMeta; +} + } // namespace android diff --git a/media/libstagefright/mpeg2ts/AnotherPacketSource.h b/media/libstagefright/mpeg2ts/AnotherPacketSource.h index e16cf78..9b193a2 100644 --- a/media/libstagefright/mpeg2ts/AnotherPacketSource.h +++ b/media/libstagefright/mpeg2ts/AnotherPacketSource.h @@ -62,6 +62,8 @@ struct AnotherPacketSource : public MediaSource { bool isFinished(int64_t duration) const; + sp<AMessage> getLatestMeta(); + protected: virtual ~AnotherPacketSource(); @@ -74,6 +76,7 @@ private: int64_t mLastQueuedTimeUs; List<sp<ABuffer> > mBuffers; status_t mEOSResult; + sp<AMessage> mLatestEnqueuedMeta; bool wasFormatChange(int32_t discontinuityType) const; diff --git a/media/libstagefright/mpeg2ts/ESQueue.cpp b/media/libstagefright/mpeg2ts/ESQueue.cpp index e0ff0d1..c0c9717 100644 --- a/media/libstagefright/mpeg2ts/ESQueue.cpp +++ b/media/libstagefright/mpeg2ts/ESQueue.cpp @@ -56,6 +56,122 @@ void ElementaryStreamQueue::clear(bool clearFormat) { } } +// Parse AC3 header assuming the current ptr is start position of syncframe, +// update metadata only applicable, and return the payload size +static unsigned parseAC3SyncFrame( + const uint8_t *ptr, size_t size, sp<MetaData> *metaData) { + static const unsigned channelCountTable[] = {2, 1, 2, 3, 3, 4, 4, 5}; + static const unsigned samplingRateTable[] = {48000, 44100, 32000}; + static const unsigned rates[] = {32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, + 320, 384, 448, 512, 576, 640}; + + static const unsigned frameSizeTable[19][3] = { + { 64, 69, 96 }, + { 80, 87, 120 }, + { 96, 104, 144 }, + { 112, 121, 168 }, + { 128, 139, 192 }, + { 160, 174, 240 }, + { 192, 208, 288 }, + { 224, 243, 336 }, + { 256, 278, 384 }, + { 320, 348, 480 }, + { 384, 417, 576 }, + { 448, 487, 672 }, + { 512, 557, 768 }, + { 640, 696, 960 }, + { 768, 835, 1152 }, + { 896, 975, 1344 }, + { 1024, 1114, 1536 }, + { 1152, 1253, 1728 }, + { 1280, 1393, 1920 }, + }; + + ABitReader bits(ptr, size); + unsigned syncStartPos = 0; // in bytes + if (bits.numBitsLeft() < 16) { + return 0; + } + if (bits.getBits(16) != 0x0B77) { + return 0; + } + + if (bits.numBitsLeft() < 16 + 2 + 6 + 5 + 3 + 3) { + ALOGV("Not enough bits left for further parsing"); + return 0; + } + bits.skipBits(16); // crc1 + + unsigned fscod = bits.getBits(2); + if (fscod == 3) { + ALOGW("Incorrect fscod in AC3 header"); + return 0; + } + + unsigned frmsizecod = bits.getBits(6); + if (frmsizecod > 37) { + ALOGW("Incorrect frmsizecod in AC3 header"); + return 0; + } + + unsigned bsid = bits.getBits(5); + if (bsid > 8) { + ALOGW("Incorrect bsid in AC3 header. Possibly E-AC-3?"); + return 0; + } + + unsigned bsmod = bits.getBits(3); + unsigned acmod = bits.getBits(3); + unsigned cmixlev = 0; + unsigned surmixlev = 0; + unsigned dsurmod = 0; + + if ((acmod & 1) > 0 && acmod != 1) { + if (bits.numBitsLeft() < 2) { + return 0; + } + cmixlev = bits.getBits(2); + } + if ((acmod & 4) > 0) { + if (bits.numBitsLeft() < 2) { + return 0; + } + surmixlev = bits.getBits(2); + } + if (acmod == 2) { + if (bits.numBitsLeft() < 2) { + return 0; + } + dsurmod = bits.getBits(2); + } + + if (bits.numBitsLeft() < 1) { + return 0; + } + unsigned lfeon = bits.getBits(1); + + unsigned samplingRate = samplingRateTable[fscod]; + unsigned payloadSize = frameSizeTable[frmsizecod >> 1][fscod]; + if (fscod == 1) { + payloadSize += frmsizecod & 1; + } + payloadSize <<= 1; // convert from 16-bit words to bytes + + unsigned channelCount = channelCountTable[acmod] + lfeon; + + if (metaData != NULL) { + (*metaData)->setCString(kKeyMIMEType, MEDIA_MIMETYPE_AUDIO_AC3); + (*metaData)->setInt32(kKeyChannelCount, channelCount); + (*metaData)->setInt32(kKeySampleRate, samplingRate); + } + + return payloadSize; +} + +static bool IsSeeminglyValidAC3Header(const uint8_t *ptr, size_t size) { + return parseAC3SyncFrame(ptr, size, NULL) > 0; +} + static bool IsSeeminglyValidADTSHeader(const uint8_t *ptr, size_t size) { if (size < 3) { // Not enough data to verify header. @@ -224,6 +340,33 @@ status_t ElementaryStreamQueue::appendData( break; } + case AC3: + { + uint8_t *ptr = (uint8_t *)data; + + ssize_t startOffset = -1; + for (size_t i = 0; i < size; ++i) { + if (IsSeeminglyValidAC3Header(&ptr[i], size - i)) { + startOffset = i; + break; + } + } + + if (startOffset < 0) { + return ERROR_MALFORMED; + } + + if (startOffset > 0) { + ALOGI("found something resembling an AC3 syncword at " + "offset %d", + startOffset); + } + + data = &ptr[startOffset]; + size -= startOffset; + break; + } + case MPEG_AUDIO: { uint8_t *ptr = (uint8_t *)data; @@ -328,6 +471,8 @@ sp<ABuffer> ElementaryStreamQueue::dequeueAccessUnit() { return dequeueAccessUnitH264(); case AAC: return dequeueAccessUnitAAC(); + case AC3: + return dequeueAccessUnitAC3(); case MPEG_VIDEO: return dequeueAccessUnitMPEGVideo(); case MPEG4_VIDEO: @@ -340,6 +485,51 @@ sp<ABuffer> ElementaryStreamQueue::dequeueAccessUnit() { } } +sp<ABuffer> ElementaryStreamQueue::dequeueAccessUnitAC3() { + unsigned syncStartPos = 0; // in bytes + unsigned payloadSize = 0; + sp<MetaData> format = new MetaData; + while (true) { + if (syncStartPos + 2 >= mBuffer->size()) { + return NULL; + } + + payloadSize = parseAC3SyncFrame( + mBuffer->data() + syncStartPos, + mBuffer->size() - syncStartPos, + &format); + if (payloadSize > 0) { + break; + } + ++syncStartPos; + } + + if (mBuffer->size() < syncStartPos + payloadSize) { + ALOGV("Not enough buffer size for AC3"); + return NULL; + } + + if (mFormat == NULL) { + mFormat = format; + } + + sp<ABuffer> accessUnit = new ABuffer(syncStartPos + payloadSize); + memcpy(accessUnit->data(), mBuffer->data(), syncStartPos + payloadSize); + + int64_t timeUs = fetchTimestamp(syncStartPos + payloadSize); + CHECK_GE(timeUs, 0ll); + accessUnit->meta()->setInt64("timeUs", timeUs); + + memmove( + mBuffer->data(), + mBuffer->data() + syncStartPos + payloadSize, + mBuffer->size() - syncStartPos - payloadSize); + + mBuffer->setRange(0, mBuffer->size() - syncStartPos - payloadSize); + + return accessUnit; +} + sp<ABuffer> ElementaryStreamQueue::dequeueAccessUnitPCMAudio() { if (mBuffer->size() < 4) { return NULL; diff --git a/media/libstagefright/mpeg2ts/ESQueue.h b/media/libstagefright/mpeg2ts/ESQueue.h index 66a8087..a2cca77 100644 --- a/media/libstagefright/mpeg2ts/ESQueue.h +++ b/media/libstagefright/mpeg2ts/ESQueue.h @@ -32,6 +32,7 @@ struct ElementaryStreamQueue { enum Mode { H264, AAC, + AC3, MPEG_AUDIO, MPEG_VIDEO, MPEG4_VIDEO, @@ -67,6 +68,7 @@ private: sp<ABuffer> dequeueAccessUnitH264(); sp<ABuffer> dequeueAccessUnitAAC(); + sp<ABuffer> dequeueAccessUnitAC3(); sp<ABuffer> dequeueAccessUnitMPEGAudio(); sp<ABuffer> dequeueAccessUnitMPEGVideo(); sp<ABuffer> dequeueAccessUnitMPEG4Video(); diff --git a/media/libstagefright/omx/GraphicBufferSource.cpp b/media/libstagefright/omx/GraphicBufferSource.cpp index b8970ad..44f0be7 100644 --- a/media/libstagefright/omx/GraphicBufferSource.cpp +++ b/media/libstagefright/omx/GraphicBufferSource.cpp @@ -42,7 +42,11 @@ GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance, mEndOfStream(false), mEndOfStreamSent(false), mRepeatAfterUs(-1ll), + mMaxTimestampGapUs(-1ll), + mPrevOriginalTimeUs(-1ll), + mPrevModifiedTimeUs(-1ll), mRepeatLastFrameGeneration(0), + mRepeatLastFrameTimestamp(-1ll), mLatestSubmittedBufferId(-1), mLatestSubmittedBufferFrameNum(0), mLatestSubmittedBufferUseCount(0), @@ -299,6 +303,32 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) { return; } +void GraphicBufferSource::codecBufferFilled(OMX_BUFFERHEADERTYPE* header) { + Mutex::Autolock autoLock(mMutex); + + if (mMaxTimestampGapUs > 0ll + && !(header->nFlags & OMX_BUFFERFLAG_CODECCONFIG)) { + ssize_t index = mOriginalTimeUs.indexOfKey(header->nTimeStamp); + if (index >= 0) { + ALOGV("OUT timestamp: %lld -> %lld", + header->nTimeStamp, mOriginalTimeUs[index]); + header->nTimeStamp = mOriginalTimeUs[index]; + mOriginalTimeUs.removeItemsAt(index); + } else { + // giving up the effort as encoder doesn't appear to preserve pts + ALOGW("giving up limiting timestamp gap (pts = %lld)", + header->nTimeStamp); + mMaxTimestampGapUs = -1ll; + } + if (mOriginalTimeUs.size() > BufferQueue::NUM_BUFFER_SLOTS) { + // something terribly wrong must have happened, giving up... + ALOGE("mOriginalTimeUs has too many entries (%d)", + mOriginalTimeUs.size()); + mMaxTimestampGapUs = -1ll; + } + } +} + void GraphicBufferSource::suspend(bool suspend) { Mutex::Autolock autoLock(mMutex); @@ -431,6 +461,7 @@ bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() { BufferQueue::BufferItem item; item.mBuf = mLatestSubmittedBufferId; item.mFrameNumber = mLatestSubmittedBufferFrameNum; + item.mTimestamp = mRepeatLastFrameTimestamp; status_t err = submitBuffer_l(item, cbi); @@ -440,6 +471,20 @@ bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() { ++mLatestSubmittedBufferUseCount; + /* repeat last frame up to kRepeatLastFrameCount times. + * in case of static scene, a single repeat might not get rid of encoder + * ghosting completely, refresh a couple more times to get better quality + */ + if (--mRepeatLastFrameCount > 0) { + mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000; + + if (mReflector != NULL) { + sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id()); + msg->setInt32("generation", ++mRepeatLastFrameGeneration); + msg->post(mRepeatAfterUs); + } + } + return true; } @@ -460,8 +505,11 @@ void GraphicBufferSource::setLatestSubmittedBuffer_l( mLatestSubmittedBufferId = item.mBuf; mLatestSubmittedBufferFrameNum = item.mFrameNumber; + mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000; + mLatestSubmittedBufferUseCount = 1; mRepeatBufferDeferred = false; + mRepeatLastFrameCount = kRepeatLastFrameCount; if (mReflector != NULL) { sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id()); @@ -497,9 +545,48 @@ status_t GraphicBufferSource::signalEndOfInputStream() { return OK; } +int64_t GraphicBufferSource::getTimestamp(const BufferQueue::BufferItem &item) { + int64_t timeUs = item.mTimestamp / 1000; + + if (mMaxTimestampGapUs > 0ll) { + /* Cap timestamp gap between adjacent frames to specified max + * + * In the scenario of cast mirroring, encoding could be suspended for + * prolonged periods. Limiting the pts gap to workaround the problem + * where encoder's rate control logic produces huge frames after a + * long period of suspension. + */ + + int64_t originalTimeUs = timeUs; + if (mPrevOriginalTimeUs >= 0ll) { + if (originalTimeUs < mPrevOriginalTimeUs) { + // Drop the frame if it's going backward in time. Bad timestamp + // could disrupt encoder's rate control completely. + ALOGW("Dropping frame that's going backward in time"); + return -1; + } + int64_t timestampGapUs = originalTimeUs - mPrevOriginalTimeUs; + timeUs = (timestampGapUs < mMaxTimestampGapUs ? + timestampGapUs : mMaxTimestampGapUs) + mPrevModifiedTimeUs; + } + mPrevOriginalTimeUs = originalTimeUs; + mPrevModifiedTimeUs = timeUs; + mOriginalTimeUs.add(timeUs, originalTimeUs); + ALOGV("IN timestamp: %lld -> %lld", originalTimeUs, timeUs); + } + + return timeUs; +} + status_t GraphicBufferSource::submitBuffer_l( const BufferQueue::BufferItem &item, int cbi) { ALOGV("submitBuffer_l cbi=%d", cbi); + + int64_t timeUs = getTimestamp(item); + if (timeUs < 0ll) { + return UNKNOWN_ERROR; + } + CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi)); codecBuffer.mGraphicBuffer = mBufferSlot[item.mBuf]; codecBuffer.mBuf = item.mBuf; @@ -515,7 +602,7 @@ status_t GraphicBufferSource::submitBuffer_l( status_t err = mNodeInstance->emptyDirectBuffer(header, 0, 4 + sizeof(buffer_handle_t), OMX_BUFFERFLAG_ENDOFFRAME, - item.mTimestamp / 1000); + timeUs); if (err != OK) { ALOGW("WARNING: emptyDirectBuffer failed: 0x%x", err); codecBuffer.mGraphicBuffer = NULL; @@ -609,6 +696,12 @@ void GraphicBufferSource::onFrameAvailable() { BufferQueue::BufferItem item; status_t err = mBufferQueue->acquireBuffer(&item, 0); if (err == OK) { + // If this is the first time we're seeing this buffer, add it to our + // slot table. + if (item.mGraphicBuffer != NULL) { + ALOGV("fillCodecBuffer_l: setting mBufferSlot %d", item.mBuf); + mBufferSlot[item.mBuf] = item.mGraphicBuffer; + } mBufferQueue->releaseBuffer(item.mBuf, item.mFrameNumber, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, item.mFence); } @@ -658,6 +751,17 @@ status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs( return OK; } +status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) { + Mutex::Autolock autoLock(mMutex); + + if (mExecuting || maxGapUs <= 0ll) { + return INVALID_OPERATION; + } + + mMaxTimestampGapUs = maxGapUs; + + return OK; +} void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) { switch (msg->what()) { case kWhatRepeatLastFrame: diff --git a/media/libstagefright/omx/GraphicBufferSource.h b/media/libstagefright/omx/GraphicBufferSource.h index 9e5eee6..3b0e454 100644 --- a/media/libstagefright/omx/GraphicBufferSource.h +++ b/media/libstagefright/omx/GraphicBufferSource.h @@ -87,6 +87,10 @@ public: // fill it with a new frame of data; otherwise, just mark it as available. void codecBufferEmptied(OMX_BUFFERHEADERTYPE* header); + // Called when omx_message::FILL_BUFFER_DONE is received. (Currently the + // buffer source will fix timestamp in the header if needed.) + void codecBufferFilled(OMX_BUFFERHEADERTYPE* header); + // This is called after the last input frame has been submitted. We // need to submit an empty buffer with the EOS flag set. If we don't // have a codec buffer ready, we just set the mEndOfStream flag. @@ -105,6 +109,15 @@ public: // state and once this behaviour is specified it cannot be reset. status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs); + // When set, the timestamp fed to the encoder will be modified such that + // the gap between two adjacent frames is capped at maxGapUs. Timestamp + // will be restored to the original when the encoded frame is returned to + // the client. + // This is to solve a problem in certain real-time streaming case, where + // encoder's rate control logic produces huge frames after a long period + // of suspension on input. + status_t setMaxTimestampGapUs(int64_t maxGapUs); + protected: // BufferQueue::ConsumerListener interface, called when a new frame of // data is available. If we're executing and a codec buffer is @@ -165,6 +178,7 @@ private: void setLatestSubmittedBuffer_l(const BufferQueue::BufferItem &item); bool repeatLatestSubmittedBuffer_l(); + int64_t getTimestamp(const BufferQueue::BufferItem &item); // Lock, covers all member variables. mutable Mutex mMutex; @@ -206,13 +220,22 @@ private: enum { kWhatRepeatLastFrame, }; - + enum { + kRepeatLastFrameCount = 10, + }; int64_t mRepeatAfterUs; + int64_t mMaxTimestampGapUs; + + KeyedVector<int64_t, int64_t> mOriginalTimeUs; + int64_t mPrevOriginalTimeUs; + int64_t mPrevModifiedTimeUs; sp<ALooper> mLooper; sp<AHandlerReflector<GraphicBufferSource> > mReflector; int32_t mRepeatLastFrameGeneration; + int64_t mRepeatLastFrameTimestamp; + int32_t mRepeatLastFrameCount; int mLatestSubmittedBufferId; uint64_t mLatestSubmittedBufferFrameNum; diff --git a/media/libstagefright/omx/OMXNodeInstance.cpp b/media/libstagefright/omx/OMXNodeInstance.cpp index 5f104fc..6c5c857 100644 --- a/media/libstagefright/omx/OMXNodeInstance.cpp +++ b/media/libstagefright/omx/OMXNodeInstance.cpp @@ -849,6 +849,7 @@ status_t OMXNodeInstance::setInternalOption( switch (type) { case IOMX::INTERNAL_OPTION_SUSPEND: case IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY: + case IOMX::INTERNAL_OPTION_MAX_TIMESTAMP_GAP: { const sp<GraphicBufferSource> &bufferSource = getGraphicBufferSource(); @@ -864,7 +865,8 @@ status_t OMXNodeInstance::setInternalOption( bool suspend = *(bool *)data; bufferSource->suspend(suspend); - } else { + } else if (type == + IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY){ if (size != sizeof(int64_t)) { return INVALID_OPERATION; } @@ -872,6 +874,14 @@ status_t OMXNodeInstance::setInternalOption( int64_t delayUs = *(int64_t *)data; return bufferSource->setRepeatPreviousFrameDelayUs(delayUs); + } else { + if (size != sizeof(int64_t)) { + return INVALID_OPERATION; + } + + int64_t maxGapUs = *(int64_t *)data; + + return bufferSource->setMaxTimestampGapUs(maxGapUs); } return OK; @@ -883,6 +893,8 @@ status_t OMXNodeInstance::setInternalOption( } void OMXNodeInstance::onMessage(const omx_message &msg) { + const sp<GraphicBufferSource>& bufferSource(getGraphicBufferSource()); + if (msg.type == omx_message::FILL_BUFFER_DONE) { OMX_BUFFERHEADERTYPE *buffer = static_cast<OMX_BUFFERHEADERTYPE *>( @@ -892,10 +904,18 @@ void OMXNodeInstance::onMessage(const omx_message &msg) { static_cast<BufferMeta *>(buffer->pAppPrivate); buffer_meta->CopyFromOMX(buffer); - } else if (msg.type == omx_message::EMPTY_BUFFER_DONE) { - const sp<GraphicBufferSource>& bufferSource(getGraphicBufferSource()); if (bufferSource != NULL) { + // fix up the buffer info (especially timestamp) if needed + bufferSource->codecBufferFilled(buffer); + + omx_message newMsg = msg; + newMsg.u.extended_buffer_data.timestamp = buffer->nTimeStamp; + mObserver->onMessage(newMsg); + return; + } + } else if (msg.type == omx_message::EMPTY_BUFFER_DONE) { + if (bufferSource != NULL) { // This is one of the buffers used exclusively by // GraphicBufferSource. // Don't dispatch a message back to ACodec, since it doesn't diff --git a/media/libstagefright/wifi-display/source/TSPacketizer.cpp b/media/libstagefright/wifi-display/source/TSPacketizer.cpp index edcc087..50d317a 100644 --- a/media/libstagefright/wifi-display/source/TSPacketizer.cpp +++ b/media/libstagefright/wifi-display/source/TSPacketizer.cpp @@ -216,7 +216,7 @@ sp<ABuffer> TSPacketizer::Track::prependADTSHeader( uint8_t *ptr = dup->data(); *ptr++ = 0xff; - *ptr++ = 0xf1; // b11110001, ID=0, layer=0, protection_absent=1 + *ptr++ = 0xf9; // b11111001, ID=1(MPEG-2), layer=0, protection_absent=1 *ptr++ = profile << 6 |