From d47dfcb5a2e5901c96fc92662cec7aa30f7f8843 Mon Sep 17 00:00:00 2001 From: Chong Zhang Date: Fri, 27 Mar 2015 15:53:45 -0700 Subject: HLS: misc bug fixes - fix no target-duration case - fix for audio-only <=> audio/video switching - disable audio-only variants if there is at least one variant with video - fix mpeg2ts PTS wraparound when bandwidth adapting - tweak up/down switch marks bug: 19567254 Change-Id: Ib46144203c56dfc96eccd6ddaa3867e8a4f2c6a9 --- media/libmedia/IStreamSource.cpp | 3 + media/libstagefright/Utils.cpp | 31 +++ media/libstagefright/httplive/LiveSession.cpp | 275 +++++++++------------ media/libstagefright/httplive/LiveSession.h | 4 +- media/libstagefright/httplive/M3UParser.cpp | 39 ++- media/libstagefright/httplive/M3UParser.h | 1 + media/libstagefright/httplive/PlaylistFetcher.cpp | 107 +++----- media/libstagefright/mpeg2ts/ATSParser.cpp | 31 ++- media/libstagefright/mpeg2ts/ATSParser.h | 1 + .../libstagefright/mpeg2ts/AnotherPacketSource.cpp | 79 +++--- media/libstagefright/mpeg2ts/AnotherPacketSource.h | 4 +- 11 files changed, 293 insertions(+), 282 deletions(-) (limited to 'media') diff --git a/media/libmedia/IStreamSource.cpp b/media/libmedia/IStreamSource.cpp index d480aef..840e453 100644 --- a/media/libmedia/IStreamSource.cpp +++ b/media/libmedia/IStreamSource.cpp @@ -35,6 +35,9 @@ const char *const IStreamListener::kKeyDiscontinuityMask = "discontinuity-mask"; // static const char *const IStreamListener::kKeyMediaTimeUs = "media-time-us"; +// static +const char *const IStreamListener::kKeyRecentMediaTimeUs = "recent-media-time-us"; + enum { // IStreamSource SET_LISTENER = IBinder::FIRST_CALL_TRANSACTION, diff --git a/media/libstagefright/Utils.cpp b/media/libstagefright/Utils.cpp index c0be136..8506e37 100644 --- a/media/libstagefright/Utils.cpp +++ b/media/libstagefright/Utils.cpp @@ -822,5 +822,36 @@ AString uriDebugString(const AString &uri, bool incognito) { return AString(""); } +HLSTime::HLSTime(const sp& meta) : + mSeq(-1), + mTimeUs(-1ll), + mMeta(meta) { + if (meta != NULL) { + CHECK(meta->findInt32("discontinuitySeq", &mSeq)); + CHECK(meta->findInt64("timeUs", &mTimeUs)); + } +} + +int64_t HLSTime::getSegmentTimeUs(bool midpoint) const { + int64_t segmentStartTimeUs = -1ll; + if (mMeta != NULL) { + CHECK(mMeta->findInt64("segmentStartTimeUs", &segmentStartTimeUs)); + if (midpoint) { + int64_t durationUs; + CHECK(mMeta->findInt64("segmentDurationUs", &durationUs)); + segmentStartTimeUs += durationUs / 2; + } + } + return segmentStartTimeUs; +} + +bool operator <(const HLSTime &t0, const HLSTime &t1) { + // we can only compare discontinuity sequence and timestamp. + // (mSegmentTimeUs is not reliable in live streaming case, it's the + // time starting from beginning of playlist but playlist could change.) + return t0.mSeq < t1.mSeq + || (t0.mSeq == t1.mSeq && t0.mTimeUs < t1.mTimeUs); +} + } // namespace android diff --git a/media/libstagefright/httplive/LiveSession.cpp b/media/libstagefright/httplive/LiveSession.cpp index 118c174..2d93152 100644 --- a/media/libstagefright/httplive/LiveSession.cpp +++ b/media/libstagefright/httplive/LiveSession.cpp @@ -52,9 +52,10 @@ namespace android { // static // Bandwidth Switch Mark Defaults -const int64_t LiveSession::kUpSwitchMarkUs = 25000000ll; -const int64_t LiveSession::kDownSwitchMarkUs = 18000000ll; +const int64_t LiveSession::kUpSwitchMarkUs = 15000000ll; +const int64_t LiveSession::kDownSwitchMarkUs = 20000000ll; const int64_t LiveSession::kUpSwitchMarginUs = 5000000ll; +const int64_t LiveSession::kResumeThresholdUs = 100000ll; // Buffer Prepare/Ready/Underflow Marks const int64_t LiveSession::kReadyMarkUs = 5000000ll; @@ -70,7 +71,7 @@ struct LiveSession::BandwidthEstimator : public RefBase { private: // Bandwidth estimation parameters static const int32_t kMaxBandwidthHistoryItems = 20; - static const int64_t kMaxBandwidthHistoryWindowUs = 3000000ll; // 3 sec + static const int64_t kMaxBandwidthHistoryWindowUs = 5000000ll; // 5 sec struct BandwidthEntry { int64_t mDelayUs; @@ -405,26 +406,30 @@ bool LiveSession::checkSwitchProgress( sp lastDequeueMeta, lastEnqueueMeta; if (delayUs > 0) { lastDequeueMeta = source->getMetaAfterLastDequeued(delayUs); + if (lastDequeueMeta == NULL) { + // this means we don't have enough cushion, try again later + ALOGV("[%s] up switching failed due to insufficient buffer", + stream == STREAMTYPE_AUDIO ? "audio" : "video"); + return false; + } } else { + // It's okay for lastDequeueMeta to be NULL here, it means the + // decoder hasn't even started dequeueing lastDequeueMeta = source->getLatestDequeuedMeta(); } // Then, trim off packets at beginning of mPacketSources2 that's before // the latest dequeued time. These samples are definitely too late. - int64_t lastTimeUs, startTimeUs; - int32_t lastSeq, startSeq; - if (lastDequeueMeta != NULL) { - CHECK(lastDequeueMeta->findInt64("timeUs", &lastTimeUs)); - CHECK(lastDequeueMeta->findInt32("discontinuitySeq", &lastSeq)); - firstNewMeta[i] = mPacketSources2.editValueAt(i) - ->trimBuffersBeforeTimeUs(lastSeq, lastTimeUs); - } + firstNewMeta[i] = mPacketSources2.editValueAt(i) + ->trimBuffersBeforeMeta(lastDequeueMeta); + // Now firstNewMeta[i] is the first sample after the trim. // If it's NULL, we failed because dequeue already past all samples // in mPacketSource2, we have to try again. if (firstNewMeta[i] == NULL) { + HLSTime dequeueTime(lastDequeueMeta); ALOGV("[%s] dequeue time (%d, %lld) past start time", stream == STREAMTYPE_AUDIO ? "audio" : "video", - lastSeq, (long long) lastTimeUs); + dequeueTime.mSeq, (long long) dequeueTime.mTimeUs); return false; } @@ -434,20 +439,16 @@ bool LiveSession::checkSwitchProgress( // lastEnqueueMeta == NULL means old fetcher stopped at a discontinuity // boundary, no need to resume as the content will look different anyways if (lastEnqueueMeta != NULL) { - CHECK(lastEnqueueMeta->findInt64("timeUs", &lastTimeUs)); - CHECK(lastEnqueueMeta->findInt32("discontinuitySeq", &lastSeq)); - CHECK(firstNewMeta[i]->findInt64("timeUs", &startTimeUs)); - CHECK(firstNewMeta[i]->findInt32("discontinuitySeq", &startSeq)); + HLSTime lastTime(lastEnqueueMeta), startTime(firstNewMeta[i]); // no need to resume old fetcher if new fetcher started in different // discontinuity sequence, as the content will look different. - *needResumeUntil |= - (startSeq == lastSeq - && startTimeUs - lastTimeUs > 100000ll); + *needResumeUntil |= (startTime.mSeq == lastTime.mSeq + && startTime.mTimeUs - lastTime.mTimeUs > kResumeThresholdUs); - // update the stopTime for resumeUntil, as we might have removed some - // packets from the head in mPacketSource2 - stopParams->setInt64(getKeyForStream(stream), startTimeUs); + // update the stopTime for resumeUntil + stopParams->setInt32("discontinuitySeq", startTime.mSeq); + stopParams->setInt64(getKeyForStream(stream), startTime.mTimeUs); } } @@ -457,18 +458,11 @@ bool LiveSession::checkSwitchProgress( for (size_t i = 0; i < kMaxStreams; ++i) { StreamType stream = indexToType(i); if (!(mSwapMask & mNewStreamMask & stream) - || (newUri != mStreams[i].mNewUri)) { - continue; - } - if (stream == STREAMTYPE_SUBTITLES) { + || (newUri != mStreams[i].mNewUri) + || stream == STREAMTYPE_SUBTITLES) { continue; } - int64_t startTimeUs; - int32_t startSeq; - CHECK(firstNewMeta[i] != NULL); - CHECK(firstNewMeta[i]->findInt64("timeUs", &startTimeUs)); - CHECK(firstNewMeta[i]->findInt32("discontinuitySeq", &startSeq)); - mPacketSources.valueFor(stream)->trimBuffersAfterTimeUs(startSeq, startTimeUs); + mPacketSources.valueFor(stream)->trimBuffersAfterMeta(firstNewMeta[i]); } // no resumeUntil if already underflow @@ -574,7 +568,7 @@ void LiveSession::onMessageReceived(const sp &msg) { { int64_t targetDurationUs; CHECK(msg->findInt64("targetDurationUs", &targetDurationUs)); - mUpSwitchMark = min(kUpSwitchMarkUs, targetDurationUs * 3); + mUpSwitchMark = min(kUpSwitchMarkUs, targetDurationUs * 7 / 4); mDownSwitchMark = min(kDownSwitchMarkUs, targetDurationUs * 9 / 4); mUpSwitchMargin = min(kUpSwitchMarginUs, targetDurationUs); break; @@ -625,15 +619,15 @@ void LiveSession::onMessageReceived(const sp &msg) { { ALOGV("kWhatStopReached"); - AString uri; - CHECK(msg->findString("uri", &uri)); + AString oldUri; + CHECK(msg->findString("uri", &oldUri)); - ssize_t index = mFetcherInfos.indexOfKey(uri); + ssize_t index = mFetcherInfos.indexOfKey(oldUri); if (index < 0) { break; } - tryToFinishBandwidthSwitch(uri); + tryToFinishBandwidthSwitch(oldUri); break; } @@ -837,6 +831,7 @@ void LiveSession::onConnect(const sp &msg) { size_t initialBandwidthIndex = 0; if (mPlaylist->isVariantPlaylist()) { + Vector itemsWithVideo; for (size_t i = 0; i < mPlaylist->size(); ++i) { BandwidthItem item; @@ -848,14 +843,22 @@ void LiveSession::onConnect(const sp &msg) { CHECK(meta->findInt32("bandwidth", (int32_t *)&item.mBandwidth)); - if (initialBandwidth == 0) { - initialBandwidth = item.mBandwidth; - } - mBandwidthItems.push(item); + if (mPlaylist->hasType(i, "video")) { + itemsWithVideo.push(item); + } + } + // remove the audio-only variants if we have at least one with video + if (!itemsWithVideo.empty() + && itemsWithVideo.size() < mBandwidthItems.size()) { + mBandwidthItems.clear(); + for (size_t i = 0; i < itemsWithVideo.size(); ++i) { + mBandwidthItems.push(itemsWithVideo[i]); + } } CHECK_GT(mBandwidthItems.size(), 0u); + initialBandwidth = mBandwidthItems[0].mBandwidth; mBandwidthItems.sort(SortByBandwidth); @@ -1090,6 +1093,9 @@ sp LiveSession::fetchPlaylist( String8 actualUrl; ssize_t err = fetchFile(url, &buffer, 0, -1, 0, NULL, &actualUrl); + // close off the connection after use + mHTTPDataSource->disconnect(); + if (err <= 0) { return NULL; } @@ -1333,22 +1339,14 @@ size_t LiveSession::getBandwidthIndex(int32_t bandwidthBps) { return index; } -int64_t LiveSession::latestMediaSegmentStartTimeUs() { - sp audioMeta = mPacketSources.valueFor(STREAMTYPE_AUDIO)->getLatestDequeuedMeta(); - int64_t minSegmentStartTimeUs = -1, videoSegmentStartTimeUs = -1; - if (audioMeta != NULL) { - audioMeta->findInt64("segmentStartTimeUs", &minSegmentStartTimeUs); - } +HLSTime LiveSession::latestMediaSegmentStartTime() const { + HLSTime audioTime(mPacketSources.valueFor( + STREAMTYPE_AUDIO)->getLatestDequeuedMeta()); - sp videoMeta = mPacketSources.valueFor(STREAMTYPE_VIDEO)->getLatestDequeuedMeta(); - if (videoMeta != NULL - && videoMeta->findInt64("segmentStartTimeUs", &videoSegmentStartTimeUs)) { - if (minSegmentStartTimeUs < 0 || videoSegmentStartTimeUs < minSegmentStartTimeUs) { - minSegmentStartTimeUs = videoSegmentStartTimeUs; - } + HLSTime videoTime(mPacketSources.valueFor( + STREAMTYPE_VIDEO)->getLatestDequeuedMeta()); - } - return minSegmentStartTimeUs; + return audioTime < videoTime ? videoTime : audioTime; } status_t LiveSession::onSeek(const sp &msg) { @@ -1459,14 +1457,9 @@ void LiveSession::changeConfiguration( } const AString &uri = mFetcherInfos.keyAt(i); + sp &fetcher = mFetcherInfos.editValueAt(i).mFetcher; - bool discardFetcher = true; - - if (timeUs < 0ll) { - // delay fetcher removal if not picking tracks - discardFetcher = pickTrack; - } - + bool discardFetcher = true, delayRemoval = false; for (size_t j = 0; j < kMaxStreams; ++j) { StreamType type = indexToType(j); if ((streamMask & type) && uri == URIs[j]) { @@ -1475,15 +1468,23 @@ void LiveSession::changeConfiguration( discardFetcher = false; } } + // Delay fetcher removal if not picking tracks, AND old fetcher + // has stream mask that overlaps new variant. (Okay to discard + // old fetcher now, if completely no overlap.) + if (discardFetcher && timeUs < 0ll && !pickTrack + && (fetcher->getStreamTypeMask() & streamMask)) { + discardFetcher = false; + delayRemoval = true; + } if (discardFetcher) { - mFetcherInfos.valueAt(i).mFetcher->stopAsync(); + fetcher->stopAsync(); } else { float threshold = -1.0f; // always finish fetching by default if (timeUs >= 0ll) { // seeking, no need to finish fetching threshold = 0.0f; - } else if (!pickTrack) { + } else if (delayRemoval) { // adapting, abort if remaining of current segment is over threshold threshold = getAbortThreshold( mOrigBandwidthIndex, mCurBandwidthIndex); @@ -1491,7 +1492,7 @@ void LiveSession::changeConfiguration( ALOGV("Pausing with threshold %.3f", threshold); - mFetcherInfos.valueAt(i).mFetcher->pauseAsync(threshold); + fetcher->pauseAsync(threshold); } } @@ -1640,17 +1641,29 @@ void LiveSession::onChangeConfiguration3(const sp &msg) { CHECK(msg->findInt32("streamMask", (int32_t *)&streamMask)); CHECK(msg->findInt32("resumeMask", (int32_t *)&resumeMask)); + mNewStreamMask = streamMask | resumeMask; + int64_t timeUs; int32_t pickTrack; bool switching = false; - bool finishSwitching = false; CHECK(msg->findInt64("timeUs", &timeUs)); CHECK(msg->findInt32("pickTrack", &pickTrack)); if (timeUs < 0ll) { if (!pickTrack) { - switching = true; - finishSwitching = (streamMask == 0); + // mSwapMask contains streams that are in both old and new variant, + // (in mNewStreamMask & mStreamMask) but with different URIs + // (not in resumeMask). + // For example, old variant has video and audio in two separate + // URIs, and new variant has only audio with unchanged URI. mSwapMask + // should be 0 as there is nothing to swap. We only need to stop video, + // and resume audio. + mSwapMask = mNewStreamMask & mStreamMask & ~resumeMask; + switching = (mSwapMask != 0); + if (!switching) { + ALOGV("#### Finishing Bandwidth Switch Early: %zd => %zd", + mOrigBandwidthIndex, mCurBandwidthIndex); + } } mRealTimeBaseUs = ALooper::GetNowUs() - mLastDequeuedTimeUs; } else { @@ -1667,11 +1680,6 @@ void LiveSession::onChangeConfiguration3(const sp &msg) { } } - mNewStreamMask = streamMask | resumeMask; - if (switching) { - mSwapMask = mStreamMask & ~resumeMask; - } - // Of all existing fetchers: // * Resume fetchers that are still needed and assign them original packet sources. // * Mark otherwise unneeded fetchers for removal. @@ -1701,14 +1709,12 @@ void LiveSession::onChangeConfiguration3(const sp &msg) { sp fetcher = addFetcher(uri.c_str()); CHECK(fetcher != NULL); - int64_t startTimeUs = -1; - int64_t segmentStartTimeUs = -1ll; - int32_t discontinuitySeq = -1; + HLSTime startTime; SeekMode seekMode = kSeekModeExactPosition; sp sources[kMaxStreams]; - if (i == kSubtitleIndex) { - segmentStartTimeUs = latestMediaSegmentStartTimeUs(); + if (i == kSubtitleIndex || (!pickTrack && !switching)) { + startTime = latestMediaSegmentStartTime(); } // TRICKY: looping from i as earlier streams are already removed from streamMask @@ -1718,25 +1724,15 @@ void LiveSession::onChangeConfiguration3(const sp &msg) { sources[j] = mPacketSources.valueFor(indexToType(j)); if (timeUs >= 0) { - startTimeUs = timeUs; + startTime.mTimeUs = timeUs; } else { int32_t type; sp meta; - if (pickTrack) { - // selecting - - // FIXME: - // This should only apply to the track that's being picked, we - // need a bitmask to indicate that. - // - // It's possible that selectTrack() gets called during a bandwidth - // switch, and we needed to fetch a new variant. The new fetcher - // should start from where old fetcher left off, not where decoder - // is dequeueing at. - + if (!switching) { + // selecting, or adapting but no swap required meta = sources[j]->getLatestDequeuedMeta(); } else { - // adapting + // adapting and swap required meta = sources[j]->getLatestEnqueuedMeta(); if (meta != NULL && mCurBandwidthIndex > mOrigBandwidthIndex) { // switching up @@ -1744,60 +1740,28 @@ void LiveSession::onChangeConfiguration3(const sp &msg) { } } - if (j != kSubtitleIndex - && meta != NULL + if (j != kSubtitleIndex && meta != NULL && !meta->findInt32("discontinuity", &type)) { - int64_t tmpUs; - int64_t tmpSegmentUs; - int32_t seq; - - CHECK(meta->findInt64("timeUs", &tmpUs)); - CHECK(meta->findInt64("segmentStartTimeUs", &tmpSegmentUs)); - CHECK(meta->findInt32("discontinuitySeq", &seq)); - // If we're switching and looking for next sample or segment, set the target - // segment start time to tmpSegmentUs + tmpDurationUs / 2, which is - // the middle point of the segment where the last sample was. - // This is needed if segments of the two variants are not perfectly - // aligned. (If the corresponding segment in new variant starts slightly - // later than that in the old variant, we still want the switching to - // start in the next one, not the current one) - if (mStreams[j].mSeekMode == kSeekModeNextSample - || mStreams[j].mSeekMode == kSeekModeNextSegment) { - int64_t tmpDurationUs; - CHECK(meta->findInt64("segmentDurationUs", &tmpDurationUs)); - tmpSegmentUs += tmpDurationUs / 2; - } - if (startTimeUs < 0 || seq > discontinuitySeq - || (seq == discontinuitySeq - && (tmpSegmentUs > segmentStartTimeUs - || (tmpSegmentUs == segmentStartTimeUs - && tmpUs > startTimeUs)))) { - startTimeUs = tmpUs; - segmentStartTimeUs = tmpSegmentUs; - discontinuitySeq = seq; + HLSTime tmpTime(meta); + if (startTime < tmpTime) { + startTime = tmpTime; } } - if (pickTrack) { - // selecting track, queue discontinuities before content + if (!switching) { + // selecting, or adapting but no swap required sources[j]->clear(); if (j == kSubtitleIndex) { break; } ALOGV("stream[%zu]: queue format change", j); - sources[j]->queueDiscontinuity( ATSParser::DISCONTINUITY_FORMAT_ONLY, NULL, true); } else { - // adapting, queue discontinuities after resume + // switching, queue discontinuities after resume sources[j] = mPacketSources2.valueFor(indexToType(j)); sources[j]->clear(); - uint32_t extraStreams = mNewStreamMask & (~mStreamMask); - if (extraStreams & indexToType(j)) { - sources[j]->queueDiscontinuity( - ATSParser::DISCONTINUITY_FORMAT_ONLY, NULL, true); - } // the new fetcher might be providing streams that used to be // provided by two different fetchers, if one of the fetcher // paused in the middle while the other somehow paused in next @@ -1812,13 +1776,19 @@ void LiveSession::onChangeConfiguration3(const sp &msg) { } } + // Set the target segment start time to the middle point of the + // segment where the last sample was. + // This gives a better guess if segments of the two variants are not + // perfectly aligned. (If the corresponding segment in new variant + // starts slightly later than that in the old variant, we still want + // to pick that segment, not the one before) fetcher->startAsync( sources[kAudioIndex], sources[kVideoIndex], sources[kSubtitleIndex], - startTimeUs < 0 ? mLastSeekTimeUs : startTimeUs, - segmentStartTimeUs, - discontinuitySeq, + startTime.mTimeUs < 0 ? mLastSeekTimeUs : startTime.mTimeUs, + startTime.getSegmentTimeUs(true /* midpoint */), + startTime.mSeq, seekMode); } @@ -1829,18 +1799,6 @@ void LiveSession::onChangeConfiguration3(const sp &msg) { mReconfigurationInProgress = false; if (switching) { mSwitchInProgress = true; - - if (finishSwitching) { - // Switch is finished now, no new fetchers are created. - // This path is hit when old variant had video and audio from - // two separate fetchers, while new variant has audio only, - // which reuses the previous audio fetcher. - for (size_t i = 0; i < kMaxStreams; ++i) { - if (mSwapMask & indexToType(i)) { - tryToFinishBandwidthSwitch(mStreams[i].mUri); - } - } - } } else { mStreamMask = mNewStreamMask; mOrigBandwidthIndex = mCurBandwidthIndex; @@ -1940,7 +1898,6 @@ void LiveSession::tryToFinishBandwidthSwitch(const AString &oldUri) { mSwitchInProgress = false; mOrigBandwidthIndex = mCurBandwidthIndex; - restartPollBuffering(); } @@ -2090,7 +2047,8 @@ bool LiveSession::checkBuffering( } if (bufferedDurationUs > mUpSwitchMark) { ++upCount; - } else if (bufferedDurationUs < mDownSwitchMark) { + } + if (bufferedDurationUs < mDownSwitchMark) { ++downCount; } } @@ -2169,15 +2127,24 @@ void LiveSession::switchBandwidthIfNeeded(bool bufferHigh, bool bufferLow) { } int32_t curBandwidth = mBandwidthItems.itemAt(mCurBandwidthIndex).mBandwidth; - bool bandwidthLow = bandwidthBps < (int32_t)curBandwidth * 8 / 10; - bool bandwidthHigh = bandwidthBps > (int32_t)curBandwidth * 12 / 10; - - if ((bufferHigh && bandwidthHigh) || (bufferLow && bandwidthLow)) { + // canSwithDown and canSwitchUp can't both be true. + // we only want to switch up when measured bw is 120% higher than current variant, + // and we only want to switch down when measured bw is below current variant. + bool canSwithDown = bufferLow + && (bandwidthBps < (int32_t)curBandwidth); + bool canSwitchUp = bufferHigh + && (bandwidthBps > (int32_t)curBandwidth * 12 / 10); + + if (canSwithDown || canSwitchUp) { ssize_t bandwidthIndex = getBandwidthIndex(bandwidthBps); + // it's possible that we're checking for canSwitchUp case, but the returned + // bandwidthIndex is < mCurBandwidthIndex, as getBandwidthIndex() only uses 70% + // of measured bw. In that case we don't want to do anything, since we have + // both enough buffer and enough bw. if (bandwidthIndex == mCurBandwidthIndex - || (bufferHigh && bandwidthIndex < mCurBandwidthIndex) - || (bufferLow && bandwidthIndex > mCurBandwidthIndex)) { + || (canSwitchUp && bandwidthIndex < mCurBandwidthIndex) + || (canSwithDown && bandwidthIndex > mCurBandwidthIndex)) { return; } diff --git a/media/libstagefright/httplive/LiveSession.h b/media/libstagefright/httplive/LiveSession.h index 3d62cab..b5e31c9 100644 --- a/media/libstagefright/httplive/LiveSession.h +++ b/media/libstagefright/httplive/LiveSession.h @@ -34,6 +34,7 @@ struct IMediaHTTPService; struct LiveDataSource; struct M3UParser; struct PlaylistFetcher; +struct HLSTime; struct LiveSession : public AHandler { enum Flags { @@ -125,6 +126,7 @@ private: static const int64_t kUpSwitchMarkUs; static const int64_t kDownSwitchMarkUs; static const int64_t kUpSwitchMarginUs; + static const int64_t kResumeThresholdUs; // Buffer Prepare/Ready/Underflow Marks static const int64_t kReadyMarkUs; @@ -271,7 +273,7 @@ private: ssize_t currentBWIndex, ssize_t targetBWIndex) const; void addBandwidthMeasurement(size_t numBytes, int64_t delayUs); size_t getBandwidthIndex(int32_t bandwidthBps); - int64_t latestMediaSegmentStartTimeUs(); + HLSTime latestMediaSegmentStartTime() const; static int SortByBandwidth(const BandwidthItem *, const BandwidthItem *); static StreamType indexToType(int idx); diff --git a/media/libstagefright/httplive/M3UParser.cpp b/media/libstagefright/httplive/M3UParser.cpp index 3c5d7cf..7bb7f2c 100644 --- a/media/libstagefright/httplive/M3UParser.cpp +++ b/media/libstagefright/httplive/M3UParser.cpp @@ -395,7 +395,9 @@ ssize_t M3UParser::getSelectedTrack(media_track_type type) const { bool M3UParser::getTypeURI(size_t index, const char *key, AString *uri) const { if (!mIsVariantPlaylist) { - *uri = mBaseURI; + if (uri != NULL) { + *uri = mBaseURI; + } // Assume media without any more specific attribute contains // audio and video, but no subtitles. @@ -408,7 +410,9 @@ bool M3UParser::getTypeURI(size_t index, const char *key, AString *uri) const { AString groupID; if (!meta->findString(key, &groupID)) { - *uri = mItems.itemAt(index).mURI; + if (uri != NULL) { + *uri = mItems.itemAt(index).mURI; + } AString codecs; if (!meta->findString("codecs", &codecs)) { @@ -434,18 +438,26 @@ bool M3UParser::getTypeURI(size_t index, const char *key, AString *uri) const { } } - sp group = mMediaGroups.valueFor(groupID); - if (!group->getActiveURI(uri)) { - return false; - } + // if uri == NULL, we're only checking if the type is present, + // don't care about the active URI (or if there is an active one) + if (uri != NULL) { + sp group = mMediaGroups.valueFor(groupID); + if (!group->getActiveURI(uri)) { + return false; + } - if ((*uri).empty()) { - *uri = mItems.itemAt(index).mURI; + if ((*uri).empty()) { + *uri = mItems.itemAt(index).mURI; + } } return true; } +bool M3UParser::hasType(size_t index, const char *key) const { + return getTypeURI(index, key, NULL /* uri */); +} + static bool MakeURL(const char *baseURL, const char *url, AString *out) { out->clear(); @@ -633,7 +645,8 @@ status_t M3UParser::parse(const void *_data, size_t size) { || !itemMeta->findInt64("durationUs", &durationUs)) { return ERROR_MALFORMED; } - itemMeta->setInt32("discontinuity-sequence", mDiscontinuitySeq + mDiscontinuityCount); + itemMeta->setInt32("discontinuity-sequence", + mDiscontinuitySeq + mDiscontinuityCount); } mItems.push(); @@ -650,6 +663,14 @@ status_t M3UParser::parse(const void *_data, size_t size) { ++lineNo; } + // error checking of all fields that's required to appear once + // (currently only checking "target-duration") + int32_t targetDurationSecs; + if (!mIsVariantPlaylist && (mMeta == NULL || !mMeta->findInt32( + "target-duration", &targetDurationSecs))) { + return ERROR_MALFORMED; + } + return OK; } diff --git a/media/libstagefright/httplive/M3UParser.h b/media/libstagefright/httplive/M3UParser.h index d475683..fef361f 100644 --- a/media/libstagefright/httplive/M3UParser.h +++ b/media/libstagefright/httplive/M3UParser.h @@ -50,6 +50,7 @@ struct M3UParser : public RefBase { ssize_t getSelectedTrack(media_track_type /* type */) const; bool getTypeURI(size_t index, const char *key, AString *uri) const; + bool hasType(size_t index, const char *key) const; protected: virtual ~M3UParser(); diff --git a/media/libstagefright/httplive/PlaylistFetcher.cpp b/media/libstagefright/httplive/PlaylistFetcher.cpp index 3ace396..368612d 100644 --- a/media/libstagefright/httplive/PlaylistFetcher.cpp +++ b/media/libstagefright/httplive/PlaylistFetcher.cpp @@ -50,10 +50,8 @@ namespace android { // static const int64_t PlaylistFetcher::kMinBufferedDurationUs = 30000000ll; const int64_t PlaylistFetcher::kMaxMonitorDelayUs = 3000000ll; -const int64_t PlaylistFetcher::kFetcherResumeThreshold = 100000ll; // LCM of 188 (size of a TS packet) & 1k works well const int32_t PlaylistFetcher::kDownloadBlockSize = 47 * 1024; -const int32_t PlaylistFetcher::kNumSkipFrames = 5; struct PlaylistFetcher::DownloadState : public RefBase { DownloadState(); @@ -676,6 +674,9 @@ void PlaylistFetcher::onStop(const sp &msg) { } } + // close off the connection after use + mHTTPDataSource->disconnect(); + mDownloadState->resetState(); mPacketSources.clear(); mStreamTypeMask = 0; @@ -690,40 +691,6 @@ status_t PlaylistFetcher::onResumeUntil(const sp &msg) { sp params; CHECK(msg->findMessage("params", ¶ms)); - size_t stopCount = 0; - for (size_t i = 0; i < mPacketSources.size(); i++) { - sp packetSource = mPacketSources.valueAt(i); - - LiveSession::StreamType streamType = mPacketSources.keyAt(i); - - if (streamType == LiveSession::STREAMTYPE_SUBTITLES) { - // the subtitle track can always be stopped - ++stopCount; - continue; - } - - const char *stopKey = LiveSession::getKeyForStream(streamType); - - // check if this stream has too little data left to be resumed - int32_t discontinuitySeq; - int64_t latestTimeUs = 0, stopTimeUs = 0; - sp latestMeta = packetSource->getLatestEnqueuedMeta(); - if (latestMeta != NULL - && latestMeta->findInt32("discontinuitySeq", &discontinuitySeq) - && discontinuitySeq == mDiscontinuitySeq - && latestMeta->findInt64("timeUs", &latestTimeUs) - && params->findInt64(stopKey, &stopTimeUs) - && stopTimeUs - latestTimeUs < kFetcherResumeThreshold) { - ++stopCount; - } - } - - // Don't resume if all streams are within a resume threshold - if (stopCount == mPacketSources.size()) { - notifyStopReached(); - return OK; - } - mStopParams = params; onDownloadNext(); @@ -982,7 +949,8 @@ bool PlaylistFetcher::initDownloadState( // timestamps coming from the media container) is used to determine the position // inside a segments. mSeqNumber = getSeqNumberForTime(mSegmentStartTimeUs); - if (mSeekMode == LiveSession::kSeekModeNextSegment) { + if (mStreamTypeMask != LiveSession::STREAMTYPE_SUBTITLES + && mSeekMode != LiveSession::kSeekModeNextSample) { // avoid double fetch/decode mSeqNumber += 1; } @@ -1219,8 +1187,10 @@ void PlaylistFetcher::onDownloadNext() { uri.c_str(), &buffer, range_offset, range_length, kDownloadBlockSize, &source, NULL, connectHTTP); - // add sample for bandwidth estimation (excluding subtitles) - if (bytesRead > 0 + // add sample for bandwidth estimation, excluding samples from subtitles (as + // its too small), or during startup/resumeUntil (when we could have more than + // one connection open which affects bandwidth) + if (!mStartup && mStopParams == NULL && bytesRead > 0 && (mStreamTypeMask & (LiveSession::STREAMTYPE_AUDIO | LiveSession::STREAMTYPE_VIDEO))) { @@ -1381,7 +1351,7 @@ void PlaylistFetcher::onDownloadNext() { // if adapting, pause after found the next starting point if (mSeekMode != LiveSession::kSeekModeExactPosition && startUp != mStartup) { - CHECK(mStartTimeUsNotify != NULL); + CHECK(mStartTimeUsNotify != NULL); mStartTimeUsNotify->post(); mStartTimeUsNotify.clear(); shouldPause = true; @@ -1424,28 +1394,22 @@ int32_t PlaylistFetcher::getSeqNumberWithAnchorTime( int32_t PlaylistFetcher::getSeqNumberForDiscontinuity(size_t discontinuitySeq) const { int32_t firstSeqNumberInPlaylist; - if (mPlaylist->meta() == NULL - || !mPlaylist->meta()->findInt32("media-sequence", &firstSeqNumberInPlaylist)) { + if (mPlaylist->meta() == NULL || !mPlaylist->meta()->findInt32( + "media-sequence", &firstSeqNumberInPlaylist)) { firstSeqNumberInPlaylist = 0; } - size_t curDiscontinuitySeq = mPlaylist->getDiscontinuitySeq(); - if (discontinuitySeq < curDiscontinuitySeq) { - return firstSeqNumberInPlaylist <= 0 ? 0 : (firstSeqNumberInPlaylist - 1); - } - size_t index = 0; while (index < mPlaylist->size()) { sp itemMeta; CHECK(mPlaylist->itemAt( index, NULL /* uri */, &itemMeta)); - - int64_t discontinuity; - if (itemMeta->findInt64("discontinuity", &discontinuity)) { - curDiscontinuitySeq++; - } - + size_t curDiscontinuitySeq; + CHECK(itemMeta->findInt32("discontinuity-sequence", (int32_t *)&curDiscontinuitySeq)); + int32_t seqNumber = firstSeqNumberInPlaylist + index; if (curDiscontinuitySeq == discontinuitySeq) { - return firstSeqNumberInPlaylist + index; + return seqNumber; + } else if (curDiscontinuitySeq > discontinuitySeq) { + return seqNumber <= 0 ? 0 : seqNumber - 1; } ++index; @@ -1521,6 +1485,12 @@ status_t PlaylistFetcher::extractAndQueueAccessUnitsFromTs(const sp &bu // ATSParser from skewing the timestamps of access units. extra->setInt64(IStreamListener::kKeyMediaTimeUs, 0); + // When adapting, signal a recent media time to the parser, + // so that PTS wrap around is handled for the new variant. + if (mStartTimeUs >= 0 && !mStartTimeUsRelative) { + extra->setInt64(IStreamListener::kKeyRecentMediaTimeUs, mStartTimeUs); + } + mTSParser->signalDiscontinuity( ATSParser::DISCONTINUITY_TIME, extra); @@ -1575,10 +1545,9 @@ status_t PlaylistFetcher::extractAndQueueAccessUnitsFromTs(const sp &bu int64_t timeUs; CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); - bool seeking = mSeekMode == LiveSession::kSeekModeExactPosition; if (mSegmentFirstPTS < 0ll) { mSegmentFirstPTS = timeUs; - if (!seeking) { + if (!mStartTimeUsRelative) { int32_t firstSeqNumberInPlaylist; if (mPlaylist->meta() == NULL || !mPlaylist->meta()->findInt32( "media-sequence", &firstSeqNumberInPlaylist)) { @@ -1604,7 +1573,7 @@ status_t PlaylistFetcher::extractAndQueueAccessUnitsFromTs(const sp &bu // timeUs to be too far ahead of mStartTimeUs because we want the old fetcher to // stop as early as possible. The definition of being "too far ahead" is // arbitrary; here we use targetDurationUs as threshold. - int64_t targetDiffUs =(mSeekMode == LiveSession::kSeekModeNextSample + int64_t targetDiffUs = (mSeekMode == LiveSession::kSeekModeNextSample ? 0 : targetDurationUs); if (mStartup && mSegmentStartTimeUs >= 0 && mSeqNumber > firstSeqNumberInPlaylist @@ -1633,25 +1602,24 @@ status_t PlaylistFetcher::extractAndQueueAccessUnitsFromTs(const sp &bu mFirstTimeUs = timeUs; mFirstPTSValid = true; } + bool startTimeReached = true; if (mStartTimeUsRelative) { timeUs -= mFirstTimeUs; if (timeUs < 0) { timeUs = 0; } + startTimeReached = (timeUs >= mStartTimeUs); } - bool startTimeReached = - seeking ? (timeUs >= mStartTimeUs) : true; - if (!startTimeReached || (isAvc && !mIDRFound)) { // buffer up to the closest preceding IDR frame in the next segement, // or the closest succeeding IDR frame after the exact position if (isAvc) { - if (IsIDR(accessUnit) && (seeking || startTimeReached)) { + if (IsIDR(accessUnit)) { mVideoBuffer->clear(); mIDRFound = true; } - if (mIDRFound && seeking && !startTimeReached) { + if (mIDRFound && mStartTimeUsRelative && !startTimeReached) { mVideoBuffer->queueAccessUnit(accessUnit); } } @@ -1662,16 +1630,9 @@ status_t PlaylistFetcher::extractAndQueueAccessUnitsFromTs(const sp &bu } if (mStartTimeUsNotify != NULL) { - int32_t seq; - if (!mStartTimeUsNotify->findInt32("discontinuitySeq", &seq)) { - mStartTimeUsNotify->setInt32("discontinuitySeq", mDiscontinuitySeq); - } - int64_t startTimeUs; - if (!mStartTimeUsNotify->findInt64(key, &startTimeUs)) { - mStartTimeUsNotify->setInt64(key, timeUs); - - uint32_t streamMask = 0; - mStartTimeUsNotify->findInt32("streamMask", (int32_t *) &streamMask); + uint32_t streamMask = 0; + mStartTimeUsNotify->findInt32("streamMask", (int32_t *) &streamMask); + if (!(streamMask & mPacketSources.keyAt(i))) { streamMask |= mPacketSources.keyAt(i); mStartTimeUsNotify->setInt32("streamMask", streamMask); @@ -1951,8 +1912,6 @@ status_t PlaylistFetcher::extractAndQueueAccessUnits( return -EAGAIN; } - mStartTimeUsNotify->setInt64("timeUsAudio", timeUs); - mStartTimeUsNotify->setInt32("discontinuitySeq", mDiscontinuitySeq); mStartTimeUsNotify->setInt32("streamMask", LiveSession::STREAMTYPE_AUDIO); mStartup = false; } diff --git a/media/libstagefright/mpeg2ts/ATSParser.cpp b/media/libstagefright/mpeg2ts/ATSParser.cpp index 6786506..ac6982e 100644 --- a/media/libstagefright/mpeg2ts/ATSParser.cpp +++ b/media/libstagefright/mpeg2ts/ATSParser.cpp @@ -48,7 +48,8 @@ namespace android { static const size_t kTSPacketSize = 188; struct ATSParser::Program : public RefBase { - Program(ATSParser *parser, unsigned programNumber, unsigned programMapPID); + Program(ATSParser *parser, unsigned programNumber, unsigned programMapPID, + int64_t lastRecoveredPTS); bool parsePSISection( unsigned pid, ABitReader *br, status_t *err); @@ -186,13 +187,14 @@ private: //////////////////////////////////////////////////////////////////////////////// ATSParser::Program::Program( - ATSParser *parser, unsigned programNumber, unsigned programMapPID) + ATSParser *parser, unsigned programNumber, unsigned programMapPID, + int64_t lastRecoveredPTS) : mParser(parser), mProgramNumber(programNumber), mProgramMapPID(programMapPID), mFirstPTSValid(false), mFirstPTS(0), - mLastRecoveredPTS(-1ll) { + mLastRecoveredPTS(lastRecoveredPTS) { ALOGV("new program number %u", programNumber); } @@ -1037,6 +1039,7 @@ ATSParser::ATSParser(uint32_t flags) mAbsoluteTimeAnchorUs(-1ll), mTimeOffsetValid(false), mTimeOffsetUs(0ll), + mLastRecoveredPTS(-1ll), mNumTSPacketsParsed(0), mNumPCRs(0) { mPSISections.add(0 /* PID */, new PSISection); @@ -1055,11 +1058,21 @@ status_t ATSParser::feedTSPacket(const void *data, size_t size) { void ATSParser::signalDiscontinuity( DiscontinuityType type, const sp &extra) { int64_t mediaTimeUs; - if ((type & DISCONTINUITY_TIME) - && extra != NULL - && extra->findInt64( - IStreamListener::kKeyMediaTimeUs, &mediaTimeUs)) { - mAbsoluteTimeAnchorUs = mediaTimeUs; + if ((type & DISCONTINUITY_TIME) && extra != NULL) { + if (extra->findInt64(IStreamListener::kKeyMediaTimeUs, &mediaTimeUs)) { + mAbsoluteTimeAnchorUs = mediaTimeUs; + } + if ((mFlags & TS_TIMESTAMPS_ARE_ABSOLUTE) + && extra->findInt64( + IStreamListener::kKeyRecentMediaTimeUs, &mediaTimeUs)) { + if (mAbsoluteTimeAnchorUs >= 0ll) { + mediaTimeUs -= mAbsoluteTimeAnchorUs; + } + if (mTimeOffsetValid) { + mediaTimeUs -= mTimeOffsetUs; + } + mLastRecoveredPTS = (mediaTimeUs * 9) / 100; + } } else if (type == DISCONTINUITY_ABSOLUTE_TIME) { int64_t timeUs; CHECK(extra->findInt64("timeUs", &timeUs)); @@ -1143,7 +1156,7 @@ void ATSParser::parseProgramAssociationTable(ABitReader *br) { if (!found) { mPrograms.push( - new Program(this, program_number, programMapPID)); + new Program(this, program_number, programMapPID, mLastRecoveredPTS)); } if (mPSISections.indexOfKey(programMapPID) < 0) { diff --git a/media/libstagefright/mpeg2ts/ATSParser.h b/media/libstagefright/mpeg2ts/ATSParser.h index 5c50747..a1405bd 100644 --- a/media/libstagefright/mpeg2ts/ATSParser.h +++ b/media/libstagefright/mpeg2ts/ATSParser.h @@ -118,6 +118,7 @@ private: bool mTimeOffsetValid; int64_t mTimeOffsetUs; + int64_t mLastRecoveredPTS; size_t mNumTSPacketsParsed; diff --git a/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp b/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp index c2f1527..c5bb41b 100644 --- a/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp +++ b/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -461,6 +462,10 @@ void AnotherPacketSource::enable(bool enable) { mEnabled = enable; } +/* + * returns the sample meta that's delayUs after queue head + * (NULL if such sample is unavailable) + */ sp AnotherPacketSource::getMetaAfterLastDequeued(int64_t delayUs) { Mutex::Autolock autoLock(mLock); int64_t firstUs = -1; @@ -490,19 +495,28 @@ sp AnotherPacketSource::getMetaAfterLastDequeued(int64_t delayUs) { } } } - return mLatestEnqueuedMeta; + return NULL; } -void AnotherPacketSource::trimBuffersAfterTimeUs( - size_t discontinuitySeq, int64_t timeUs) { - ALOGV("trimBuffersAfterTimeUs: discontinuitySeq %zu, timeUs %lld", - discontinuitySeq, (long long)timeUs); +/* + * removes samples with time equal or after meta + */ +void AnotherPacketSource::trimBuffersAfterMeta( + const sp &meta) { + if (meta == NULL) { + ALOGW("trimming with NULL meta, ignoring"); + return; + } Mutex::Autolock autoLock(mLock); if (mBuffers.empty()) { return; } + HLSTime stopTime(meta); + ALOGV("trimBuffersAfterMeta: discontinuitySeq %zu, timeUs %lld", + stopTime.mSeq, (long long)stopTime.mTimeUs); + List >::iterator it; sp newLatestEnqueuedMeta = NULL; int64_t newLastQueuedTimeUs = 0; @@ -514,20 +528,15 @@ void AnotherPacketSource::trimBuffersAfterTimeUs( newDiscontinuityCount++; continue; } - size_t curDiscontinuitySeq; - int64_t curTimeUs; - CHECK(buffer->meta()->findInt32( - "discontinuitySeq", (int32_t*)&curDiscontinuitySeq)); - CHECK(buffer->meta()->findInt64("timeUs", &curTimeUs)); - if ((curDiscontinuitySeq > discontinuitySeq - || (curDiscontinuitySeq == discontinuitySeq - && curTimeUs >= timeUs))) { - ALOGI("trimming from %lld (inclusive) to end", - (long long)curTimeUs); + + HLSTime curTime(buffer->meta()); + if (!(curTime < stopTime)) { + ALOGV("trimming from %lld (inclusive) to end", + (long long)curTime.mTimeUs); break; } newLatestEnqueuedMeta = buffer->meta(); - newLastQueuedTimeUs = curTimeUs; + newLastQueuedTimeUs = curTime.mTimeUs; } mBuffers.erase(it, mBuffers.end()); mLatestEnqueuedMeta = newLatestEnqueuedMeta; @@ -535,11 +544,20 @@ void AnotherPacketSource::trimBuffersAfterTimeUs( mQueuedDiscontinuityCount = newDiscontinuityCount; } -sp AnotherPacketSource::trimBuffersBeforeTimeUs( - size_t discontinuitySeq, int64_t timeUs) { - ALOGV("trimBuffersBeforeTimeUs: discontinuitySeq %zu, timeUs %lld", - discontinuitySeq, (long long)timeUs); - sp meta; +/* + * removes samples with time equal or before meta; + * returns first sample left in the queue. + * + * (for AVC, if trim happens, the samples left will always start + * at next IDR.) + */ +sp AnotherPacketSource::trimBuffersBeforeMeta( + const sp &meta) { + HLSTime startTime(meta); + ALOGV("trimBuffersBeforeMeta: discontinuitySeq %zu, timeUs %lld", + startTime.mSeq, (long long)startTime.mTimeUs); + + sp firstMeta; Mutex::Autolock autoLock(mLock); if (mBuffers.empty()) { return NULL; @@ -572,24 +590,19 @@ sp AnotherPacketSource::trimBuffersBeforeTimeUs( if (isAvc && !IsIDR(buffer)) { continue; } - size_t curDiscontinuitySeq; - int64_t curTimeUs; - CHECK(buffer->meta()->findInt32( - "discontinuitySeq", (int32_t*)&curDiscontinuitySeq)); - CHECK(buffer->meta()->findInt64("timeUs", &curTimeUs)); - if ((curDiscontinuitySeq > discontinuitySeq - || (curDiscontinuitySeq == discontinuitySeq - && curTimeUs > timeUs))) { - ALOGI("trimming from beginning to %lld (not inclusive)", - (long long)curTimeUs); - meta = buffer->meta(); + + HLSTime curTime(buffer->meta()); + if (startTime < curTime) { + ALOGV("trimming from beginning to %lld (not inclusive)", + (long long)curTime.mTimeUs); + firstMeta = buffer->meta(); break; } } mBuffers.erase(mBuffers.begin(), it); mQueuedDiscontinuityCount -= discontinuityCount; mLatestDequeuedMeta = NULL; - return meta; + return firstMeta; } } // namespace android diff --git a/media/libstagefright/mpeg2ts/AnotherPacketSource.h b/media/libstagefright/mpeg2ts/AnotherPacketSource.h index e126006..fa7dd6a 100644 --- a/media/libstagefright/mpeg2ts/AnotherPacketSource.h +++ b/media/libstagefright/mpeg2ts/AnotherPacketSource.h @@ -76,8 +76,8 @@ struct AnotherPacketSource : public MediaSource { sp getLatestDequeuedMeta(); sp getMetaAfterLastDequeued(int64_t delayUs); - void trimBuffersAfterTimeUs(size_t discontinuitySeq, int64_t timeUs); - sp trimBuffersBeforeTimeUs(size_t discontinuitySeq, int64_t timeUs); + void trimBuffersAfterMeta(const sp &meta); + sp trimBuffersBeforeMeta(const sp &meta); protected: virtual ~AnotherPacketSource(); -- cgit v1.1