From 3fe62150fa3dd6d25cb84aad80bc9e27ddd16c45 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Fri, 16 Sep 2011 15:09:22 -0700 Subject: In order to recover from video lagging behind audio, drop avc frames that are not referenced by other frames before feeding them into the decoder. Change-Id: I822190af8f8329567bff8da1ea23136d0a765481 --- media/libmediaplayerservice/nuplayer/NuPlayer.cpp | 117 ++++++++++++++------- media/libmediaplayerservice/nuplayer/NuPlayer.h | 4 + .../nuplayer/NuPlayerDriver.cpp | 28 +++++ .../nuplayer/NuPlayerDriver.h | 7 +- .../nuplayer/NuPlayerRenderer.cpp | 18 ++-- .../nuplayer/NuPlayerRenderer.h | 2 + media/libstagefright/avc_utils.cpp | 22 ++++ media/libstagefright/include/avc_utils.h | 1 + 8 files changed, 155 insertions(+), 44 deletions(-) diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp index 6f34ba7..7218faf 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp @@ -34,17 +34,21 @@ #include #include #include +#include #include #include #include #include +#include "avc_utils.h" + namespace android { //////////////////////////////////////////////////////////////////////////////// NuPlayer::NuPlayer() : mUIDValid(false), + mVideoIsAVC(false), mAudioEOS(false), mVideoEOS(false), mScanSourcesPending(false), @@ -52,7 +56,12 @@ NuPlayer::NuPlayer() mFlushingAudio(NONE), mFlushingVideo(NONE), mResetInProgress(false), - mResetPostponed(false) { + mResetPostponed(false), + mSkipRenderingAudioUntilMediaTimeUs(-1ll), + mSkipRenderingVideoUntilMediaTimeUs(-1ll), + mVideoLateByUs(0ll), + mNumFramesTotal(0ll), + mNumFramesDropped(0ll) { } NuPlayer::~NuPlayer() { @@ -185,10 +194,14 @@ void NuPlayer::onMessageReceived(const sp &msg) { { LOGV("kWhatStart"); + mVideoIsAVC = false; mAudioEOS = false; mVideoEOS = false; mSkipRenderingAudioUntilMediaTimeUs = -1; mSkipRenderingVideoUntilMediaTimeUs = -1; + mVideoLateByUs = 0; + mNumFramesTotal = 0; + mNumFramesDropped = 0; mSource->start(); @@ -269,6 +282,8 @@ void NuPlayer::onMessageReceived(const sp &msg) { } else { CHECK(IsFlushingState(mFlushingVideo, &needShutdown)); mFlushingVideo = FLUSHED; + + mVideoLateByUs = 0; } LOGV("decoder %s flush completed", audio ? "audio" : "video"); @@ -397,13 +412,18 @@ void NuPlayer::onMessageReceived(const sp &msg) { int64_t positionUs; CHECK(msg->findInt64("positionUs", &positionUs)); + CHECK(msg->findInt64("videoLateByUs", &mVideoLateByUs)); + if (mDriver != NULL) { sp driver = mDriver.promote(); if (driver != NULL) { driver->notifyPosition(positionUs); + + driver->notifyFrameStats( + mNumFramesTotal, mNumFramesDropped); } } - } else { + } else if (what == Renderer::kWhatFlushComplete) { CHECK_EQ(what, (int32_t)Renderer::kWhatFlushComplete); int32_t audio; @@ -565,6 +585,12 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp *decoder) { return -EWOULDBLOCK; } + if (!audio) { + const char *mime; + CHECK(meta->findCString(kKeyMIMEType, &mime)); + mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime); + } + sp notify = new AMessage(audio ? kWhatAudioNotify : kWhatVideoNotify, id()); @@ -598,53 +624,70 @@ status_t NuPlayer::feedDecoderInputData(bool audio, const sp &msg) { } sp accessUnit; - status_t err = mSource->dequeueAccessUnit(audio, &accessUnit); - if (err == -EWOULDBLOCK) { - return err; - } else if (err != OK) { - if (err == INFO_DISCONTINUITY) { - int32_t type; - CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); + bool dropAccessUnit; + do { + status_t err = mSource->dequeueAccessUnit(audio, &accessUnit); - bool formatChange = - type == ATSParser::DISCONTINUITY_FORMATCHANGE; + if (err == -EWOULDBLOCK) { + return err; + } else if (err != OK) { + if (err == INFO_DISCONTINUITY) { + int32_t type; + CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); - LOGV("%s discontinuity (formatChange=%d)", - audio ? "audio" : "video", formatChange); + bool formatChange = + type == ATSParser::DISCONTINUITY_FORMATCHANGE; - if (audio) { - mSkipRenderingAudioUntilMediaTimeUs = -1; - } else { - mSkipRenderingVideoUntilMediaTimeUs = -1; - } + LOGV("%s discontinuity (formatChange=%d)", + audio ? "audio" : "video", formatChange); - sp extra; - if (accessUnit->meta()->findMessage("extra", &extra) - && extra != NULL) { - int64_t resumeAtMediaTimeUs; - if (extra->findInt64( - "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) { - LOGI("suppressing rendering of %s until %lld us", - audio ? "audio" : "video", resumeAtMediaTimeUs); + if (audio) { + mSkipRenderingAudioUntilMediaTimeUs = -1; + } else { + mSkipRenderingVideoUntilMediaTimeUs = -1; + } - if (audio) { - mSkipRenderingAudioUntilMediaTimeUs = - resumeAtMediaTimeUs; - } else { - mSkipRenderingVideoUntilMediaTimeUs = - resumeAtMediaTimeUs; + sp extra; + if (accessUnit->meta()->findMessage("extra", &extra) + && extra != NULL) { + int64_t resumeAtMediaTimeUs; + if (extra->findInt64( + "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) { + LOGI("suppressing rendering of %s until %lld us", + audio ? "audio" : "video", resumeAtMediaTimeUs); + + if (audio) { + mSkipRenderingAudioUntilMediaTimeUs = + resumeAtMediaTimeUs; + } else { + mSkipRenderingVideoUntilMediaTimeUs = + resumeAtMediaTimeUs; + } } } + + flushDecoder(audio, formatChange); } - flushDecoder(audio, formatChange); + reply->setInt32("err", err); + reply->post(); + return OK; } - reply->setInt32("err", err); - reply->post(); - return OK; - } + if (!audio) { + ++mNumFramesTotal; + } + + dropAccessUnit = false; + if (!audio + && mVideoLateByUs > 100000ll + && mVideoIsAVC + && !IsAVCReferenceFrame(accessUnit)) { + dropAccessUnit = true; + ++mNumFramesDropped; + } + } while (dropAccessUnit); // LOGV("returned a valid buffer of %s data", audio ? "audio" : "video"); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h index 41e015b..a5382b4 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h @@ -92,6 +92,7 @@ private: sp mNativeWindow; sp mAudioSink; sp mVideoDecoder; + bool mVideoIsAVC; sp mAudioDecoder; sp mRenderer; @@ -119,6 +120,9 @@ private: int64_t mSkipRenderingAudioUntilMediaTimeUs; int64_t mSkipRenderingVideoUntilMediaTimeUs; + int64_t mVideoLateByUs; + int64_t mNumFramesTotal, mNumFramesDropped; + status_t instantiateDecoder(bool audio, sp *decoder); status_t feedDecoderInputData(bool audio, const sp &msg); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp index c6fca2c..b1e917d 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp @@ -31,6 +31,8 @@ NuPlayerDriver::NuPlayerDriver() : mResetInProgress(false), mDurationUs(-1), mPositionUs(-1), + mNumFramesTotal(0), + mNumFramesDropped(0), mLooper(new ALooper), mState(UNINITIALIZED), mStartupSeekTimeUs(-1) { @@ -292,4 +294,30 @@ void NuPlayerDriver::notifySeekComplete() { sendEvent(MEDIA_SEEK_COMPLETE); } +void NuPlayerDriver::notifyFrameStats( + int64_t numFramesTotal, int64_t numFramesDropped) { + Mutex::Autolock autoLock(mLock); + mNumFramesTotal = numFramesTotal; + mNumFramesDropped = numFramesDropped; +} + +status_t NuPlayerDriver::dump(int fd, const Vector &args) const { + Mutex::Autolock autoLock(mLock); + + FILE *out = fdopen(dup(fd), "w"); + + fprintf(out, " NuPlayer\n"); + fprintf(out, " numFramesTotal(%lld), numFramesDropped(%lld), " + "percentageDropped(%.2f)\n", + mNumFramesTotal, + mNumFramesDropped, + mNumFramesTotal == 0 + ? 0.0 : (double)mNumFramesDropped / mNumFramesTotal); + + fclose(out); + out = NULL; + + return OK; +} + } // namespace android diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h index 1bb7ca2..181c37d 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h @@ -60,16 +60,19 @@ struct NuPlayerDriver : public MediaPlayerInterface { virtual status_t getMetadata( const media::Metadata::Filter& ids, Parcel *records); + virtual status_t dump(int fd, const Vector &args) const; + void notifyResetComplete(); void notifyDuration(int64_t durationUs); void notifyPosition(int64_t positionUs); void notifySeekComplete(); + void notifyFrameStats(int64_t numFramesTotal, int64_t numFramesDropped); protected: virtual ~NuPlayerDriver(); private: - Mutex mLock; + mutable Mutex mLock; Condition mCondition; // The following are protected through "mLock" @@ -77,6 +80,8 @@ private: bool mResetInProgress; int64_t mDurationUs; int64_t mPositionUs; + int64_t mNumFramesTotal; + int64_t mNumFramesDropped; // <<< sp mLooper; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp index 3b3fca2..07e347e 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp @@ -47,7 +47,8 @@ NuPlayer::Renderer::Renderer( mHasVideo(false), mSyncQueues(false), mPaused(false), - mLastPositionUpdateUs(-1ll) { + mLastPositionUpdateUs(-1ll), + mVideoLateByUs(0ll) { } NuPlayer::Renderer::~Renderer() { @@ -357,22 +358,26 @@ void NuPlayer::Renderer::onDrainVideoQueue() { mVideoQueue.erase(mVideoQueue.begin()); entry = NULL; + + mVideoLateByUs = 0ll; + + notifyPosition(); return; } -#if 0 int64_t mediaTimeUs; CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); int64_t realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs; - int64_t lateByUs = ALooper::GetNowUs() - realTimeUs; + mVideoLateByUs = ALooper::GetNowUs() - realTimeUs; - if (lateByUs > 40000) { - LOGI("video late by %lld us (%.2f secs)", lateByUs, lateByUs / 1E6); + bool tooLate = (mVideoLateByUs > 40000); + + if (tooLate) { + LOGV("video late by %lld us (%.2f secs)", lateByUs, lateByUs / 1E6); } else { LOGV("rendering video at media time %.2f secs", mediaTimeUs / 1E6); } -#endif entry->mNotifyConsumed->setInt32("render", true); entry->mNotifyConsumed->post(); @@ -604,6 +609,7 @@ void NuPlayer::Renderer::notifyPosition() { sp notify = mNotify->dup(); notify->setInt32("what", kWhatPosition); notify->setInt64("positionUs", positionUs); + notify->setInt64("videoLateByUs", mVideoLateByUs); notify->post(); } diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h index 44c5d44..268628b 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h @@ -101,6 +101,7 @@ private: bool mPaused; int64_t mLastPositionUpdateUs; + int64_t mVideoLateByUs; bool onDrainAudioQueue(); void postDrainAudioQueue(int64_t delayUs = 0); @@ -118,6 +119,7 @@ private: void notifyEOS(bool audio, status_t finalResult); void notifyFlushComplete(bool audio); void notifyPosition(); + void notifyVideoLateBy(int64_t lateByUs); void flushQueue(List *queue); bool dropBufferWhileFlushing(bool audio, const sp &msg); diff --git a/media/libstagefright/avc_utils.cpp b/media/libstagefright/avc_utils.cpp index 8a42e8b..07aa140 100644 --- a/media/libstagefright/avc_utils.cpp +++ b/media/libstagefright/avc_utils.cpp @@ -329,6 +329,28 @@ bool IsIDR(const sp &buffer) { return foundIDR; } +bool IsAVCReferenceFrame(const sp &accessUnit) { + const uint8_t *data = accessUnit->data(); + size_t size = accessUnit->size(); + + const uint8_t *nalStart; + size_t nalSize; + while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) { + CHECK_GT(nalSize, 0u); + + unsigned nalType = nalStart[0] & 0x1f; + + if (nalType == 5) { + return true; + } else if (nalType == 1) { + unsigned nal_ref_idc = (nalStart[0] >> 5) & 3; + return nal_ref_idc != 0; + } + } + + return true; +} + sp MakeAACCodecSpecificData( unsigned profile, unsigned sampling_freq_index, unsigned channel_configuration) { diff --git a/media/libstagefright/include/avc_utils.h b/media/libstagefright/include/avc_utils.h index 15cd4d4..e418822 100644 --- a/media/libstagefright/include/avc_utils.h +++ b/media/libstagefright/include/avc_utils.h @@ -50,6 +50,7 @@ struct MetaData; sp MakeAVCCodecSpecificData(const sp &accessUnit); bool IsIDR(const sp &accessUnit); +bool IsAVCReferenceFrame(const sp &accessUnit); const char *AVCProfileToString(uint8_t profile); -- cgit v1.1