In order to recover from video lagging behind audio, drop avc frames

that are not referenced by other frames before feeding them into the decoder.

Change-Id: I822190af8f8329567bff8da1ea23136d0a765481

8 files changed, 155 insertions, 44 deletions

diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
index 6f34ba7..7218faf 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
@@ -34,17 +34,21 @@
 #include <media/stagefright/foundation/ADebug.h>
 #include <media/stagefright/foundation/AMessage.h>
 #include <media/stagefright/ACodec.h>
+#include <media/stagefright/MediaDefs.h>
 #include <media/stagefright/MediaErrors.h>
 #include <media/stagefright/MetaData.h>
 #include <surfaceflinger/Surface.h>
 #include <gui/ISurfaceTexture.h>
 
+#include "avc_utils.h"
+
 namespace android {
 
 ////////////////////////////////////////////////////////////////////////////////
 
 NuPlayer::NuPlayer()
     : mUIDValid(false),
+      mVideoIsAVC(false),
       mAudioEOS(false),
       mVideoEOS(false),
       mScanSourcesPending(false),
@@ -52,7 +56,12 @@ NuPlayer::NuPlayer()
       mFlushingAudio(NONE),
       mFlushingVideo(NONE),
       mResetInProgress(false),
-      mResetPostponed(false) {
+      mResetPostponed(false),
+      mSkipRenderingAudioUntilMediaTimeUs(-1ll),
+      mSkipRenderingVideoUntilMediaTimeUs(-1ll),
+      mVideoLateByUs(0ll),
+      mNumFramesTotal(0ll),
+      mNumFramesDropped(0ll) {
 }
 
 NuPlayer::~NuPlayer() {
@@ -185,10 +194,14 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
         {
             LOGV("kWhatStart");
 
+            mVideoIsAVC = false;
             mAudioEOS = false;
             mVideoEOS = false;
             mSkipRenderingAudioUntilMediaTimeUs = -1;
             mSkipRenderingVideoUntilMediaTimeUs = -1;
+            mVideoLateByUs = 0;
+            mNumFramesTotal = 0;
+            mNumFramesDropped = 0;
 
             mSource->start();
 
@@ -269,6 +282,8 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
                 } else {
                     CHECK(IsFlushingState(mFlushingVideo, &needShutdown));
                     mFlushingVideo = FLUSHED;
+
+                    mVideoLateByUs = 0;
                 }
 
                 LOGV("decoder %s flush completed", audio ? "audio" : "video");
@@ -397,13 +412,18 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
                 int64_t positionUs;
                 CHECK(msg->findInt64("positionUs", &positionUs));
 
+                CHECK(msg->findInt64("videoLateByUs", &mVideoLateByUs));
+
                 if (mDriver != NULL) {
                     sp<NuPlayerDriver> driver = mDriver.promote();
                     if (driver != NULL) {
                         driver->notifyPosition(positionUs);
+
+                        driver->notifyFrameStats(
+                                mNumFramesTotal, mNumFramesDropped);
                     }
                 }
-            } else {
+            } else if (what == Renderer::kWhatFlushComplete) {
                 CHECK_EQ(what, (int32_t)Renderer::kWhatFlushComplete);
 
                 int32_t audio;
@@ -565,6 +585,12 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
         return -EWOULDBLOCK;
     }
 
+    if (!audio) {
+        const char *mime;
+        CHECK(meta->findCString(kKeyMIMEType, &mime));
+        mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime);
+    }
+
     sp<AMessage> notify =
         new AMessage(audio ? kWhatAudioNotify : kWhatVideoNotify,
                      id());
@@ -598,53 +624,70 @@ status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
     }
 
     sp<ABuffer> accessUnit;
-    status_t err = mSource->dequeueAccessUnit(audio, &accessUnit);
 
-    if (err == -EWOULDBLOCK) {
-        return err;
-    } else if (err != OK) {
-        if (err == INFO_DISCONTINUITY) {
-            int32_t type;
-            CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
+    bool dropAccessUnit;
+    do {
+        status_t err = mSource->dequeueAccessUnit(audio, &accessUnit);
 
-            bool formatChange =
-                type == ATSParser::DISCONTINUITY_FORMATCHANGE;
+        if (err == -EWOULDBLOCK) {
+            return err;
+        } else if (err != OK) {
+            if (err == INFO_DISCONTINUITY) {
+                int32_t type;
+                CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
 
-            LOGV("%s discontinuity (formatChange=%d)",
-                 audio ? "audio" : "video", formatChange);
+                bool formatChange =
+                    type == ATSParser::DISCONTINUITY_FORMATCHANGE;
 
-            if (audio) {
-                mSkipRenderingAudioUntilMediaTimeUs = -1;
-            } else {
-                mSkipRenderingVideoUntilMediaTimeUs = -1;
-            }
+                LOGV("%s discontinuity (formatChange=%d)",
+                     audio ? "audio" : "video", formatChange);
 
-            sp<AMessage> extra;
-            if (accessUnit->meta()->findMessage("extra", &extra)
-                    && extra != NULL) {
-                int64_t resumeAtMediaTimeUs;
-                if (extra->findInt64(
-                            "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
-                    LOGI("suppressing rendering of %s until %lld us",
-                            audio ? "audio" : "video", resumeAtMediaTimeUs);
+                if (audio) {
+                    mSkipRenderingAudioUntilMediaTimeUs = -1;
+                } else {
+                    mSkipRenderingVideoUntilMediaTimeUs = -1;
+                }
 
-                    if (audio) {
-                        mSkipRenderingAudioUntilMediaTimeUs =
-                            resumeAtMediaTimeUs;
-                    } else {
-                        mSkipRenderingVideoUntilMediaTimeUs =
-                            resumeAtMediaTimeUs;
+                sp<AMessage> extra;
+                if (accessUnit->meta()->findMessage("extra", &extra)
+                        && extra != NULL) {
+                    int64_t resumeAtMediaTimeUs;
+                    if (extra->findInt64(
+                                "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
+                        LOGI("suppressing rendering of %s until %lld us",
+                                audio ? "audio" : "video", resumeAtMediaTimeUs);
+
+                        if (audio) {
+                            mSkipRenderingAudioUntilMediaTimeUs =
+                                resumeAtMediaTimeUs;
+                        } else {
+                            mSkipRenderingVideoUntilMediaTimeUs =
+                                resumeAtMediaTimeUs;
+                        }
                     }
                 }
+
+                flushDecoder(audio, formatChange);
             }
 
-            flushDecoder(audio, formatChange);
+            reply->setInt32("err", err);
+            reply->post();
+            return OK;
         }
 
-        reply->setInt32("err", err);
-        reply->post();
-        return OK;
-    }
+        if (!audio) {
+            ++mNumFramesTotal;
+        }
+
+        dropAccessUnit = false;
+        if (!audio
+                && mVideoLateByUs > 100000ll
+                && mVideoIsAVC
+                && !IsAVCReferenceFrame(accessUnit)) {
+            dropAccessUnit = true;
+            ++mNumFramesDropped;
+        }
+    } while (dropAccessUnit);
 
     // LOGV("returned a valid buffer of %s data", audio ? "audio" : "video");
 
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h
index 41e015b..a5382b4 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h
@@ -92,6 +92,7 @@ private:
     sp<NativeWindowWrapper> mNativeWindow;
     sp<MediaPlayerBase::AudioSink> mAudioSink;
     sp<Decoder> mVideoDecoder;
+    bool mVideoIsAVC;
     sp<Decoder> mAudioDecoder;
     sp<Renderer> mRenderer;
 
@@ -119,6 +120,9 @@ private:
     int64_t mSkipRenderingAudioUntilMediaTimeUs;
     int64_t mSkipRenderingVideoUntilMediaTimeUs;
 
+    int64_t mVideoLateByUs;
+    int64_t mNumFramesTotal, mNumFramesDropped;
+
     status_t instantiateDecoder(bool audio, sp<Decoder> *decoder);
 
     status_t feedDecoderInputData(bool audio, const sp<AMessage> &msg);
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
index c6fca2c..b1e917d 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
@@ -31,6 +31,8 @@ NuPlayerDriver::NuPlayerDriver()
     : mResetInProgress(false),
       mDurationUs(-1),
       mPositionUs(-1),
+      mNumFramesTotal(0),
+      mNumFramesDropped(0),
       mLooper(new ALooper),
       mState(UNINITIALIZED),
       mStartupSeekTimeUs(-1) {
@@ -292,4 +294,30 @@ void NuPlayerDriver::notifySeekComplete() {
     sendEvent(MEDIA_SEEK_COMPLETE);
 }
 
+void NuPlayerDriver::notifyFrameStats(
+        int64_t numFramesTotal, int64_t numFramesDropped) {
+    Mutex::Autolock autoLock(mLock);
+    mNumFramesTotal = numFramesTotal;
+    mNumFramesDropped = numFramesDropped;
+}
+
+status_t NuPlayerDriver::dump(int fd, const Vector<String16> &args) const {
+    Mutex::Autolock autoLock(mLock);
+
+    FILE *out = fdopen(dup(fd), "w");
+
+    fprintf(out, " NuPlayer\n");
+    fprintf(out, "  numFramesTotal(%lld), numFramesDropped(%lld), "
+                 "percentageDropped(%.2f)\n",
+                 mNumFramesTotal,
+                 mNumFramesDropped,
+                 mNumFramesTotal == 0
+                    ? 0.0 : (double)mNumFramesDropped / mNumFramesTotal);
+
+    fclose(out);
+    out = NULL;
+
+    return OK;
+}
+
 }  // namespace android
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h
index 1bb7ca2..181c37d 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h
@@ -60,16 +60,19 @@ struct NuPlayerDriver : public MediaPlayerInterface {
     virtual status_t getMetadata(
             const media::Metadata::Filter& ids, Parcel *records);
 
+    virtual status_t dump(int fd, const Vector<String16> &args) const;
+
     void notifyResetComplete();
     void notifyDuration(int64_t durationUs);
     void notifyPosition(int64_t positionUs);
     void notifySeekComplete();
+    void notifyFrameStats(int64_t numFramesTotal, int64_t numFramesDropped);
 
 protected:
     virtual ~NuPlayerDriver();
 
 private:
-    Mutex mLock;
+    mutable Mutex mLock;
     Condition mCondition;
 
     // The following are protected through "mLock"
@@ -77,6 +80,8 @@ private:
     bool mResetInProgress;
     int64_t mDurationUs;
     int64_t mPositionUs;
+    int64_t mNumFramesTotal;
+    int64_t mNumFramesDropped;
     // <<<
 
     sp<ALooper> mLooper;
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
index 3b3fca2..07e347e 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
@@ -47,7 +47,8 @@ NuPlayer::Renderer::Renderer(
       mHasVideo(false),
       mSyncQueues(false),
       mPaused(false),
-      mLastPositionUpdateUs(-1ll) {
+      mLastPositionUpdateUs(-1ll),
+      mVideoLateByUs(0ll) {
 }
 
 NuPlayer::Renderer::~Renderer() {
@@ -357,22 +358,26 @@ void NuPlayer::Renderer::onDrainVideoQueue() {
 
         mVideoQueue.erase(mVideoQueue.begin());
         entry = NULL;
+
+        mVideoLateByUs = 0ll;
+
+        notifyPosition();
         return;
     }
 
-#if 0
     int64_t mediaTimeUs;
     CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
 
     int64_t realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs;
-    int64_t lateByUs = ALooper::GetNowUs() - realTimeUs;
+    mVideoLateByUs = ALooper::GetNowUs() - realTimeUs;
 
-    if (lateByUs > 40000) {
-        LOGI("video late by %lld us (%.2f secs)", lateByUs, lateByUs / 1E6);
+    bool tooLate = (mVideoLateByUs > 40000);
+
+    if (tooLate) {
+        LOGV("video late by %lld us (%.2f secs)", lateByUs, lateByUs / 1E6);
     } else {
         LOGV("rendering video at media time %.2f secs", mediaTimeUs / 1E6);
     }
-#endif
 
     entry->mNotifyConsumed->setInt32("render", true);
     entry->mNotifyConsumed->post();
@@ -604,6 +609,7 @@ void NuPlayer::Renderer::notifyPosition() {
     sp<AMessage> notify = mNotify->dup();
     notify->setInt32("what", kWhatPosition);
     notify->setInt64("positionUs", positionUs);
+    notify->setInt64("videoLateByUs", mVideoLateByUs);
     notify->post();
 }
 
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h
index 44c5d44..268628b 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h
@@ -101,6 +101,7 @@ private:
     bool mPaused;
 
     int64_t mLastPositionUpdateUs;
+    int64_t mVideoLateByUs;
 
     bool onDrainAudioQueue();
     void postDrainAudioQueue(int64_t delayUs = 0);
@@ -118,6 +119,7 @@ private:
     void notifyEOS(bool audio, status_t finalResult);
     void notifyFlushComplete(bool audio);
     void notifyPosition();
+    void notifyVideoLateBy(int64_t lateByUs);
 
     void flushQueue(List<QueueEntry> *queue);
     bool dropBufferWhileFlushing(bool audio, const sp<AMessage> &msg);
diff --git a/media/libstagefright/avc_utils.cpp b/media/libstagefright/avc_utils.cpp
index 8a42e8b..07aa140 100644
--- a/media/libstagefright/avc_utils.cpp
+++ b/media/libstagefright/avc_utils.cpp
@@ -329,6 +329,28 @@ bool IsIDR(const sp<ABuffer> &buffer) {
     return foundIDR;
 }
 
+bool IsAVCReferenceFrame(const sp<ABuffer> &accessUnit) {
+    const uint8_t *data = accessUnit->data();
+    size_t size = accessUnit->size();
+
+    const uint8_t *nalStart;
+    size_t nalSize;
+    while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) {
+        CHECK_GT(nalSize, 0u);
+
+        unsigned nalType = nalStart[0] & 0x1f;
+
+        if (nalType == 5) {
+            return true;
+        } else if (nalType == 1) {
+            unsigned nal_ref_idc = (nalStart[0] >> 5) & 3;
+            return nal_ref_idc != 0;
+        }
+    }
+
+    return true;
+}
+
 sp<MetaData> MakeAACCodecSpecificData(
         unsigned profile, unsigned sampling_freq_index,
         unsigned channel_configuration) {
diff --git a/media/libstagefright/include/avc_utils.h b/media/libstagefright/include/avc_utils.h
index 15cd4d4..e418822 100644
--- a/media/libstagefright/include/avc_utils.h
+++ b/media/libstagefright/include/avc_utils.h
@@ -50,6 +50,7 @@ struct MetaData;
 sp<MetaData> MakeAVCCodecSpecificData(const sp<ABuffer> &accessUnit);
 
 bool IsIDR(const sp<ABuffer> &accessUnit);
+bool IsAVCReferenceFrame(const sp<ABuffer> &accessUnit);
 
 const char *AVCProfileToString(uint8_t profile);