Merge "DO NOT MERGE: Cherry picked hacks to better hide A/V sync issues." into honeycomb

2 files changed, 153 insertions, 61 deletions

diff --git a/media/libstagefright/AwesomePlayer.cpp b/media/libstagefright/AwesomePlayer.cpp
index 89b3dab..1a8e86d 100644
--- a/media/libstagefright/AwesomePlayer.cpp
+++ b/media/libstagefright/AwesomePlayer.cpp
@@ -52,7 +52,7 @@
 #include "include/LiveSession.h"
 
 #define USE_SURFACE_ALLOC 1
-#define FRAME_DROP_FREQ 7
+#define FRAME_DROP_FREQ 0
 
 namespace android {
 
@@ -477,7 +477,7 @@ void AwesomePlayer::reset_l() {
     mTimeSourceDeltaUs = 0;
     mVideoTimeUs = 0;
 
-    mSeeking = false;
+    mSeeking = NO_SEEK;
     mSeekNotificationSent = false;
     mSeekTimeUs = 0;
 
@@ -783,34 +783,33 @@ status_t AwesomePlayer::play_l() {
                 mAudioPlayer = new AudioPlayer(mAudioSink, this);
                 mAudioPlayer->setSource(mAudioSource);
 
-                // We've already started the MediaSource in order to enable
-                // the prefetcher to read its data.
-                status_t err = mAudioPlayer->start(
-                        true /* sourceAlreadyStarted */);
+                mTimeSource = mAudioPlayer;
 
-                if (err != OK) {
-                    delete mAudioPlayer;
-                    mAudioPlayer = NULL;
+                deferredAudioSeek = true;
 
-                    mFlags &= ~(PLAYING | FIRST_FRAME);
+                mWatchForAudioSeekComplete = false;
+                mWatchForAudioEOS = true;
+            }
+        }
 
-                    if (mDecryptHandle != NULL) {
-                        mDrmManagerClient->setPlaybackStatus(mDecryptHandle,
-                                 Playback::STOP, 0);
-                    }
+        CHECK(!(mFlags & AUDIO_RUNNING));
 
-                    return err;
-                }
+        if (mVideoSource == NULL) {
+            status_t err = startAudioPlayer_l();
 
-                mTimeSource = mAudioPlayer;
+            if (err != OK) {
+                delete mAudioPlayer;
+                mAudioPlayer = NULL;
 
-                deferredAudioSeek = true;
+                mFlags &= ~(PLAYING | FIRST_FRAME);
 
-                mWatchForAudioSeekComplete = false;
-                mWatchForAudioEOS = true;
+                if (mDecryptHandle != NULL) {
+                    mDrmManagerClient->setPlaybackStatus(
+                            mDecryptHandle, Playback::STOP, 0);
+                }
+
+                return err;
             }
-        } else {
-            mAudioPlayer->resume();
         }
     }
 
@@ -842,6 +841,36 @@ status_t AwesomePlayer::play_l() {
     return OK;
 }
 
+status_t AwesomePlayer::startAudioPlayer_l() {
+    CHECK(!(mFlags & AUDIO_RUNNING));
+
+    if (mAudioSource == NULL || mAudioPlayer == NULL) {
+        return OK;
+    }
+
+    if (!(mFlags & AUDIOPLAYER_STARTED)) {
+        mFlags |= AUDIOPLAYER_STARTED;
+
+        // We've already started the MediaSource in order to enable
+        // the prefetcher to read its data.
+        status_t err = mAudioPlayer->start(
+                true /* sourceAlreadyStarted */);
+
+        if (err != OK) {
+            notifyListener_l(MEDIA_ERROR, MEDIA_ERROR_UNKNOWN, err);
+            return err;
+        }
+    } else {
+        mAudioPlayer->resume();
+    }
+
+    mFlags |= AUDIO_RUNNING;
+
+    mWatchForAudioEOS = true;
+
+    return OK;
+}
+
 void AwesomePlayer::notifyVideoSize_l() {
     sp<MetaData> meta = mVideoSource->getFormat();
 
@@ -943,7 +972,7 @@ status_t AwesomePlayer::pause_l(bool at_eos) {
 
     cancelPlayerEvents(true /* keepBufferingGoing */);
 
-    if (mAudioPlayer != NULL) {
+    if (mAudioPlayer != NULL && (mFlags & AUDIO_RUNNING)) {
         if (at_eos) {
             // If we played the audio stream to completion we
             // want to make sure that all samples remaining in the audio
@@ -952,6 +981,8 @@ status_t AwesomePlayer::pause_l(bool at_eos) {
         } else {
             mAudioPlayer->pause();
         }
+
+        mFlags &= ~AUDIO_RUNNING;
     }
 
     mFlags &= ~PLAYING;
@@ -1009,7 +1040,7 @@ status_t AwesomePlayer::getPosition(int64_t *positionUs) {
     if (mRTSPController != NULL) {
         *positionUs = mRTSPController->getNormalPlayTimeUs();
     }
-    else if (mSeeking) {
+    else if (mSeeking != NO_SEEK) {
         *positionUs = mSeekTimeUs;
     } else if (mVideoSource != NULL) {
         Mutex::Autolock autoLock(mMiscStateLock);
@@ -1053,7 +1084,7 @@ status_t AwesomePlayer::seekTo_l(int64_t timeUs) {
         play_l();
     }
 
-    mSeeking = true;
+    mSeeking = SEEK;
     mSeekNotificationSent = false;
     mSeekTimeUs = timeUs;
     mFlags &= ~(AT_EOS | AUDIO_AT_EOS | VIDEO_AT_EOS);
@@ -1072,7 +1103,7 @@ status_t AwesomePlayer::seekTo_l(int64_t timeUs) {
 }
 
 void AwesomePlayer::seekAudioIfNecessary_l() {
-    if (mSeeking && mVideoSource == NULL && mAudioPlayer != NULL) {
+    if (mSeeking != NO_SEEK && mVideoSource == NULL && mAudioPlayer != NULL) {
         mAudioPlayer->seekTo(mSeekTimeUs);
 
         mWatchForAudioSeekComplete = true;
@@ -1168,7 +1199,12 @@ status_t AwesomePlayer::initVideoDecoder(uint32_t flags) {
 }
 
 void AwesomePlayer::finishSeekIfNecessary(int64_t videoTimeUs) {
-    if (!mSeeking) {
+    if (mSeeking == SEEK_VIDEO_ONLY) {
+        mSeeking = NO_SEEK;
+        return;
+    }
+
+    if (mSeeking == NO_SEEK) {
         return;
     }
 
@@ -1179,9 +1215,7 @@ void AwesomePlayer::finishSeekIfNecessary(int64_t videoTimeUs) {
         // requested seek time instead.
 
         mAudioPlayer->seekTo(videoTimeUs < 0 ? mSeekTimeUs : videoTimeUs);
-        mAudioPlayer->resume();
         mWatchForAudioSeekComplete = true;
-        mWatchForAudioEOS = true;
     } else if (!mSeekNotificationSent) {
         // If we're playing video only, report seek complete now,
         // otherwise audio player will notify us later.
@@ -1189,7 +1223,7 @@ void AwesomePlayer::finishSeekIfNecessary(int64_t videoTimeUs) {
     }
 
     mFlags |= FIRST_FRAME;
-    mSeeking = false;
+    mSeeking = NO_SEEK;
     mSeekNotificationSent = false;
 
     if (mDecryptHandle != NULL) {
@@ -1209,13 +1243,13 @@ void AwesomePlayer::onVideoEvent() {
     }
     mVideoEventPending = false;
 
-    if (mSeeking) {
+    if (mSeeking != NO_SEEK) {
         if (mVideoBuffer) {
             mVideoBuffer->release();
             mVideoBuffer = NULL;
         }
 
-        if (mCachedSource != NULL && mAudioSource != NULL) {
+        if (mSeeking == SEEK && mCachedSource != NULL && mAudioSource != NULL) {
             // We're going to seek the video source first, followed by
             // the audio source.
             // In order to avoid jumps in the DataSource offset caused by
@@ -1224,8 +1258,10 @@ void AwesomePlayer::onVideoEvent() {
             // locations, we'll "pause" the audio source, causing it to
             // stop reading input data until a subsequent seek.
 
-            if (mAudioPlayer != NULL) {
+            if (mAudioPlayer != NULL && (mFlags & AUDIO_RUNNING)) {
                 mAudioPlayer->pause();
+
+                mFlags &= ~AUDIO_RUNNING;
             }
             mAudioSource->pause();
         }
@@ -1233,11 +1269,14 @@ void AwesomePlayer::onVideoEvent() {
 
     if (!mVideoBuffer) {
         MediaSource::ReadOptions options;
-        if (mSeeking) {
+        if (mSeeking != NO_SEEK) {
             LOGV("seeking to %lld us (%.2f secs)", mSeekTimeUs, mSeekTimeUs / 1E6);
 
             options.setSeekTo(
-                    mSeekTimeUs, MediaSource::ReadOptions::SEEK_CLOSEST_SYNC);
+                    mSeekTimeUs,
+                    mSeeking == SEEK_VIDEO_ONLY
+                        ? MediaSource::ReadOptions::SEEK_NEXT_SYNC
+                        : MediaSource::ReadOptions::SEEK_CLOSEST_SYNC);
         }
         for (;;) {
             status_t err = mVideoSource->read(&mVideoBuffer, &options);
@@ -1261,7 +1300,7 @@ void AwesomePlayer::onVideoEvent() {
                 // So video playback is complete, but we may still have
                 // a seek request pending that needs to be applied
                 // to the audio track.
-                if (mSeeking) {
+                if (mSeeking != NO_SEEK) {
                     LOGV("video stream ended while seeking!");
                 }
                 finishSeekIfNecessary(-1);
@@ -1287,14 +1326,29 @@ void AwesomePlayer::onVideoEvent() {
     int64_t timeUs;
     CHECK(mVideoBuffer->meta_data()->findInt64(kKeyTime, &timeUs));
 
+    if (mSeeking == SEEK_VIDEO_ONLY) {
+        if (mSeekTimeUs > timeUs) {
+            LOGI("XXX mSeekTimeUs = %lld us, timeUs = %lld us",
+                 mSeekTimeUs, timeUs);
+        }
+    }
+
     {
         Mutex::Autolock autoLock(mMiscStateLock);
         mVideoTimeUs = timeUs;
     }
 
-    bool wasSeeking = mSeeking;
+    SeekType wasSeeking = mSeeking;
     finishSeekIfNecessary(timeUs);
 
+    if (mAudioPlayer != NULL && !(mFlags & AUDIO_RUNNING)) {
+        status_t err = startAudioPlayer_l();
+        if (err != OK) {
+            LOGE("Startung the audio player failed w/ err %d", err);
+            return;
+        }
+    }
+
     TimeSource *ts = (mFlags & AUDIO_AT_EOS) ? &mSystemTimeSource : mTimeSource;
 
     if (mFlags & FIRST_FRAME) {
@@ -1309,41 +1363,68 @@ void AwesomePlayer::onVideoEvent() {
         mTimeSourceDeltaUs = realTimeUs - mediaTimeUs;
     }
 
-    int64_t nowUs = ts->getRealTimeUs() - mTimeSourceDeltaUs;
+    if (wasSeeking == SEEK_VIDEO_ONLY) {
+        int64_t nowUs = ts->getRealTimeUs() - mTimeSourceDeltaUs;
 
-    int64_t latenessUs = nowUs - timeUs;
+        int64_t latenessUs = nowUs - timeUs;
 
-    if (wasSeeking) {
-        // Let's display the first frame after seeking right away.
-        latenessUs = 0;
+        if (latenessUs > 0) {
+            LOGI("after SEEK_VIDEO_ONLY we're late by %.2f secs", latenessUs / 1E6);
+        }
     }
 
-    if (mRTPSession != NULL) {
-        // We'll completely ignore timestamps for gtalk videochat
-        // and we'll play incoming video as fast as we get it.
-        latenessUs = 0;
-    }
+    if (wasSeeking == NO_SEEK) {
+        // Let's display the first frame after seeking right away.
+
+        int64_t nowUs = ts->getRealTimeUs() - mTimeSourceDeltaUs;
+
+        int64_t latenessUs = nowUs - timeUs;
+
+        if (latenessUs > 500000ll
+                && mRTSPController == NULL
+                && mAudioPlayer != NULL
+                && mAudioPlayer->getMediaTimeMapping(
+                    &realTimeUs, &mediaTimeUs)) {
+            LOGI("we're much too late (%.2f secs), video skipping ahead",
+                 latenessUs / 1E6);
 
-    if (latenessUs > 40000) {
-        // We're more than 40ms late.
-        LOGV("we're late by %lld us (%.2f secs)", latenessUs, latenessUs / 1E6);
-        if ( mSinceLastDropped > FRAME_DROP_FREQ)
-        {
-            LOGV("we're late by %lld us (%.2f secs) dropping one after %d frames", latenessUs, latenessUs / 1E6, mSinceLastDropped);
-            mSinceLastDropped = 0;
             mVideoBuffer->release();
             mVideoBuffer = NULL;
 
+            mSeeking = SEEK_VIDEO_ONLY;
+            mSeekTimeUs = mediaTimeUs;
+
             postVideoEvent_l();
             return;
         }
-    }
 
-    if (latenessUs < -10000) {
-        // We're more than 10ms early.
+        if (latenessUs > 40000) {
+            // We're more than 40ms late.
 
-        postVideoEvent_l(10000);
-        return;
+            LOGV("we're late by %lld us (%.2f secs)",
+                 latenessUs, latenessUs / 1E6);
+
+            if ( mSinceLastDropped > FRAME_DROP_FREQ)
+            {
+                LOGV("we're late by %lld us (%.2f secs) dropping one "
+                     "after %d frames",
+                     latenessUs, latenessUs / 1E6, mSinceLastDropped);
+
+                mSinceLastDropped = 0;
+                mVideoBuffer->release();
+                mVideoBuffer = NULL;
+
+                postVideoEvent_l();
+                return;
+            }
+        }
+
+        if (latenessUs < -10000) {
+            // We're more than 10ms early.
+
+            postVideoEvent_l(10000);
+            return;
+        }
     }
 
     if (mVideoRendererIsPreview || mVideoRenderer == NULL) {
@@ -1424,7 +1505,7 @@ void AwesomePlayer::onCheckAudioStatus() {
             mSeekNotificationSent = true;
         }
 
-        mSeeking = false;
+        mSeeking = NO_SEEK;
     }
 
     status_t finalStatus;
diff --git a/media/libstagefright/include/AwesomePlayer.h b/media/libstagefright/include/AwesomePlayer.h
index 41ef181..d5819a4 100644
--- a/media/libstagefright/include/AwesomePlayer.h
+++ b/media/libstagefright/include/AwesomePlayer.h
@@ -119,6 +119,9 @@ private:
         // sufficient data to begin playback and finish the preparation phase
         // for good.
         PREPARING_CONNECTED = 2048,
+
+        AUDIO_RUNNING       = 8192,
+        AUDIOPLAYER_STARTED = 16384,
     };
 
     mutable Mutex mLock;
@@ -160,7 +163,13 @@ private:
     int64_t mTimeSourceDeltaUs;
     int64_t mVideoTimeUs;
 
-    bool mSeeking;
+    enum SeekType {
+        NO_SEEK,
+        SEEK,
+        SEEK_VIDEO_ONLY
+    };
+    SeekType mSeeking;
+
     bool mSeekNotificationSent;
     int64_t mSeekTimeUs;
 
@@ -256,6 +265,8 @@ private:
     void finishSeekIfNecessary(int64_t videoTimeUs);
     void ensureCacheIsFetching_l();
 
+    status_t startAudioPlayer_l();
+
     AwesomePlayer(const AwesomePlayer &);
     AwesomePlayer &operator=(const AwesomePlayer &);
 };