stagefright: support setting/getting playback/sync config in MediaSync

Bug: 18249558
Bug: 19666434
Bug: 20057497
Change-Id: I5868b17423d7c20cfaf4a399f3eb67bfba440605

5 files changed, 251 insertions, 25 deletions

diff --git a/media/libstagefright/AudioPlayer.cpp b/media/libstagefright/AudioPlayer.cpp
index e24824b..dd9d393 100644
--- a/media/libstagefright/AudioPlayer.cpp
+++ b/media/libstagefright/AudioPlayer.cpp
@@ -408,11 +408,22 @@ void AudioPlayer::notifyAudioEOS() {
     }
 }
 
-status_t AudioPlayer::setPlaybackRatePermille(int32_t ratePermille) {
+status_t AudioPlayer::setPlaybackRate(const AudioPlaybackRate &rate) {
     if (mAudioSink.get() != NULL) {
-        return mAudioSink->setPlaybackRatePermille(ratePermille);
+        return mAudioSink->setPlaybackRate(rate);
     } else if (mAudioTrack != 0){
-        return mAudioTrack->setSampleRate(ratePermille * mSampleRate / 1000);
+        return mAudioTrack->setPlaybackRate(rate);
+    } else {
+        return NO_INIT;
+    }
+}
+
+status_t AudioPlayer::getPlaybackRate(AudioPlaybackRate *rate /* nonnull */) {
+    if (mAudioSink.get() != NULL) {
+        return mAudioSink->getPlaybackRate(rate);
+    } else if (mAudioTrack != 0) {
+        *rate = mAudioTrack->getPlaybackRate();
+        return OK;
     } else {
         return NO_INIT;
     }
diff --git a/media/libstagefright/AwesomePlayer.cpp b/media/libstagefright/AwesomePlayer.cpp
index c14625d..df01e7c 100644
--- a/media/libstagefright/AwesomePlayer.cpp
+++ b/media/libstagefright/AwesomePlayer.cpp
@@ -241,6 +241,8 @@ AwesomePlayer::AwesomePlayer()
 
     mClockEstimator = new WindowedLinearFitEstimator();
 
+    mPlaybackSettings = AUDIO_PLAYBACK_RATE_DEFAULT;
+
     reset();
 }
 
@@ -1009,6 +1011,10 @@ status_t AwesomePlayer::play_l() {
                 return err;
             }
         }
+
+        if (mAudioPlayer != NULL) {
+            mAudioPlayer->setPlaybackRate(mPlaybackSettings);
+        }
     }
 
     if (mTimeSource == NULL && mAudioPlayer == NULL) {
@@ -1131,6 +1137,10 @@ status_t AwesomePlayer::startAudioPlayer_l(bool sendErrorNotification) {
     }
 
     if (err == OK) {
+        err = mAudioPlayer->setPlaybackRate(mPlaybackSettings);
+    }
+
+    if (err == OK) {
         modifyFlags(AUDIO_RUNNING, SET);
 
         mWatchForAudioEOS = true;
@@ -2553,14 +2563,6 @@ status_t AwesomePlayer::setParameter(int key, const Parcel &request) {
         {
             return setCacheStatCollectFreq(request);
         }
-        case KEY_PARAMETER_PLAYBACK_RATE_PERMILLE:
-        {
-            if (mAudioPlayer != NULL) {
-                return mAudioPlayer->setPlaybackRatePermille(request.readInt32());
-            } else {
-                return NO_INIT;
-            }
-        }
         default:
         {
             return ERROR_UNSUPPORTED;
@@ -2597,6 +2599,58 @@ status_t AwesomePlayer::getParameter(int key, Parcel *reply) {
     }
 }
 
+status_t AwesomePlayer::setPlaybackSettings(const AudioPlaybackRate &rate) {
+    Mutex::Autolock autoLock(mLock);
+    // cursory sanity check for non-audio and paused cases
+    if ((rate.mSpeed != 0.f && rate.mSpeed < AUDIO_TIMESTRETCH_SPEED_MIN)
+        || rate.mSpeed > AUDIO_TIMESTRETCH_SPEED_MAX
+        || rate.mPitch < AUDIO_TIMESTRETCH_SPEED_MIN
+        || rate.mPitch > AUDIO_TIMESTRETCH_SPEED_MAX) {
+        return BAD_VALUE;
+    }
+
+    status_t err = OK;
+    if (rate.mSpeed == 0.f) {
+        if (mFlags & PLAYING) {
+            modifyFlags(CACHE_UNDERRUN, CLEAR); // same as pause
+            err = pause_l();
+        }
+        if (err == OK) {
+            // save settings (using old speed) in case player is resumed
+            AudioPlaybackRate newRate = rate;
+            newRate.mSpeed = mPlaybackSettings.mSpeed;
+            mPlaybackSettings = newRate;
+        }
+        return err;
+    }
+    if (mAudioPlayer != NULL) {
+        err = mAudioPlayer->setPlaybackRate(rate);
+    }
+    if (err == OK) {
+        mPlaybackSettings = rate;
+        if (!(mFlags & PLAYING)) {
+            play_l();
+        }
+    }
+    return err;
+}
+
+status_t AwesomePlayer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
+    if (mAudioPlayer != NULL) {
+        status_t err = mAudioPlayer->getPlaybackRate(rate);
+        if (err == OK) {
+            mPlaybackSettings = *rate;
+            Mutex::Autolock autoLock(mLock);
+            if (!(mFlags & PLAYING)) {
+                rate->mSpeed = 0.f;
+            }
+        }
+        return err;
+    }
+    *rate = mPlaybackSettings;
+    return OK;
+}
+
 status_t AwesomePlayer::getTrackInfo(Parcel *reply) const {
     Mutex::Autolock autoLock(mLock);
     size_t trackCount = mExtractor->countTracks();
diff --git a/media/libstagefright/MediaSync.cpp b/media/libstagefright/MediaSync.cpp
index 4350b59..85027ce 100644
--- a/media/libstagefright/MediaSync.cpp
+++ b/media/libstagefright/MediaSync.cpp
@@ -56,6 +56,10 @@ MediaSync::MediaSync()
         mPlaybackRate(0.0) {
     mMediaClock = new MediaClock;
 
+    // initialize settings
+    mPlaybackSettings = AUDIO_PLAYBACK_RATE_DEFAULT;
+    mPlaybackSettings.mSpeed = mPlaybackRate;
+
     mLooper = new ALooper;
     mLooper->setName("MediaSync");
     mLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
@@ -84,6 +88,11 @@ status_t MediaSync::configureSurface(const sp<IGraphicBufferProducer> &output) {
         return INVALID_OPERATION;
     }
 
+    if (output == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_VSYNC) {
+        ALOGE("configureSurface: output surface is used as sync source and cannot be removed.");
+        return INVALID_OPERATION;
+    }
+
     if (output != NULL) {
         IGraphicBufferProducer::QueueBufferOutput queueBufferOutput;
         sp<OutputListener> listener(new OutputListener(this));
@@ -105,8 +114,7 @@ status_t MediaSync::configureSurface(const sp<IGraphicBufferProducer> &output) {
 }
 
 // |audioTrack| is used only for querying information.
-status_t MediaSync::configureAudioTrack(
-        const sp<AudioTrack> &audioTrack, uint32_t nativeSampleRateInHz) {
+status_t MediaSync::configureAudioTrack(const sp<AudioTrack> &audioTrack) {
     Mutex::Autolock lock(mMutex);
 
     // TODO: support audio track change.
@@ -115,15 +123,35 @@ status_t MediaSync::configureAudioTrack(
         return INVALID_OPERATION;
     }
 
-    if (audioTrack != NULL && nativeSampleRateInHz <= 0) {
-        ALOGE("configureAudioTrack: native sample rate should be positive.");
-        return BAD_VALUE;
+    if (audioTrack == NULL && mSyncSettings.mSource == AVSYNC_SOURCE_AUDIO) {
+        ALOGE("configureAudioTrack: audioTrack is used as sync source and cannot be removed.");
+        return INVALID_OPERATION;
     }
 
-    mAudioTrack = audioTrack;
-    mNativeSampleRateInHz = nativeSampleRateInHz;
+    if (audioTrack != NULL) {
+        // check if audio track supports the playback settings
+        if (mPlaybackSettings.mSpeed != 0.f
+                && audioTrack->setPlaybackRate(mPlaybackSettings) != OK) {
+            ALOGE("playback settings are not supported by the audio track");
+            return INVALID_OPERATION;
+        }
+        uint32_t nativeSampleRateInHz = audioTrack->getOriginalSampleRate();
+        if (nativeSampleRateInHz <= 0) {
+            ALOGE("configureAudioTrack: native sample rate should be positive.");
+            return BAD_VALUE;
+        }
+        mAudioTrack = audioTrack;
+        mNativeSampleRateInHz = nativeSampleRateInHz;
+        (void)setPlaybackSettings_l(mPlaybackSettings);
+    }
+    else {
+        mAudioTrack = NULL;
+        mNativeSampleRateInHz = 0;
+    }
 
-    return NO_ERROR;
+    // potentially resync to new source
+    resync_l();
+    return OK;
 }
 
 status_t MediaSync::createInputSurface(
@@ -158,21 +186,27 @@ status_t MediaSync::createInputSurface(
     return status;
 }
 
-status_t MediaSync::setPlaybackRate(float rate) {
-    if (rate < 0.0) {
-        return BAD_VALUE;
+void MediaSync::resync_l() {
+    AVSyncSource src = mSyncSettings.mSource;
+    if (src == AVSYNC_SOURCE_DEFAULT) {
+        if (mAudioTrack != NULL) {
+            src = AVSYNC_SOURCE_AUDIO;
+        } else {
+            src = AVSYNC_SOURCE_SYSTEM_CLOCK;
+        }
     }
 
-    Mutex::Autolock lock(mMutex);
+    // TODO: resync ourselves to the current clock (e.g. on sync source change)
+    updatePlaybackRate_l(mPlaybackRate);
+}
 
+void MediaSync::updatePlaybackRate_l(float rate) {
     if (rate > mPlaybackRate) {
         mNextBufferItemMediaUs = -1;
     }
     mPlaybackRate = rate;
     mMediaClock->setPlaybackRate(rate);
     onDrainVideo_l();
-
-    return OK;
 }
 
 sp<const MediaClock> MediaSync::getMediaClock() {
@@ -264,6 +298,95 @@ void MediaSync::setName(const AString &name) {
     mInput->setConsumerName(String8(name.c_str()));
 }
 
+status_t MediaSync::setVideoFrameRateHint(float rate) {
+    // ignored until we add the FrameScheduler
+    return rate >= 0.f ? OK : BAD_VALUE;
+}
+
+float MediaSync::getVideoFrameRate() {
+    // we don't know the frame rate
+    return -1.f;
+}
+
+status_t MediaSync::setSyncSettings(const AVSyncSettings &syncSettings) {
+    // validate settings
+    if (syncSettings.mSource >= AVSYNC_SOURCE_MAX
+            || syncSettings.mAudioAdjustMode >= AVSYNC_AUDIO_ADJUST_MODE_MAX
+            || syncSettings.mTolerance < 0.f
+            || syncSettings.mTolerance >= AVSYNC_TOLERANCE_MAX) {
+        return BAD_VALUE;
+    }
+
+    Mutex::Autolock lock(mMutex);
+
+    // verify that we have the sync source
+    switch (syncSettings.mSource) {
+        case AVSYNC_SOURCE_AUDIO:
+            if (mAudioTrack == NULL) {
+                ALOGE("setSyncSettings: audio sync source requires an audio track");
+                return BAD_VALUE;
+            }
+            break;
+        case AVSYNC_SOURCE_VSYNC:
+            if (mOutput == NULL) {
+                ALOGE("setSyncSettings: vsync sync source requires an output surface");
+                return BAD_VALUE;
+            }
+            break;
+        default:
+            break;
+    }
+
+    mSyncSettings = syncSettings;
+    resync_l();
+    return OK;
+}
+
+void MediaSync::getSyncSettings(AVSyncSettings *syncSettings) {
+    Mutex::Autolock lock(mMutex);
+    *syncSettings = mSyncSettings;
+}
+
+status_t MediaSync::setPlaybackSettings(const AudioPlaybackRate &rate) {
+    Mutex::Autolock lock(mMutex);
+
+    status_t err = setPlaybackSettings_l(rate);
+    if (err == OK) {
+        // TODO: adjust rate if using VSYNC as source
+        updatePlaybackRate_l(rate.mSpeed);
+    }
+    return err;
+}
+
+status_t MediaSync::setPlaybackSettings_l(const AudioPlaybackRate &rate) {
+    if (rate.mSpeed < 0.f || rate.mPitch < 0.f) {
+        // We don't validate other audio settings.
+        // They will be validated when/if audiotrack is set.
+        return BAD_VALUE;
+    }
+
+    if (mAudioTrack != NULL) {
+        if (rate.mSpeed == 0.f) {
+            mAudioTrack->pause();
+        } else {
+            status_t err = mAudioTrack->setPlaybackRate(rate);
+            if (err != OK) {
+                return BAD_VALUE;
+            }
+
+            // ignore errors
+            (void)mAudioTrack->start();
+        }
+    }
+    mPlaybackSettings = rate;
+    return OK;
+}
+
+void MediaSync::getPlaybackSettings(AudioPlaybackRate *rate) {
+    Mutex::Autolock lock(mMutex);
+    *rate = mPlaybackSettings;
+}
+
 int64_t MediaSync::getRealTime(int64_t mediaTimeUs, int64_t nowUs) {
     int64_t realUs;
     if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
diff --git a/media/libstagefright/Utils.cpp b/media/libstagefright/Utils.cpp
index 7b089b0..413628d 100644
--- a/media/libstagefright/Utils.cpp
+++ b/media/libstagefright/Utils.cpp
@@ -901,5 +901,39 @@ bool operator <(const HLSTime &t0, const HLSTime &t1) {
             || (t0.mSeq == t1.mSeq && t0.mTimeUs < t1.mTimeUs);
 }
 
+void writeToAMessage(sp<AMessage> msg, const AudioPlaybackRate &rate) {
+    msg->setFloat("speed", rate.mSpeed);
+    msg->setFloat("pitch", rate.mPitch);
+    msg->setInt32("audio-fallback-mode", rate.mFallbackMode);
+    msg->setInt32("audio-stretch-mode", rate.mStretchMode);
+}
+
+void readFromAMessage(const sp<AMessage> &msg, AudioPlaybackRate *rate /* nonnull */) {
+    *rate = AUDIO_PLAYBACK_RATE_DEFAULT;
+    CHECK(msg->findFloat("speed", &rate->mSpeed));
+    CHECK(msg->findFloat("pitch", &rate->mPitch));
+    CHECK(msg->findInt32("audio-fallback-mode", (int32_t *)&rate->mFallbackMode));
+    CHECK(msg->findInt32("audio-stretch-mode", (int32_t *)&rate->mStretchMode));
+}
+
+void writeToAMessage(sp<AMessage> msg, const AVSyncSettings &sync, float videoFpsHint) {
+    msg->setInt32("sync-source", sync.mSource);
+    msg->setInt32("audio-adjust-mode", sync.mAudioAdjustMode);
+    msg->setFloat("tolerance", sync.mTolerance);
+    msg->setFloat("video-fps", videoFpsHint);
+}
+
+void readFromAMessage(
+        const sp<AMessage> &msg,
+        AVSyncSettings *sync /* nonnull */,
+        float *videoFps /* nonnull */) {
+    AVSyncSettings settings;
+    CHECK(msg->findInt32("sync-source", (int32_t *)&settings.mSource));
+    CHECK(msg->findInt32("audio-adjust-mode", (int32_t *)&settings.mAudioAdjustMode));
+    CHECK(msg->findFloat("tolerance", &settings.mTolerance));
+    CHECK(msg->findFloat("video-fps", videoFps));
+    *sync = settings;
+}
+
 }  // namespace android
 
diff --git a/media/libstagefright/include/AwesomePlayer.h b/media/libstagefright/include/AwesomePlayer.h
index 8bba804..758b2c9 100644
--- a/media/libstagefright/include/AwesomePlayer.h
+++ b/media/libstagefright/include/AwesomePlayer.h
@@ -21,6 +21,7 @@
 #include "HTTPBase.h"
 #include "TimedEventQueue.h"
 
+#include <media/AudioResamplerPublic.h>
 #include <media/MediaPlayerInterface.h>
 #include <media/stagefright/DataSource.h>
 #include <media/stagefright/OMXClient.h>
@@ -93,6 +94,8 @@ struct AwesomePlayer {
 
     status_t setParameter(int key, const Parcel &request);
     status_t getParameter(int key, Parcel *reply);
+    status_t setPlaybackSettings(const AudioPlaybackRate &rate);
+    status_t getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */);
     status_t invoke(const Parcel &request, Parcel *reply);
     status_t setCacheStatCollectFreq(const Parcel &request);
 
@@ -180,6 +183,7 @@ private:
     sp<MediaSource> mOmxSource;
     sp<MediaSource> mAudioSource;
     AudioPlayer *mAudioPlayer;
+    AudioPlaybackRate mPlaybackSettings;
     int64_t mDurationUs;
 
     int32_t mDisplayWidth;