stagefright: support setting/getting playback/sync config in MediaSync

Bug: 18249558
Bug: 19666434
Bug: 20057497
Change-Id: I5868b17423d7c20cfaf4a399f3eb67bfba440605

8 files changed, 168 insertions, 22 deletions

diff --git a/include/media/AVSyncSettings.h b/include/media/AVSyncSettings.h
new file mode 100644
index 0000000..10e3bcc
--- /dev/null
+++ b/include/media/AVSyncSettings.h
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2015 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_AV_SYNC_SETTINGS_H
+#define ANDROID_AV_SYNC_SETTINGS_H
+
+namespace android {
+
+enum AVSyncSource : unsigned {
+    // let the system decide the best sync source
+    AVSYNC_SOURCE_DEFAULT = 0,
+    // sync to the system clock
+    AVSYNC_SOURCE_SYSTEM_CLOCK = 1,
+    // sync to the audio track
+    AVSYNC_SOURCE_AUDIO = 2,
+    // sync to the display vsync
+    AVSYNC_SOURCE_VSYNC = 3,
+    AVSYNC_SOURCE_MAX,
+};
+
+enum AVSyncAudioAdjustMode : unsigned {
+    // let the system decide the best audio adjust mode
+    AVSYNC_AUDIO_ADJUST_MODE_DEFAULT = 0,
+    // adjust audio by time stretching
+    AVSYNC_AUDIO_ADJUST_MODE_STRETCH = 1,
+    // adjust audio by resampling
+    AVSYNC_AUDIO_ADJUST_MODE_RESAMPLE = 2,
+    AVSYNC_AUDIO_ADJUST_MODE_MAX,
+};
+
+// max tolerance when adjusting playback speed to desired playback speed
+#define AVSYNC_TOLERANCE_MAX 1.0f
+
+struct AVSyncSettings {
+    AVSyncSource mSource;
+    AVSyncAudioAdjustMode mAudioAdjustMode;
+    float mTolerance;
+    AVSyncSettings()
+        : mSource(AVSYNC_SOURCE_DEFAULT),
+          mAudioAdjustMode(AVSYNC_AUDIO_ADJUST_MODE_DEFAULT),
+          mTolerance(.044f) { }
+};
+
+} // namespace android
+
+// ---------------------------------------------------------------------------
+
+#endif // ANDROID_AV_SYNC_SETTINGS_H
diff --git a/include/media/AudioTrack.h b/include/media/AudioTrack.h
index d361901..51d40bb 100644
--- a/include/media/AudioTrack.h
+++ b/include/media/AudioTrack.h
@@ -360,6 +360,11 @@ public:
     /* Return current source sample rate in Hz */
             uint32_t    getSampleRate() const;
 
+    /* Return the original source sample rate in Hz. This corresponds to the sample rate
+     * if playback rate had normal speed and pitch.
+     */
+            uint32_t    getOriginalSampleRate() const;
+
     /* Set source playback rate for timestretch
      * 1.0 is normal speed: < 1.0 is slower, > 1.0 is faster
      * 1.0 is normal pitch: < 1.0 is lower pitch, > 1.0 is higher pitch
@@ -749,6 +754,7 @@ protected:
     float                   mVolume[2];
     float                   mSendLevel;
     mutable uint32_t        mSampleRate;            // mutable because getSampleRate() can update it
+    uint32_t                mOriginalSampleRate;
     AudioPlaybackRate       mPlaybackRate;
     size_t                  mFrameCount;            // corresponds to current IAudioTrack, value is
                                                     // reported back by AudioFlinger to the client
diff --git a/include/media/IMediaPlayer.h b/include/media/IMediaPlayer.h
index df6130d..0fd8933 100644
--- a/include/media/IMediaPlayer.h
+++ b/include/media/IMediaPlayer.h
@@ -35,6 +35,8 @@ class IDataSource;
 struct IStreamSource;
 class IGraphicBufferProducer;
 struct IMediaHTTPService;
+struct AudioPlaybackRate;
+struct AVSyncSettings;
 
 class IMediaPlayer: public IInterface
 {
@@ -58,7 +60,11 @@ public:
     virtual status_t        stop() = 0;
     virtual status_t        pause() = 0;
     virtual status_t        isPlaying(bool* state) = 0;
-    virtual status_t        setPlaybackRate(float rate) = 0;
+    virtual status_t        setPlaybackSettings(const AudioPlaybackRate& rate) = 0;
+    virtual status_t        getPlaybackSettings(AudioPlaybackRate* rate /* nonnull */) = 0;
+    virtual status_t        setSyncSettings(const AVSyncSettings& sync, float videoFpsHint) = 0;
+    virtual status_t        getSyncSettings(AVSyncSettings* sync /* nonnull */,
+                                            float* videoFps /* nonnull */) = 0;
     virtual status_t        seekTo(int msec) = 0;
     virtual status_t        getCurrentPosition(int* msec) = 0;
     virtual status_t        getDuration(int* msec) = 0;
diff --git a/include/media/MediaPlayerInterface.h b/include/media/MediaPlayerInterface.h
index 824762a..fa917f9 100644
--- a/include/media/MediaPlayerInterface.h
+++ b/include/media/MediaPlayerInterface.h
@@ -26,8 +26,10 @@
 #include <utils/RefBase.h>
 
 #include <media/mediaplayer.h>
+#include <media/AudioResamplerPublic.h>
 #include <media/AudioSystem.h>
 #include <media/AudioTimestamp.h>
+#include <media/AVSyncSettings.h>
 #include <media/Metadata.h>
 
 // Fwd decl to make sure everyone agrees that the scope of struct sockaddr_in is
@@ -132,7 +134,8 @@ public:
         virtual void        pause() = 0;
         virtual void        close() = 0;
 
-        virtual status_t    setPlaybackRatePermille(int32_t /* rate */) { return INVALID_OPERATION;}
+        virtual status_t    setPlaybackRate(const AudioPlaybackRate& rate) = 0;
+        virtual status_t    getPlaybackRate(AudioPlaybackRate* rate /* nonnull */) = 0;
         virtual bool        needsTrailingPadding() { return true; }
 
         virtual status_t    setParameters(const String8& /* keyValuePairs */) { return NO_ERROR; }
@@ -173,7 +176,31 @@ public:
     virtual status_t    stop() = 0;
     virtual status_t    pause() = 0;
     virtual bool        isPlaying() = 0;
-    virtual status_t    setPlaybackRate(float /* rate */) { return INVALID_OPERATION; }
+    virtual status_t    setPlaybackSettings(const AudioPlaybackRate& rate) {
+        // by default, players only support setting rate to the default
+        if (!isAudioPlaybackRateEqual(rate, AUDIO_PLAYBACK_RATE_DEFAULT)) {
+            return BAD_VALUE;
+        }
+        return OK;
+    }
+    virtual status_t    getPlaybackSettings(AudioPlaybackRate* rate /* nonnull */) {
+        *rate = AUDIO_PLAYBACK_RATE_DEFAULT;
+        return OK;
+    }
+    virtual status_t    setSyncSettings(const AVSyncSettings& sync, float /* videoFps */) {
+        // By default, players only support setting sync source to default; all other sync
+        // settings are ignored. There is no requirement for getters to return set values.
+        if (sync.mSource != AVSYNC_SOURCE_DEFAULT) {
+            return BAD_VALUE;
+        }
+        return OK;
+    }
+    virtual status_t    getSyncSettings(
+                                AVSyncSettings* sync /* nonnull */, float* videoFps /* nonnull */) {
+        *sync = AVSyncSettings();
+        *videoFps = -1.f;
+        return OK;
+    }
     virtual status_t    seekTo(int msec) = 0;
     virtual status_t    getCurrentPosition(int *msec) = 0;
     virtual status_t    getDuration(int *msec) = 0;
diff --git a/include/media/mediaplayer.h b/include/media/mediaplayer.h
index 256fa9a..3fe749c 100644
--- a/include/media/mediaplayer.h
+++ b/include/media/mediaplayer.h
@@ -20,6 +20,8 @@
 #include <arpa/inet.h>
 
 #include <binder/IMemory.h>
+
+#include <media/AudioResamplerPublic.h>
 #include <media/IMediaPlayerClient.h>
 #include <media/IMediaPlayer.h>
 #include <media/IMediaDeathNotifier.h>
@@ -32,8 +34,9 @@ class ANativeWindow;
 
 namespace android {
 
-class Surface;
+struct AVSyncSettings;
 class IGraphicBufferProducer;
+class Surface;
 
 enum media_event_type {
     MEDIA_NOP               = 0, // interface test message
@@ -223,7 +226,12 @@ public:
             status_t        stop();
             status_t        pause();
             bool            isPlaying();
-            status_t        setPlaybackRate(float rate);
+            status_t        setPlaybackSettings(const AudioPlaybackRate& rate);
+            status_t        getPlaybackSettings(AudioPlaybackRate* rate /* nonnull */);
+            status_t        setSyncSettings(const AVSyncSettings& sync, float videoFpsHint);
+            status_t        getSyncSettings(
+                                    AVSyncSettings* sync /* nonnull */,
+                                    float* videoFps /* nonnull */);
             status_t        getVideoWidth(int *w);
             status_t        getVideoHeight(int *h);
             status_t        seekTo(int msec);
@@ -278,7 +286,6 @@ private:
     int                         mVideoWidth;
     int                         mVideoHeight;
     int                         mAudioSessionId;
-    float                       mPlaybackRate;
     float                       mSendLevel;
     struct sockaddr_in          mRetransmitEndpoint;
     bool                        mRetransmitEndpointValid;
diff --git a/include/media/stagefright/AudioPlayer.h b/include/media/stagefright/AudioPlayer.h
index 98c4fa7..e0cd965 100644
--- a/include/media/stagefright/AudioPlayer.h
+++ b/include/media/stagefright/AudioPlayer.h
@@ -25,9 +25,10 @@
 
 namespace android {
 
-class MediaSource;
+struct AudioPlaybackRate;
 class AudioTrack;
 struct AwesomePlayer;
+class MediaSource;
 
 class AudioPlayer : public TimeSource {
 public:
@@ -73,7 +74,8 @@ public:
     bool isSeeking();
     bool reachedEOS(status_t *finalStatus);
 
-    status_t setPlaybackRatePermille(int32_t ratePermille);
+    status_t setPlaybackRate(const AudioPlaybackRate &rate);
+    status_t getPlaybackRate(AudioPlaybackRate *rate /* nonnull */);
 
     void notifyAudioEOS();
 
diff --git a/include/media/stagefright/MediaSync.h b/include/media/stagefright/MediaSync.h
index e071b65..a349986 100644
--- a/include/media/stagefright/MediaSync.h
+++ b/include/media/stagefright/MediaSync.h
@@ -20,6 +20,8 @@
 #include <gui/IConsumerListener.h>
 #include <gui/IProducerListener.h>
 
+#include <media/AudioResamplerPublic.h>
+#include <media/AVSyncSettings.h>
 #include <media/stagefright/foundation/AHandler.h>
 
 #include <utils/Condition.h>
@@ -77,10 +79,7 @@ public:
 
     // Called when audio track is used as media clock source. It should be
     // called before updateQueuedAudioData().
-    // |nativeSampleRateInHz| is the sample rate of audio data fed into audio
-    // track. It's the same number used to create AudioTrack.
-    status_t configureAudioTrack(
-            const sp<AudioTrack> &audioTrack, uint32_t nativeSampleRateInHz);
+    status_t configureAudioTrack(const sp<AudioTrack> &audioTrack);
 
     // Create a surface for client to render video frames. This is the surface
     // on which the client should render video frames. Those video frames will
@@ -99,21 +98,31 @@ public:
     // Set the consumer name of the input queue.
     void setName(const AString &name);
 
-    // Set the playback in a desired speed.
-    // This method can be called any time.
-    // |rate| is the ratio between desired speed and the normal one, and should
-    // be non-negative. The meaning of rate values:
-    // 1.0 -- normal playback
-    // 0.0 -- stop or pause
-    // larger than 1.0 -- faster than normal speed
-    // between 0.0 and 1.0 -- slower than normal speed
-    status_t setPlaybackRate(float rate);
-
     // Get the media clock used by the MediaSync so that the client can obtain
     // corresponding media time or real time via
     // MediaClock::getMediaTime() and MediaClock::getRealTimeFor().
     sp<const MediaClock> getMediaClock();
 
+    // Set the video frame rate hint - this is used by the video FrameScheduler
+    status_t setVideoFrameRateHint(float rate);
+
+    // Get the video frame rate measurement from the FrameScheduler
+    // returns -1 if there is no measurement
+    float getVideoFrameRate();
+
+    // Set the sync settings parameters.
+    status_t setSyncSettings(const AVSyncSettings &syncSettings);
+
+    // Gets the sync settings parameters.
+    void getSyncSettings(AVSyncSettings *syncSettings /* nonnull */);
+
+    // Sets the playback rate using playback settings.
+    // This method can be called any time.
+    status_t setPlaybackSettings(const AudioPlaybackRate &rate);
+
+    // Gets the playback rate (playback settings parameters).
+    void getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */);
+
     // Get the play time for pending audio frames in audio sink.
     status_t getPlayTimeForPendingAudioFrames(int64_t *outTimeUs);
 
@@ -201,6 +210,9 @@ private:
     sp<ALooper> mLooper;
     float mPlaybackRate;
 
+    AudioPlaybackRate mPlaybackSettings;
+    AVSyncSettings mSyncSettings;
+
     sp<MediaClock> mMediaClock;
 
     MediaSync();
@@ -239,6 +251,22 @@ private:
     // up. This must be called with mMutex locked.
     void onAbandoned_l(bool isInput);
 
+    // Set the playback in a desired speed.
+    // This method can be called any time.
+    // |rate| is the ratio between desired speed and the normal one, and should
+    // be non-negative. The meaning of rate values:
+    // 1.0 -- normal playback
+    // 0.0 -- stop or pause
+    // larger than 1.0 -- faster than normal speed
+    // between 0.0 and 1.0 -- slower than normal speed
+    void updatePlaybackRate_l(float rate);
+
+    // apply new sync settings
+    void resync_l();
+
+    // apply playback settings only - without resyncing or updating playback rate
+    status_t setPlaybackSettings_l(const AudioPlaybackRate &rate);
+
     // helper.
     bool isPlaying() { return mPlaybackRate != 0.0; }
 
diff --git a/include/media/stagefright/Utils.h b/include/media/stagefright/Utils.h
index 0ce1603..5e9d7d4 100644
--- a/include/media/stagefright/Utils.h
+++ b/include/media/stagefright/Utils.h
@@ -76,6 +76,15 @@ struct HLSTime {
 
 bool operator <(const HLSTime &t0, const HLSTime &t1);
 
+// read and write various object to/from AMessage
+
+void writeToAMessage(sp<AMessage> msg, const AudioPlaybackRate &rate);
+void readFromAMessage(const sp<AMessage> &msg, AudioPlaybackRate *rate /* nonnull */);
+
+void writeToAMessage(sp<AMessage> msg, const AVSyncSettings &sync, float videoFpsHint);
+void readFromAMessage(
+        const sp<AMessage> &msg, AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */);
+
 }  // namespace android
 
 #endif  // UTILS_H_