Add playback rate to AudioMixer

Bug: 19196501
Change-Id: I42d1f90e6297cf3f1304860d1691a5dfedd4c37d

4 files changed, 270 insertions, 1 deletions

diff --git a/services/audioflinger/AudioMixer.cpp b/services/audioflinger/AudioMixer.cpp
index cb90ece..c2c791f 100644
--- a/services/audioflinger/AudioMixer.cpp
+++ b/services/audioflinger/AudioMixer.cpp
@@ -123,6 +123,7 @@ AudioMixer::AudioMixer(size_t frameCount, uint32_t sampleRate, uint32_t maxNumTr
         t->resampler = NULL;
         t->downmixerBufferProvider = NULL;
         t->mReformatBufferProvider = NULL;
+        t->mTimestretchBufferProvider = NULL;
         t++;
     }
 
@@ -135,6 +136,7 @@ AudioMixer::~AudioMixer()
         delete t->resampler;
         delete t->downmixerBufferProvider;
         delete t->mReformatBufferProvider;
+        delete t->mTimestretchBufferProvider;
         t++;
     }
     delete [] mState.outputTemp;
@@ -213,6 +215,7 @@ int AudioMixer::getTrackName(audio_channel_mask_t channelMask,
         t->mReformatBufferProvider = NULL;
         t->downmixerBufferProvider = NULL;
         t->mPostDownmixReformatBufferProvider = NULL;
+        t->mTimestretchBufferProvider = NULL;
         t->mMixerFormat = AUDIO_FORMAT_PCM_16_BIT;
         t->mFormat = format;
         t->mMixerInFormat = selectMixerInFormat(format);
@@ -220,6 +223,8 @@ int AudioMixer::getTrackName(audio_channel_mask_t channelMask,
         t->mMixerChannelMask = audio_channel_mask_from_representation_and_bits(
                 AUDIO_CHANNEL_REPRESENTATION_POSITION, AUDIO_CHANNEL_OUT_STEREO);
         t->mMixerChannelCount = audio_channel_count_from_out_mask(t->mMixerChannelMask);
+        t->mSpeed = AUDIO_TIMESTRETCH_SPEED_NORMAL;
+        t->mPitch = AUDIO_TIMESTRETCH_PITCH_NORMAL;
         // Check the downmixing (or upmixing) requirements.
         status_t status = t->prepareForDownmix();
         if (status != OK) {
@@ -412,6 +417,10 @@ void AudioMixer::track_t::reconfigureBufferProviders()
         mPostDownmixReformatBufferProvider->setBufferProvider(bufferProvider);
         bufferProvider = mPostDownmixReformatBufferProvider;
     }
+    if (mTimestretchBufferProvider) {
+        mTimestretchBufferProvider->setBufferProvider(bufferProvider);
+        bufferProvider = mTimestretchBufferProvider;
+    }
 }
 
 void AudioMixer::deleteTrackName(int name)
@@ -432,7 +441,9 @@ void AudioMixer::deleteTrackName(int name)
     mState.tracks[name].unprepareForDownmix();
     // delete the reformatter
     mState.tracks[name].unprepareForReformat();
-
+    // delete the timestretch provider
+    delete track.mTimestretchBufferProvider;
+    track.mTimestretchBufferProvider = NULL;
     mTrackNames &= ~(1<<name);
 }
 
@@ -654,6 +665,26 @@ void AudioMixer::setParameter(int name, int target, int param, void *value)
             }
         }
         break;
+        case TIMESTRETCH:
+            switch (param) {
+            case PLAYBACK_RATE: {
+                const float speed = reinterpret_cast<float*>(value)[0];
+                const float pitch = reinterpret_cast<float*>(value)[1];
+                ALOG_ASSERT(AUDIO_TIMESTRETCH_SPEED_MIN <= speed
+                        && speed <= AUDIO_TIMESTRETCH_SPEED_MAX,
+                        "bad speed %f", speed);
+                ALOG_ASSERT(AUDIO_TIMESTRETCH_PITCH_MIN <= pitch
+                        && pitch <= AUDIO_TIMESTRETCH_PITCH_MAX,
+                        "bad pitch %f", pitch);
+                if (track.setPlaybackRate(speed, pitch)) {
+                    ALOGV("setParameter(TIMESTRETCH, PLAYBACK_RATE, %f %f", speed, pitch);
+                    // invalidateState(1 << name);
+                }
+                } break;
+            default:
+                LOG_ALWAYS_FATAL("setParameter timestretch: bad param %d", param);
+            }
+            break;
 
     default:
         LOG_ALWAYS_FATAL("setParameter: bad target %d", target);
@@ -699,6 +730,28 @@ bool AudioMixer::track_t::setResampler(uint32_t trackSampleRate, uint32_t devSam
     return false;
 }
 
+bool AudioMixer::track_t::setPlaybackRate(float speed, float pitch)
+{
+    if (speed == mSpeed && pitch == mPitch) {
+        return false;
+    }
+    mSpeed = speed;
+    mPitch = pitch;
+    if (mTimestretchBufferProvider == NULL) {
+        // TODO: Remove MONO_HACK. Resampler sees #channels after the downmixer
+        // but if none exists, it is the channel count (1 for mono).
+        const int timestretchChannelCount = downmixerBufferProvider != NULL
+                ? mMixerChannelCount : channelCount;
+        mTimestretchBufferProvider = new TimestretchBufferProvider(timestretchChannelCount,
+                mMixerInFormat, sampleRate, speed, pitch);
+        reconfigureBufferProviders();
+    } else {
+        reinterpret_cast<TimestretchBufferProvider*>(mTimestretchBufferProvider)
+                ->setPlaybackRate(speed, pitch);
+    }
+    return true;
+}
+
 /* Checks to see if the volume ramp has completed and clears the increment
  * variables appropriately.
  *
@@ -777,6 +830,8 @@ void AudioMixer::setBufferProvider(int name, AudioBufferProvider* bufferProvider
         mState.tracks[name].downmixerBufferProvider->reset();
     } else if (mState.tracks[name].mPostDownmixReformatBufferProvider != NULL) {
         mState.tracks[name].mPostDownmixReformatBufferProvider->reset();
+    } else if (mState.tracks[name].mTimestretchBufferProvider != NULL) {
+        mState.tracks[name].mTimestretchBufferProvider->reset();
     }
 
     mState.tracks[name].mInputBufferProvider = bufferProvider;
diff --git a/services/audioflinger/AudioMixer.h b/services/audioflinger/AudioMixer.h
index e283b83..e27a0d1 100644
--- a/services/audioflinger/AudioMixer.h
+++ b/services/audioflinger/AudioMixer.h
@@ -73,6 +73,7 @@ public:
         RESAMPLE        = 0x3001,
         RAMP_VOLUME     = 0x3002, // ramp to new volume
         VOLUME          = 0x3003, // don't ramp
+        TIMESTRETCH     = 0x3004,
 
         // set Parameter names
         // for target TRACK
@@ -100,6 +101,9 @@ public:
         VOLUME0         = 0x4200,
         VOLUME1         = 0x4201,
         AUXLEVEL        = 0x4210,
+        // for target TIMESTRETCH
+        PLAYBACK_RATE   = 0x4300, // Configure timestretch on this track name;
+                                  // parameter 'value' is a pointer to the new playback rate.
     };
 
 
@@ -214,6 +218,9 @@ private:
 
         /* Buffer providers are constructed to translate the track input data as needed.
          *
+         * TODO: perhaps make a single PlaybackConverterProvider class to move
+         * all pre-mixer track buffer conversions outside the AudioMixer class.
+         *
          * 1) mInputBufferProvider: The AudioTrack buffer provider.
          * 2) mReformatBufferProvider: If not NULL, performs the audio reformat to
          *    match either mMixerInFormat or mDownmixRequiresFormat, if the downmixer
@@ -223,11 +230,13 @@ private:
          *    the number of channels required by the mixer sink.
          * 4) mPostDownmixReformatBufferProvider: If not NULL, performs reformatting from
          *    the downmixer requirements to the mixer engine input requirements.
+         * 5) mTimestretchBufferProvider: Adds timestretching for playback rate
          */
         AudioBufferProvider*     mInputBufferProvider;    // externally provided buffer provider.
         PassthruBufferProvider*  mReformatBufferProvider; // provider wrapper for reformatting.
         PassthruBufferProvider*  downmixerBufferProvider; // wrapper for channel conversion.
         PassthruBufferProvider*  mPostDownmixReformatBufferProvider;
+        PassthruBufferProvider*  mTimestretchBufferProvider;
 
         int32_t     sessionId;
 
@@ -250,6 +259,9 @@ private:
         audio_channel_mask_t mMixerChannelMask;
         uint32_t             mMixerChannelCount;
 
+        float          mSpeed;
+        float          mPitch;
+
         bool        needsRamp() { return (volumeInc[0] | volumeInc[1] | auxInc) != 0; }
         bool        setResampler(uint32_t trackSampleRate, uint32_t devSampleRate);
         bool        doesResample() const { return resampler != NULL; }
@@ -262,6 +274,7 @@ private:
         void        unprepareForDownmix();
         status_t    prepareForReformat();
         void        unprepareForReformat();
+        bool        setPlaybackRate(float speed, float pitch);
         void        reconfigureBufferProviders();
     };
 
diff --git a/services/audioflinger/BufferProviders.cpp b/services/audioflinger/BufferProviders.cpp
index e143805..e058e6c 100644
--- a/services/audioflinger/BufferProviders.cpp
+++ b/services/audioflinger/BufferProviders.cpp
@@ -20,7 +20,9 @@
 #include <audio_effects/effect_downmix.h>
 #include <audio_utils/primitives.h>
 #include <audio_utils/format.h>
+#include <media/AudioResamplerPublic.h>
 #include <media/EffectsFactoryApi.h>
+
 #include <utils/Log.h>
 
 #include "Configuration.h"
@@ -358,5 +360,165 @@ void ReformatBufferProvider::copyFrames(void *dst, const void *src, size_t frame
     memcpy_by_audio_format(dst, mOutputFormat, src, mInputFormat, frames * mChannelCount);
 }
 
+TimestretchBufferProvider::TimestretchBufferProvider(int32_t channelCount,
+        audio_format_t format, uint32_t sampleRate, float speed, float pitch) :
+        mChannelCount(channelCount),
+        mFormat(format),
+        mSampleRate(sampleRate),
+        mFrameSize(channelCount * audio_bytes_per_sample(format)),
+        mSpeed(speed),
+        mPitch(pitch),
+        mLocalBufferFrameCount(0),
+        mLocalBufferData(NULL),
+        mRemaining(0)
+{
+    ALOGV("TimestretchBufferProvider(%p)(%u, %#x, %u %f %f)",
+            this, channelCount, format, sampleRate, speed, pitch);
+    mBuffer.frameCount = 0;
+}
+
+TimestretchBufferProvider::~TimestretchBufferProvider()
+{
+    ALOGV("~TimestretchBufferProvider(%p)", this);
+    if (mBuffer.frameCount != 0) {
+        mTrackBufferProvider->releaseBuffer(&mBuffer);
+    }
+    free(mLocalBufferData);
+}
+
+status_t TimestretchBufferProvider::getNextBuffer(
+        AudioBufferProvider::Buffer *pBuffer, int64_t pts)
+{
+    ALOGV("TimestretchBufferProvider(%p)::getNextBuffer(%p (%zu), %lld)",
+            this, pBuffer, pBuffer->frameCount, pts);
+
+    // BYPASS
+    //return mTrackBufferProvider->getNextBuffer(pBuffer, pts);
+
+    // check if previously processed data is sufficient.
+    if (pBuffer->frameCount <= mRemaining) {
+        ALOGV("previous sufficient");
+        pBuffer->raw = mLocalBufferData;
+        return OK;
+    }
+
+    // do we need to resize our buffer?
+    if (pBuffer->frameCount > mLocalBufferFrameCount) {
+        void *newmem;
+        if (posix_memalign(&newmem, 32, pBuffer->frameCount * mFrameSize) == OK) {
+            if (mRemaining != 0) {
+                memcpy(newmem, mLocalBufferData, mRemaining * mFrameSize);
+            }
+            free(mLocalBufferData);
+            mLocalBufferData = newmem;
+            mLocalBufferFrameCount = pBuffer->frameCount;
+        }
+    }
+
+    // need to fetch more data
+    const size_t outputDesired = pBuffer->frameCount - mRemaining;
+    mBuffer.frameCount = mSpeed == AUDIO_TIMESTRETCH_SPEED_NORMAL
+            ? outputDesired : outputDesired * mSpeed + 1;
+
+    status_t res = mTrackBufferProvider->getNextBuffer(&mBuffer, pts);
+
+    ALOG_ASSERT(res == OK || mBuffer.frameCount == 0);
+    if (res != OK || mBuffer.frameCount == 0) { // not needed by API spec, but to be safe.
+        ALOGD("buffer error");
+        if (mRemaining == 0) {
+            pBuffer->raw = NULL;
+            pBuffer->frameCount = 0;
+            return res;
+        } else { // return partial count
+            pBuffer->raw = mLocalBufferData;
+            pBuffer->frameCount = mRemaining;
+            return OK;
+        }
+    }
+
+    // time-stretch the data
+    size_t dstAvailable = min(mLocalBufferFrameCount - mRemaining, outputDesired);
+    size_t srcAvailable = mBuffer.frameCount;
+    processFrames((uint8_t*)mLocalBufferData + mRemaining * mFrameSize, &dstAvailable,
+            mBuffer.raw, &srcAvailable);
+
+    // release all data consumed
+    mBuffer.frameCount = srcAvailable;
+    mTrackBufferProvider->releaseBuffer(&mBuffer);
+
+    // update buffer vars with the actual data processed and return with buffer
+    mRemaining += dstAvailable;
+
+    pBuffer->raw = mLocalBufferData;
+    pBuffer->frameCount = mRemaining;
+
+    return OK;
+}
+
+void TimestretchBufferProvider::releaseBuffer(AudioBufferProvider::Buffer *pBuffer)
+{
+    ALOGV("TimestretchBufferProvider(%p)::releaseBuffer(%p (%zu))",
+       this, pBuffer, pBuffer->frameCount);
+
+    // BYPASS
+    //return mTrackBufferProvider->releaseBuffer(pBuffer);
+
+    // LOG_ALWAYS_FATAL_IF(pBuffer->frameCount == 0, "Invalid framecount");
+    if (pBuffer->frameCount < mRemaining) {
+        memcpy(mLocalBufferData,
+                (uint8_t*)mLocalBufferData + pBuffer->frameCount * mFrameSize,
+                (mRemaining - pBuffer->frameCount) * mFrameSize);
+        mRemaining -= pBuffer->frameCount;
+    } else if (pBuffer->frameCount == mRemaining) {
+        mRemaining = 0;
+    } else {
+        LOG_ALWAYS_FATAL("Releasing more frames(%zu) than available(%zu)",
+                pBuffer->frameCount, mRemaining);
+    }
+
+    pBuffer->raw = NULL;
+    pBuffer->frameCount = 0;
+}
+
+void TimestretchBufferProvider::reset()
+{
+    mRemaining = 0;
+}
+
+status_t TimestretchBufferProvider::setPlaybackRate(float speed, float pitch)
+{
+    mSpeed = speed;
+    mPitch = pitch;
+    return OK;
+}
+
+void TimestretchBufferProvider::processFrames(void *dstBuffer, size_t *dstFrames,
+        const void *srcBuffer, size_t *srcFrames)
+{
+    ALOGV("processFrames(%zu %zu)  remaining(%zu)", *dstFrames, *srcFrames, mRemaining);
+    // Note dstFrames is the required number of frames.
+
+    // Ensure consumption from src is as expected.
+    const size_t targetSrc = *dstFrames * mSpeed;
+    if (*srcFrames < targetSrc) { // limit dst frames to that possible
+        *dstFrames = *srcFrames / mSpeed;
+    } else if (*srcFrames > targetSrc + 1) {
+        *srcFrames = targetSrc + 1;
+    }
+
+    // Do the time stretch by memory copy without any local buffer.
+    if (*dstFrames <= *srcFrames) {
+        size_t copySize = mFrameSize * *dstFrames;
+        memcpy(dstBuffer, srcBuffer, copySize);
+    } else {
+        // cyclically repeat the source.
+        for (size_t count = 0; count < *dstFrames; count += *srcFrames) {
+            size_t remaining = min(*srcFrames, *dstFrames - count);
+            memcpy((uint8_t*)dstBuffer + mFrameSize * count,
+                    srcBuffer, mFrameSize * *srcFrames);
+        }
+    }
+}
+
 // ----------------------------------------------------------------------------
 } // namespace android
diff --git a/services/audioflinger/BufferProviders.h b/services/audioflinger/BufferProviders.h
index 7145b80..2b6ea47 100644
--- a/services/audioflinger/BufferProviders.h
+++ b/services/audioflinger/BufferProviders.h
@@ -146,6 +146,45 @@ protected:
     const audio_format_t mOutputFormat;
 };
 
+// TimestretchBufferProvider derives from PassthruBufferProvider for time stretching
+class TimestretchBufferProvider : public PassthruBufferProvider {
+public:
+    TimestretchBufferProvider(int32_t channelCount,
+            audio_format_t format, uint32_t sampleRate, float speed, float pitch);
+    virtual ~TimestretchBufferProvider();
+
+    // Overrides AudioBufferProvider methods
+    virtual status_t getNextBuffer(Buffer* buffer, int64_t pts);
+    virtual void releaseBuffer(Buffer* buffer);
+
+    // Overrides PassthruBufferProvider
+    virtual void reset();
+
+    virtual status_t setPlaybackRate(float speed, float pitch);
+
+    // processes frames
+    // dstBuffer is where to place the data
+    // dstFrames [in/out] is the desired frames (return with actual placed in buffer)
+    // srcBuffer is the source data
+    // srcFrames [in/out] is the available source frames (return with consumed)
+    virtual void processFrames(void *dstBuffer, size_t *dstFrames,
+            const void *srcBuffer, size_t *srcFrames);
+
+protected:
+    const uint32_t       mChannelCount;
+    const audio_format_t mFormat;
+    const uint32_t       mSampleRate; // const for now (TODO change this)
+    const size_t         mFrameSize;
+    float                mSpeed;
+    float                mPitch;
+
+private:
+    AudioBufferProvider::Buffer mBuffer;
+    size_t               mLocalBufferFrameCount;
+    void                *mLocalBufferData;
+    size_t               mRemaining;
+};
+
 // ----------------------------------------------------------------------------
 } // namespace android