From c5656cc900aeb4a705e27508dd82c70030a97709 Mon Sep 17 00:00:00 2001 From: Andy Hung Date: Thu, 26 Mar 2015 19:04:33 -0700 Subject: Add playback rate to AudioMixer Bug: 19196501 Change-Id: I42d1f90e6297cf3f1304860d1691a5dfedd4c37d --- services/audioflinger/AudioMixer.cpp | 57 ++++++++++- services/audioflinger/AudioMixer.h | 13 +++ services/audioflinger/BufferProviders.cpp | 162 ++++++++++++++++++++++++++++++ services/audioflinger/BufferProviders.h | 39 +++++++ 4 files changed, 270 insertions(+), 1 deletion(-) (limited to 'services') diff --git a/services/audioflinger/AudioMixer.cpp b/services/audioflinger/AudioMixer.cpp index cb90ece..c2c791f 100644 --- a/services/audioflinger/AudioMixer.cpp +++ b/services/audioflinger/AudioMixer.cpp @@ -123,6 +123,7 @@ AudioMixer::AudioMixer(size_t frameCount, uint32_t sampleRate, uint32_t maxNumTr t->resampler = NULL; t->downmixerBufferProvider = NULL; t->mReformatBufferProvider = NULL; + t->mTimestretchBufferProvider = NULL; t++; } @@ -135,6 +136,7 @@ AudioMixer::~AudioMixer() delete t->resampler; delete t->downmixerBufferProvider; delete t->mReformatBufferProvider; + delete t->mTimestretchBufferProvider; t++; } delete [] mState.outputTemp; @@ -213,6 +215,7 @@ int AudioMixer::getTrackName(audio_channel_mask_t channelMask, t->mReformatBufferProvider = NULL; t->downmixerBufferProvider = NULL; t->mPostDownmixReformatBufferProvider = NULL; + t->mTimestretchBufferProvider = NULL; t->mMixerFormat = AUDIO_FORMAT_PCM_16_BIT; t->mFormat = format; t->mMixerInFormat = selectMixerInFormat(format); @@ -220,6 +223,8 @@ int AudioMixer::getTrackName(audio_channel_mask_t channelMask, t->mMixerChannelMask = audio_channel_mask_from_representation_and_bits( AUDIO_CHANNEL_REPRESENTATION_POSITION, AUDIO_CHANNEL_OUT_STEREO); t->mMixerChannelCount = audio_channel_count_from_out_mask(t->mMixerChannelMask); + t->mSpeed = AUDIO_TIMESTRETCH_SPEED_NORMAL; + t->mPitch = AUDIO_TIMESTRETCH_PITCH_NORMAL; // Check the downmixing (or upmixing) requirements. status_t status = t->prepareForDownmix(); if (status != OK) { @@ -412,6 +417,10 @@ void AudioMixer::track_t::reconfigureBufferProviders() mPostDownmixReformatBufferProvider->setBufferProvider(bufferProvider); bufferProvider = mPostDownmixReformatBufferProvider; } + if (mTimestretchBufferProvider) { + mTimestretchBufferProvider->setBufferProvider(bufferProvider); + bufferProvider = mTimestretchBufferProvider; + } } void AudioMixer::deleteTrackName(int name) @@ -432,7 +441,9 @@ void AudioMixer::deleteTrackName(int name) mState.tracks[name].unprepareForDownmix(); // delete the reformatter mState.tracks[name].unprepareForReformat(); - + // delete the timestretch provider + delete track.mTimestretchBufferProvider; + track.mTimestretchBufferProvider = NULL; mTrackNames &= ~(1<(value)[0]; + const float pitch = reinterpret_cast(value)[1]; + ALOG_ASSERT(AUDIO_TIMESTRETCH_SPEED_MIN <= speed + && speed <= AUDIO_TIMESTRETCH_SPEED_MAX, + "bad speed %f", speed); + ALOG_ASSERT(AUDIO_TIMESTRETCH_PITCH_MIN <= pitch + && pitch <= AUDIO_TIMESTRETCH_PITCH_MAX, + "bad pitch %f", pitch); + if (track.setPlaybackRate(speed, pitch)) { + ALOGV("setParameter(TIMESTRETCH, PLAYBACK_RATE, %f %f", speed, pitch); + // invalidateState(1 << name); + } + } break; + default: + LOG_ALWAYS_FATAL("setParameter timestretch: bad param %d", param); + } + break; default: LOG_ALWAYS_FATAL("setParameter: bad target %d", target); @@ -699,6 +730,28 @@ bool AudioMixer::track_t::setResampler(uint32_t trackSampleRate, uint32_t devSam return false; } +bool AudioMixer::track_t::setPlaybackRate(float speed, float pitch) +{ + if (speed == mSpeed && pitch == mPitch) { + return false; + } + mSpeed = speed; + mPitch = pitch; + if (mTimestretchBufferProvider == NULL) { + // TODO: Remove MONO_HACK. Resampler sees #channels after the downmixer + // but if none exists, it is the channel count (1 for mono). + const int timestretchChannelCount = downmixerBufferProvider != NULL + ? mMixerChannelCount : channelCount; + mTimestretchBufferProvider = new TimestretchBufferProvider(timestretchChannelCount, + mMixerInFormat, sampleRate, speed, pitch); + reconfigureBufferProviders(); + } else { + reinterpret_cast(mTimestretchBufferProvider) + ->setPlaybackRate(speed, pitch); + } + return true; +} + /* Checks to see if the volume ramp has completed and clears the increment * variables appropriately. * @@ -777,6 +830,8 @@ void AudioMixer::setBufferProvider(int name, AudioBufferProvider* bufferProvider mState.tracks[name].downmixerBufferProvider->reset(); } else if (mState.tracks[name].mPostDownmixReformatBufferProvider != NULL) { mState.tracks[name].mPostDownmixReformatBufferProvider->reset(); + } else if (mState.tracks[name].mTimestretchBufferProvider != NULL) { + mState.tracks[name].mTimestretchBufferProvider->reset(); } mState.tracks[name].mInputBufferProvider = bufferProvider; diff --git a/services/audioflinger/AudioMixer.h b/services/audioflinger/AudioMixer.h index e283b83..e27a0d1 100644 --- a/services/audioflinger/AudioMixer.h +++ b/services/audioflinger/AudioMixer.h @@ -73,6 +73,7 @@ public: RESAMPLE = 0x3001, RAMP_VOLUME = 0x3002, // ramp to new volume VOLUME = 0x3003, // don't ramp + TIMESTRETCH = 0x3004, // set Parameter names // for target TRACK @@ -100,6 +101,9 @@ public: VOLUME0 = 0x4200, VOLUME1 = 0x4201, AUXLEVEL = 0x4210, + // for target TIMESTRETCH + PLAYBACK_RATE = 0x4300, // Configure timestretch on this track name; + // parameter 'value' is a pointer to the new playback rate. }; @@ -214,6 +218,9 @@ private: /* Buffer providers are constructed to translate the track input data as needed. * + * TODO: perhaps make a single PlaybackConverterProvider class to move + * all pre-mixer track buffer conversions outside the AudioMixer class. + * * 1) mInputBufferProvider: The AudioTrack buffer provider. * 2) mReformatBufferProvider: If not NULL, performs the audio reformat to * match either mMixerInFormat or mDownmixRequiresFormat, if the downmixer @@ -223,11 +230,13 @@ private: * the number of channels required by the mixer sink. * 4) mPostDownmixReformatBufferProvider: If not NULL, performs reformatting from * the downmixer requirements to the mixer engine input requirements. + * 5) mTimestretchBufferProvider: Adds timestretching for playback rate */ AudioBufferProvider* mInputBufferProvider; // externally provided buffer provider. PassthruBufferProvider* mReformatBufferProvider; // provider wrapper for reformatting. PassthruBufferProvider* downmixerBufferProvider; // wrapper for channel conversion. PassthruBufferProvider* mPostDownmixReformatBufferProvider; + PassthruBufferProvider* mTimestretchBufferProvider; int32_t sessionId; @@ -250,6 +259,9 @@ private: audio_channel_mask_t mMixerChannelMask; uint32_t mMixerChannelCount; + float mSpeed; + float mPitch; + bool needsRamp() { return (volumeInc[0] | volumeInc[1] | auxInc) != 0; } bool setResampler(uint32_t trackSampleRate, uint32_t devSampleRate); bool doesResample() const { return resampler != NULL; } @@ -262,6 +274,7 @@ private: void unprepareForDownmix(); status_t prepareForReformat(); void unprepareForReformat(); + bool setPlaybackRate(float speed, float pitch); void reconfigureBufferProviders(); }; diff --git a/services/audioflinger/BufferProviders.cpp b/services/audioflinger/BufferProviders.cpp index e143805..e058e6c 100644 --- a/services/audioflinger/BufferProviders.cpp +++ b/services/audioflinger/BufferProviders.cpp @@ -20,7 +20,9 @@ #include #include #include +#include #include + #include #include "Configuration.h" @@ -358,5 +360,165 @@ void ReformatBufferProvider::copyFrames(void *dst, const void *src, size_t frame memcpy_by_audio_format(dst, mOutputFormat, src, mInputFormat, frames * mChannelCount); } +TimestretchBufferProvider::TimestretchBufferProvider(int32_t channelCount, + audio_format_t format, uint32_t sampleRate, float speed, float pitch) : + mChannelCount(channelCount), + mFormat(format), + mSampleRate(sampleRate), + mFrameSize(channelCount * audio_bytes_per_sample(format)), + mSpeed(speed), + mPitch(pitch), + mLocalBufferFrameCount(0), + mLocalBufferData(NULL), + mRemaining(0) +{ + ALOGV("TimestretchBufferProvider(%p)(%u, %#x, %u %f %f)", + this, channelCount, format, sampleRate, speed, pitch); + mBuffer.frameCount = 0; +} + +TimestretchBufferProvider::~TimestretchBufferProvider() +{ + ALOGV("~TimestretchBufferProvider(%p)", this); + if (mBuffer.frameCount != 0) { + mTrackBufferProvider->releaseBuffer(&mBuffer); + } + free(mLocalBufferData); +} + +status_t TimestretchBufferProvider::getNextBuffer( + AudioBufferProvider::Buffer *pBuffer, int64_t pts) +{ + ALOGV("TimestretchBufferProvider(%p)::getNextBuffer(%p (%zu), %lld)", + this, pBuffer, pBuffer->frameCount, pts); + + // BYPASS + //return mTrackBufferProvider->getNextBuffer(pBuffer, pts); + + // check if previously processed data is sufficient. + if (pBuffer->frameCount <= mRemaining) { + ALOGV("previous sufficient"); + pBuffer->raw = mLocalBufferData; + return OK; + } + + // do we need to resize our buffer? + if (pBuffer->frameCount > mLocalBufferFrameCount) { + void *newmem; + if (posix_memalign(&newmem, 32, pBuffer->frameCount * mFrameSize) == OK) { + if (mRemaining != 0) { + memcpy(newmem, mLocalBufferData, mRemaining * mFrameSize); + } + free(mLocalBufferData); + mLocalBufferData = newmem; + mLocalBufferFrameCount = pBuffer->frameCount; + } + } + + // need to fetch more data + const size_t outputDesired = pBuffer->frameCount - mRemaining; + mBuffer.frameCount = mSpeed == AUDIO_TIMESTRETCH_SPEED_NORMAL + ? outputDesired : outputDesired * mSpeed + 1; + + status_t res = mTrackBufferProvider->getNextBuffer(&mBuffer, pts); + + ALOG_ASSERT(res == OK || mBuffer.frameCount == 0); + if (res != OK || mBuffer.frameCount == 0) { // not needed by API spec, but to be safe. + ALOGD("buffer error"); + if (mRemaining == 0) { + pBuffer->raw = NULL; + pBuffer->frameCount = 0; + return res; + } else { // return partial count + pBuffer->raw = mLocalBufferData; + pBuffer->frameCount = mRemaining; + return OK; + } + } + + // time-stretch the data + size_t dstAvailable = min(mLocalBufferFrameCount - mRemaining, outputDesired); + size_t srcAvailable = mBuffer.frameCount; + processFrames((uint8_t*)mLocalBufferData + mRemaining * mFrameSize, &dstAvailable, + mBuffer.raw, &srcAvailable); + + // release all data consumed + mBuffer.frameCount = srcAvailable; + mTrackBufferProvider->releaseBuffer(&mBuffer); + + // update buffer vars with the actual data processed and return with buffer + mRemaining += dstAvailable; + + pBuffer->raw = mLocalBufferData; + pBuffer->frameCount = mRemaining; + + return OK; +} + +void TimestretchBufferProvider::releaseBuffer(AudioBufferProvider::Buffer *pBuffer) +{ + ALOGV("TimestretchBufferProvider(%p)::releaseBuffer(%p (%zu))", + this, pBuffer, pBuffer->frameCount); + + // BYPASS + //return mTrackBufferProvider->releaseBuffer(pBuffer); + + // LOG_ALWAYS_FATAL_IF(pBuffer->frameCount == 0, "Invalid framecount"); + if (pBuffer->frameCount < mRemaining) { + memcpy(mLocalBufferData, + (uint8_t*)mLocalBufferData + pBuffer->frameCount * mFrameSize, + (mRemaining - pBuffer->frameCount) * mFrameSize); + mRemaining -= pBuffer->frameCount; + } else if (pBuffer->frameCount == mRemaining) { + mRemaining = 0; + } else { + LOG_ALWAYS_FATAL("Releasing more frames(%zu) than available(%zu)", + pBuffer->frameCount, mRemaining); + } + + pBuffer->raw = NULL; + pBuffer->frameCount = 0; +} + +void TimestretchBufferProvider::reset() +{ + mRemaining = 0; +} + +status_t TimestretchBufferProvider::setPlaybackRate(float speed, float pitch) +{ + mSpeed = speed; + mPitch = pitch; + return OK; +} + +void TimestretchBufferProvider::processFrames(void *dstBuffer, size_t *dstFrames, + const void *srcBuffer, size_t *srcFrames) +{ + ALOGV("processFrames(%zu %zu) remaining(%zu)", *dstFrames, *srcFrames, mRemaining); + // Note dstFrames is the required number of frames. + + // Ensure consumption from src is as expected. + const size_t targetSrc = *dstFrames * mSpeed; + if (*srcFrames < targetSrc) { // limit dst frames to that possible + *dstFrames = *srcFrames / mSpeed; + } else if (*srcFrames > targetSrc + 1) { + *srcFrames = targetSrc + 1; + } + + // Do the time stretch by memory copy without any local buffer. + if (*dstFrames <= *srcFrames) { + size_t copySize = mFrameSize * *dstFrames; + memcpy(dstBuffer, srcBuffer, copySize); + } else { + // cyclically repeat the source. + for (size_t count = 0; count < *dstFrames; count += *srcFrames) { + size_t remaining = min(*srcFrames, *dstFrames - count); + memcpy((uint8_t*)dstBuffer + mFrameSize * count, + srcBuffer, mFrameSize * *srcFrames); + } + } +} + // ---------------------------------------------------------------------------- } // namespace android diff --git a/services/audioflinger/BufferProviders.h b/services/audioflinger/BufferProviders.h index 7145b80..2b6ea47 100644 --- a/services/audioflinger/BufferProviders.h +++ b/services/audioflinger/BufferProviders.h @@ -146,6 +146,45 @@ protected: const audio_format_t mOutputFormat; }; +// TimestretchBufferProvider derives from PassthruBufferProvider for time stretching +class TimestretchBufferProvider : public PassthruBufferProvider { +public: + TimestretchBufferProvider(int32_t channelCount, + audio_format_t format, uint32_t sampleRate, float speed, float pitch); + virtual ~TimestretchBufferProvider(); + + // Overrides AudioBufferProvider methods + virtual status_t getNextBuffer(Buffer* buffer, int64_t pts); + virtual void releaseBuffer(Buffer* buffer); + + // Overrides PassthruBufferProvider + virtual void reset(); + + virtual status_t setPlaybackRate(float speed, float pitch); + + // processes frames + // dstBuffer is where to place the data + // dstFrames [in/out] is the desired frames (return with actual placed in buffer) + // srcBuffer is the source data + // srcFrames [in/out] is the available source frames (return with consumed) + virtual void processFrames(void *dstBuffer, size_t *dstFrames, + const void *srcBuffer, size_t *srcFrames); + +protected: + const uint32_t mChannelCount; + const audio_format_t mFormat; + const uint32_t mSampleRate; // const for now (TODO change this) + const size_t mFrameSize; + float mSpeed; + float mPitch; + +private: + AudioBufferProvider::Buffer mBuffer; + size_t mLocalBufferFrameCount; + void *mLocalBufferData; + size_t mRemaining; +}; + // ---------------------------------------------------------------------------- } // namespace android -- cgit v1.1