diff options
26 files changed, 517 insertions, 287 deletions
diff --git a/include/media/AudioResamplerPublic.h b/include/media/AudioResamplerPublic.h index 07d946d..53b8c13 100644 --- a/include/media/AudioResamplerPublic.h +++ b/include/media/AudioResamplerPublic.h @@ -18,6 +18,9 @@ #define ANDROID_AUDIO_RESAMPLER_PUBLIC_H #include <stdint.h> +#include <math.h> + +namespace android { // AUDIO_RESAMPLER_DOWN_RATIO_MAX is the maximum ratio between the original // audio sample rate and the target rate when downsampling, @@ -34,13 +37,76 @@ // an int32_t of the phase increments, making the resulting sample rate inexact. #define AUDIO_RESAMPLER_UP_RATIO_MAX 65536 -#define AUDIO_TIMESTRETCH_SPEED_MIN 0.5f -#define AUDIO_TIMESTRETCH_SPEED_MAX 2.0f +// AUDIO_TIMESTRETCH_SPEED_MIN and AUDIO_TIMESTRETCH_SPEED_MAX define the min and max time stretch +// speeds supported by the system. These are enforced by the system and values outside this range +// will result in a runtime error. +// Depending on the AudioPlaybackRate::mStretchMode, the effective limits might be narrower than +// the ones specified here +// AUDIO_TIMESTRETCH_SPEED_MIN_DELTA is the minimum absolute speed difference that might trigger a +// parameter update +#define AUDIO_TIMESTRETCH_SPEED_MIN 0.01f +#define AUDIO_TIMESTRETCH_SPEED_MAX 20.0f #define AUDIO_TIMESTRETCH_SPEED_NORMAL 1.0f +#define AUDIO_TIMESTRETCH_SPEED_MIN_DELTA 0.0001f -#define AUDIO_TIMESTRETCH_PITCH_MIN 0.5f -#define AUDIO_TIMESTRETCH_PITCH_MAX 2.0f +// AUDIO_TIMESTRETCH_PITCH_MIN and AUDIO_TIMESTRETCH_PITCH_MAX define the min and max time stretch +// pitch shifting supported by the system. These are not enforced by the system and values +// outside this range might result in a pitch different than the one requested. +// Depending on the AudioPlaybackRate::mStretchMode, the effective limits might be narrower than +// the ones specified here. +// AUDIO_TIMESTRETCH_PITCH_MIN_DELTA is the minimum absolute pitch difference that might trigger a +// parameter update +#define AUDIO_TIMESTRETCH_PITCH_MIN 0.25f +#define AUDIO_TIMESTRETCH_PITCH_MAX 4.0f #define AUDIO_TIMESTRETCH_PITCH_NORMAL 1.0f +#define AUDIO_TIMESTRETCH_PITCH_MIN_DELTA 0.0001f + + +//Determines the current algorithm used for stretching +enum AudioTimestretchStretchMode : int32_t { + AUDIO_TIMESTRETCH_STRETCH_DEFAULT = 0, + AUDIO_TIMESTRETCH_STRETCH_SPEECH = 1, + //TODO: add more stretch modes/algorithms +}; + +//Limits for AUDIO_TIMESTRETCH_STRETCH_SPEECH mode +#define TIMESTRETCH_SONIC_SPEED_MIN 0.1f +#define TIMESTRETCH_SONIC_SPEED_MAX 6.0f + +//Determines behavior of Timestretch if current algorithm can't perform +//with current parameters. +// FALLBACK_CUT_REPEAT: (internal only) for speed <1.0 will truncate frames +// for speed > 1.0 will repeat frames +// FALLBACK_MUTE: will set all processed frames to zero +// FALLBACK_FAIL: will stop program execution and log a fatal error +enum AudioTimestretchFallbackMode : int32_t { + AUDIO_TIMESTRETCH_FALLBACK_CUT_REPEAT = -1, + AUDIO_TIMESTRETCH_FALLBACK_DEFAULT = 0, + AUDIO_TIMESTRETCH_FALLBACK_MUTE = 1, + AUDIO_TIMESTRETCH_FALLBACK_FAIL = 2, +}; + +struct AudioPlaybackRate { + float mSpeed; + float mPitch; + enum AudioTimestretchStretchMode mStretchMode; + enum AudioTimestretchFallbackMode mFallbackMode; +}; + +static const AudioPlaybackRate AUDIO_PLAYBACK_RATE_DEFAULT = { + AUDIO_TIMESTRETCH_SPEED_NORMAL, + AUDIO_TIMESTRETCH_PITCH_NORMAL, + AUDIO_TIMESTRETCH_STRETCH_DEFAULT, + AUDIO_TIMESTRETCH_FALLBACK_DEFAULT +}; + +static inline bool isAudioPlaybackRateEqual(const AudioPlaybackRate &pr1, + const AudioPlaybackRate &pr2) { + return fabs(pr1.mSpeed - pr2.mSpeed) < AUDIO_TIMESTRETCH_SPEED_MIN_DELTA && + fabs(pr1.mPitch - pr2.mPitch) < AUDIO_TIMESTRETCH_PITCH_MIN_DELTA && + pr2.mStretchMode == pr2.mStretchMode && + pr2.mFallbackMode == pr2.mFallbackMode; +} // TODO: Consider putting these inlines into a class scope @@ -77,4 +143,8 @@ static inline size_t sourceFramesNeededWithTimestretch( return required * (double)speed + 1 + 1; // accounting for rounding dependencies } +} // namespace android + +// --------------------------------------------------------------------------- + #endif // ANDROID_AUDIO_RESAMPLER_PUBLIC_H diff --git a/include/media/AudioTrack.h b/include/media/AudioTrack.h index a06197f..2d34c02 100644 --- a/include/media/AudioTrack.h +++ b/include/media/AudioTrack.h @@ -21,6 +21,7 @@ #include <media/AudioSystem.h> #include <media/AudioTimestamp.h> #include <media/IAudioTrack.h> +#include <media/AudioResamplerPublic.h> #include <utils/threads.h> namespace android { @@ -369,10 +370,10 @@ public: * Speed increases the playback rate of media, but does not alter pitch. * Pitch increases the "tonal frequency" of media, but does not affect the playback rate. */ - status_t setPlaybackRate(float speed, float pitch); + status_t setPlaybackRate(const AudioPlaybackRate &playbackRate); /* Return current playback rate */ - void getPlaybackRate(float *speed, float *pitch) const; + const AudioPlaybackRate& getPlaybackRate() const; /* Enables looping and sets the start and end points of looping. * Only supported for static buffer mode. @@ -748,8 +749,7 @@ protected: float mVolume[2]; float mSendLevel; mutable uint32_t mSampleRate; // mutable because getSampleRate() can update it - float mSpeed; // timestretch: 1.0f for normal speed. - float mPitch; // timestretch: 1.0f for normal pitch. + AudioPlaybackRate mPlaybackRate; size_t mFrameCount; // corresponds to current IAudioTrack, value is // reported back by AudioFlinger to the client size_t mReqFrameCount; // frame count to request the first or next time diff --git a/include/private/media/AudioTrackShared.h b/include/private/media/AudioTrackShared.h index 6cc2e2b..1e5064f 100644 --- a/include/private/media/AudioTrackShared.h +++ b/include/private/media/AudioTrackShared.h @@ -114,13 +114,7 @@ struct AudioTrackSharedStatic { mPosLoopQueue; }; - -struct AudioTrackPlaybackRate { - float mSpeed; - float mPitch; -}; - -typedef SingleStateQueue<AudioTrackPlaybackRate> AudioTrackPlaybackRateQueue; +typedef SingleStateQueue<AudioPlaybackRate> PlaybackRateQueue; // ---------------------------------------------------------------------------- @@ -168,7 +162,7 @@ private: uint32_t mSampleRate; // AudioTrack only: client's requested sample rate in Hz // or 0 == default. Write-only client, read-only server. - AudioTrackPlaybackRateQueue::Shared mPlaybackRateQueue; + PlaybackRateQueue::Shared mPlaybackRateQueue; // client write-only, server read-only uint16_t mSendLevel; // Fixed point U4.12 so 0x1000 means 1.0 @@ -345,10 +339,7 @@ public: mCblk->mSampleRate = sampleRate; } - void setPlaybackRate(float speed, float pitch) { - AudioTrackPlaybackRate playbackRate; - playbackRate.mSpeed = speed; - playbackRate.mPitch = pitch; + void setPlaybackRate(const AudioPlaybackRate& playbackRate) { mPlaybackRateMutator.push(playbackRate); } @@ -365,7 +356,7 @@ public: status_t waitStreamEndDone(const struct timespec *requested); private: - AudioTrackPlaybackRateQueue::Mutator mPlaybackRateMutator; + PlaybackRateQueue::Mutator mPlaybackRateMutator; }; class StaticAudioTrackClientProxy : public AudioTrackClientProxy { @@ -483,8 +474,7 @@ public: : ServerProxy(cblk, buffers, frameCount, frameSize, true /*isOut*/, clientInServer), mPlaybackRateObserver(&cblk->mPlaybackRateQueue) { mCblk->mSampleRate = sampleRate; - mPlaybackRate.mSpeed = AUDIO_TIMESTRETCH_SPEED_NORMAL; - mPlaybackRate.mPitch = AUDIO_TIMESTRETCH_PITCH_NORMAL; + mPlaybackRate = AUDIO_PLAYBACK_RATE_DEFAULT; } protected: virtual ~AudioTrackServerProxy() { } @@ -520,11 +510,11 @@ public: virtual size_t framesReleased() const { return mCblk->mServer; } // Return the playback speed and pitch read atomically. Not multi-thread safe on server side. - void getPlaybackRate(float *speed, float *pitch); + AudioPlaybackRate getPlaybackRate(); private: - AudioTrackPlaybackRate mPlaybackRate; // last observed playback rate - AudioTrackPlaybackRateQueue::Observer mPlaybackRateObserver; + AudioPlaybackRate mPlaybackRate; // last observed playback rate + PlaybackRateQueue::Observer mPlaybackRateObserver; }; class StaticAudioTrackServerProxy : public AudioTrackServerProxy { diff --git a/media/libmedia/AudioTrack.cpp b/media/libmedia/AudioTrack.cpp index 055556f..8555983 100644 --- a/media/libmedia/AudioTrack.cpp +++ b/media/libmedia/AudioTrack.cpp @@ -393,8 +393,7 @@ status_t AudioTrack::set( return BAD_VALUE; } mSampleRate = sampleRate; - mSpeed = AUDIO_TIMESTRETCH_SPEED_NORMAL; - mPitch = AUDIO_TIMESTRETCH_PITCH_NORMAL; + mPlaybackRate = AUDIO_PLAYBACK_RATE_DEFAULT; // Make copy of input parameter offloadInfo so that in the future: // (a) createTrack_l doesn't need it as an input parameter @@ -722,7 +721,7 @@ status_t AudioTrack::setSampleRate(uint32_t rate) return NO_INIT; } // pitch is emulated by adjusting speed and sampleRate - const uint32_t effectiveSampleRate = adjustSampleRate(rate, mPitch); + const uint32_t effectiveSampleRate = adjustSampleRate(rate, mPlaybackRate.mPitch); if (rate == 0 || effectiveSampleRate > afSamplingRate * AUDIO_RESAMPLER_DOWN_RATIO_MAX) { return BAD_VALUE; } @@ -757,10 +756,10 @@ uint32_t AudioTrack::getSampleRate() const return mSampleRate; } -status_t AudioTrack::setPlaybackRate(float speed, float pitch) +status_t AudioTrack::setPlaybackRate(const AudioPlaybackRate &playbackRate) { AutoMutex lock(mLock); - if (speed == mSpeed && pitch == mPitch) { + if (isAudioPlaybackRateEqual(playbackRate, mPlaybackRate)) { return NO_ERROR; } if (mIsTimed || isOffloadedOrDirect_l()) { @@ -770,32 +769,37 @@ status_t AudioTrack::setPlaybackRate(float speed, float pitch) return INVALID_OPERATION; } // pitch is emulated by adjusting speed and sampleRate - const uint32_t effectiveRate = adjustSampleRate(mSampleRate, pitch); - const float effectiveSpeed = adjustSpeed(speed, pitch); - const float effectivePitch = adjustPitch(pitch); + const uint32_t effectiveRate = adjustSampleRate(mSampleRate, playbackRate.mPitch); + const float effectiveSpeed = adjustSpeed(playbackRate.mSpeed, playbackRate.mPitch); + const float effectivePitch = adjustPitch(playbackRate.mPitch); if (effectiveSpeed < AUDIO_TIMESTRETCH_SPEED_MIN || effectiveSpeed > AUDIO_TIMESTRETCH_SPEED_MAX || effectivePitch < AUDIO_TIMESTRETCH_PITCH_MIN || effectivePitch > AUDIO_TIMESTRETCH_PITCH_MAX) { return BAD_VALUE; + //TODO: add function in AudioResamplerPublic.h to check for validity. } // Check if the buffer size is compatible. if (!isSampleRateSpeedAllowed_l(effectiveRate, effectiveSpeed)) { - ALOGV("setPlaybackRate(%f, %f) failed", speed, pitch); + ALOGV("setPlaybackRate(%f, %f) failed", playbackRate.mSpeed, playbackRate.mPitch); return BAD_VALUE; } - mSpeed = speed; - mPitch = pitch; - mProxy->setPlaybackRate(effectiveSpeed, effectivePitch); + mPlaybackRate = playbackRate; + mProxy->setPlaybackRate(playbackRate); + + //modify this + AudioPlaybackRate playbackRateTemp = playbackRate; + playbackRateTemp.mSpeed = effectiveSpeed; + playbackRateTemp.mPitch = effectivePitch; + mProxy->setPlaybackRate(playbackRateTemp); mProxy->setSampleRate(effectiveRate); // FIXME: not quite "atomic" with setPlaybackRate return NO_ERROR; } -void AudioTrack::getPlaybackRate(float *speed, float *pitch) const +const AudioPlaybackRate& AudioTrack::getPlaybackRate() const { AutoMutex lock(mLock); - *speed = mSpeed; - *pitch = mPitch; + return mPlaybackRate; } status_t AudioTrack::setLoop(uint32_t loopStart, uint32_t loopEnd, int loopCount) @@ -1014,10 +1018,9 @@ status_t AudioTrack::setOutputDevice(audio_port_handle_t deviceId) { AutoMutex lock(mLock); if (mSelectedDeviceId != deviceId) { mSelectedDeviceId = deviceId; - return restoreTrack_l("setOutputDevice() restart"); - } else { - return NO_ERROR; + android_atomic_or(CBLK_INVALID, &mCblk->mFlags); } + return NO_ERROR; } audio_port_handle_t AudioTrack::getOutputDevice() { @@ -1170,7 +1173,8 @@ status_t AudioTrack::createTrack_l() if ((mFlags & AUDIO_OUTPUT_FLAG_FAST) == 0) { // for normal tracks precompute the frame count based on speed. const size_t minFrameCount = calculateMinFrameCount( - afLatency, afFrameCount, afSampleRate, mSampleRate, mSpeed); + afLatency, afFrameCount, afSampleRate, mSampleRate, + mPlaybackRate.mSpeed); if (frameCount < minFrameCount) { frameCount = minFrameCount; } @@ -1342,11 +1346,15 @@ status_t AudioTrack::createTrack_l() gain_from_float(mVolume[AUDIO_INTERLEAVE_RIGHT]))); mProxy->setSendLevel(mSendLevel); - const uint32_t effectiveSampleRate = adjustSampleRate(mSampleRate, mPitch); - const float effectiveSpeed = adjustSpeed(mSpeed, mPitch); - const float effectivePitch = adjustPitch(mPitch); + const uint32_t effectiveSampleRate = adjustSampleRate(mSampleRate, mPlaybackRate.mPitch); + const float effectiveSpeed = adjustSpeed(mPlaybackRate.mSpeed, mPlaybackRate.mPitch); + const float effectivePitch = adjustPitch(mPlaybackRate.mPitch); mProxy->setSampleRate(effectiveSampleRate); - mProxy->setPlaybackRate(effectiveSpeed, effectivePitch); + + AudioPlaybackRate playbackRateTemp = mPlaybackRate; + playbackRateTemp.mSpeed = effectiveSpeed; + playbackRateTemp.mPitch = effectivePitch; + mProxy->setPlaybackRate(playbackRateTemp); mProxy->setMinimum(mNotificationFramesAct); mDeathNotifier = new DeathNotifier(this); @@ -1716,7 +1724,7 @@ nsecs_t AudioTrack::processAudioBuffer() // Cache other fields that will be needed soon uint32_t sampleRate = mSampleRate; - float speed = mSpeed; + float speed = mPlaybackRate.mSpeed; uint32_t notificationFrames = mNotificationFramesAct; if (mRefreshRemaining) { mRefreshRemaining = false; @@ -2138,7 +2146,7 @@ status_t AudioTrack::getTimestamp(AudioTimestamp& timestamp) } const int64_t deltaTimeUs = timestampTimeUs - mStartUs; const int64_t deltaPositionByUs = (double)timestamp.mPosition * 1000000 - / ((double)mSampleRate * mSpeed); + / ((double)mSampleRate * mPlaybackRate.mSpeed); if (deltaPositionByUs > deltaTimeUs + kTimeJitterUs) { // Verify that the counter can't count faster than the sample rate @@ -2226,7 +2234,7 @@ status_t AudioTrack::dump(int fd, const Vector<String16>& args __unused) const mChannelCount, mFrameCount); result.append(buffer); snprintf(buffer, 255, " sample rate(%u), speed(%f), status(%d)\n", - mSampleRate, mSpeed, mStatus); + mSampleRate, mPlaybackRate.mSpeed, mStatus); result.append(buffer); snprintf(buffer, 255, " state(%d), latency (%d)\n", mState, mLatency); result.append(buffer); diff --git a/media/libmedia/AudioTrackShared.cpp b/media/libmedia/AudioTrackShared.cpp index aee9fc2..1d7aed2 100644 --- a/media/libmedia/AudioTrackShared.cpp +++ b/media/libmedia/AudioTrackShared.cpp @@ -794,14 +794,10 @@ void AudioTrackServerProxy::tallyUnderrunFrames(uint32_t frameCount) (void) android_atomic_or(CBLK_UNDERRUN, &cblk->mFlags); } -void AudioTrackServerProxy::getPlaybackRate(float *speed, float *pitch) +AudioPlaybackRate AudioTrackServerProxy::getPlaybackRate() { // do not call from multiple threads without holding lock - AudioTrackPlaybackRate playbackRate; - if (mPlaybackRateObserver.poll(playbackRate)) { - mPlaybackRate = playbackRate; - } - *speed = mPlaybackRate.mSpeed; - *pitch = mPlaybackRate.mPitch; + mPlaybackRateObserver.poll(mPlaybackRate); + return mPlaybackRate; } // --------------------------------------------------------------------------- diff --git a/media/libmediaplayerservice/MediaPlayerService.cpp b/media/libmediaplayerservice/MediaPlayerService.cpp index 87003c5..3bc763f 100644 --- a/media/libmediaplayerservice/MediaPlayerService.cpp +++ b/media/libmediaplayerservice/MediaPlayerService.cpp @@ -746,7 +746,7 @@ status_t MediaPlayerService::Client::setDataSource(int fd, int64_t offset, int64 return UNKNOWN_ERROR; } - ALOGV("st_dev = %llu", sb.st_dev); + ALOGV("st_dev = %llu", static_cast<uint64_t>(sb.st_dev)); ALOGV("st_mode = %u", sb.st_mode); ALOGV("st_uid = %lu", static_cast<unsigned long>(sb.st_uid)); ALOGV("st_gid = %lu", static_cast<unsigned long>(sb.st_gid)); diff --git a/media/libmediaplayerservice/MetadataRetrieverClient.cpp b/media/libmediaplayerservice/MetadataRetrieverClient.cpp index 6ef4c1f..9a37302 100644 --- a/media/libmediaplayerservice/MetadataRetrieverClient.cpp +++ b/media/libmediaplayerservice/MetadataRetrieverClient.cpp @@ -141,7 +141,7 @@ status_t MetadataRetrieverClient::setDataSource(int fd, int64_t offset, int64_t ALOGE("fstat(%d) failed: %d, %s", fd, ret, strerror(errno)); return BAD_VALUE; } - ALOGV("st_dev = %llu", sb.st_dev); + ALOGV("st_dev = %llu", static_cast<uint64_t>(sb.st_dev)); ALOGV("st_mode = %u", sb.st_mode); ALOGV("st_uid = %lu", static_cast<unsigned long>(sb.st_uid)); ALOGV("st_gid = %lu", static_cast<unsigned long>(sb.st_gid)); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp index f8be16a..f229452 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp @@ -621,7 +621,8 @@ bool NuPlayer::Renderer::onDrainAudioQueue() { return false; } - if (entry->mOffset == 0) { + // ignore 0-sized buffer which could be EOS marker with no data + if (entry->mOffset == 0 && entry->mBuffer->size() > 0) { int64_t mediaTimeUs; CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6); diff --git a/media/libstagefright/CallbackDataSource.cpp b/media/libstagefright/CallbackDataSource.cpp index 2e0745f..41f0175 100644 --- a/media/libstagefright/CallbackDataSource.cpp +++ b/media/libstagefright/CallbackDataSource.cpp @@ -70,9 +70,10 @@ ssize_t CallbackDataSource::readAt(off64_t offset, void* data, size_t size) { if (numRead == 0) { return totalNumRead; } - // Sanity check. - CHECK((size_t)numRead <= numToRead && numRead >= 0 && - (size_t)numRead <= bufferSize); + if ((size_t)numRead > numToRead) { + return ERROR_OUT_OF_RANGE; + } + CHECK(numRead >= 0 && (size_t)numRead <= bufferSize); memcpy(((uint8_t*)data) + totalNumRead, mMemory->pointer(), numRead); numLeft -= numRead; totalNumRead += numRead; @@ -94,4 +95,49 @@ status_t CallbackDataSource::getSize(off64_t *size) { return OK; } +TinyCacheSource::TinyCacheSource(const sp<DataSource>& source) + : mSource(source), mCachedOffset(0), mCachedSize(0) { +} + +status_t TinyCacheSource::initCheck() const { + return mSource->initCheck(); +} + +ssize_t TinyCacheSource::readAt(off64_t offset, void* data, size_t size) { + if (size >= kCacheSize) { + return mSource->readAt(offset, data, size); + } + + // Check if the cache satisfies the read. + if (offset >= mCachedOffset && offset + size <= mCachedOffset + mCachedSize) { + memcpy(data, &mCache[offset - mCachedOffset], size); + return size; + } + + // Fill the cache and copy to the caller. + const ssize_t numRead = mSource->readAt(offset, mCache, kCacheSize); + if (numRead <= 0) { + return numRead; + } + if ((size_t)numRead > kCacheSize) { + return ERROR_OUT_OF_RANGE; + } + + mCachedSize = numRead; + mCachedOffset = offset; + CHECK(mCachedSize <= kCacheSize && mCachedOffset >= 0); + const size_t numToReturn = std::min(size, (size_t)numRead); + memcpy(data, mCache, numToReturn); + + return numToReturn; +} + +status_t TinyCacheSource::getSize(off64_t *size) { + return mSource->getSize(size); +} + +uint32_t TinyCacheSource::flags() { + return mSource->flags(); +} + } // namespace android diff --git a/media/libstagefright/DataSource.cpp b/media/libstagefright/DataSource.cpp index 6a89154..75ef288 100644 --- a/media/libstagefright/DataSource.cpp +++ b/media/libstagefright/DataSource.cpp @@ -283,7 +283,7 @@ sp<DataSource> DataSource::CreateMediaHTTP(const sp<IMediaHTTPService> &httpServ } sp<DataSource> DataSource::CreateFromIDataSource(const sp<IDataSource> &source) { - return new CallbackDataSource(source); + return new TinyCacheSource(new CallbackDataSource(source)); } String8 DataSource::getMIMEType() const { diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp index 5ae79e6..93864e4 100644 --- a/media/libstagefright/MediaCodec.cpp +++ b/media/libstagefright/MediaCodec.cpp @@ -236,11 +236,7 @@ void MediaCodec::ResourceManagerServiceProxy::init() { ALOGE("Failed to get ResourceManagerService"); return; } - if (IInterface::asBinder(mService)->linkToDeath(this) != OK) { - mService.clear(); - ALOGE("Failed to linkToDeath to ResourceManagerService."); - return; - } + IInterface::asBinder(mService)->linkToDeath(this); } void MediaCodec::ResourceManagerServiceProxy::binderDied(const wp<IBinder>& /*who*/) { diff --git a/media/libstagefright/httplive/LiveSession.cpp b/media/libstagefright/httplive/LiveSession.cpp index 764ff82..9ac764c 100644 --- a/media/libstagefright/httplive/LiveSession.cpp +++ b/media/libstagefright/httplive/LiveSession.cpp @@ -71,8 +71,9 @@ struct LiveSession::BandwidthEstimator : public RefBase { private: // Bandwidth estimation parameters - static const int32_t kMaxBandwidthHistoryItems = 20; - static const int64_t kMaxBandwidthHistoryWindowUs = 5000000ll; // 5 sec + static const int32_t kMinBandwidthHistoryItems = 20; + static const int64_t kMinBandwidthHistoryWindowUs = 5000000ll; // 5 sec + static const int64_t kMaxBandwidthHistoryWindowUs = 30000000ll; // 30 sec struct BandwidthEntry { int64_t mDelayUs; @@ -109,11 +110,21 @@ void LiveSession::BandwidthEstimator::addBandwidthMeasurement( mBandwidthHistory.push_back(entry); mHasNewSample = true; + // Remove no more than 10% of total transfer time at a time + // to avoid sudden jump on bandwidth estimation. There might + // be long blocking reads that takes up signification time, + // we have to keep a longer window in that case. + int64_t bandwidthHistoryWindowUs = mTotalTransferTimeUs * 9 / 10; + if (bandwidthHistoryWindowUs < kMinBandwidthHistoryWindowUs) { + bandwidthHistoryWindowUs = kMinBandwidthHistoryWindowUs; + } else if (bandwidthHistoryWindowUs > kMaxBandwidthHistoryWindowUs) { + bandwidthHistoryWindowUs = kMaxBandwidthHistoryWindowUs; + } // trim old samples, keeping at least kMaxBandwidthHistoryItems samples, // and total transfer time at least kMaxBandwidthHistoryWindowUs. - while (mBandwidthHistory.size() > kMaxBandwidthHistoryItems) { + while (mBandwidthHistory.size() > kMinBandwidthHistoryItems) { List<BandwidthEntry>::iterator it = mBandwidthHistory.begin(); - if (mTotalTransferTimeUs - it->mDelayUs < kMaxBandwidthHistoryWindowUs) { + if (mTotalTransferTimeUs - it->mDelayUs < bandwidthHistoryWindowUs) { break; } mTotalTransferTimeUs -= it->mDelayUs; @@ -122,7 +133,8 @@ void LiveSession::BandwidthEstimator::addBandwidthMeasurement( } } -bool LiveSession::BandwidthEstimator::estimateBandwidth(int32_t *bandwidthBps, bool *isStable) { +bool LiveSession::BandwidthEstimator::estimateBandwidth( + int32_t *bandwidthBps, bool *isStable) { AutoMutex autoLock(mLock); if (mBandwidthHistory.size() < 2) { @@ -159,6 +171,29 @@ bool LiveSession::BandwidthEstimator::estimateBandwidth(int32_t *bandwidthBps, b if (isStable) { *isStable = mIsStable; } +#if 0 + { + char dumpStr[1024] = {0}; + size_t itemIdx = 0; + size_t histSize = mBandwidthHistory.size(); + sprintf(dumpStr, "estimate bps=%d stable=%d history (n=%d): {", + *bandwidthBps, mIsStable, histSize); + List<BandwidthEntry>::iterator it = mBandwidthHistory.begin(); + for (; it != mBandwidthHistory.end(); ++it) { + if (itemIdx > 50) { + sprintf(dumpStr + strlen(dumpStr), + "...(%zd more items)... }", histSize - itemIdx); + break; + } + sprintf(dumpStr + strlen(dumpStr), "%dk/%.3fs%s", + it->mNumBytes / 1024, + (double)it->mDelayUs * 1.0e-6, + (it == (--mBandwidthHistory.end())) ? "}" : ", "); + itemIdx++; + } + ALOGE(dumpStr); + } +#endif return true; } @@ -338,7 +373,8 @@ status_t LiveSession::dequeueAccessUnit( offsetTimeUs = 0; } - if (mDiscontinuityAbsStartTimesUs.indexOfKey(strm.mCurDiscontinuitySeq) >= 0) { + if (mDiscontinuityAbsStartTimesUs.indexOfKey(strm.mCurDiscontinuitySeq) >= 0 + && strm.mLastDequeuedTimeUs >= 0) { int64_t firstTimeUs; firstTimeUs = mDiscontinuityAbsStartTimesUs.valueFor(strm.mCurDiscontinuitySeq); offsetTimeUs += strm.mLastDequeuedTimeUs - firstTimeUs; @@ -1731,7 +1767,7 @@ void LiveSession::onChangeConfiguration2(const sp<AMessage> &msg) { } for (size_t i = 0; i < kMaxStreams; ++i) { - mStreams[i].mCurDiscontinuitySeq = 0; + mStreams[i].reset(); } mDiscontinuityOffsetTimesUs.clear(); @@ -2226,8 +2262,9 @@ bool LiveSession::checkBuffering( continue; } + status_t finalResult; int64_t bufferedDurationUs = - mPacketSources[i]->getEstimatedDurationUs(); + mPacketSources[i]->getBufferedDurationUs(&finalResult); ALOGV("[%s] buffered %lld us", getNameForStream(mPacketSources.keyAt(i)), (long long)bufferedDurationUs); diff --git a/media/libstagefright/httplive/LiveSession.h b/media/libstagefright/httplive/LiveSession.h index 56cd702..4e7ccac 100644 --- a/media/libstagefright/httplive/LiveSession.h +++ b/media/libstagefright/httplive/LiveSession.h @@ -166,10 +166,14 @@ private: : StreamItem("") {} StreamItem(const char *type) : mType(type), - mSeekMode(kSeekModeExactPosition), - mCurDiscontinuitySeq(0), - mLastDequeuedTimeUs(0), - mLastSampleDurationUs(0) {} + mSeekMode(kSeekModeExactPosition) { + reset(); + } + void reset() { + mCurDiscontinuitySeq = 0; + mLastDequeuedTimeUs = -1ll; + mLastSampleDurationUs = 0ll; + } AString uriKey() { AString key(mType); key.append("URI"); diff --git a/media/libstagefright/include/CallbackDataSource.h b/media/libstagefright/include/CallbackDataSource.h index 678eb2e..1a21dd3 100644 --- a/media/libstagefright/include/CallbackDataSource.h +++ b/media/libstagefright/include/CallbackDataSource.h @@ -44,6 +44,36 @@ private: DISALLOW_EVIL_CONSTRUCTORS(CallbackDataSource); }; + +// A caching DataSource that wraps a CallbackDataSource. For reads smaller +// than kCacheSize it will read up to kCacheSize ahead and cache it. +// This reduces the number of binder round trips to the IDataSource and has a significant +// impact on time taken for filetype sniffing and metadata extraction. +class TinyCacheSource : public DataSource { +public: + TinyCacheSource(const sp<DataSource>& source); + + virtual status_t initCheck() const; + virtual ssize_t readAt(off64_t offset, void* data, size_t size); + virtual status_t getSize(off64_t* size); + virtual uint32_t flags(); + +private: + // 2kb comes from experimenting with the time-to-first-frame from a MediaPlayer + // with an in-memory MediaDataSource source on a Nexus 5. Beyond 2kb there was + // no improvement. + enum { + kCacheSize = 2048, + }; + + sp<DataSource> mSource; + uint8_t mCache[kCacheSize]; + off64_t mCachedOffset; + size_t mCachedSize; + + DISALLOW_EVIL_CONSTRUCTORS(TinyCacheSource); +}; + }; // namespace android #endif // ANDROID_CALLBACKDATASOURCE_H diff --git a/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp b/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp index 87ec860..f74b859 100644 --- a/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp +++ b/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp @@ -46,9 +46,10 @@ AnotherPacketSource::AnotherPacketSource(const sp<MetaData> &meta) mLastQueuedTimeUs(0), mEOSResult(OK), mLatestEnqueuedMeta(NULL), - mLatestDequeuedMeta(NULL), - mQueuedDiscontinuityCount(0) { + mLatestDequeuedMeta(NULL) { setFormat(meta); + + mDiscontinuitySegments.push_back(DiscontinuitySegment()); } void AnotherPacketSource::setFormat(const sp<MetaData> &meta) { @@ -129,11 +130,20 @@ status_t AnotherPacketSource::dequeueAccessUnit(sp<ABuffer> *buffer) { mFormat.clear(); } - --mQueuedDiscontinuityCount; + mDiscontinuitySegments.erase(mDiscontinuitySegments.begin()); + // CHECK(!mDiscontinuitySegments.empty()); return INFO_DISCONTINUITY; } + // CHECK(!mDiscontinuitySegments.empty()); + DiscontinuitySegment &seg = *mDiscontinuitySegments.begin(); + + int64_t timeUs; mLatestDequeuedMeta = (*buffer)->meta()->dup(); + CHECK(mLatestDequeuedMeta->findInt64("timeUs", &timeUs)); + if (timeUs > seg.mMaxDequeTimeUs) { + seg.mMaxDequeTimeUs = timeUs; + } sp<RefBase> object; if ((*buffer)->meta()->findObject("format", &object)) { @@ -172,6 +182,8 @@ status_t AnotherPacketSource::read( mFormat.clear(); } + mDiscontinuitySegments.erase(mDiscontinuitySegments.begin()); + // CHECK(!mDiscontinuitySegments.empty()); return INFO_DISCONTINUITY; } @@ -184,6 +196,11 @@ status_t AnotherPacketSource::read( int64_t timeUs; CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); + // CHECK(!mDiscontinuitySegments.empty()); + DiscontinuitySegment &seg = *mDiscontinuitySegments.begin(); + if (timeUs > seg.mMaxDequeTimeUs) { + seg.mMaxDequeTimeUs = timeUs; + } MediaBuffer *mediaBuffer = new MediaBuffer(buffer); @@ -226,12 +243,14 @@ void AnotherPacketSource::queueAccessUnit(const sp<ABuffer> &buffer) { mCondition.signal(); int32_t discontinuity; - if (buffer->meta()->findInt32("discontinuity", &discontinuity)) { - // discontinuity handling needs to be consistent with queueDiscontinuity() - ++mQueuedDiscontinuityCount; + if (buffer->meta()->findInt32("discontinuity", &discontinuity)){ + ALOGV("queueing a discontinuity with queueAccessUnit"); + mLastQueuedTimeUs = 0ll; mEOSResult = OK; mLatestEnqueuedMeta = NULL; + + mDiscontinuitySegments.push_back(DiscontinuitySegment()); return; } @@ -241,6 +260,15 @@ void AnotherPacketSource::queueAccessUnit(const sp<ABuffer> &buffer) { ALOGV("queueAccessUnit timeUs=%" PRIi64 " us (%.2f secs)", mLastQueuedTimeUs, mLastQueuedTimeUs / 1E6); + // CHECK(!mDiscontinuitySegments.empty()); + DiscontinuitySegment &tailSeg = *(--mDiscontinuitySegments.end()); + if (lastQueuedTimeUs > tailSeg.mMaxEnqueTimeUs) { + tailSeg.mMaxEnqueTimeUs = lastQueuedTimeUs; + } + if (tailSeg.mMaxDequeTimeUs == -1) { + tailSeg.mMaxDequeTimeUs = lastQueuedTimeUs; + } + if (mLatestEnqueuedMeta == NULL) { mLatestEnqueuedMeta = buffer->meta()->dup(); } else { @@ -264,7 +292,9 @@ void AnotherPacketSource::clear() { mBuffers.clear(); mEOSResult = OK; - mQueuedDiscontinuityCount = 0; + + mDiscontinuitySegments.clear(); + mDiscontinuitySegments.push_back(DiscontinuitySegment()); mFormat = NULL; mLatestEnqueuedMeta = NULL; @@ -291,6 +321,14 @@ void AnotherPacketSource::queueDiscontinuity( ++it; } + + for (List<DiscontinuitySegment>::iterator it2 = mDiscontinuitySegments.begin(); + it2 != mDiscontinuitySegments.end(); + ++it2) { + DiscontinuitySegment &seg = *it2; + seg.clear(); + } + } mEOSResult = OK; @@ -301,7 +339,8 @@ void AnotherPacketSource::queueDiscontinuity( return; } - ++mQueuedDiscontinuityCount; + mDiscontinuitySegments.push_back(DiscontinuitySegment()); + sp<ABuffer> buffer = new ABuffer(0); buffer->meta()->setInt32("discontinuity", static_cast<int32_t>(type)); buffer->meta()->setMessage("extra", extra); @@ -352,95 +391,19 @@ bool AnotherPacketSource::hasDataBufferAvailable(status_t *finalResult) { int64_t AnotherPacketSource::getBufferedDurationUs(status_t *finalResult) { Mutex::Autolock autoLock(mLock); - return getBufferedDurationUs_l(finalResult); -} - -int64_t AnotherPacketSource::getBufferedDurationUs_l(status_t *finalResult) { *finalResult = mEOSResult; - if (mBuffers.empty()) { - return 0; - } - - int64_t time1 = -1; - int64_t time2 = -1; int64_t durationUs = 0; - - List<sp<ABuffer> >::iterator it; - for (it = mBuffers.begin(); it != mBuffers.end(); it++) { - const sp<ABuffer> &buffer = *it; - - int32_t discard; - if (buffer->meta()->findInt32("discard", &discard) && discard) { - continue; - } - - int64_t timeUs; - if (buffer->meta()->findInt64("timeUs", &timeUs)) { - if (time1 < 0 || timeUs < time1) { - time1 = timeUs; - } - - if (time2 < 0 || timeUs > time2) { - time2 = timeUs; - } - } else { - // This is a discontinuity, reset everything. - durationUs += time2 - time1; - time1 = time2 = -1; - } - } - - return durationUs + (time2 - time1); -} - -// A cheaper but less precise version of getBufferedDurationUs that we would like to use in -// LiveSession::dequeueAccessUnit to trigger downwards adaptation. -int64_t AnotherPacketSource::getEstimatedDurationUs() { - Mutex::Autolock autoLock(mLock); - if (mBuffers.empty()) { - return 0; - } - - if (mQueuedDiscontinuityCount > 0) { - status_t finalResult; - return getBufferedDurationUs_l(&finalResult); - } - - sp<ABuffer> buffer; - int32_t discard; - int64_t startTimeUs = -1ll; - List<sp<ABuffer> >::iterator it; - for (it = mBuffers.begin(); it != mBuffers.end(); it++) { - buffer = *it; - if (buffer->meta()->findInt32("discard", &discard) && discard) { - continue; - } - buffer->meta()->findInt64("timeUs", &startTimeUs); - break; + for (List<DiscontinuitySegment>::iterator it = mDiscontinuitySegments.begin(); + it != mDiscontinuitySegments.end(); + ++it) { + const DiscontinuitySegment &seg = *it; + // dequeued access units should be a subset of enqueued access units + // CHECK(seg.maxEnqueTimeUs >= seg.mMaxDequeTimeUs); + durationUs += (seg.mMaxEnqueTimeUs - seg.mMaxDequeTimeUs); } - if (startTimeUs < 0) { - return 0; - } - - it = mBuffers.end(); - --it; - buffer = *it; - - int64_t endTimeUs; - buffer->meta()->findInt64("timeUs", &endTimeUs); - if (endTimeUs < 0) { - return 0; - } - - int64_t diffUs; - if (endTimeUs > startTimeUs) { - diffUs = endTimeUs - startTimeUs; - } else { - diffUs = startTimeUs - endTimeUs; - } - return diffUs; + return durationUs; } status_t AnotherPacketSource::nextBufferTime(int64_t *timeUs) { @@ -540,14 +503,15 @@ void AnotherPacketSource::trimBuffersAfterMeta( stopTime.mSeq, (long long)stopTime.mTimeUs); List<sp<ABuffer> >::iterator it; + List<DiscontinuitySegment >::iterator it2; sp<AMessage> newLatestEnqueuedMeta = NULL; int64_t newLastQueuedTimeUs = 0; - size_t newDiscontinuityCount = 0; - for (it = mBuffers.begin(); it != mBuffers.end(); ++it) { + for (it = mBuffers.begin(), it2 = mDiscontinuitySegments.begin(); it != mBuffers.end(); ++it) { const sp<ABuffer> &buffer = *it; int32_t discontinuity; if (buffer->meta()->findInt32("discontinuity", &discontinuity)) { - newDiscontinuityCount++; + // CHECK(it2 != mDiscontinuitySegments.end()); + ++it2; continue; } @@ -560,10 +524,18 @@ void AnotherPacketSource::trimBuffersAfterMeta( newLatestEnqueuedMeta = buffer->meta(); newLastQueuedTimeUs = curTime.mTimeUs; } + mBuffers.erase(it, mBuffers.end()); mLatestEnqueuedMeta = newLatestEnqueuedMeta; mLastQueuedTimeUs = newLastQueuedTimeUs; - mQueuedDiscontinuityCount = newDiscontinuityCount; + + DiscontinuitySegment &seg = *it2; + if (newLatestEnqueuedMeta != NULL) { + seg.mMaxEnqueTimeUs = newLastQueuedTimeUs; + } else { + seg.clear(); + } + mDiscontinuitySegments.erase(++it2, mDiscontinuitySegments.end()); } /* @@ -580,6 +552,7 @@ sp<AMessage> AnotherPacketSource::trimBuffersBeforeMeta( startTime.mSeq, (long long)startTime.mTimeUs); sp<AMessage> firstMeta; + int64_t firstTimeUs = -1; Mutex::Autolock autoLock(mLock); if (mBuffers.empty()) { return NULL; @@ -589,14 +562,14 @@ sp<AMessage> AnotherPacketSource::trimBuffersBeforeMeta( bool isAvc = false; List<sp<ABuffer> >::iterator it; - size_t discontinuityCount = 0; for (it = mBuffers.begin(); it != mBuffers.end(); ++it) { const sp<ABuffer> &buffer = *it; int32_t discontinuity; if (buffer->meta()->findInt32("discontinuity", &discontinuity)) { + mDiscontinuitySegments.erase(mDiscontinuitySegments.begin()); + // CHECK(!mDiscontinuitySegments.empty()); format = NULL; isAvc = false; - discontinuityCount++; continue; } if (format == NULL) { @@ -618,12 +591,21 @@ sp<AMessage> AnotherPacketSource::trimBuffersBeforeMeta( ALOGV("trimming from beginning to %lld (not inclusive)", (long long)curTime.mTimeUs); firstMeta = buffer->meta(); + firstTimeUs = curTime.mTimeUs; break; } } mBuffers.erase(mBuffers.begin(), it); - mQueuedDiscontinuityCount -= discontinuityCount; mLatestDequeuedMeta = NULL; + + // CHECK(!mDiscontinuitySegments.empty()); + DiscontinuitySegment &seg = *mDiscontinuitySegments.begin(); + if (firstTimeUs >= 0) { + seg.mMaxDequeTimeUs = firstTimeUs; + } else { + seg.clear(); + } + return firstMeta; } diff --git a/media/libstagefright/mpeg2ts/AnotherPacketSource.h b/media/libstagefright/mpeg2ts/AnotherPacketSource.h index 08cd92e..eb9dc9b 100644 --- a/media/libstagefright/mpeg2ts/AnotherPacketSource.h +++ b/media/libstagefright/mpeg2ts/AnotherPacketSource.h @@ -53,8 +53,6 @@ struct AnotherPacketSource : public MediaSource { // presentation timestamps since the last discontinuity (if any). int64_t getBufferedDurationUs(status_t *finalResult); - int64_t getEstimatedDurationUs(); - status_t nextBufferTime(int64_t *timeUs); void queueAccessUnit(const sp<ABuffer> &buffer); @@ -84,6 +82,25 @@ protected: virtual ~AnotherPacketSource(); private: + + struct DiscontinuitySegment { + int64_t mMaxDequeTimeUs, mMaxEnqueTimeUs; + DiscontinuitySegment() + : mMaxDequeTimeUs(-1), + mMaxEnqueTimeUs(-1) { + }; + + void clear() { + mMaxDequeTimeUs = mMaxEnqueTimeUs = -1; + } + }; + + // Discontinuity segments are consecutive access units between + // discontinuity markers. There should always be at least _ONE_ + // discontinuity segment, hence the various CHECKs in + // AnotherPacketSource.cpp for non-empty()-ness. + List<DiscontinuitySegment> mDiscontinuitySegments; + Mutex mLock; Condition mCondition; @@ -97,10 +114,7 @@ private: sp<AMessage> mLatestEnqueuedMeta; sp<AMessage> mLatestDequeuedMeta; - size_t mQueuedDiscontinuityCount; - bool wasFormatChange(int32_t discontinuityType) const; - int64_t getBufferedDurationUs_l(status_t *finalResult); DISALLOW_EVIL_CONSTRUCTORS(AnotherPacketSource); }; diff --git a/services/audioflinger/AudioMixer.cpp b/services/audioflinger/AudioMixer.cpp index f28c625..7040af4 100644 --- a/services/audioflinger/AudioMixer.cpp +++ b/services/audioflinger/AudioMixer.cpp @@ -38,7 +38,6 @@ #include <audio_utils/format.h> #include <common_time/local_clock.h> #include <common_time/cc_helper.h> -#include <media/AudioResamplerPublic.h> #include "AudioMixerOps.h" #include "AudioMixer.h" @@ -223,8 +222,7 @@ int AudioMixer::getTrackName(audio_channel_mask_t channelMask, t->mMixerChannelMask = audio_channel_mask_from_representation_and_bits( AUDIO_CHANNEL_REPRESENTATION_POSITION, AUDIO_CHANNEL_OUT_STEREO); t->mMixerChannelCount = audio_channel_count_from_out_mask(t->mMixerChannelMask); - t->mSpeed = AUDIO_TIMESTRETCH_SPEED_NORMAL; - t->mPitch = AUDIO_TIMESTRETCH_PITCH_NORMAL; + t->mPlaybackRate = AUDIO_PLAYBACK_RATE_DEFAULT; // Check the downmixing (or upmixing) requirements. status_t status = t->prepareForDownmix(); if (status != OK) { @@ -668,19 +666,25 @@ void AudioMixer::setParameter(int name, int target, int param, void *value) case TIMESTRETCH: switch (param) { case PLAYBACK_RATE: { - const float speed = reinterpret_cast<float*>(value)[0]; - const float pitch = reinterpret_cast<float*>(value)[1]; - ALOG_ASSERT(AUDIO_TIMESTRETCH_SPEED_MIN <= speed - && speed <= AUDIO_TIMESTRETCH_SPEED_MAX, - "bad speed %f", speed); - ALOG_ASSERT(AUDIO_TIMESTRETCH_PITCH_MIN <= pitch - && pitch <= AUDIO_TIMESTRETCH_PITCH_MAX, - "bad pitch %f", pitch); - if (track.setPlaybackRate(speed, pitch)) { - ALOGV("setParameter(TIMESTRETCH, PLAYBACK_RATE, %f %f", speed, pitch); + const AudioPlaybackRate *playbackRate = + reinterpret_cast<AudioPlaybackRate*>(value); + ALOG_ASSERT(AUDIO_TIMESTRETCH_SPEED_MIN <= playbackRate->mSpeed + && playbackRate->mSpeed <= AUDIO_TIMESTRETCH_SPEED_MAX, + "bad speed %f", playbackRate->mSpeed); + ALOG_ASSERT(AUDIO_TIMESTRETCH_PITCH_MIN <= playbackRate->mPitch + && playbackRate->mPitch <= AUDIO_TIMESTRETCH_PITCH_MAX, + "bad pitch %f", playbackRate->mPitch); + //TODO: use function from AudioResamplerPublic.h to test validity. + if (track.setPlaybackRate(*playbackRate)) { + ALOGV("setParameter(TIMESTRETCH, PLAYBACK_RATE, STRETCH_MODE, FALLBACK_MODE " + "%f %f %d %d", + playbackRate->mSpeed, + playbackRate->mPitch, + playbackRate->mStretchMode, + playbackRate->mFallbackMode); // invalidateState(1 << name); } - } break; + } break; default: LOG_ALWAYS_FATAL("setParameter timestretch: bad param %d", param); } @@ -730,24 +734,26 @@ bool AudioMixer::track_t::setResampler(uint32_t trackSampleRate, uint32_t devSam return false; } -bool AudioMixer::track_t::setPlaybackRate(float speed, float pitch) +bool AudioMixer::track_t::setPlaybackRate(const AudioPlaybackRate &playbackRate) { - if (speed == mSpeed && pitch == mPitch) { + if ((mTimestretchBufferProvider == NULL && + fabs(playbackRate.mSpeed - mPlaybackRate.mSpeed) < AUDIO_TIMESTRETCH_SPEED_MIN_DELTA && + fabs(playbackRate.mPitch - mPlaybackRate.mPitch) < AUDIO_TIMESTRETCH_PITCH_MIN_DELTA) || + isAudioPlaybackRateEqual(playbackRate, mPlaybackRate)) { return false; } - mSpeed = speed; - mPitch = pitch; + mPlaybackRate = playbackRate; if (mTimestretchBufferProvider == NULL) { // TODO: Remove MONO_HACK. Resampler sees #channels after the downmixer // but if none exists, it is the channel count (1 for mono). const int timestretchChannelCount = downmixerBufferProvider != NULL ? mMixerChannelCount : channelCount; mTimestretchBufferProvider = new TimestretchBufferProvider(timestretchChannelCount, - mMixerInFormat, sampleRate, speed, pitch); + mMixerInFormat, sampleRate, playbackRate); reconfigureBufferProviders(); } else { reinterpret_cast<TimestretchBufferProvider*>(mTimestretchBufferProvider) - ->setPlaybackRate(speed, pitch); + ->setPlaybackRate(playbackRate); } return true; } diff --git a/services/audioflinger/AudioMixer.h b/services/audioflinger/AudioMixer.h index e27a0d1..7165c6c 100644 --- a/services/audioflinger/AudioMixer.h +++ b/services/audioflinger/AudioMixer.h @@ -23,6 +23,7 @@ #include <hardware/audio_effect.h> #include <media/AudioBufferProvider.h> +#include <media/AudioResamplerPublic.h> #include <media/nbaio/NBLog.h> #include <system/audio.h> #include <utils/Compat.h> @@ -259,8 +260,7 @@ private: audio_channel_mask_t mMixerChannelMask; uint32_t mMixerChannelCount; - float mSpeed; - float mPitch; + AudioPlaybackRate mPlaybackRate; bool needsRamp() { return (volumeInc[0] | volumeInc[1] | auxInc) != 0; } bool setResampler(uint32_t trackSampleRate, uint32_t devSampleRate); @@ -274,7 +274,7 @@ private: void unprepareForDownmix(); status_t prepareForReformat(); void unprepareForReformat(); - bool setPlaybackRate(float speed, float pitch); + bool setPlaybackRate(const AudioPlaybackRate &playbackRate); void reconfigureBufferProviders(); }; diff --git a/services/audioflinger/BufferProviders.cpp b/services/audioflinger/BufferProviders.cpp index dcae5e7..77bf4ac 100644 --- a/services/audioflinger/BufferProviders.cpp +++ b/services/audioflinger/BufferProviders.cpp @@ -361,25 +361,25 @@ void ReformatBufferProvider::copyFrames(void *dst, const void *src, size_t frame } TimestretchBufferProvider::TimestretchBufferProvider(int32_t channelCount, - audio_format_t format, uint32_t sampleRate, float speed, float pitch) : + audio_format_t format, uint32_t sampleRate, const AudioPlaybackRate &playbackRate) : mChannelCount(channelCount), mFormat(format), mSampleRate(sampleRate), mFrameSize(channelCount * audio_bytes_per_sample(format)), - mSpeed(speed), - mPitch(pitch), mLocalBufferFrameCount(0), mLocalBufferData(NULL), mRemaining(0), - mSonicStream(sonicCreateStream(sampleRate, mChannelCount)) + mSonicStream(sonicCreateStream(sampleRate, mChannelCount)), + mFallbackFailErrorShown(false) { - ALOGV("TimestretchBufferProvider(%p)(%u, %#x, %u %f %f)", - this, channelCount, format, sampleRate, speed, pitch); - mBuffer.frameCount = 0; - LOG_ALWAYS_FATAL_IF(mSonicStream == NULL, "TimestretchBufferProvider can't allocate Sonic stream"); - sonicSetSpeed(mSonicStream, speed); + + setPlaybackRate(playbackRate); + ALOGV("TimestretchBufferProvider(%p)(%u, %#x, %u %f %f %d %d)", + this, channelCount, format, sampleRate, playbackRate.mSpeed, + playbackRate.mPitch, playbackRate.mStretchMode, playbackRate.mFallbackMode); + mBuffer.frameCount = 0; } TimestretchBufferProvider::~TimestretchBufferProvider() @@ -423,8 +423,8 @@ status_t TimestretchBufferProvider::getNextBuffer( // need to fetch more data const size_t outputDesired = pBuffer->frameCount - mRemaining; - mBuffer.frameCount = mSpeed == AUDIO_TIMESTRETCH_SPEED_NORMAL - ? outputDesired : outputDesired * mSpeed + 1; + mBuffer.frameCount = mPlaybackRate.mSpeed == AUDIO_TIMESTRETCH_SPEED_NORMAL + ? outputDesired : outputDesired * mPlaybackRate.mSpeed + 1; status_t res = mTrackBufferProvider->getNextBuffer(&mBuffer, pts); @@ -491,13 +491,13 @@ void TimestretchBufferProvider::reset() mRemaining = 0; } -status_t TimestretchBufferProvider::setPlaybackRate(float speed, float pitch) +status_t TimestretchBufferProvider::setPlaybackRate(const AudioPlaybackRate &playbackRate) { - mSpeed = speed; - mPitch = pitch; - - sonicSetSpeed(mSonicStream, speed); + mPlaybackRate = playbackRate; + mFallbackFailErrorShown = false; + sonicSetSpeed(mSonicStream, mPlaybackRate.mSpeed); //TODO: pitch is ignored for now + //TODO: optimize: if parameters are the same, don't do any extra computation. return OK; } @@ -508,33 +508,68 @@ void TimestretchBufferProvider::processFrames(void *dstBuffer, size_t *dstFrames // Note dstFrames is the required number of frames. // Ensure consumption from src is as expected. - const size_t targetSrc = *dstFrames * mSpeed; + //TODO: add logic to track "very accurate" consumption related to speed, original sampling + //rate, actual frames processed. + const size_t targetSrc = *dstFrames * mPlaybackRate.mSpeed; if (*srcFrames < targetSrc) { // limit dst frames to that possible - *dstFrames = *srcFrames / mSpeed; + *dstFrames = *srcFrames / mPlaybackRate.mSpeed; } else if (*srcFrames > targetSrc + 1) { *srcFrames = targetSrc + 1; } - switch (mFormat) { - case AUDIO_FORMAT_PCM_FLOAT: - if (sonicWriteFloatToStream(mSonicStream, (float*)srcBuffer, *srcFrames) != 1) { - ALOGE("sonicWriteFloatToStream cannot realloc"); - *srcFrames = 0; // cannot consume all of srcBuffer + if (mPlaybackRate.mSpeed< TIMESTRETCH_SONIC_SPEED_MIN || + mPlaybackRate.mSpeed > TIMESTRETCH_SONIC_SPEED_MAX ) { + //fallback mode + if (*dstFrames > 0) { + switch(mPlaybackRate.mFallbackMode) { + case AUDIO_TIMESTRETCH_FALLBACK_CUT_REPEAT: + if (*dstFrames <= *srcFrames) { + size_t copySize = mFrameSize * *dstFrames; + memcpy(dstBuffer, srcBuffer, copySize); + } else { + // cyclically repeat the source. + for (size_t count = 0; count < *dstFrames; count += *srcFrames) { + size_t remaining = min(*srcFrames, *dstFrames - count); + memcpy((uint8_t*)dstBuffer + mFrameSize * count, + srcBuffer, mFrameSize * remaining); + } + } + break; + case AUDIO_TIMESTRETCH_FALLBACK_DEFAULT: + case AUDIO_TIMESTRETCH_FALLBACK_MUTE: + memset(dstBuffer,0, mFrameSize * *dstFrames); + break; + case AUDIO_TIMESTRETCH_FALLBACK_FAIL: + default: + if(!mFallbackFailErrorShown) { + ALOGE("invalid parameters in TimestretchBufferProvider fallbackMode:%d", + mPlaybackRate.mFallbackMode); + mFallbackFailErrorShown = true; + } + break; + } } - *dstFrames = sonicReadFloatFromStream(mSonicStream, (float*)dstBuffer, *dstFrames); - break; - case AUDIO_FORMAT_PCM_16_BIT: - if (sonicWriteShortToStream(mSonicStream, (short*)srcBuffer, *srcFrames) != 1) { - ALOGE("sonicWriteShortToStream cannot realloc"); - *srcFrames = 0; // cannot consume all of srcBuffer + } else { + switch (mFormat) { + case AUDIO_FORMAT_PCM_FLOAT: + if (sonicWriteFloatToStream(mSonicStream, (float*)srcBuffer, *srcFrames) != 1) { + ALOGE("sonicWriteFloatToStream cannot realloc"); + *srcFrames = 0; // cannot consume all of srcBuffer + } + *dstFrames = sonicReadFloatFromStream(mSonicStream, (float*)dstBuffer, *dstFrames); + break; + case AUDIO_FORMAT_PCM_16_BIT: + if (sonicWriteShortToStream(mSonicStream, (short*)srcBuffer, *srcFrames) != 1) { + ALOGE("sonicWriteShortToStream cannot realloc"); + *srcFrames = 0; // cannot consume all of srcBuffer + } + *dstFrames = sonicReadShortFromStream(mSonicStream, (short*)dstBuffer, *dstFrames); + break; + default: + // could also be caught on construction + LOG_ALWAYS_FATAL("invalid format %#x for TimestretchBufferProvider", mFormat); } - *dstFrames = sonicReadShortFromStream(mSonicStream, (short*)dstBuffer, *dstFrames); - break; - default: - // could also be caught on construction - LOG_ALWAYS_FATAL("invalid format %#x for TimestretchBufferProvider", mFormat); } } - // ---------------------------------------------------------------------------- } // namespace android diff --git a/services/audioflinger/BufferProviders.h b/services/audioflinger/BufferProviders.h index 42030c0..4970b6c 100644 --- a/services/audioflinger/BufferProviders.h +++ b/services/audioflinger/BufferProviders.h @@ -151,7 +151,8 @@ protected: class TimestretchBufferProvider : public PassthruBufferProvider { public: TimestretchBufferProvider(int32_t channelCount, - audio_format_t format, uint32_t sampleRate, float speed, float pitch); + audio_format_t format, uint32_t sampleRate, + const AudioPlaybackRate &playbackRate); virtual ~TimestretchBufferProvider(); // Overrides AudioBufferProvider methods @@ -161,7 +162,7 @@ public: // Overrides PassthruBufferProvider virtual void reset(); - virtual status_t setPlaybackRate(float speed, float pitch); + virtual status_t setPlaybackRate(const AudioPlaybackRate &playbackRate); // processes frames // dstBuffer is where to place the data @@ -176,15 +177,17 @@ protected: const audio_format_t mFormat; const uint32_t mSampleRate; // const for now (TODO change this) const size_t mFrameSize; - float mSpeed; - float mPitch; + AudioPlaybackRate mPlaybackRate; private: - AudioBufferProvider::Buffer mBuffer; - size_t mLocalBufferFrameCount; - void *mLocalBufferData; - size_t mRemaining; - sonicStream mSonicStream; + AudioBufferProvider::Buffer mBuffer; // for upstream request + size_t mLocalBufferFrameCount; // size of local buffer + void *mLocalBufferData; // internally allocated buffer for data returned + // to caller + size_t mRemaining; // remaining data in local buffer + sonicStream mSonicStream; // handle to sonic timestretch object + //FIXME: this dependency should be abstracted out + bool mFallbackFailErrorShown; // log fallback error only once }; // ---------------------------------------------------------------------------- diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp index 373d643..d3d77cd 100644 --- a/services/audioflinger/Threads.cpp +++ b/services/audioflinger/Threads.cpp @@ -3599,11 +3599,10 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac // during last round size_t desiredFrames; const uint32_t sampleRate = track->mAudioTrackServerProxy->getSampleRate(); - float speed, pitch; - track->mAudioTrackServerProxy->getPlaybackRate(&speed, &pitch); + AudioPlaybackRate playbackRate = track->mAudioTrackServerProxy->getPlaybackRate(); desiredFrames = sourceFramesNeededWithTimestretch( - sampleRate, mNormalFrameCount, mSampleRate, speed); + sampleRate, mNormalFrameCount, mSampleRate, playbackRate.mSpeed); // TODO: ONLY USED FOR LEGACY RESAMPLERS, remove when they are removed. // add frames already consumed but not yet released by the resampler // because mAudioTrackServerProxy->framesReady() will include these frames @@ -3772,15 +3771,12 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac AudioMixer::SAMPLE_RATE, (void *)(uintptr_t)reqSampleRate); - // set the playback rate as an float array {speed, pitch} - float playbackRate[2]; - track->mAudioTrackServerProxy->getPlaybackRate( - &playbackRate[0] /*speed*/, &playbackRate[1] /*pitch*/); + AudioPlaybackRate playbackRate = track->mAudioTrackServerProxy->getPlaybackRate(); mAudioMixer->setParameter( name, AudioMixer::TIMESTRETCH, AudioMixer::PLAYBACK_RATE, - playbackRate); + &playbackRate); /* * Select the appropriate output buffer for the track. diff --git a/services/audioflinger/Tracks.cpp b/services/audioflinger/Tracks.cpp index da2d634..c6e9745 100644 --- a/services/audioflinger/Tracks.cpp +++ b/services/audioflinger/Tracks.cpp @@ -906,11 +906,9 @@ status_t AudioFlinger::PlaybackThread::Track::getTimestamp(AudioTimestamp& times // FIXME Not accurate under dynamic changes of sample rate and speed. // Do not use track's mSampleRate as it is not current for mixer tracks. uint32_t sampleRate = mAudioTrackServerProxy->getSampleRate(); - float speed, pitch; - mAudioTrackServerProxy->getPlaybackRate(&speed, &pitch); - uint32_t unpresentedFrames = - ((double) playbackThread->mLatchQ.mUnpresentedFrames * sampleRate * speed) - / playbackThread->mSampleRate; + AudioPlaybackRate playbackRate = mAudioTrackServerProxy->getPlaybackRate(); + uint32_t unpresentedFrames = ((double) playbackThread->mLatchQ.mUnpresentedFrames * + sampleRate * playbackRate.mSpeed)/ playbackThread->mSampleRate; // FIXME Since we're using a raw pointer as the key, it is theoretically possible // for a brand new track to share the same address as a recently destroyed // track, and thus for us to get the frames released of the wrong track. diff --git a/services/audiopolicy/common/managerdefinitions/src/DeviceDescriptor.cpp b/services/audiopolicy/common/managerdefinitions/src/DeviceDescriptor.cpp index 9573583..0715eea 100644 --- a/services/audiopolicy/common/managerdefinitions/src/DeviceDescriptor.cpp +++ b/services/audiopolicy/common/managerdefinitions/src/DeviceDescriptor.cpp @@ -52,6 +52,9 @@ bool DeviceDescriptor::equals(const sp<DeviceDescriptor>& other) const // - are of the same type (a device type cannot be AUDIO_DEVICE_NONE) // - have the same address or one device does not specify the address // - have the same channel mask or one device does not specify the channel mask + if (other == 0) { + return false; + } return (mDeviceType == other->mDeviceType) && (mAddress == "" || other->mAddress == "" || mAddress == other->mAddress) && (mChannelMask == 0 || other->mChannelMask == 0 || diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp index 3ea6a11..01f2b61 100644 --- a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp +++ b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp @@ -969,6 +969,8 @@ status_t AudioPolicyManager::startOutput(audio_io_handle_t output, audio_devices_t newDevice; if (outputDesc->mPolicyMix != NULL) { newDevice = AUDIO_DEVICE_OUT_REMOTE_SUBMIX; + } else if (mOutputRoutes.hasRouteChanged(session)) { + newDevice = getNewOutputDevice(outputDesc, false /*fromCache*/); } else { newDevice = AUDIO_DEVICE_NONE; } @@ -1026,7 +1028,7 @@ status_t AudioPolicyManager::startSource(sp<AudioOutputDescriptor> outputDesc, // necessary for a correct control of hardware output routing by startOutput() and stopOutput() outputDesc->changeRefCount(stream, 1); - if (outputDesc->mRefCount[stream] == 1) { + if (outputDesc->mRefCount[stream] == 1 || device != AUDIO_DEVICE_NONE) { // starting an output being rerouted? if (device == AUDIO_DEVICE_NONE) { device = getNewOutputDevice(outputDesc, false /*fromCache*/); @@ -2462,14 +2464,14 @@ status_t AudioPolicyManager::acquireSoundTriggerSession(audio_session_t *session return mSoundTriggerSessions.acquireSession(*session, *ioHandle); } -status_t AudioPolicyManager::startAudioSource(const struct audio_port_config *source, - const audio_attributes_t *attributes, - audio_io_handle_t *handle) +status_t AudioPolicyManager::startAudioSource(const struct audio_port_config *source __unused, + const audio_attributes_t *attributes __unused, + audio_io_handle_t *handle __unused) { return INVALID_OPERATION; } -status_t AudioPolicyManager::stopAudioSource(audio_io_handle_t handle) +status_t AudioPolicyManager::stopAudioSource(audio_io_handle_t handle __unused) { return INVALID_OPERATION; } @@ -4511,18 +4513,36 @@ bool AudioPolicyManager::SessionRouteMap::hasRoute(audio_session_t session) return indexOfKey(session) >= 0 && valueFor(session)->mDeviceDescriptor != 0; } +bool AudioPolicyManager::SessionRouteMap::hasRouteChanged(audio_session_t session) +{ + if (indexOfKey(session) >= 0) { + if (valueFor(session)->mChanged) { + valueFor(session)->mChanged = false; + return true; + } + } + return false; +} + void AudioPolicyManager::SessionRouteMap::addRoute(audio_session_t session, audio_stream_type_t streamType, sp<DeviceDescriptor> deviceDescriptor) { sp<SessionRoute> route = indexOfKey(session) >= 0 ? valueFor(session) : 0; if (route != NULL) { + if ((route->mDeviceDescriptor == 0 && deviceDescriptor != 0) || + (!route->mDeviceDescriptor->equals(deviceDescriptor))) { + route->mChanged = true; + } route->mRefCount++; route->mDeviceDescriptor = deviceDescriptor; } else { route = new AudioPolicyManager::SessionRoute(session, streamType, deviceDescriptor); route->mRefCount++; add(session, route); + if (deviceDescriptor != 0) { + route->mChanged = true; + } } } diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.h b/services/audiopolicy/managerdefault/AudioPolicyManager.h index 146a7af..521f6c4 100644 --- a/services/audiopolicy/managerdefault/AudioPolicyManager.h +++ b/services/audiopolicy/managerdefault/AudioPolicyManager.h @@ -244,7 +244,10 @@ protected: mStreamType(streamType), mDeviceDescriptor(deviceDescriptor), mRefCount(0), - mActivityCount(0) {} + mActivityCount(0), + mChanged(false) {} + + void log(const char* prefix); audio_session_t mSession; audio_stream_type_t mStreamType; @@ -252,10 +255,9 @@ protected: sp<DeviceDescriptor> mDeviceDescriptor; // "reference" counting - int mRefCount; // +/- on references - int mActivityCount; // +/- on start/stop - - void log(const char* prefix); + int mRefCount; // +/- on references + int mActivityCount; // +/- on start/stop + bool mChanged; }; class SessionRouteMap: public KeyedVector<audio_session_t, sp<SessionRoute>> @@ -268,7 +270,7 @@ protected: int incRouteActivity(audio_session_t session); int decRouteActivity(audio_session_t session); - + bool hasRouteChanged(audio_session_t session); // also clears the changed flag void log(const char* caption); }; @@ -510,6 +512,8 @@ protected: void updateCallRouting(audio_devices_t rxDevice, int delayMs = 0); + // if argument "device" is different from AUDIO_DEVICE_NONE, startSource() will force + // the re-evaluation of the output device. status_t startSource(sp<AudioOutputDescriptor> outputDesc, audio_stream_type_t stream, audio_devices_t device, diff --git a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp index 5f501a5..e764eda 100644 --- a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp +++ b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp @@ -546,9 +546,6 @@ status_t AudioPolicyService::listAudioPorts(audio_port_role_t role, unsigned int *generation) { Mutex::Autolock _l(mLock); - if(!modifyAudioRoutingAllowed()) { - return PERMISSION_DENIED; - } if (mAudioPolicyManager == NULL) { return NO_INIT; } @@ -559,9 +556,6 @@ status_t AudioPolicyService::listAudioPorts(audio_port_role_t role, status_t AudioPolicyService::getAudioPort(struct audio_port *port) { Mutex::Autolock _l(mLock); - if(!modifyAudioRoutingAllowed()) { - return PERMISSION_DENIED; - } if (mAudioPolicyManager == NULL) { return NO_INIT; } @@ -602,9 +596,6 @@ status_t AudioPolicyService::listAudioPatches(unsigned int *num_patches, unsigned int *generation) { Mutex::Autolock _l(mLock); - if(!modifyAudioRoutingAllowed()) { - return PERMISSION_DENIED; - } if (mAudioPolicyManager == NULL) { return NO_INIT; } |