From d33a4ccb5c3b58406cee45a4d933d40219177f8c Mon Sep 17 00:00:00 2001 From: James Dong Date: Tue, 15 Feb 2011 10:08:07 -0800 Subject: Decouple AudioRecord read and audio encoding bug - 3313754 Change-Id: I951dd0e21e34aa1412c391f003bc32103d0424b0 --- include/media/stagefright/AudioSource.h | 24 ++- media/libstagefright/AudioSource.cpp | 273 +++++++++++++++++--------------- 2 files changed, 163 insertions(+), 134 deletions(-) diff --git a/include/media/stagefright/AudioSource.h b/include/media/stagefright/AudioSource.h index d484d60..b35a6e6 100644 --- a/include/media/stagefright/AudioSource.h +++ b/include/media/stagefright/AudioSource.h @@ -18,15 +18,17 @@ #define AUDIO_SOURCE_H_ +#include #include #include +#include +#include namespace android { class AudioRecord; -struct MediaBufferGroup; -struct AudioSource : public MediaSource { +struct AudioSource : public MediaSource, public MediaBufferObserver { // Note that the "channels" parameter is _not_ the number of channels, // but a bitmask of AudioSystem::audio_channels constants. AudioSource( @@ -45,6 +47,9 @@ struct AudioSource : public MediaSource { virtual status_t read( MediaBuffer **buffer, const ReadOptions *options = NULL); + status_t dataCallbackTimestamp(const AudioRecord::Buffer& buffer, int64_t timeUs); + virtual void signalBufferReturned(MediaBuffer *buffer); + protected: virtual ~AudioSource(); @@ -61,20 +66,24 @@ private: kAutoRampStartUs = 1000000, }; + Mutex mLock; + Condition mFrameAvailableCondition; + Condition mFrameEncodingCompletionCondition; + AudioRecord *mRecord; status_t mInitCheck; bool mStarted; + int32_t mSampleRate; - bool mCollectStats; bool mTrackMaxAmplitude; int64_t mStartTimeUs; int16_t mMaxAmplitude; int64_t mPrevSampleTimeUs; - int64_t mTotalLostFrames; - int64_t mPrevLostBytes; int64_t mInitialReadTimeUs; + int64_t mNumFramesReceived; + int64_t mNumClientOwnedBuffers; - MediaBufferGroup *mGroup; + List mBuffersReceived; void trackMaxAmplitude(int16_t *data, int nSamples); @@ -84,6 +93,9 @@ private: int32_t startFrame, int32_t rampDurationFrames, uint8_t *data, size_t bytes); + void releaseQueuedFrames_l(); + void waitOutstandingEncodingFrames_l(); + AudioSource(const AudioSource &); AudioSource &operator=(const AudioSource &); }; diff --git a/media/libstagefright/AudioSource.cpp b/media/libstagefright/AudioSource.cpp index 7a1d73b..cd0e021 100644 --- a/media/libstagefright/AudioSource.cpp +++ b/media/libstagefright/AudioSource.cpp @@ -18,38 +18,54 @@ #define LOG_TAG "AudioSource" #include -#include - #include -#include -#include +#include +#include #include #include +#include #include #include namespace android { +static void AudioRecordCallbackFunction(int event, void *user, void *info) { + AudioSource *source = (AudioSource *) user; + switch (event) { + case AudioRecord::EVENT_MORE_DATA: { + source->dataCallbackTimestamp(*((AudioRecord::Buffer *) info), systemTime() / 1000); + break; + } + case AudioRecord::EVENT_OVERRUN: { + LOGW("AudioRecord reported overrun!"); + break; + } + default: + // does nothing + break; + } +} + AudioSource::AudioSource( int inputSource, uint32_t sampleRate, uint32_t channels) : mStarted(false), - mCollectStats(false), + mSampleRate(sampleRate), mPrevSampleTimeUs(0), - mTotalLostFrames(0), - mPrevLostBytes(0), - mGroup(NULL) { + mNumFramesReceived(0), + mNumClientOwnedBuffers(0) { LOGV("sampleRate: %d, channels: %d", sampleRate, channels); CHECK(channels == 1 || channels == 2); uint32_t flags = AudioRecord::RECORD_AGC_ENABLE | AudioRecord::RECORD_NS_ENABLE | AudioRecord::RECORD_IIR_ENABLE; - mRecord = new AudioRecord( inputSource, sampleRate, AudioSystem::PCM_16_BIT, channels > 1? AudioSystem::CHANNEL_IN_STEREO: AudioSystem::CHANNEL_IN_MONO, 4 * kMaxBufferSize / sizeof(int16_t), /* Enable ping-pong buffers */ - flags); + flags, + AudioRecordCallbackFunction, + this); mInitCheck = mRecord->initCheck(); } @@ -68,6 +84,7 @@ status_t AudioSource::initCheck() const { } status_t AudioSource::start(MetaData *params) { + Mutex::Autolock autoLock(mLock); if (mStarted) { return UNKNOWN_ERROR; } @@ -76,12 +93,6 @@ status_t AudioSource::start(MetaData *params) { return NO_INIT; } - char value[PROPERTY_VALUE_MAX]; - if (property_get("media.stagefright.record-stats", value, NULL) - && (!strcmp(value, "1") || !strcasecmp(value, "true"))) { - mCollectStats = true; - } - mTrackMaxAmplitude = false; mMaxAmplitude = 0; mInitialReadTimeUs = 0; @@ -92,9 +103,6 @@ status_t AudioSource::start(MetaData *params) { } status_t err = mRecord->start(); if (err == OK) { - mGroup = new MediaBufferGroup; - mGroup->add_buffer(new MediaBuffer(kMaxBufferSize)); - mStarted = true; } else { delete mRecord; @@ -105,7 +113,25 @@ status_t AudioSource::start(MetaData *params) { return err; } +void AudioSource::releaseQueuedFrames_l() { + LOGV("releaseQueuedFrames_l"); + List::iterator it; + while (!mBuffersReceived.empty()) { + it = mBuffersReceived.begin(); + (*it)->release(); + mBuffersReceived.erase(it); + } +} + +void AudioSource::waitOutstandingEncodingFrames_l() { + LOGV("waitOutstandingEncodingFrames_l: %lld", mNumClientOwnedBuffers); + while (mNumClientOwnedBuffers > 0) { + mFrameEncodingCompletionCondition.wait(mLock); + } +} + status_t AudioSource::stop() { + Mutex::Autolock autoLock(mLock); if (!mStarted) { return UNKNOWN_ERROR; } @@ -114,29 +140,23 @@ status_t AudioSource::stop() { return NO_INIT; } - mRecord->stop(); - - delete mGroup; - mGroup = NULL; - mStarted = false; - - if (mCollectStats) { - LOGI("Total lost audio frames: %lld", - mTotalLostFrames + (mPrevLostBytes >> 1)); - } + mRecord->stop(); + waitOutstandingEncodingFrames_l(); + releaseQueuedFrames_l(); return OK; } sp AudioSource::getFormat() { + Mutex::Autolock autoLock(mLock); if (mInitCheck != OK) { return 0; } sp meta = new MetaData; meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_AUDIO_RAW); - meta->setInt32(kKeySampleRate, mRecord->getSampleRate()); + meta->setInt32(kKeySampleRate, mSampleRate); meta->setInt32(kKeyChannelCount, mRecord->channelCount()); meta->setInt32(kKeyMaxInputSize, kMaxBufferSize); @@ -177,121 +197,118 @@ void AudioSource::rampVolume( status_t AudioSource::read( MediaBuffer **out, const ReadOptions *options) { + Mutex::Autolock autoLock(mLock); + *out = NULL; if (mInitCheck != OK) { return NO_INIT; } - int64_t readTimeUs = systemTime() / 1000; - *out = NULL; - - MediaBuffer *buffer; - CHECK_EQ(mGroup->acquire_buffer(&buffer), OK); - - int err = 0; - if (mStarted) { - - uint32_t numFramesRecorded; - mRecord->getPosition(&numFramesRecorded); + while (mStarted && mBuffersReceived.empty()) { + mFrameAvailableCondition.wait(mLock); + } + if (!mStarted) { + return OK; + } + MediaBuffer *buffer = *mBuffersReceived.begin(); + mBuffersReceived.erase(mBuffersReceived.begin()); + ++mNumClientOwnedBuffers; + buffer->setObserver(this); + buffer->add_ref(); + + // Mute/suppress the recording sound + int64_t timeUs; + CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs)); + int64_t elapsedTimeUs = timeUs - mStartTimeUs; + if (elapsedTimeUs < kAutoRampStartUs) { + memset((uint8_t *) buffer->data(), 0, buffer->range_length()); + } else if (elapsedTimeUs < kAutoRampStartUs + kAutoRampDurationUs) { + int32_t autoRampDurationFrames = + (kAutoRampDurationUs * mSampleRate + 500000LL) / 1000000LL; + + int32_t autoRampStartFrames = + (kAutoRampStartUs * mSampleRate + 500000LL) / 1000000LL; + + int32_t nFrames = mNumFramesReceived - autoRampStartFrames; + rampVolume(nFrames, autoRampDurationFrames, + (uint8_t *) buffer->data(), buffer->range_length()); + } + // Track the max recording signal amplitude. + if (mTrackMaxAmplitude) { + trackMaxAmplitude( + (int16_t *) buffer->data(), buffer->range_length() >> 1); + } - if (numFramesRecorded == 0 && mPrevSampleTimeUs == 0) { - mInitialReadTimeUs = readTimeUs; - // Initial delay - if (mStartTimeUs > 0) { - mStartTimeUs = readTimeUs - mStartTimeUs; - } else { - // Assume latency is constant. - mStartTimeUs += mRecord->latency() * 1000; - } - mPrevSampleTimeUs = mStartTimeUs; - } + *out = buffer; + return OK; +} - uint32_t sampleRate = mRecord->getSampleRate(); +void AudioSource::signalBufferReturned(MediaBuffer *buffer) { + LOGV("signalBufferReturned: %p", buffer->data()); + Mutex::Autolock autoLock(mLock); + --mNumClientOwnedBuffers; + buffer->setObserver(0); + buffer->release(); + mFrameEncodingCompletionCondition.signal(); + return; +} - // Insert null frames when lost frames are detected. - int64_t timestampUs = mPrevSampleTimeUs; - uint32_t numLostBytes = mRecord->getInputFramesLost() << 1; - numLostBytes += mPrevLostBytes; -#if 0 - // Simulate lost frames - numLostBytes = ((rand() * 1.0 / RAND_MAX)) * 2 * kMaxBufferSize; - numLostBytes &= 0xFFFFFFFE; // Alignment requirement +status_t AudioSource::dataCallbackTimestamp( + const AudioRecord::Buffer& audioBuffer, int64_t timeUs) { + LOGV("dataCallbackTimestamp: %lld us", timeUs); + Mutex::Autolock autoLock(mLock); + if (!mStarted) { + LOGW("Spurious callback from AudioRecord. Drop the audio data."); + return OK; + } - // Reduce the chance to lose - if (rand() * 1.0 / RAND_MAX >= 0.05) { - numLostBytes = 0; - } -#endif - if (numLostBytes > 0) { - if (numLostBytes > kMaxBufferSize) { - mPrevLostBytes = numLostBytes - kMaxBufferSize; - numLostBytes = kMaxBufferSize; - } else { - mPrevLostBytes = 0; - } - - CHECK_EQ(numLostBytes & 1, 0); - timestampUs += ((1000000LL * (numLostBytes >> 1)) + - (sampleRate >> 1)) / sampleRate; - - if (mCollectStats) { - mTotalLostFrames += (numLostBytes >> 1); - } - memset(buffer->data(), 0, numLostBytes); - buffer->set_range(0, numLostBytes); - if (numFramesRecorded == 0) { - buffer->meta_data()->setInt64(kKeyAnchorTime, mStartTimeUs); - } - buffer->meta_data()->setInt64(kKeyTime, mStartTimeUs + mPrevSampleTimeUs); - buffer->meta_data()->setInt64(kKeyDriftTime, readTimeUs - mInitialReadTimeUs); - mPrevSampleTimeUs = timestampUs; - *out = buffer; - return OK; + if (mNumFramesReceived == 0 && mPrevSampleTimeUs == 0) { + mInitialReadTimeUs = timeUs; + // Initial delay + if (mStartTimeUs > 0) { + mStartTimeUs = timeUs - mStartTimeUs; + } else { + // Assume latency is constant. + mStartTimeUs += mRecord->latency() * 1000; } + mPrevSampleTimeUs = mStartTimeUs; + } - ssize_t n = mRecord->read(buffer->data(), buffer->size()); - if (n <= 0) { + int64_t timestampUs = mPrevSampleTimeUs; + + size_t numLostBytes = mRecord->getInputFramesLost(); + CHECK_EQ(numLostBytes & 1, 0u); + CHECK_EQ(audioBuffer.size & 1, 0u); + size_t bufferSize = numLostBytes + audioBuffer.size; + MediaBuffer *buffer = new MediaBuffer(bufferSize); + if (numLostBytes > 0) { + memset(buffer->data(), 0, numLostBytes); + memcpy((uint8_t *) buffer->data() + numLostBytes, + audioBuffer.i16, audioBuffer.size); + } else { + if (audioBuffer.size == 0) { + LOGW("Nothing is available from AudioRecord callback buffer"); buffer->release(); - LOGE("Read from AudioRecord returns %d", n); - return UNKNOWN_ERROR; - } - - int64_t recordDurationUs = (1000000LL * n >> 1) / sampleRate; - timestampUs += recordDurationUs; - - if (mPrevSampleTimeUs - mStartTimeUs < kAutoRampStartUs) { - // Mute the initial video recording signal - memset((uint8_t *) buffer->data(), 0, n); - } else if (mPrevSampleTimeUs - mStartTimeUs < kAutoRampStartUs + kAutoRampDurationUs) { - int32_t autoRampDurationFrames = - (kAutoRampDurationUs * sampleRate + 500000LL) / 1000000LL; - - int32_t autoRampStartFrames = - (kAutoRampStartUs * sampleRate + 500000LL) / 1000000LL; - - int32_t nFrames = numFramesRecorded - autoRampStartFrames; - rampVolume(nFrames, autoRampDurationFrames, (uint8_t *) buffer->data(), n); - } - if (mTrackMaxAmplitude) { - trackMaxAmplitude((int16_t *) buffer->data(), n >> 1); - } - - if (numFramesRecorded == 0) { - buffer->meta_data()->setInt64(kKeyAnchorTime, mStartTimeUs); + return OK; } + memcpy((uint8_t *) buffer->data(), + audioBuffer.i16, audioBuffer.size); + } - buffer->meta_data()->setInt64(kKeyTime, mStartTimeUs + mPrevSampleTimeUs); - buffer->meta_data()->setInt64(kKeyDriftTime, readTimeUs - mInitialReadTimeUs); - mPrevSampleTimeUs = timestampUs; - LOGV("initial delay: %lld, sample rate: %d, timestamp: %lld", - mStartTimeUs, sampleRate, timestampUs); - - buffer->set_range(0, n); + buffer->set_range(0, bufferSize); + timestampUs += ((1000000LL * (bufferSize >> 1)) + + (mSampleRate >> 1)) / mSampleRate; - *out = buffer; - return OK; + if (mNumFramesReceived == 0) { + buffer->meta_data()->setInt64(kKeyAnchorTime, mStartTimeUs); } + buffer->meta_data()->setInt64(kKeyTime, mPrevSampleTimeUs); + buffer->meta_data()->setInt64(kKeyDriftTime, timeUs - mInitialReadTimeUs); + mPrevSampleTimeUs = timestampUs; + mNumFramesReceived += buffer->range_length() / sizeof(int16_t); + mBuffersReceived.push_back(buffer); + mFrameAvailableCondition.signal(); return OK; } -- cgit v1.1