diff options
Diffstat (limited to 'services')
49 files changed, 2316 insertions, 667 deletions
diff --git a/services/audioflinger/Android.mk b/services/audioflinger/Android.mk index fee2347..c359be5 100644 --- a/services/audioflinger/Android.mk +++ b/services/audioflinger/Android.mk @@ -44,12 +44,13 @@ LOCAL_SRC_FILES:= \ SpdifStreamOut.cpp \ Effects.cpp \ AudioMixer.cpp.arm \ - PatchPanel.cpp - -LOCAL_SRC_FILES += StateQueue.cpp + BufferProviders.cpp \ + PatchPanel.cpp \ + StateQueue.cpp LOCAL_C_INCLUDES := \ $(TOPDIR)frameworks/av/services/audiopolicy \ + $(TOPDIR)external/sonic \ $(call include-path-for, audio-effects) \ $(call include-path-for, audio-utils) @@ -68,7 +69,8 @@ LOCAL_SHARED_LIBRARIES := \ libhardware_legacy \ libeffects \ libpowermanager \ - libserviceutility + libserviceutility \ + libsonic LOCAL_STATIC_LIBRARIES := \ libscheduling_policy \ diff --git a/services/audioflinger/AudioHwDevice.cpp b/services/audioflinger/AudioHwDevice.cpp index 09d86ea..3191598 100644 --- a/services/audioflinger/AudioHwDevice.cpp +++ b/services/audioflinger/AudioHwDevice.cpp @@ -44,7 +44,7 @@ status_t AudioHwDevice::openOutputStream( AudioStreamOut *outputStream = new AudioStreamOut(this, flags); // Try to open the HAL first using the current format. - ALOGV("AudioHwDevice::openOutputStream(), try " + ALOGV("openOutputStream(), try " " sampleRate %d, Format %#x, " "channelMask %#x", config->sample_rate, @@ -59,7 +59,7 @@ status_t AudioHwDevice::openOutputStream( // FIXME Look at any modification to the config. // The HAL might modify the config to suggest a wrapped format. // Log this so we can see what the HALs are doing. - ALOGI("AudioHwDevice::openOutputStream(), HAL returned" + ALOGI("openOutputStream(), HAL returned" " sampleRate %d, Format %#x, " "channelMask %#x, status %d", config->sample_rate, @@ -72,16 +72,19 @@ status_t AudioHwDevice::openOutputStream( && ((flags & AUDIO_OUTPUT_FLAG_DIRECT) != 0) && ((flags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) == 0); - // FIXME - Add isEncodingSupported() query to SPDIF wrapper then - // call it from here. if (wrapperNeeded) { - outputStream = new SpdifStreamOut(this, flags); - status = outputStream->open(handle, devices, &originalConfig, address); - if (status != NO_ERROR) { - ALOGE("ERROR - AudioHwDevice::openOutputStream(), SPDIF open returned %d", - status); - delete outputStream; - outputStream = NULL; + if (SPDIFEncoder::isFormatSupported(originalConfig.format)) { + outputStream = new SpdifStreamOut(this, flags, originalConfig.format); + status = outputStream->open(handle, devices, &originalConfig, address); + if (status != NO_ERROR) { + ALOGE("ERROR - openOutputStream(), SPDIF open returned %d", + status); + delete outputStream; + outputStream = NULL; + } + } else { + ALOGE("ERROR - openOutputStream(), SPDIFEncoder does not support format 0x%08x", + originalConfig.format); } } } diff --git a/services/audioflinger/AudioMixer.cpp b/services/audioflinger/AudioMixer.cpp index dddca02..c2c791f 100644 --- a/services/audioflinger/AudioMixer.cpp +++ b/services/audioflinger/AudioMixer.cpp @@ -38,9 +38,7 @@ #include <audio_utils/format.h> #include <common_time/local_clock.h> #include <common_time/cc_helper.h> - -#include <media/EffectsFactoryApi.h> -#include <audio_effects/effect_downmix.h> +#include <media/AudioResamplerPublic.h> #include "AudioMixerOps.h" #include "AudioMixer.h" @@ -91,323 +89,6 @@ T min(const T& a, const T& b) return a < b ? a : b; } -AudioMixer::CopyBufferProvider::CopyBufferProvider(size_t inputFrameSize, - size_t outputFrameSize, size_t bufferFrameCount) : - mInputFrameSize(inputFrameSize), - mOutputFrameSize(outputFrameSize), - mLocalBufferFrameCount(bufferFrameCount), - mLocalBufferData(NULL), - mConsumed(0) -{ - ALOGV("CopyBufferProvider(%p)(%zu, %zu, %zu)", this, - inputFrameSize, outputFrameSize, bufferFrameCount); - LOG_ALWAYS_FATAL_IF(inputFrameSize < outputFrameSize && bufferFrameCount == 0, - "Requires local buffer if inputFrameSize(%zu) < outputFrameSize(%zu)", - inputFrameSize, outputFrameSize); - if (mLocalBufferFrameCount) { - (void)posix_memalign(&mLocalBufferData, 32, mLocalBufferFrameCount * mOutputFrameSize); - } - mBuffer.frameCount = 0; -} - -AudioMixer::CopyBufferProvider::~CopyBufferProvider() -{ - ALOGV("~CopyBufferProvider(%p)", this); - if (mBuffer.frameCount != 0) { - mTrackBufferProvider->releaseBuffer(&mBuffer); - } - free(mLocalBufferData); -} - -status_t AudioMixer::CopyBufferProvider::getNextBuffer(AudioBufferProvider::Buffer *pBuffer, - int64_t pts) -{ - //ALOGV("CopyBufferProvider(%p)::getNextBuffer(%p (%zu), %lld)", - // this, pBuffer, pBuffer->frameCount, pts); - if (mLocalBufferFrameCount == 0) { - status_t res = mTrackBufferProvider->getNextBuffer(pBuffer, pts); - if (res == OK) { - copyFrames(pBuffer->raw, pBuffer->raw, pBuffer->frameCount); - } - return res; - } - if (mBuffer.frameCount == 0) { - mBuffer.frameCount = pBuffer->frameCount; - status_t res = mTrackBufferProvider->getNextBuffer(&mBuffer, pts); - // At one time an upstream buffer provider had - // res == OK and mBuffer.frameCount == 0, doesn't seem to happen now 7/18/2014. - // - // By API spec, if res != OK, then mBuffer.frameCount == 0. - // but there may be improper implementations. - ALOG_ASSERT(res == OK || mBuffer.frameCount == 0); - if (res != OK || mBuffer.frameCount == 0) { // not needed by API spec, but to be safe. - pBuffer->raw = NULL; - pBuffer->frameCount = 0; - return res; - } - mConsumed = 0; - } - ALOG_ASSERT(mConsumed < mBuffer.frameCount); - size_t count = min(mLocalBufferFrameCount, mBuffer.frameCount - mConsumed); - count = min(count, pBuffer->frameCount); - pBuffer->raw = mLocalBufferData; - pBuffer->frameCount = count; - copyFrames(pBuffer->raw, (uint8_t*)mBuffer.raw + mConsumed * mInputFrameSize, - pBuffer->frameCount); - return OK; -} - -void AudioMixer::CopyBufferProvider::releaseBuffer(AudioBufferProvider::Buffer *pBuffer) -{ - //ALOGV("CopyBufferProvider(%p)::releaseBuffer(%p(%zu))", - // this, pBuffer, pBuffer->frameCount); - if (mLocalBufferFrameCount == 0) { - mTrackBufferProvider->releaseBuffer(pBuffer); - return; - } - // LOG_ALWAYS_FATAL_IF(pBuffer->frameCount == 0, "Invalid framecount"); - mConsumed += pBuffer->frameCount; // TODO: update for efficiency to reuse existing content - if (mConsumed != 0 && mConsumed >= mBuffer.frameCount) { - mTrackBufferProvider->releaseBuffer(&mBuffer); - ALOG_ASSERT(mBuffer.frameCount == 0); - } - pBuffer->raw = NULL; - pBuffer->frameCount = 0; -} - -void AudioMixer::CopyBufferProvider::reset() -{ - if (mBuffer.frameCount != 0) { - mTrackBufferProvider->releaseBuffer(&mBuffer); - } - mConsumed = 0; -} - -AudioMixer::DownmixerBufferProvider::DownmixerBufferProvider( - audio_channel_mask_t inputChannelMask, - audio_channel_mask_t outputChannelMask, audio_format_t format, - uint32_t sampleRate, int32_t sessionId, size_t bufferFrameCount) : - CopyBufferProvider( - audio_bytes_per_sample(format) * audio_channel_count_from_out_mask(inputChannelMask), - audio_bytes_per_sample(format) * audio_channel_count_from_out_mask(outputChannelMask), - bufferFrameCount) // set bufferFrameCount to 0 to do in-place -{ - ALOGV("DownmixerBufferProvider(%p)(%#x, %#x, %#x %u %d)", - this, inputChannelMask, outputChannelMask, format, - sampleRate, sessionId); - if (!sIsMultichannelCapable - || EffectCreate(&sDwnmFxDesc.uuid, - sessionId, - SESSION_ID_INVALID_AND_IGNORED, - &mDownmixHandle) != 0) { - ALOGE("DownmixerBufferProvider() error creating downmixer effect"); - mDownmixHandle = NULL; - return; - } - // channel input configuration will be overridden per-track - mDownmixConfig.inputCfg.channels = inputChannelMask; // FIXME: Should be bits - mDownmixConfig.outputCfg.channels = outputChannelMask; // FIXME: should be bits - mDownmixConfig.inputCfg.format = format; - mDownmixConfig.outputCfg.format = format; - mDownmixConfig.inputCfg.samplingRate = sampleRate; - mDownmixConfig.outputCfg.samplingRate = sampleRate; - mDownmixConfig.inputCfg.accessMode = EFFECT_BUFFER_ACCESS_READ; - mDownmixConfig.outputCfg.accessMode = EFFECT_BUFFER_ACCESS_WRITE; - // input and output buffer provider, and frame count will not be used as the downmix effect - // process() function is called directly (see DownmixerBufferProvider::getNextBuffer()) - mDownmixConfig.inputCfg.mask = EFFECT_CONFIG_SMP_RATE | EFFECT_CONFIG_CHANNELS | - EFFECT_CONFIG_FORMAT | EFFECT_CONFIG_ACC_MODE; - mDownmixConfig.outputCfg.mask = mDownmixConfig.inputCfg.mask; - - int cmdStatus; - uint32_t replySize = sizeof(int); - - // Configure downmixer - status_t status = (*mDownmixHandle)->command(mDownmixHandle, - EFFECT_CMD_SET_CONFIG /*cmdCode*/, sizeof(effect_config_t) /*cmdSize*/, - &mDownmixConfig /*pCmdData*/, - &replySize, &cmdStatus /*pReplyData*/); - if (status != 0 || cmdStatus != 0) { - ALOGE("DownmixerBufferProvider() error %d cmdStatus %d while configuring downmixer", - status, cmdStatus); - EffectRelease(mDownmixHandle); - mDownmixHandle = NULL; - return; - } - - // Enable downmixer - replySize = sizeof(int); - status = (*mDownmixHandle)->command(mDownmixHandle, - EFFECT_CMD_ENABLE /*cmdCode*/, 0 /*cmdSize*/, NULL /*pCmdData*/, - &replySize, &cmdStatus /*pReplyData*/); - if (status != 0 || cmdStatus != 0) { - ALOGE("DownmixerBufferProvider() error %d cmdStatus %d while enabling downmixer", - status, cmdStatus); - EffectRelease(mDownmixHandle); - mDownmixHandle = NULL; - return; - } - - // Set downmix type - // parameter size rounded for padding on 32bit boundary - const int psizePadded = ((sizeof(downmix_params_t) - 1)/sizeof(int) + 1) * sizeof(int); - const int downmixParamSize = - sizeof(effect_param_t) + psizePadded + sizeof(downmix_type_t); - effect_param_t * const param = (effect_param_t *) malloc(downmixParamSize); - param->psize = sizeof(downmix_params_t); - const downmix_params_t downmixParam = DOWNMIX_PARAM_TYPE; - memcpy(param->data, &downmixParam, param->psize); - const downmix_type_t downmixType = DOWNMIX_TYPE_FOLD; - param->vsize = sizeof(downmix_type_t); - memcpy(param->data + psizePadded, &downmixType, param->vsize); - replySize = sizeof(int); - status = (*mDownmixHandle)->command(mDownmixHandle, - EFFECT_CMD_SET_PARAM /* cmdCode */, downmixParamSize /* cmdSize */, - param /*pCmdData*/, &replySize, &cmdStatus /*pReplyData*/); - free(param); - if (status != 0 || cmdStatus != 0) { - ALOGE("DownmixerBufferProvider() error %d cmdStatus %d while setting downmix type", - status, cmdStatus); - EffectRelease(mDownmixHandle); - mDownmixHandle = NULL; - return; - } - ALOGV("DownmixerBufferProvider() downmix type set to %d", (int) downmixType); -} - -AudioMixer::DownmixerBufferProvider::~DownmixerBufferProvider() -{ - ALOGV("~DownmixerBufferProvider (%p)", this); - EffectRelease(mDownmixHandle); - mDownmixHandle = NULL; -} - -void AudioMixer::DownmixerBufferProvider::copyFrames(void *dst, const void *src, size_t frames) -{ - mDownmixConfig.inputCfg.buffer.frameCount = frames; - mDownmixConfig.inputCfg.buffer.raw = const_cast<void *>(src); - mDownmixConfig.outputCfg.buffer.frameCount = frames; - mDownmixConfig.outputCfg.buffer.raw = dst; - // may be in-place if src == dst. - status_t res = (*mDownmixHandle)->process(mDownmixHandle, - &mDownmixConfig.inputCfg.buffer, &mDownmixConfig.outputCfg.buffer); - ALOGE_IF(res != OK, "DownmixBufferProvider error %d", res); -} - -/* call once in a pthread_once handler. */ -/*static*/ status_t AudioMixer::DownmixerBufferProvider::init() -{ - // find multichannel downmix effect if we have to play multichannel content - uint32_t numEffects = 0; - int ret = EffectQueryNumberEffects(&numEffects); - if (ret != 0) { - ALOGE("AudioMixer() error %d querying number of effects", ret); - return NO_INIT; - } - ALOGV("EffectQueryNumberEffects() numEffects=%d", numEffects); - - for (uint32_t i = 0 ; i < numEffects ; i++) { - if (EffectQueryEffect(i, &sDwnmFxDesc) == 0) { - ALOGV("effect %d is called %s", i, sDwnmFxDesc.name); - if (memcmp(&sDwnmFxDesc.type, EFFECT_UIID_DOWNMIX, sizeof(effect_uuid_t)) == 0) { - ALOGI("found effect \"%s\" from %s", - sDwnmFxDesc.name, sDwnmFxDesc.implementor); - sIsMultichannelCapable = true; - break; - } - } - } - ALOGW_IF(!sIsMultichannelCapable, "unable to find downmix effect"); - return NO_INIT; -} - -/*static*/ bool AudioMixer::DownmixerBufferProvider::sIsMultichannelCapable = false; -/*static*/ effect_descriptor_t AudioMixer::DownmixerBufferProvider::sDwnmFxDesc; - -AudioMixer::RemixBufferProvider::RemixBufferProvider(audio_channel_mask_t inputChannelMask, - audio_channel_mask_t outputChannelMask, audio_format_t format, - size_t bufferFrameCount) : - CopyBufferProvider( - audio_bytes_per_sample(format) - * audio_channel_count_from_out_mask(inputChannelMask), - audio_bytes_per_sample(format) - * audio_channel_count_from_out_mask(outputChannelMask), - bufferFrameCount), - mFormat(format), - mSampleSize(audio_bytes_per_sample(format)), - mInputChannels(audio_channel_count_from_out_mask(inputChannelMask)), - mOutputChannels(audio_channel_count_from_out_mask(outputChannelMask)) -{ - ALOGV("RemixBufferProvider(%p)(%#x, %#x, %#x) %zu %zu", - this, format, inputChannelMask, outputChannelMask, - mInputChannels, mOutputChannels); - - const audio_channel_representation_t inputRepresentation = - audio_channel_mask_get_representation(inputChannelMask); - const audio_channel_representation_t outputRepresentation = - audio_channel_mask_get_representation(outputChannelMask); - const uint32_t inputBits = audio_channel_mask_get_bits(inputChannelMask); - const uint32_t outputBits = audio_channel_mask_get_bits(outputChannelMask); - - switch (inputRepresentation) { - case AUDIO_CHANNEL_REPRESENTATION_POSITION: - switch (outputRepresentation) { - case AUDIO_CHANNEL_REPRESENTATION_POSITION: - memcpy_by_index_array_initialization(mIdxAry, ARRAY_SIZE(mIdxAry), - outputBits, inputBits); - return; - case AUDIO_CHANNEL_REPRESENTATION_INDEX: - // TODO: output channel index mask not currently allowed - // fall through - default: - break; - } - break; - case AUDIO_CHANNEL_REPRESENTATION_INDEX: - switch (outputRepresentation) { - case AUDIO_CHANNEL_REPRESENTATION_POSITION: - memcpy_by_index_array_initialization_src_index(mIdxAry, ARRAY_SIZE(mIdxAry), - outputBits, inputBits); - return; - case AUDIO_CHANNEL_REPRESENTATION_INDEX: - // TODO: output channel index mask not currently allowed - // fall through - default: - break; - } - break; - default: - break; - } - LOG_ALWAYS_FATAL("invalid channel mask conversion from %#x to %#x", - inputChannelMask, outputChannelMask); -} - -void AudioMixer::RemixBufferProvider::copyFrames(void *dst, const void *src, size_t frames) -{ - memcpy_by_index_array(dst, mOutputChannels, - src, mInputChannels, mIdxAry, mSampleSize, frames); -} - -AudioMixer::ReformatBufferProvider::ReformatBufferProvider(int32_t channels, - audio_format_t inputFormat, audio_format_t outputFormat, - size_t bufferFrameCount) : - CopyBufferProvider( - channels * audio_bytes_per_sample(inputFormat), - channels * audio_bytes_per_sample(outputFormat), - bufferFrameCount), - mChannels(channels), - mInputFormat(inputFormat), - mOutputFormat(outputFormat) -{ - ALOGV("ReformatBufferProvider(%p)(%d, %#x, %#x)", this, channels, inputFormat, outputFormat); -} - -void AudioMixer::ReformatBufferProvider::copyFrames(void *dst, const void *src, size_t frames) -{ - memcpy_by_audio_format(dst, mOutputFormat, src, mInputFormat, frames * mChannels); -} - // ---------------------------------------------------------------------------- // Ensure mConfiguredNames bitmask is initialized properly on all architectures. @@ -442,6 +123,7 @@ AudioMixer::AudioMixer(size_t frameCount, uint32_t sampleRate, uint32_t maxNumTr t->resampler = NULL; t->downmixerBufferProvider = NULL; t->mReformatBufferProvider = NULL; + t->mTimestretchBufferProvider = NULL; t++; } @@ -454,6 +136,7 @@ AudioMixer::~AudioMixer() delete t->resampler; delete t->downmixerBufferProvider; delete t->mReformatBufferProvider; + delete t->mTimestretchBufferProvider; t++; } delete [] mState.outputTemp; @@ -532,6 +215,7 @@ int AudioMixer::getTrackName(audio_channel_mask_t channelMask, t->mReformatBufferProvider = NULL; t->downmixerBufferProvider = NULL; t->mPostDownmixReformatBufferProvider = NULL; + t->mTimestretchBufferProvider = NULL; t->mMixerFormat = AUDIO_FORMAT_PCM_16_BIT; t->mFormat = format; t->mMixerInFormat = selectMixerInFormat(format); @@ -539,6 +223,8 @@ int AudioMixer::getTrackName(audio_channel_mask_t channelMask, t->mMixerChannelMask = audio_channel_mask_from_representation_and_bits( AUDIO_CHANNEL_REPRESENTATION_POSITION, AUDIO_CHANNEL_OUT_STEREO); t->mMixerChannelCount = audio_channel_count_from_out_mask(t->mMixerChannelMask); + t->mSpeed = AUDIO_TIMESTRETCH_SPEED_NORMAL; + t->mPitch = AUDIO_TIMESTRETCH_PITCH_NORMAL; // Check the downmixing (or upmixing) requirements. status_t status = t->prepareForDownmix(); if (status != OK) { @@ -731,6 +417,10 @@ void AudioMixer::track_t::reconfigureBufferProviders() mPostDownmixReformatBufferProvider->setBufferProvider(bufferProvider); bufferProvider = mPostDownmixReformatBufferProvider; } + if (mTimestretchBufferProvider) { + mTimestretchBufferProvider->setBufferProvider(bufferProvider); + bufferProvider = mTimestretchBufferProvider; + } } void AudioMixer::deleteTrackName(int name) @@ -751,7 +441,9 @@ void AudioMixer::deleteTrackName(int name) mState.tracks[name].unprepareForDownmix(); // delete the reformatter mState.tracks[name].unprepareForReformat(); - + // delete the timestretch provider + delete track.mTimestretchBufferProvider; + track.mTimestretchBufferProvider = NULL; mTrackNames &= ~(1<<name); } @@ -973,6 +665,26 @@ void AudioMixer::setParameter(int name, int target, int param, void *value) } } break; + case TIMESTRETCH: + switch (param) { + case PLAYBACK_RATE: { + const float speed = reinterpret_cast<float*>(value)[0]; + const float pitch = reinterpret_cast<float*>(value)[1]; + ALOG_ASSERT(AUDIO_TIMESTRETCH_SPEED_MIN <= speed + && speed <= AUDIO_TIMESTRETCH_SPEED_MAX, + "bad speed %f", speed); + ALOG_ASSERT(AUDIO_TIMESTRETCH_PITCH_MIN <= pitch + && pitch <= AUDIO_TIMESTRETCH_PITCH_MAX, + "bad pitch %f", pitch); + if (track.setPlaybackRate(speed, pitch)) { + ALOGV("setParameter(TIMESTRETCH, PLAYBACK_RATE, %f %f", speed, pitch); + // invalidateState(1 << name); + } + } break; + default: + LOG_ALWAYS_FATAL("setParameter timestretch: bad param %d", param); + } + break; default: LOG_ALWAYS_FATAL("setParameter: bad target %d", target); @@ -1018,6 +730,28 @@ bool AudioMixer::track_t::setResampler(uint32_t trackSampleRate, uint32_t devSam return false; } +bool AudioMixer::track_t::setPlaybackRate(float speed, float pitch) +{ + if (speed == mSpeed && pitch == mPitch) { + return false; + } + mSpeed = speed; + mPitch = pitch; + if (mTimestretchBufferProvider == NULL) { + // TODO: Remove MONO_HACK. Resampler sees #channels after the downmixer + // but if none exists, it is the channel count (1 for mono). + const int timestretchChannelCount = downmixerBufferProvider != NULL + ? mMixerChannelCount : channelCount; + mTimestretchBufferProvider = new TimestretchBufferProvider(timestretchChannelCount, + mMixerInFormat, sampleRate, speed, pitch); + reconfigureBufferProviders(); + } else { + reinterpret_cast<TimestretchBufferProvider*>(mTimestretchBufferProvider) + ->setPlaybackRate(speed, pitch); + } + return true; +} + /* Checks to see if the volume ramp has completed and clears the increment * variables appropriately. * @@ -1096,6 +830,8 @@ void AudioMixer::setBufferProvider(int name, AudioBufferProvider* bufferProvider mState.tracks[name].downmixerBufferProvider->reset(); } else if (mState.tracks[name].mPostDownmixReformatBufferProvider != NULL) { mState.tracks[name].mPostDownmixReformatBufferProvider->reset(); + } else if (mState.tracks[name].mTimestretchBufferProvider != NULL) { + mState.tracks[name].mTimestretchBufferProvider->reset(); } mState.tracks[name].mInputBufferProvider = bufferProvider; diff --git a/services/audioflinger/AudioMixer.h b/services/audioflinger/AudioMixer.h index 381036b..e27a0d1 100644 --- a/services/audioflinger/AudioMixer.h +++ b/services/audioflinger/AudioMixer.h @@ -29,6 +29,7 @@ #include <utils/threads.h> #include "AudioResampler.h" +#include "BufferProviders.h" // FIXME This is actually unity gain, which might not be max in future, expressed in U.12 #define MAX_GAIN_INT AudioMixer::UNITY_GAIN_INT @@ -72,6 +73,7 @@ public: RESAMPLE = 0x3001, RAMP_VOLUME = 0x3002, // ramp to new volume VOLUME = 0x3003, // don't ramp + TIMESTRETCH = 0x3004, // set Parameter names // for target TRACK @@ -99,6 +101,9 @@ public: VOLUME0 = 0x4200, VOLUME1 = 0x4201, AUXLEVEL = 0x4210, + // for target TIMESTRETCH + PLAYBACK_RATE = 0x4300, // Configure timestretch on this track name; + // parameter 'value' is a pointer to the new playback rate. }; @@ -159,7 +164,6 @@ private: struct state_t; struct track_t; - class CopyBufferProvider; typedef void (*hook_t)(track_t* t, int32_t* output, size_t numOutFrames, int32_t* temp, int32_t* aux); @@ -214,6 +218,9 @@ private: /* Buffer providers are constructed to translate the track input data as needed. * + * TODO: perhaps make a single PlaybackConverterProvider class to move + * all pre-mixer track buffer conversions outside the AudioMixer class. + * * 1) mInputBufferProvider: The AudioTrack buffer provider. * 2) mReformatBufferProvider: If not NULL, performs the audio reformat to * match either mMixerInFormat or mDownmixRequiresFormat, if the downmixer @@ -223,13 +230,14 @@ private: * the number of channels required by the mixer sink. * 4) mPostDownmixReformatBufferProvider: If not NULL, performs reformatting from * the downmixer requirements to the mixer engine input requirements. + * 5) mTimestretchBufferProvider: Adds timestretching for playback rate */ AudioBufferProvider* mInputBufferProvider; // externally provided buffer provider. - CopyBufferProvider* mReformatBufferProvider; // provider wrapper for reformatting. - CopyBufferProvider* downmixerBufferProvider; // wrapper for channel conversion. - CopyBufferProvider* mPostDownmixReformatBufferProvider; + PassthruBufferProvider* mReformatBufferProvider; // provider wrapper for reformatting. + PassthruBufferProvider* downmixerBufferProvider; // wrapper for channel conversion. + PassthruBufferProvider* mPostDownmixReformatBufferProvider; + PassthruBufferProvider* mTimestretchBufferProvider; - // 16-byte boundary int32_t sessionId; audio_format_t mMixerFormat; // output mix format: AUDIO_FORMAT_PCM_(FLOAT|16_BIT) @@ -251,6 +259,9 @@ private: audio_channel_mask_t mMixerChannelMask; uint32_t mMixerChannelCount; + float mSpeed; + float mPitch; + bool needsRamp() { return (volumeInc[0] | volumeInc[1] | auxInc) != 0; } bool setResampler(uint32_t trackSampleRate, uint32_t devSampleRate); bool doesResample() const { return resampler != NULL; } @@ -263,6 +274,7 @@ private: void unprepareForDownmix(); status_t prepareForReformat(); void unprepareForReformat(); + bool setPlaybackRate(float speed, float pitch); void reconfigureBufferProviders(); }; @@ -282,112 +294,6 @@ private: track_t tracks[MAX_NUM_TRACKS] __attribute__((aligned(32))); }; - // Base AudioBufferProvider class used for DownMixerBufferProvider, RemixBufferProvider, - // and ReformatBufferProvider. - // It handles a private buffer for use in converting format or channel masks from the - // input data to a form acceptable by the mixer. - // TODO: Make a ResamplerBufferProvider when integers are entirely removed from the - // processing pipeline. - class CopyBufferProvider : public AudioBufferProvider { - public: - // Use a private buffer of bufferFrameCount frames (each frame is outputFrameSize bytes). - // If bufferFrameCount is 0, no private buffer is created and in-place modification of - // the upstream buffer provider's buffers is performed by copyFrames(). - CopyBufferProvider(size_t inputFrameSize, size_t outputFrameSize, - size_t bufferFrameCount); - virtual ~CopyBufferProvider(); - - // Overrides AudioBufferProvider methods - virtual status_t getNextBuffer(Buffer* buffer, int64_t pts); - virtual void releaseBuffer(Buffer* buffer); - - // Other public methods - - // call this to release the buffer to the upstream provider. - // treat it as an audio discontinuity for future samples. - virtual void reset(); - - // this function should be supplied by the derived class. It converts - // #frames in the *src pointer to the *dst pointer. It is public because - // some providers will allow this to work on arbitrary buffers outside - // of the internal buffers. - virtual void copyFrames(void *dst, const void *src, size_t frames) = 0; - - // set the upstream buffer provider. Consider calling "reset" before this function. - void setBufferProvider(AudioBufferProvider *p) { - mTrackBufferProvider = p; - } - - protected: - AudioBufferProvider* mTrackBufferProvider; - const size_t mInputFrameSize; - const size_t mOutputFrameSize; - private: - AudioBufferProvider::Buffer mBuffer; - const size_t mLocalBufferFrameCount; - void* mLocalBufferData; - size_t mConsumed; - }; - - // DownmixerBufferProvider wraps a track AudioBufferProvider to provide - // position dependent downmixing by an Audio Effect. - class DownmixerBufferProvider : public CopyBufferProvider { - public: - DownmixerBufferProvider(audio_channel_mask_t inputChannelMask, - audio_channel_mask_t outputChannelMask, audio_format_t format, - uint32_t sampleRate, int32_t sessionId, size_t bufferFrameCount); - virtual ~DownmixerBufferProvider(); - virtual void copyFrames(void *dst, const void *src, size_t frames); - bool isValid() const { return mDownmixHandle != NULL; } - - static status_t init(); - static bool isMultichannelCapable() { return sIsMultichannelCapable; } - - protected: - effect_handle_t mDownmixHandle; - effect_config_t mDownmixConfig; - - // effect descriptor for the downmixer used by the mixer - static effect_descriptor_t sDwnmFxDesc; - // indicates whether a downmix effect has been found and is usable by this mixer - static bool sIsMultichannelCapable; - // FIXME: should we allow effects outside of the framework? - // We need to here. A special ioId that must be <= -2 so it does not map to a session. - static const int32_t SESSION_ID_INVALID_AND_IGNORED = -2; - }; - - // RemixBufferProvider wraps a track AudioBufferProvider to perform an - // upmix or downmix to the proper channel count and mask. - class RemixBufferProvider : public CopyBufferProvider { - public: - RemixBufferProvider(audio_channel_mask_t inputChannelMask, - audio_channel_mask_t outputChannelMask, audio_format_t format, - size_t bufferFrameCount); - virtual void copyFrames(void *dst, const void *src, size_t frames); - - protected: - const audio_format_t mFormat; - const size_t mSampleSize; - const size_t mInputChannels; - const size_t mOutputChannels; - int8_t mIdxAry[sizeof(uint32_t)*8]; // 32 bits => channel indices - }; - - // ReformatBufferProvider wraps a track AudioBufferProvider to convert the input data - // to an acceptable mixer input format type. - class ReformatBufferProvider : public CopyBufferProvider { - public: - ReformatBufferProvider(int32_t channels, - audio_format_t inputFormat, audio_format_t outputFormat, - size_t bufferFrameCount); - virtual void copyFrames(void *dst, const void *src, size_t frames); - - protected: - const int32_t mChannels; - const audio_format_t mInputFormat; - const audio_format_t mOutputFormat; - }; - // bitmask of allocated track names, where bit 0 corresponds to TRACK0 etc. uint32_t mTrackNames; diff --git a/services/audioflinger/BufferProviders.cpp b/services/audioflinger/BufferProviders.cpp new file mode 100644 index 0000000..dcae5e7 --- /dev/null +++ b/services/audioflinger/BufferProviders.cpp @@ -0,0 +1,540 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "BufferProvider" +//#define LOG_NDEBUG 0 + +#include <audio_effects/effect_downmix.h> +#include <audio_utils/primitives.h> +#include <audio_utils/format.h> +#include <media/AudioResamplerPublic.h> +#include <media/EffectsFactoryApi.h> + +#include <utils/Log.h> + +#include "Configuration.h" +#include "BufferProviders.h" + +#ifndef ARRAY_SIZE +#define ARRAY_SIZE(x) (sizeof(x)/sizeof((x)[0])) +#endif + +namespace android { + +// ---------------------------------------------------------------------------- + +template <typename T> +static inline T min(const T& a, const T& b) +{ + return a < b ? a : b; +} + +CopyBufferProvider::CopyBufferProvider(size_t inputFrameSize, + size_t outputFrameSize, size_t bufferFrameCount) : + mInputFrameSize(inputFrameSize), + mOutputFrameSize(outputFrameSize), + mLocalBufferFrameCount(bufferFrameCount), + mLocalBufferData(NULL), + mConsumed(0) +{ + ALOGV("CopyBufferProvider(%p)(%zu, %zu, %zu)", this, + inputFrameSize, outputFrameSize, bufferFrameCount); + LOG_ALWAYS_FATAL_IF(inputFrameSize < outputFrameSize && bufferFrameCount == 0, + "Requires local buffer if inputFrameSize(%zu) < outputFrameSize(%zu)", + inputFrameSize, outputFrameSize); + if (mLocalBufferFrameCount) { + (void)posix_memalign(&mLocalBufferData, 32, mLocalBufferFrameCount * mOutputFrameSize); + } + mBuffer.frameCount = 0; +} + +CopyBufferProvider::~CopyBufferProvider() +{ + ALOGV("~CopyBufferProvider(%p)", this); + if (mBuffer.frameCount != 0) { + mTrackBufferProvider->releaseBuffer(&mBuffer); + } + free(mLocalBufferData); +} + +status_t CopyBufferProvider::getNextBuffer(AudioBufferProvider::Buffer *pBuffer, + int64_t pts) +{ + //ALOGV("CopyBufferProvider(%p)::getNextBuffer(%p (%zu), %lld)", + // this, pBuffer, pBuffer->frameCount, pts); + if (mLocalBufferFrameCount == 0) { + status_t res = mTrackBufferProvider->getNextBuffer(pBuffer, pts); + if (res == OK) { + copyFrames(pBuffer->raw, pBuffer->raw, pBuffer->frameCount); + } + return res; + } + if (mBuffer.frameCount == 0) { + mBuffer.frameCount = pBuffer->frameCount; + status_t res = mTrackBufferProvider->getNextBuffer(&mBuffer, pts); + // At one time an upstream buffer provider had + // res == OK and mBuffer.frameCount == 0, doesn't seem to happen now 7/18/2014. + // + // By API spec, if res != OK, then mBuffer.frameCount == 0. + // but there may be improper implementations. + ALOG_ASSERT(res == OK || mBuffer.frameCount == 0); + if (res != OK || mBuffer.frameCount == 0) { // not needed by API spec, but to be safe. + pBuffer->raw = NULL; + pBuffer->frameCount = 0; + return res; + } + mConsumed = 0; + } + ALOG_ASSERT(mConsumed < mBuffer.frameCount); + size_t count = min(mLocalBufferFrameCount, mBuffer.frameCount - mConsumed); + count = min(count, pBuffer->frameCount); + pBuffer->raw = mLocalBufferData; + pBuffer->frameCount = count; + copyFrames(pBuffer->raw, (uint8_t*)mBuffer.raw + mConsumed * mInputFrameSize, + pBuffer->frameCount); + return OK; +} + +void CopyBufferProvider::releaseBuffer(AudioBufferProvider::Buffer *pBuffer) +{ + //ALOGV("CopyBufferProvider(%p)::releaseBuffer(%p(%zu))", + // this, pBuffer, pBuffer->frameCount); + if (mLocalBufferFrameCount == 0) { + mTrackBufferProvider->releaseBuffer(pBuffer); + return; + } + // LOG_ALWAYS_FATAL_IF(pBuffer->frameCount == 0, "Invalid framecount"); + mConsumed += pBuffer->frameCount; // TODO: update for efficiency to reuse existing content + if (mConsumed != 0 && mConsumed >= mBuffer.frameCount) { + mTrackBufferProvider->releaseBuffer(&mBuffer); + ALOG_ASSERT(mBuffer.frameCount == 0); + } + pBuffer->raw = NULL; + pBuffer->frameCount = 0; +} + +void CopyBufferProvider::reset() +{ + if (mBuffer.frameCount != 0) { + mTrackBufferProvider->releaseBuffer(&mBuffer); + } + mConsumed = 0; +} + +DownmixerBufferProvider::DownmixerBufferProvider( + audio_channel_mask_t inputChannelMask, + audio_channel_mask_t outputChannelMask, audio_format_t format, + uint32_t sampleRate, int32_t sessionId, size_t bufferFrameCount) : + CopyBufferProvider( + audio_bytes_per_sample(format) * audio_channel_count_from_out_mask(inputChannelMask), + audio_bytes_per_sample(format) * audio_channel_count_from_out_mask(outputChannelMask), + bufferFrameCount) // set bufferFrameCount to 0 to do in-place +{ + ALOGV("DownmixerBufferProvider(%p)(%#x, %#x, %#x %u %d)", + this, inputChannelMask, outputChannelMask, format, + sampleRate, sessionId); + if (!sIsMultichannelCapable + || EffectCreate(&sDwnmFxDesc.uuid, + sessionId, + SESSION_ID_INVALID_AND_IGNORED, + &mDownmixHandle) != 0) { + ALOGE("DownmixerBufferProvider() error creating downmixer effect"); + mDownmixHandle = NULL; + return; + } + // channel input configuration will be overridden per-track + mDownmixConfig.inputCfg.channels = inputChannelMask; // FIXME: Should be bits + mDownmixConfig.outputCfg.channels = outputChannelMask; // FIXME: should be bits + mDownmixConfig.inputCfg.format = format; + mDownmixConfig.outputCfg.format = format; + mDownmixConfig.inputCfg.samplingRate = sampleRate; + mDownmixConfig.outputCfg.samplingRate = sampleRate; + mDownmixConfig.inputCfg.accessMode = EFFECT_BUFFER_ACCESS_READ; + mDownmixConfig.outputCfg.accessMode = EFFECT_BUFFER_ACCESS_WRITE; + // input and output buffer provider, and frame count will not be used as the downmix effect + // process() function is called directly (see DownmixerBufferProvider::getNextBuffer()) + mDownmixConfig.inputCfg.mask = EFFECT_CONFIG_SMP_RATE | EFFECT_CONFIG_CHANNELS | + EFFECT_CONFIG_FORMAT | EFFECT_CONFIG_ACC_MODE; + mDownmixConfig.outputCfg.mask = mDownmixConfig.inputCfg.mask; + + int cmdStatus; + uint32_t replySize = sizeof(int); + + // Configure downmixer + status_t status = (*mDownmixHandle)->command(mDownmixHandle, + EFFECT_CMD_SET_CONFIG /*cmdCode*/, sizeof(effect_config_t) /*cmdSize*/, + &mDownmixConfig /*pCmdData*/, + &replySize, &cmdStatus /*pReplyData*/); + if (status != 0 || cmdStatus != 0) { + ALOGE("DownmixerBufferProvider() error %d cmdStatus %d while configuring downmixer", + status, cmdStatus); + EffectRelease(mDownmixHandle); + mDownmixHandle = NULL; + return; + } + + // Enable downmixer + replySize = sizeof(int); + status = (*mDownmixHandle)->command(mDownmixHandle, + EFFECT_CMD_ENABLE /*cmdCode*/, 0 /*cmdSize*/, NULL /*pCmdData*/, + &replySize, &cmdStatus /*pReplyData*/); + if (status != 0 || cmdStatus != 0) { + ALOGE("DownmixerBufferProvider() error %d cmdStatus %d while enabling downmixer", + status, cmdStatus); + EffectRelease(mDownmixHandle); + mDownmixHandle = NULL; + return; + } + + // Set downmix type + // parameter size rounded for padding on 32bit boundary + const int psizePadded = ((sizeof(downmix_params_t) - 1)/sizeof(int) + 1) * sizeof(int); + const int downmixParamSize = + sizeof(effect_param_t) + psizePadded + sizeof(downmix_type_t); + effect_param_t * const param = (effect_param_t *) malloc(downmixParamSize); + param->psize = sizeof(downmix_params_t); + const downmix_params_t downmixParam = DOWNMIX_PARAM_TYPE; + memcpy(param->data, &downmixParam, param->psize); + const downmix_type_t downmixType = DOWNMIX_TYPE_FOLD; + param->vsize = sizeof(downmix_type_t); + memcpy(param->data + psizePadded, &downmixType, param->vsize); + replySize = sizeof(int); + status = (*mDownmixHandle)->command(mDownmixHandle, + EFFECT_CMD_SET_PARAM /* cmdCode */, downmixParamSize /* cmdSize */, + param /*pCmdData*/, &replySize, &cmdStatus /*pReplyData*/); + free(param); + if (status != 0 || cmdStatus != 0) { + ALOGE("DownmixerBufferProvider() error %d cmdStatus %d while setting downmix type", + status, cmdStatus); + EffectRelease(mDownmixHandle); + mDownmixHandle = NULL; + return; + } + ALOGV("DownmixerBufferProvider() downmix type set to %d", (int) downmixType); +} + +DownmixerBufferProvider::~DownmixerBufferProvider() +{ + ALOGV("~DownmixerBufferProvider (%p)", this); + EffectRelease(mDownmixHandle); + mDownmixHandle = NULL; +} + +void DownmixerBufferProvider::copyFrames(void *dst, const void *src, size_t frames) +{ + mDownmixConfig.inputCfg.buffer.frameCount = frames; + mDownmixConfig.inputCfg.buffer.raw = const_cast<void *>(src); + mDownmixConfig.outputCfg.buffer.frameCount = frames; + mDownmixConfig.outputCfg.buffer.raw = dst; + // may be in-place if src == dst. + status_t res = (*mDownmixHandle)->process(mDownmixHandle, + &mDownmixConfig.inputCfg.buffer, &mDownmixConfig.outputCfg.buffer); + ALOGE_IF(res != OK, "DownmixBufferProvider error %d", res); +} + +/* call once in a pthread_once handler. */ +/*static*/ status_t DownmixerBufferProvider::init() +{ + // find multichannel downmix effect if we have to play multichannel content + uint32_t numEffects = 0; + int ret = EffectQueryNumberEffects(&numEffects); + if (ret != 0) { + ALOGE("AudioMixer() error %d querying number of effects", ret); + return NO_INIT; + } + ALOGV("EffectQueryNumberEffects() numEffects=%d", numEffects); + + for (uint32_t i = 0 ; i < numEffects ; i++) { + if (EffectQueryEffect(i, &sDwnmFxDesc) == 0) { + ALOGV("effect %d is called %s", i, sDwnmFxDesc.name); + if (memcmp(&sDwnmFxDesc.type, EFFECT_UIID_DOWNMIX, sizeof(effect_uuid_t)) == 0) { + ALOGI("found effect \"%s\" from %s", + sDwnmFxDesc.name, sDwnmFxDesc.implementor); + sIsMultichannelCapable = true; + break; + } + } + } + ALOGW_IF(!sIsMultichannelCapable, "unable to find downmix effect"); + return NO_INIT; +} + +/*static*/ bool DownmixerBufferProvider::sIsMultichannelCapable = false; +/*static*/ effect_descriptor_t DownmixerBufferProvider::sDwnmFxDesc; + +RemixBufferProvider::RemixBufferProvider(audio_channel_mask_t inputChannelMask, + audio_channel_mask_t outputChannelMask, audio_format_t format, + size_t bufferFrameCount) : + CopyBufferProvider( + audio_bytes_per_sample(format) + * audio_channel_count_from_out_mask(inputChannelMask), + audio_bytes_per_sample(format) + * audio_channel_count_from_out_mask(outputChannelMask), + bufferFrameCount), + mFormat(format), + mSampleSize(audio_bytes_per_sample(format)), + mInputChannels(audio_channel_count_from_out_mask(inputChannelMask)), + mOutputChannels(audio_channel_count_from_out_mask(outputChannelMask)) +{ + ALOGV("RemixBufferProvider(%p)(%#x, %#x, %#x) %zu %zu", + this, format, inputChannelMask, outputChannelMask, + mInputChannels, mOutputChannels); + + const audio_channel_representation_t inputRepresentation = + audio_channel_mask_get_representation(inputChannelMask); + const audio_channel_representation_t outputRepresentation = + audio_channel_mask_get_representation(outputChannelMask); + const uint32_t inputBits = audio_channel_mask_get_bits(inputChannelMask); + const uint32_t outputBits = audio_channel_mask_get_bits(outputChannelMask); + + switch (inputRepresentation) { + case AUDIO_CHANNEL_REPRESENTATION_POSITION: + switch (outputRepresentation) { + case AUDIO_CHANNEL_REPRESENTATION_POSITION: + memcpy_by_index_array_initialization(mIdxAry, ARRAY_SIZE(mIdxAry), + outputBits, inputBits); + return; + case AUDIO_CHANNEL_REPRESENTATION_INDEX: + // TODO: output channel index mask not currently allowed + // fall through + default: + break; + } + break; + case AUDIO_CHANNEL_REPRESENTATION_INDEX: + switch (outputRepresentation) { + case AUDIO_CHANNEL_REPRESENTATION_POSITION: + memcpy_by_index_array_initialization_src_index(mIdxAry, ARRAY_SIZE(mIdxAry), + outputBits, inputBits); + return; + case AUDIO_CHANNEL_REPRESENTATION_INDEX: + // TODO: output channel index mask not currently allowed + // fall through + default: + break; + } + break; + default: + break; + } + LOG_ALWAYS_FATAL("invalid channel mask conversion from %#x to %#x", + inputChannelMask, outputChannelMask); +} + +void RemixBufferProvider::copyFrames(void *dst, const void *src, size_t frames) +{ + memcpy_by_index_array(dst, mOutputChannels, + src, mInputChannels, mIdxAry, mSampleSize, frames); +} + +ReformatBufferProvider::ReformatBufferProvider(int32_t channelCount, + audio_format_t inputFormat, audio_format_t outputFormat, + size_t bufferFrameCount) : + CopyBufferProvider( + channelCount * audio_bytes_per_sample(inputFormat), + channelCount * audio_bytes_per_sample(outputFormat), + bufferFrameCount), + mChannelCount(channelCount), + mInputFormat(inputFormat), + mOutputFormat(outputFormat) +{ + ALOGV("ReformatBufferProvider(%p)(%u, %#x, %#x)", + this, channelCount, inputFormat, outputFormat); +} + +void ReformatBufferProvider::copyFrames(void *dst, const void *src, size_t frames) +{ + memcpy_by_audio_format(dst, mOutputFormat, src, mInputFormat, frames * mChannelCount); +} + +TimestretchBufferProvider::TimestretchBufferProvider(int32_t channelCount, + audio_format_t format, uint32_t sampleRate, float speed, float pitch) : + mChannelCount(channelCount), + mFormat(format), + mSampleRate(sampleRate), + mFrameSize(channelCount * audio_bytes_per_sample(format)), + mSpeed(speed), + mPitch(pitch), + mLocalBufferFrameCount(0), + mLocalBufferData(NULL), + mRemaining(0), + mSonicStream(sonicCreateStream(sampleRate, mChannelCount)) +{ + ALOGV("TimestretchBufferProvider(%p)(%u, %#x, %u %f %f)", + this, channelCount, format, sampleRate, speed, pitch); + mBuffer.frameCount = 0; + + LOG_ALWAYS_FATAL_IF(mSonicStream == NULL, + "TimestretchBufferProvider can't allocate Sonic stream"); + sonicSetSpeed(mSonicStream, speed); +} + +TimestretchBufferProvider::~TimestretchBufferProvider() +{ + ALOGV("~TimestretchBufferProvider(%p)", this); + sonicDestroyStream(mSonicStream); + if (mBuffer.frameCount != 0) { + mTrackBufferProvider->releaseBuffer(&mBuffer); + } + free(mLocalBufferData); +} + +status_t TimestretchBufferProvider::getNextBuffer( + AudioBufferProvider::Buffer *pBuffer, int64_t pts) +{ + ALOGV("TimestretchBufferProvider(%p)::getNextBuffer(%p (%zu), %lld)", + this, pBuffer, pBuffer->frameCount, pts); + + // BYPASS + //return mTrackBufferProvider->getNextBuffer(pBuffer, pts); + + // check if previously processed data is sufficient. + if (pBuffer->frameCount <= mRemaining) { + ALOGV("previous sufficient"); + pBuffer->raw = mLocalBufferData; + return OK; + } + + // do we need to resize our buffer? + if (pBuffer->frameCount > mLocalBufferFrameCount) { + void *newmem; + if (posix_memalign(&newmem, 32, pBuffer->frameCount * mFrameSize) == OK) { + if (mRemaining != 0) { + memcpy(newmem, mLocalBufferData, mRemaining * mFrameSize); + } + free(mLocalBufferData); + mLocalBufferData = newmem; + mLocalBufferFrameCount = pBuffer->frameCount; + } + } + + // need to fetch more data + const size_t outputDesired = pBuffer->frameCount - mRemaining; + mBuffer.frameCount = mSpeed == AUDIO_TIMESTRETCH_SPEED_NORMAL + ? outputDesired : outputDesired * mSpeed + 1; + + status_t res = mTrackBufferProvider->getNextBuffer(&mBuffer, pts); + + ALOG_ASSERT(res == OK || mBuffer.frameCount == 0); + if (res != OK || mBuffer.frameCount == 0) { // not needed by API spec, but to be safe. + ALOGD("buffer error"); + if (mRemaining == 0) { + pBuffer->raw = NULL; + pBuffer->frameCount = 0; + return res; + } else { // return partial count + pBuffer->raw = mLocalBufferData; + pBuffer->frameCount = mRemaining; + return OK; + } + } + + // time-stretch the data + size_t dstAvailable = min(mLocalBufferFrameCount - mRemaining, outputDesired); + size_t srcAvailable = mBuffer.frameCount; + processFrames((uint8_t*)mLocalBufferData + mRemaining * mFrameSize, &dstAvailable, + mBuffer.raw, &srcAvailable); + + // release all data consumed + mBuffer.frameCount = srcAvailable; + mTrackBufferProvider->releaseBuffer(&mBuffer); + + // update buffer vars with the actual data processed and return with buffer + mRemaining += dstAvailable; + + pBuffer->raw = mLocalBufferData; + pBuffer->frameCount = mRemaining; + + return OK; +} + +void TimestretchBufferProvider::releaseBuffer(AudioBufferProvider::Buffer *pBuffer) +{ + ALOGV("TimestretchBufferProvider(%p)::releaseBuffer(%p (%zu))", + this, pBuffer, pBuffer->frameCount); + + // BYPASS + //return mTrackBufferProvider->releaseBuffer(pBuffer); + + // LOG_ALWAYS_FATAL_IF(pBuffer->frameCount == 0, "Invalid framecount"); + if (pBuffer->frameCount < mRemaining) { + memcpy(mLocalBufferData, + (uint8_t*)mLocalBufferData + pBuffer->frameCount * mFrameSize, + (mRemaining - pBuffer->frameCount) * mFrameSize); + mRemaining -= pBuffer->frameCount; + } else if (pBuffer->frameCount == mRemaining) { + mRemaining = 0; + } else { + LOG_ALWAYS_FATAL("Releasing more frames(%zu) than available(%zu)", + pBuffer->frameCount, mRemaining); + } + + pBuffer->raw = NULL; + pBuffer->frameCount = 0; +} + +void TimestretchBufferProvider::reset() +{ + mRemaining = 0; +} + +status_t TimestretchBufferProvider::setPlaybackRate(float speed, float pitch) +{ + mSpeed = speed; + mPitch = pitch; + + sonicSetSpeed(mSonicStream, speed); + //TODO: pitch is ignored for now + return OK; +} + +void TimestretchBufferProvider::processFrames(void *dstBuffer, size_t *dstFrames, + const void *srcBuffer, size_t *srcFrames) +{ + ALOGV("processFrames(%zu %zu) remaining(%zu)", *dstFrames, *srcFrames, mRemaining); + // Note dstFrames is the required number of frames. + + // Ensure consumption from src is as expected. + const size_t targetSrc = *dstFrames * mSpeed; + if (*srcFrames < targetSrc) { // limit dst frames to that possible + *dstFrames = *srcFrames / mSpeed; + } else if (*srcFrames > targetSrc + 1) { + *srcFrames = targetSrc + 1; + } + + switch (mFormat) { + case AUDIO_FORMAT_PCM_FLOAT: + if (sonicWriteFloatToStream(mSonicStream, (float*)srcBuffer, *srcFrames) != 1) { + ALOGE("sonicWriteFloatToStream cannot realloc"); + *srcFrames = 0; // cannot consume all of srcBuffer + } + *dstFrames = sonicReadFloatFromStream(mSonicStream, (float*)dstBuffer, *dstFrames); + break; + case AUDIO_FORMAT_PCM_16_BIT: + if (sonicWriteShortToStream(mSonicStream, (short*)srcBuffer, *srcFrames) != 1) { + ALOGE("sonicWriteShortToStream cannot realloc"); + *srcFrames = 0; // cannot consume all of srcBuffer + } + *dstFrames = sonicReadShortFromStream(mSonicStream, (short*)dstBuffer, *dstFrames); + break; + default: + // could also be caught on construction + LOG_ALWAYS_FATAL("invalid format %#x for TimestretchBufferProvider", mFormat); + } +} + +// ---------------------------------------------------------------------------- +} // namespace android diff --git a/services/audioflinger/BufferProviders.h b/services/audioflinger/BufferProviders.h new file mode 100644 index 0000000..42030c0 --- /dev/null +++ b/services/audioflinger/BufferProviders.h @@ -0,0 +1,193 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_BUFFER_PROVIDERS_H +#define ANDROID_BUFFER_PROVIDERS_H + +#include <stdint.h> +#include <sys/types.h> + +#include <hardware/audio_effect.h> +#include <media/AudioBufferProvider.h> +#include <system/audio.h> +#include <sonic.h> + +namespace android { + +// ---------------------------------------------------------------------------- + +class PassthruBufferProvider : public AudioBufferProvider { +public: + PassthruBufferProvider() : mTrackBufferProvider(NULL) { } + + virtual ~PassthruBufferProvider() { } + + // call this to release the buffer to the upstream provider. + // treat it as an audio discontinuity for future samples. + virtual void reset() { } + + // set the upstream buffer provider. Consider calling "reset" before this function. + virtual void setBufferProvider(AudioBufferProvider *p) { + mTrackBufferProvider = p; + } + +protected: + AudioBufferProvider *mTrackBufferProvider; +}; + +// Base AudioBufferProvider class used for DownMixerBufferProvider, RemixBufferProvider, +// and ReformatBufferProvider. +// It handles a private buffer for use in converting format or channel masks from the +// input data to a form acceptable by the mixer. +// TODO: Make a ResamplerBufferProvider when integers are entirely removed from the +// processing pipeline. +class CopyBufferProvider : public PassthruBufferProvider { +public: + // Use a private buffer of bufferFrameCount frames (each frame is outputFrameSize bytes). + // If bufferFrameCount is 0, no private buffer is created and in-place modification of + // the upstream buffer provider's buffers is performed by copyFrames(). + CopyBufferProvider(size_t inputFrameSize, size_t outputFrameSize, + size_t bufferFrameCount); + virtual ~CopyBufferProvider(); + + // Overrides AudioBufferProvider methods + virtual status_t getNextBuffer(Buffer *buffer, int64_t pts); + virtual void releaseBuffer(Buffer *buffer); + + // Overrides PassthruBufferProvider + virtual void reset(); + + // this function should be supplied by the derived class. It converts + // #frames in the *src pointer to the *dst pointer. It is public because + // some providers will allow this to work on arbitrary buffers outside + // of the internal buffers. + virtual void copyFrames(void *dst, const void *src, size_t frames) = 0; + +protected: + const size_t mInputFrameSize; + const size_t mOutputFrameSize; +private: + AudioBufferProvider::Buffer mBuffer; + const size_t mLocalBufferFrameCount; + void *mLocalBufferData; + size_t mConsumed; +}; + +// DownmixerBufferProvider derives from CopyBufferProvider to provide +// position dependent downmixing by an Audio Effect. +class DownmixerBufferProvider : public CopyBufferProvider { +public: + DownmixerBufferProvider(audio_channel_mask_t inputChannelMask, + audio_channel_mask_t outputChannelMask, audio_format_t format, + uint32_t sampleRate, int32_t sessionId, size_t bufferFrameCount); + virtual ~DownmixerBufferProvider(); + //Overrides + virtual void copyFrames(void *dst, const void *src, size_t frames); + + bool isValid() const { return mDownmixHandle != NULL; } + static status_t init(); + static bool isMultichannelCapable() { return sIsMultichannelCapable; } + +protected: + effect_handle_t mDownmixHandle; + effect_config_t mDownmixConfig; + + // effect descriptor for the downmixer used by the mixer + static effect_descriptor_t sDwnmFxDesc; + // indicates whether a downmix effect has been found and is usable by this mixer + static bool sIsMultichannelCapable; + // FIXME: should we allow effects outside of the framework? + // We need to here. A special ioId that must be <= -2 so it does not map to a session. + static const int32_t SESSION_ID_INVALID_AND_IGNORED = -2; +}; + +// RemixBufferProvider derives from CopyBufferProvider to perform an +// upmix or downmix to the proper channel count and mask. +class RemixBufferProvider : public CopyBufferProvider { +public: + RemixBufferProvider(audio_channel_mask_t inputChannelMask, + audio_channel_mask_t outputChannelMask, audio_format_t format, + size_t bufferFrameCount); + //Overrides + virtual void copyFrames(void *dst, const void *src, size_t frames); + +protected: + const audio_format_t mFormat; + const size_t mSampleSize; + const size_t mInputChannels; + const size_t mOutputChannels; + int8_t mIdxAry[sizeof(uint32_t) * 8]; // 32 bits => channel indices +}; + +// ReformatBufferProvider derives from CopyBufferProvider to convert the input data +// to an acceptable mixer input format type. +class ReformatBufferProvider : public CopyBufferProvider { +public: + ReformatBufferProvider(int32_t channelCount, + audio_format_t inputFormat, audio_format_t outputFormat, + size_t bufferFrameCount); + virtual void copyFrames(void *dst, const void *src, size_t frames); + +protected: + const uint32_t mChannelCount; + const audio_format_t mInputFormat; + const audio_format_t mOutputFormat; +}; + +// TimestretchBufferProvider derives from PassthruBufferProvider for time stretching +class TimestretchBufferProvider : public PassthruBufferProvider { +public: + TimestretchBufferProvider(int32_t channelCount, + audio_format_t format, uint32_t sampleRate, float speed, float pitch); + virtual ~TimestretchBufferProvider(); + + // Overrides AudioBufferProvider methods + virtual status_t getNextBuffer(Buffer* buffer, int64_t pts); + virtual void releaseBuffer(Buffer* buffer); + + // Overrides PassthruBufferProvider + virtual void reset(); + + virtual status_t setPlaybackRate(float speed, float pitch); + + // processes frames + // dstBuffer is where to place the data + // dstFrames [in/out] is the desired frames (return with actual placed in buffer) + // srcBuffer is the source data + // srcFrames [in/out] is the available source frames (return with consumed) + virtual void processFrames(void *dstBuffer, size_t *dstFrames, + const void *srcBuffer, size_t *srcFrames); + +protected: + const uint32_t mChannelCount; + const audio_format_t mFormat; + const uint32_t mSampleRate; // const for now (TODO change this) + const size_t mFrameSize; + float mSpeed; + float mPitch; + +private: + AudioBufferProvider::Buffer mBuffer; + size_t mLocalBufferFrameCount; + void *mLocalBufferData; + size_t mRemaining; + sonicStream mSonicStream; +}; + +// ---------------------------------------------------------------------------- +} // namespace android + +#endif // ANDROID_BUFFER_PROVIDERS_H diff --git a/services/audioflinger/ServiceUtilities.cpp b/services/audioflinger/ServiceUtilities.cpp index fae19a1..8246fef 100644 --- a/services/audioflinger/ServiceUtilities.cpp +++ b/services/audioflinger/ServiceUtilities.cpp @@ -50,13 +50,6 @@ bool captureHotwordAllowed() { return ok; } -bool captureFmTunerAllowed() { - static const String16 sCaptureFmTunerAllowed("android.permission.ACCESS_FM_RADIO"); - bool ok = checkCallingPermission(sCaptureFmTunerAllowed); - if (!ok) ALOGE("android.permission.ACCESS_FM_RADIO"); - return ok; -} - bool settingsAllowed() { if (getpid_cached == IPCThreadState::self()->getCallingPid()) return true; static const String16 sAudioSettings("android.permission.MODIFY_AUDIO_SETTINGS"); diff --git a/services/audioflinger/ServiceUtilities.h b/services/audioflinger/ServiceUtilities.h index ce18a90..df6f6f4 100644 --- a/services/audioflinger/ServiceUtilities.h +++ b/services/audioflinger/ServiceUtilities.h @@ -23,7 +23,6 @@ extern pid_t getpid_cached; bool recordingAllowed(); bool captureAudioOutputAllowed(); bool captureHotwordAllowed(); -bool captureFmTunerAllowed(); bool settingsAllowed(); bool modifyAudioRoutingAllowed(); bool dumpAllowed(); diff --git a/services/audioflinger/SpdifStreamOut.cpp b/services/audioflinger/SpdifStreamOut.cpp index d23588e..45b541a 100644 --- a/services/audioflinger/SpdifStreamOut.cpp +++ b/services/audioflinger/SpdifStreamOut.cpp @@ -32,10 +32,12 @@ namespace android { * If the AudioFlinger is processing encoded data and the HAL expects * PCM then we need to wrap the data in an SPDIF wrapper. */ -SpdifStreamOut::SpdifStreamOut(AudioHwDevice *dev, audio_output_flags_t flags) +SpdifStreamOut::SpdifStreamOut(AudioHwDevice *dev, + audio_output_flags_t flags, + audio_format_t format) : AudioStreamOut(dev,flags) , mRateMultiplier(1) - , mSpdifEncoder(this) + , mSpdifEncoder(this, format) , mRenderPositionHal(0) , mPreviousHalPosition32(0) { @@ -49,15 +51,15 @@ status_t SpdifStreamOut::open( { struct audio_config customConfig = *config; - customConfig.format = AUDIO_FORMAT_PCM_16_BIT; - customConfig.channel_mask = AUDIO_CHANNEL_OUT_STEREO; - // Some data bursts run at a higher sample rate. + // TODO Move this into the audio_utils as a static method. switch(config->format) { case AUDIO_FORMAT_E_AC3: mRateMultiplier = 4; break; case AUDIO_FORMAT_AC3: + case AUDIO_FORMAT_DTS: + case AUDIO_FORMAT_DTS_HD: mRateMultiplier = 1; break; default: @@ -67,6 +69,9 @@ status_t SpdifStreamOut::open( } customConfig.sample_rate = config->sample_rate * mRateMultiplier; + customConfig.format = AUDIO_FORMAT_PCM_16_BIT; + customConfig.channel_mask = AUDIO_CHANNEL_OUT_STEREO; + // Always print this because otherwise it could be very confusing if the // HAL and AudioFlinger are using different formats. // Print before open() because HAL may modify customConfig. diff --git a/services/audioflinger/SpdifStreamOut.h b/services/audioflinger/SpdifStreamOut.h index cb82ac7..d81c064 100644 --- a/services/audioflinger/SpdifStreamOut.h +++ b/services/audioflinger/SpdifStreamOut.h @@ -38,7 +38,8 @@ namespace android { class SpdifStreamOut : public AudioStreamOut { public: - SpdifStreamOut(AudioHwDevice *dev, audio_output_flags_t flags); + SpdifStreamOut(AudioHwDevice *dev, audio_output_flags_t flags, + audio_format_t format); virtual ~SpdifStreamOut() { } @@ -77,8 +78,9 @@ private: class MySPDIFEncoder : public SPDIFEncoder { public: - MySPDIFEncoder(SpdifStreamOut *spdifStreamOut) - : mSpdifStreamOut(spdifStreamOut) + MySPDIFEncoder(SpdifStreamOut *spdifStreamOut, audio_format_t format) + : SPDIFEncoder(format) + , mSpdifStreamOut(spdifStreamOut) { } diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp index 1a20fae..b30fd20 100644 --- a/services/audioflinger/Threads.cpp +++ b/services/audioflinger/Threads.cpp @@ -1608,13 +1608,19 @@ sp<AudioFlinger::PlaybackThread::Track> AudioFlinger::PlaybackThread::createTrac // If you change this calculation, also review the start threshold which is related. if (!(*flags & IAudioFlinger::TRACK_FAST) && audio_is_linear_pcm(format) && sharedBuffer == 0) { + // this must match AudioTrack.cpp calculateMinFrameCount(). + // TODO: Move to a common library uint32_t latencyMs = mOutput->stream->get_latency(mOutput->stream); uint32_t minBufCount = latencyMs / ((1000 * mNormalFrameCount) / mSampleRate); if (minBufCount < 2) { minBufCount = 2; } + // For normal mixing tracks, if speed is > 1.0f (normal), AudioTrack + // or the client should compute and pass in a larger buffer request. size_t minFrameCount = - minBufCount * sourceFramesNeeded(sampleRate, mNormalFrameCount, mSampleRate); + minBufCount * sourceFramesNeededWithTimestretch( + sampleRate, mNormalFrameCount, + mSampleRate, AUDIO_TIMESTRETCH_SPEED_NORMAL /*speed*/); if (frameCount < minFrameCount) { // including frameCount == 0 frameCount = minFrameCount; } @@ -3592,21 +3598,17 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac // hence the test on (mMixerStatus == MIXER_TRACKS_READY) meaning the track was mixed // during last round size_t desiredFrames; - uint32_t sr = track->sampleRate(); - if (sr == mSampleRate) { - desiredFrames = mNormalFrameCount; - } else { - desiredFrames = sourceFramesNeeded(sr, mNormalFrameCount, mSampleRate); - // add frames already consumed but not yet released by the resampler - // because mAudioTrackServerProxy->framesReady() will include these frames - desiredFrames += mAudioMixer->getUnreleasedFrames(track->name()); -#if 0 - // the minimum track buffer size is normally twice the number of frames necessary - // to fill one buffer and the resampler should not leave more than one buffer worth - // of unreleased frames after each pass, but just in case... - ALOG_ASSERT(desiredFrames <= cblk->frameCount_); -#endif - } + const uint32_t sampleRate = track->mAudioTrackServerProxy->getSampleRate(); + float speed, pitch; + track->mAudioTrackServerProxy->getPlaybackRate(&speed, &pitch); + + desiredFrames = sourceFramesNeededWithTimestretch( + sampleRate, mNormalFrameCount, mSampleRate, speed); + // TODO: ONLY USED FOR LEGACY RESAMPLERS, remove when they are removed. + // add frames already consumed but not yet released by the resampler + // because mAudioTrackServerProxy->framesReady() will include these frames + desiredFrames += mAudioMixer->getUnreleasedFrames(track->name()); + uint32_t minFrames = 1; if ((track->sharedBuffer() == 0) && !track->isStopped() && !track->isPausing() && (mMixerStatusIgnoringFastTracks == MIXER_TRACKS_READY)) { @@ -3769,6 +3771,17 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac AudioMixer::RESAMPLE, AudioMixer::SAMPLE_RATE, (void *)(uintptr_t)reqSampleRate); + + // set the playback rate as an float array {speed, pitch} + float playbackRate[2]; + track->mAudioTrackServerProxy->getPlaybackRate( + &playbackRate[0] /*speed*/, &playbackRate[1] /*pitch*/); + mAudioMixer->setParameter( + name, + AudioMixer::TIMESTRETCH, + AudioMixer::PLAYBACK_RATE, + playbackRate); + /* * Select the appropriate output buffer for the track. * diff --git a/services/audioflinger/Tracks.cpp b/services/audioflinger/Tracks.cpp index 1566b1f..da2d634 100644 --- a/services/audioflinger/Tracks.cpp +++ b/services/audioflinger/Tracks.cpp @@ -903,9 +903,14 @@ status_t AudioFlinger::PlaybackThread::Track::getTimestamp(AudioTimestamp& times mPreviousTimestampValid = false; return INVALID_OPERATION; } + // FIXME Not accurate under dynamic changes of sample rate and speed. + // Do not use track's mSampleRate as it is not current for mixer tracks. + uint32_t sampleRate = mAudioTrackServerProxy->getSampleRate(); + float speed, pitch; + mAudioTrackServerProxy->getPlaybackRate(&speed, &pitch); uint32_t unpresentedFrames = - ((int64_t) playbackThread->mLatchQ.mUnpresentedFrames * mSampleRate) / - playbackThread->mSampleRate; + ((double) playbackThread->mLatchQ.mUnpresentedFrames * sampleRate * speed) + / playbackThread->mSampleRate; // FIXME Since we're using a raw pointer as the key, it is theoretically possible // for a brand new track to share the same address as a recently destroyed // track, and thus for us to get the frames released of the wrong track. diff --git a/services/audioflinger/tests/Android.mk b/services/audioflinger/tests/Android.mk index 8604ef5..536eb93 100644 --- a/services/audioflinger/tests/Android.mk +++ b/services/audioflinger/tests/Android.mk @@ -39,11 +39,13 @@ endif LOCAL_SRC_FILES:= \ test-mixer.cpp \ ../AudioMixer.cpp.arm \ + ../BufferProviders.cpp LOCAL_C_INCLUDES := \ $(call include-path-for, audio-effects) \ $(call include-path-for, audio-utils) \ - frameworks/av/services/audioflinger + frameworks/av/services/audioflinger \ + external/sonic LOCAL_STATIC_LIBRARIES := \ libsndfile @@ -57,7 +59,8 @@ LOCAL_SHARED_LIBRARIES := \ libdl \ libcutils \ libutils \ - liblog + liblog \ + libsonic LOCAL_MODULE:= test-mixer diff --git a/services/audiopolicy/common/managerdefinitions/include/AudioPort.h b/services/audiopolicy/common/managerdefinitions/include/AudioPort.h index dea1b8a..1c2c27e 100644 --- a/services/audiopolicy/common/managerdefinitions/include/AudioPort.h +++ b/services/audiopolicy/common/managerdefinitions/include/AudioPort.h @@ -62,8 +62,12 @@ public: // searches for an exact match status_t checkExactChannelMask(audio_channel_mask_t channelMask) const; // searches for a compatible match, currently implemented for input channel masks only - status_t checkCompatibleChannelMask(audio_channel_mask_t channelMask) const; - status_t checkFormat(audio_format_t format) const; + status_t checkCompatibleChannelMask(audio_channel_mask_t channelMask, + audio_channel_mask_t *updatedChannelMask) const; + + status_t checkExactFormat(audio_format_t format) const; + // searches for a compatible match, currently implemented for input formats only + status_t checkCompatibleFormat(audio_format_t format, audio_format_t *updatedFormat) const; status_t checkGain(const struct audio_gain_config *gainConfig, int index) const; uint32_t pickSamplingRate() const; @@ -71,6 +75,11 @@ public: audio_format_t pickFormat() const; static const audio_format_t sPcmFormatCompareTable[]; + static int compareFormatsGoodToBad( + const audio_format_t *format1, const audio_format_t *format2) { + // compareFormats sorts from bad to good, we reverse it here + return compareFormats(*format2, *format1); + } static int compareFormats(audio_format_t format1, audio_format_t format2); audio_module_handle_t getModuleHandle() const; diff --git a/services/audiopolicy/common/managerdefinitions/include/IOProfile.h b/services/audiopolicy/common/managerdefinitions/include/IOProfile.h index 022257e..ab6fcc1 100644 --- a/services/audiopolicy/common/managerdefinitions/include/IOProfile.h +++ b/services/audiopolicy/common/managerdefinitions/include/IOProfile.h @@ -45,7 +45,9 @@ public: uint32_t samplingRate, uint32_t *updatedSamplingRate, audio_format_t format, + audio_format_t *updatedFormat, audio_channel_mask_t channelMask, + audio_channel_mask_t *updatedChannelMask, uint32_t flags) const; void dump(int fd); diff --git a/services/audiopolicy/common/managerdefinitions/src/AudioPort.cpp b/services/audiopolicy/common/managerdefinitions/src/AudioPort.cpp index e8191dd..f3978ec 100644 --- a/services/audiopolicy/common/managerdefinitions/src/AudioPort.cpp +++ b/services/audiopolicy/common/managerdefinitions/src/AudioPort.cpp @@ -16,7 +16,7 @@ #define LOG_TAG "APM::AudioPort" //#define LOG_NDEBUG 0 - +#include <media/AudioResamplerPublic.h> #include "AudioPort.h" #include "HwModule.h" #include "AudioGain.h" @@ -216,6 +216,7 @@ void AudioPort::loadFormats(char *name) } str = strtok(NULL, "|"); } + mFormats.sort(compareFormatsGoodToBad); } void AudioPort::loadInChannels(char *name) @@ -358,6 +359,9 @@ status_t AudioPort::checkCompatibleSamplingRate(uint32_t samplingRate, uint32_t *updatedSamplingRate) const { if (mSamplingRates.isEmpty()) { + if (updatedSamplingRate != NULL) { + *updatedSamplingRate = samplingRate; + } return NO_ERROR; } @@ -387,16 +391,11 @@ status_t AudioPort::checkCompatibleSamplingRate(uint32_t samplingRate, } } } - // This uses hard-coded knowledge about AudioFlinger resampling ratios. - // TODO Move these assumptions out. - static const uint32_t kMaxDownSampleRatio = 6; // beyond this aliasing occurs - static const uint32_t kMaxUpSampleRatio = 256; // beyond this sample rate inaccuracies occur - // due to approximation by an int32_t of the - // phase increments + // Prefer to down-sample from a higher sampling rate, as we get the desired frequency spectrum. if (minAbove >= 0) { candidate = mSamplingRates[minAbove]; - if (candidate / kMaxDownSampleRatio <= samplingRate) { + if (candidate / AUDIO_RESAMPLER_DOWN_RATIO_MAX <= samplingRate) { if (updatedSamplingRate != NULL) { *updatedSamplingRate = candidate; } @@ -406,7 +405,7 @@ status_t AudioPort::checkCompatibleSamplingRate(uint32_t samplingRate, // But if we have to up-sample from a lower sampling rate, that's OK. if (maxBelow >= 0) { candidate = mSamplingRates[maxBelow]; - if (candidate * kMaxUpSampleRatio >= samplingRate) { + if (candidate * AUDIO_RESAMPLER_UP_RATIO_MAX >= samplingRate) { if (updatedSamplingRate != NULL) { *updatedSamplingRate = candidate; } @@ -431,10 +430,13 @@ status_t AudioPort::checkExactChannelMask(audio_channel_mask_t channelMask) cons return BAD_VALUE; } -status_t AudioPort::checkCompatibleChannelMask(audio_channel_mask_t channelMask) - const +status_t AudioPort::checkCompatibleChannelMask(audio_channel_mask_t channelMask, + audio_channel_mask_t *updatedChannelMask) const { if (mChannelMasks.isEmpty()) { + if (updatedChannelMask != NULL) { + *updatedChannelMask = channelMask; + } return NO_ERROR; } @@ -443,6 +445,9 @@ status_t AudioPort::checkCompatibleChannelMask(audio_channel_mask_t channelMask) // FIXME Does not handle multi-channel automatic conversions yet audio_channel_mask_t supported = mChannelMasks[i]; if (supported == channelMask) { + if (updatedChannelMask != NULL) { + *updatedChannelMask = channelMask; + } return NO_ERROR; } if (isRecordThread) { @@ -452,6 +457,9 @@ status_t AudioPort::checkCompatibleChannelMask(audio_channel_mask_t channelMask) && channelMask == AUDIO_CHANNEL_IN_MONO) || (supported == AUDIO_CHANNEL_IN_MONO && (channelMask == AUDIO_CHANNEL_IN_FRONT_BACK || channelMask == AUDIO_CHANNEL_IN_STEREO))) { + if (updatedChannelMask != NULL) { + *updatedChannelMask = supported; + } return NO_ERROR; } } @@ -459,7 +467,7 @@ status_t AudioPort::checkCompatibleChannelMask(audio_channel_mask_t channelMask) return BAD_VALUE; } -status_t AudioPort::checkFormat(audio_format_t format) const +status_t AudioPort::checkExactFormat(audio_format_t format) const { if (mFormats.isEmpty()) { return NO_ERROR; @@ -473,6 +481,33 @@ status_t AudioPort::checkFormat(audio_format_t format) const return BAD_VALUE; } +status_t AudioPort::checkCompatibleFormat(audio_format_t format, audio_format_t *updatedFormat) + const +{ + if (mFormats.isEmpty()) { + if (updatedFormat != NULL) { + *updatedFormat = format; + } + return NO_ERROR; + } + + const bool checkInexact = // when port is input and format is linear pcm + mType == AUDIO_PORT_TYPE_MIX && mRole == AUDIO_PORT_ROLE_SINK + && audio_is_linear_pcm(format); + + for (size_t i = 0; i < mFormats.size(); ++i) { + if (mFormats[i] == format || + (checkInexact && audio_is_linear_pcm(mFormats[i]))) { + // for inexact checks we take the first linear pcm format since + // mFormats is sorted from best PCM format to worst PCM format. + if (updatedFormat != NULL) { + *updatedFormat = mFormats[i]; + } + return NO_ERROR; + } + } + return BAD_VALUE; +} uint32_t AudioPort::pickSamplingRate() const { @@ -756,7 +791,7 @@ status_t AudioPortConfig::applyAudioPortConfig( mChannelMask = config->channel_mask; } if (config->config_mask & AUDIO_PORT_CONFIG_FORMAT) { - status = audioport->checkFormat(config->format); + status = audioport->checkExactFormat(config->format); if (status != NO_ERROR) { goto exit; } diff --git a/services/audiopolicy/common/managerdefinitions/src/IOProfile.cpp b/services/audiopolicy/common/managerdefinitions/src/IOProfile.cpp index de6539c..7b6d51d 100644 --- a/services/audiopolicy/common/managerdefinitions/src/IOProfile.cpp +++ b/services/audiopolicy/common/managerdefinitions/src/IOProfile.cpp @@ -40,7 +40,9 @@ bool IOProfile::isCompatibleProfile(audio_devices_t device, uint32_t samplingRate, uint32_t *updatedSamplingRate, audio_format_t format, + audio_format_t *updatedFormat, audio_channel_mask_t channelMask, + audio_channel_mask_t *updatedChannelMask, uint32_t flags) const { const bool isPlaybackThread = mType == AUDIO_PORT_TYPE_MIX && mRole == AUDIO_PORT_ROLE_SOURCE; @@ -71,7 +73,14 @@ bool IOProfile::isCompatibleProfile(audio_devices_t device, return false; } - if (!audio_is_valid_format(format) || checkFormat(format) != NO_ERROR) { + if (!audio_is_valid_format(format)) { + return false; + } + if (isPlaybackThread && checkExactFormat(format) != NO_ERROR) { + return false; + } + audio_format_t myUpdatedFormat = format; + if (isRecordThread && checkCompatibleFormat(format, &myUpdatedFormat) != NO_ERROR) { return false; } @@ -79,8 +88,9 @@ bool IOProfile::isCompatibleProfile(audio_devices_t device, checkExactChannelMask(channelMask) != NO_ERROR)) { return false; } + audio_channel_mask_t myUpdatedChannelMask = channelMask; if (isRecordThread && (!audio_is_input_channel(channelMask) || - checkCompatibleChannelMask(channelMask) != NO_ERROR)) { + checkCompatibleChannelMask(channelMask, &myUpdatedChannelMask) != NO_ERROR)) { return false; } @@ -99,6 +109,12 @@ bool IOProfile::isCompatibleProfile(audio_devices_t device, if (updatedSamplingRate != NULL) { *updatedSamplingRate = myUpdatedSamplingRate; } + if (updatedFormat != NULL) { + *updatedFormat = myUpdatedFormat; + } + if (updatedChannelMask != NULL) { + *updatedChannelMask = myUpdatedChannelMask; + } return true; } diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp index 35e80f7..ba9f996 100644 --- a/services/audiopolicy/managerdefault/AudioPolicyManager.cpp +++ b/services/audiopolicy/managerdefault/AudioPolicyManager.cpp @@ -585,8 +585,10 @@ sp<IOProfile> AudioPolicyManager::getProfileForDirectOutput( } for (size_t j = 0; j < mHwModules[i]->mOutputProfiles.size(); j++) { sp<IOProfile> profile = mHwModules[i]->mOutputProfiles[j]; - bool found = profile->isCompatibleProfile(device, String8(""), samplingRate, - NULL /*updatedSamplingRate*/, format, channelMask, + bool found = profile->isCompatibleProfile(device, String8(""), + samplingRate, NULL /*updatedSamplingRate*/, + format, NULL /*updatedFormat*/, + channelMask, NULL /*updatedChannelMask*/, flags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD ? AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD : AUDIO_OUTPUT_FLAG_DIRECT); if (found && (mAvailableOutputDevices.types() & profile->mSupportedDevices.types())) { @@ -1303,20 +1305,25 @@ status_t AudioPolicyManager::getInputForAttr(const audio_attributes_t *attr, } } - sp<IOProfile> profile = getInputProfile(device, address, - samplingRate, format, channelMask, - flags); - if (profile == 0) { - //retry without flags - audio_input_flags_t log_flags = flags; - flags = AUDIO_INPUT_FLAG_NONE; + // find a compatible input profile (not necessarily identical in parameters) + sp<IOProfile> profile; + // samplingRate and flags may be updated by getInputProfile + uint32_t profileSamplingRate = samplingRate; + audio_format_t profileFormat = format; + audio_channel_mask_t profileChannelMask = channelMask; + audio_input_flags_t profileFlags = flags; + for (;;) { profile = getInputProfile(device, address, - samplingRate, format, channelMask, - flags); - if (profile == 0) { + profileSamplingRate, profileFormat, profileChannelMask, + profileFlags); + if (profile != 0) { + break; // success + } else if (profileFlags != AUDIO_INPUT_FLAG_NONE) { + profileFlags = AUDIO_INPUT_FLAG_NONE; // retry + } else { // fail ALOGW("getInputForAttr() could not find profile for device 0x%X, samplingRate %u," "format %#x, channelMask 0x%X, flags %#x", - device, samplingRate, format, channelMask, log_flags); + device, samplingRate, format, channelMask, flags); return BAD_VALUE; } } @@ -1327,9 +1334,9 @@ status_t AudioPolicyManager::getInputForAttr(const audio_attributes_t *attr, } audio_config_t config = AUDIO_CONFIG_INITIALIZER; - config.sample_rate = samplingRate; - config.channel_mask = channelMask; - config.format = format; + config.sample_rate = profileSamplingRate; + config.channel_mask = profileChannelMask; + config.format = profileFormat; status_t status = mpClientInterface->openInput(profile->getModuleHandle(), input, @@ -1337,14 +1344,15 @@ status_t AudioPolicyManager::getInputForAttr(const audio_attributes_t *attr, &device, address, halInputSource, - flags); + profileFlags); // only accept input with the exact requested set of parameters if (status != NO_ERROR || *input == AUDIO_IO_HANDLE_NONE || - (samplingRate != config.sample_rate) || - (format != config.format) || - (channelMask != config.channel_mask)) { - ALOGW("getInputForAttr() failed opening input: samplingRate %d, format %d, channelMask %x", + (profileSamplingRate != config.sample_rate) || + (profileFormat != config.format) || + (profileChannelMask != config.channel_mask)) { + ALOGW("getInputForAttr() failed opening input: samplingRate %d, format %d," + " channelMask %x", samplingRate, format, channelMask); if (*input != AUDIO_IO_HANDLE_NONE) { mpClientInterface->closeInput(*input); @@ -1356,9 +1364,9 @@ status_t AudioPolicyManager::getInputForAttr(const audio_attributes_t *attr, inputDesc->mInputSource = inputSource; inputDesc->mRefCount = 0; inputDesc->mOpenRefCount = 1; - inputDesc->mSamplingRate = samplingRate; - inputDesc->mFormat = format; - inputDesc->mChannelMask = channelMask; + inputDesc->mSamplingRate = profileSamplingRate; + inputDesc->mFormat = profileFormat; + inputDesc->mChannelMask = profileChannelMask; inputDesc->mDevice = device; inputDesc->mSessions.add(session); inputDesc->mIsSoundTrigger = isSoundTrigger; @@ -2122,9 +2130,12 @@ status_t AudioPolicyManager::createAudioPatch(const struct audio_patch *patch, patch->sources[0].sample_rate, NULL, // updatedSamplingRate patch->sources[0].format, + NULL, // updatedFormat patch->sources[0].channel_mask, + NULL, // updatedChannelMask AUDIO_OUTPUT_FLAG_NONE /*FIXME*/)) { - ALOGV("createAudioPatch() profile not supported for device %08x", devDesc->type()); + ALOGV("createAudioPatch() profile not supported for device %08x", + devDesc->type()); return INVALID_OPERATION; } devices.add(devDesc); @@ -2176,7 +2187,9 @@ status_t AudioPolicyManager::createAudioPatch(const struct audio_patch *patch, patch->sinks[0].sample_rate, NULL, /*updatedSampleRate*/ patch->sinks[0].format, + NULL, /*updatedFormat*/ patch->sinks[0].channel_mask, + NULL, /*updatedChannelMask*/ // FIXME for the parameter type, // and the NONE (audio_output_flags_t) @@ -4201,12 +4214,15 @@ status_t AudioPolicyManager::resetInputDevice(audio_io_handle_t input, sp<IOProfile> AudioPolicyManager::getInputProfile(audio_devices_t device, String8 address, uint32_t& samplingRate, - audio_format_t format, - audio_channel_mask_t channelMask, + audio_format_t& format, + audio_channel_mask_t& channelMask, audio_input_flags_t flags) { // Choose an input profile based on the requested capture parameters: select the first available // profile supporting all requested parameters. + // + // TODO: perhaps isCompatibleProfile should return a "matching" score so we can return + // the best matching profile, not the first one. for (size_t i = 0; i < mHwModules.size(); i++) { @@ -4219,7 +4235,11 @@ sp<IOProfile> AudioPolicyManager::getInputProfile(audio_devices_t device, // profile->log(); if (profile->isCompatibleProfile(device, address, samplingRate, &samplingRate /*updatedSamplingRate*/, - format, channelMask, (audio_output_flags_t) flags)) { + format, + &format /*updatedFormat*/, + channelMask, + &channelMask /*updatedChannelMask*/, + (audio_output_flags_t) flags)) { return profile; } diff --git a/services/audiopolicy/managerdefault/AudioPolicyManager.h b/services/audiopolicy/managerdefault/AudioPolicyManager.h index 11fd5ff..fe6b986 100644 --- a/services/audiopolicy/managerdefault/AudioPolicyManager.h +++ b/services/audiopolicy/managerdefault/AudioPolicyManager.h @@ -470,12 +470,12 @@ protected: audio_io_handle_t selectOutput(const SortedVector<audio_io_handle_t>& outputs, audio_output_flags_t flags, audio_format_t format); - // samplingRate parameter is an in/out and so may be modified + // samplingRate, format, channelMask are in/out and so may be modified sp<IOProfile> getInputProfile(audio_devices_t device, String8 address, uint32_t& samplingRate, - audio_format_t format, - audio_channel_mask_t channelMask, + audio_format_t& format, + audio_channel_mask_t& channelMask, audio_input_flags_t flags); sp<IOProfile> getProfileForDirectOutput(audio_devices_t device, uint32_t samplingRate, diff --git a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp index a763151..9510727 100644 --- a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp +++ b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp @@ -262,8 +262,7 @@ status_t AudioPolicyService::getInputForAttr(const audio_attributes_t *attr, return BAD_VALUE; } - if (((attr->source == AUDIO_SOURCE_HOTWORD) && !captureHotwordAllowed()) || - ((attr->source == AUDIO_SOURCE_FM_TUNER) && !captureFmTunerAllowed())) { + if ((attr->source == AUDIO_SOURCE_HOTWORD) && !captureHotwordAllowed()) { return BAD_VALUE; } sp<AudioPolicyEffects>audioPolicyEffects; diff --git a/services/audiopolicy/service/AudioPolicyInterfaceImplLegacy.cpp b/services/audiopolicy/service/AudioPolicyInterfaceImplLegacy.cpp index 372a9fa..e4ca5dc 100644 --- a/services/audiopolicy/service/AudioPolicyInterfaceImplLegacy.cpp +++ b/services/audiopolicy/service/AudioPolicyInterfaceImplLegacy.cpp @@ -255,8 +255,7 @@ status_t AudioPolicyService::getInputForAttr(const audio_attributes_t *attr, inputSource = AUDIO_SOURCE_MIC; } - if (((inputSource == AUDIO_SOURCE_HOTWORD) && !captureHotwordAllowed()) || - ((inputSource == AUDIO_SOURCE_FM_TUNER) && !captureFmTunerAllowed())) { + if ((inputSource == AUDIO_SOURCE_HOTWORD) && !captureHotwordAllowed()) { return BAD_VALUE; } diff --git a/services/camera/libcameraservice/CameraService.cpp b/services/camera/libcameraservice/CameraService.cpp index e9c96c6..414d563 100644 --- a/services/camera/libcameraservice/CameraService.cpp +++ b/services/camera/libcameraservice/CameraService.cpp @@ -122,7 +122,7 @@ static void torch_mode_status_change( // should be ok for now. static CameraService *gCameraService; -CameraService::CameraService() : mEventLog(DEFAULT_EVICTION_LOG_LENGTH), +CameraService::CameraService() : mEventLog(DEFAULT_EVENT_LOG_LENGTH), mLastUserId(DEFAULT_LAST_USER_ID), mSoundRef(0), mModule(0), mFlashlight(0) { ALOGI("CameraService started (pid=%d)", getpid()); gCameraService = this; @@ -242,6 +242,8 @@ void CameraService::onDeviceStatusChanged(camera_device_status_t cameraId, } if (newStatus == CAMERA_DEVICE_STATUS_NOT_PRESENT) { + logDeviceRemoved(id, String8::format("Device status changed from %d to %d", oldStatus, + newStatus)); sp<BasicClient> clientToDisconnect; { // Don't do this in updateStatus to avoid deadlock over mServiceLock @@ -274,6 +276,10 @@ void CameraService::onDeviceStatusChanged(camera_device_status_t cameraId, } } else { + if (oldStatus == ICameraServiceListener::Status::STATUS_NOT_PRESENT) { + logDeviceAdded(id, String8::format("Device status changed from %d to %d", oldStatus, + newStatus)); + } updateStatus(static_cast<ICameraServiceListener::Status>(newStatus), id); } @@ -765,8 +771,8 @@ status_t CameraService::validateConnectLocked(const String8& cameraId, /*inout*/ } else { // We only trust our own process to forward client UIDs if (callingPid != getpid()) { - ALOGE("CameraService::connect X (PID %d) rejected (don't trust clientUid)", - callingPid); + ALOGE("CameraService::connect X (PID %d) rejected (don't trust clientUid %d)", + callingPid, clientUid); return PERMISSION_DENIED; } } @@ -796,10 +802,12 @@ status_t CameraService::validateConnectLocked(const String8& cameraId, /*inout*/ return -EACCES; } - // Only allow clients who are being used by the current foreground device user. - if (mLastUserId != clientUserId && mLastUserId != DEFAULT_LAST_USER_ID) { - ALOGE("CameraService::connect X (PID %d) rejected (cannot connect from non-foreground " - "device user)", callingPid); + // Only allow clients who are being used by the current foreground device user, unless calling + // from our own process. + if (callingPid != getpid() && + (mLastUserId != clientUserId && mLastUserId != DEFAULT_LAST_USER_ID)) { + ALOGE("CameraService::connect X (PID %d) rejected (cannot connect from previous " + "device user %d, current device user %d)", callingPid, clientUserId, mLastUserId); return PERMISSION_DENIED; } @@ -858,7 +866,7 @@ status_t CameraService::handleEvictionsLocked(const String8& cameraId, int clien std::shared_ptr<resource_policy::ClientDescriptor<String8, sp<BasicClient>>>* partial) { status_t ret = NO_ERROR; - std::vector<sp<BasicClient>> evictedClients; + std::vector<DescriptorPtr> evictedClients; DescriptorPtr clientDescriptor; { if (effectiveApiLevel == API_1) { @@ -934,7 +942,7 @@ status_t CameraService::handleEvictionsLocked(const String8& cameraId, int clien mActiveClientManager.getIncompatibleClients(clientDescriptor); String8 msg = String8::format("%s : DENIED connect device %s client for package %s " - "(PID %d, priority %d)", curTime.string(), + "(PID %d, priority %d) due to eviction policy", curTime.string(), cameraId.string(), packageName.string(), clientPid, getCameraPriorityFromProcState(priorities[priorities.size() - 1])); @@ -946,6 +954,7 @@ status_t CameraService::handleEvictionsLocked(const String8& cameraId, int clien } // Log the client's attempt + Mutex::Autolock l(mLogLock); mEventLog.add(msg); return -EBUSY; @@ -965,14 +974,12 @@ status_t CameraService::handleEvictionsLocked(const String8& cameraId, int clien ALOGE("CameraService::connect evicting conflicting client for camera ID %s", i->getKey().string()); - evictedClients.push_back(clientSp); - - String8 curTime = getFormattedCurrentTime(); + evictedClients.push_back(i); // Log the clients evicted - mEventLog.add(String8::format("%s : EVICT device %s client for package %s (PID %" - PRId32 ", priority %" PRId32 ")\n - Evicted by device %s client for " - "package %s (PID %d, priority %" PRId32 ")", curTime.string(), + logEvent(String8::format("EVICT device %s client held by package %s (PID" + " %" PRId32 ", priority %" PRId32 ")\n - Evicted by device %s client for" + " package %s (PID %d, priority %" PRId32 ")", i->getKey().string(), String8{clientSp->getPackageName()}.string(), i->getOwnerId(), i->getPriority(), cameraId.string(), packageName.string(), clientPid, @@ -994,12 +1001,31 @@ status_t CameraService::handleEvictionsLocked(const String8& cameraId, int clien // Destroy evicted clients for (auto& i : evictedClients) { // Disconnect is blocking, and should only have returned when HAL has cleaned up - i->disconnect(); // Clients will remove themselves from the active client list here + i->getValue()->disconnect(); // Clients will remove themselves from the active client list } - evictedClients.clear(); IPCThreadState::self()->restoreCallingIdentity(token); + for (const auto& i : evictedClients) { + ALOGV("%s: Waiting for disconnect to complete for client for device %s (PID %" PRId32 ")", + __FUNCTION__, i->getKey().string(), i->getOwnerId()); + ret = mActiveClientManager.waitUntilRemoved(i, DEFAULT_DISCONNECT_TIMEOUT_NS); + if (ret == TIMED_OUT) { + ALOGE("%s: Timed out waiting for client for device %s to disconnect, " + "current clients:\n%s", __FUNCTION__, i->getKey().string(), + mActiveClientManager.toString().string()); + return -EBUSY; + } + if (ret != NO_ERROR) { + ALOGE("%s: Received error waiting for client for device %s to disconnect: %s (%d), " + "current clients:\n%s", __FUNCTION__, i->getKey().string(), strerror(-ret), + ret, mActiveClientManager.toString().string()); + return ret; + } + } + + evictedClients.clear(); + // Once clients have been disconnected, relock mServiceLock.lock(); @@ -1027,6 +1053,8 @@ status_t CameraService::connect( clientPackageName, clientUid, API_1, false, false, /*out*/client); if(ret != NO_ERROR) { + logRejected(id, getCallingPid(), String8(clientPackageName), + String8::format("%s (%d)", strerror(-ret), ret)); return ret; } @@ -1042,6 +1070,7 @@ status_t CameraService::connectLegacy( /*out*/ sp<ICamera>& device) { + String8 id = String8::format("%d", cameraId); int apiVersion = mModule->getModuleApiVersion(); if (halVersion != CAMERA_HAL_API_VERSION_UNSPECIFIED && apiVersion < CAMERA_MODULE_API_VERSION_2_3) { @@ -1053,16 +1082,19 @@ status_t CameraService::connectLegacy( */ ALOGE("%s: camera HAL module version %x doesn't support connecting to legacy HAL devices!", __FUNCTION__, apiVersion); + logRejected(id, getCallingPid(), String8(clientPackageName), + String8("HAL module version doesn't support legacy HAL connections")); return INVALID_OPERATION; } status_t ret = NO_ERROR; - String8 id = String8::format("%d", cameraId); sp<Client> client = nullptr; ret = connectHelper<ICameraClient,Client>(cameraClient, id, halVersion, clientPackageName, clientUid, API_1, true, false, /*out*/client); if(ret != NO_ERROR) { + logRejected(id, getCallingPid(), String8(clientPackageName), + String8::format("%s (%d)", strerror(-ret), ret)); return ret; } @@ -1086,6 +1118,8 @@ status_t CameraService::connectDevice( /*out*/client); if(ret != NO_ERROR) { + logRejected(id, getCallingPid(), String8(clientPackageName), + String8::format("%s (%d)", strerror(-ret), ret)); return ret; } @@ -1426,6 +1460,8 @@ void CameraService::doUserSwitch(int newUserId) { newUserId = DEFAULT_LAST_USER_ID; } + logUserSwitch(mLastUserId, newUserId); + mLastUserId = newUserId; // Current user has switched, evict all current clients. @@ -1444,12 +1480,12 @@ void CameraService::doUserSwitch(int newUserId) { ALOGE("Evicting conflicting client for camera ID %s due to user change", i->getKey().string()); + // Log the clients evicted - mEventLog.add(String8::format("%s : EVICT device %s client for package %s (PID %" + logEvent(String8::format("EVICT device %s client held by package %s (PID %" PRId32 ", priority %" PRId32 ")\n - Evicted due to user switch.", - curTime.string(), i->getKey().string(), - String8{clientSp->getPackageName()}.string(), i->getOwnerId(), - i->getPriority())); + i->getKey().string(), String8{clientSp->getPackageName()}.string(), + i->getOwnerId(), i->getPriority())); } @@ -1470,22 +1506,52 @@ void CameraService::doUserSwitch(int newUserId) { mServiceLock.lock(); } -void CameraService::logDisconnected(const String8& cameraId, int clientPid, - const String8& clientPackage) { - +void CameraService::logEvent(const char* event) { String8 curTime = getFormattedCurrentTime(); - // Log the clients evicted - mEventLog.add(String8::format("%s : DISCONNECT device %s client for package %s (PID %d)", - curTime.string(), cameraId.string(), clientPackage.string(), clientPid)); + Mutex::Autolock l(mLogLock); + mEventLog.add(String8::format("%s : %s", curTime.string(), event)); } -void CameraService::logConnected(const String8& cameraId, int clientPid, - const String8& clientPackage) { +void CameraService::logDisconnected(const char* cameraId, int clientPid, + const char* clientPackage) { + // Log the clients evicted + logEvent(String8::format("DISCONNECT device %s client for package %s (PID %d)", cameraId, + clientPackage, clientPid)); +} - String8 curTime = getFormattedCurrentTime(); +void CameraService::logConnected(const char* cameraId, int clientPid, + const char* clientPackage) { // Log the clients evicted - mEventLog.add(String8::format("%s : CONNECT device %s client for package %s (PID %d)", - curTime.string(), cameraId.string(), clientPackage.string(), clientPid)); + logEvent(String8::format("CONNECT device %s client for package %s (PID %d)", cameraId, + clientPackage, clientPid)); +} + +void CameraService::logRejected(const char* cameraId, int clientPid, + const char* clientPackage, const char* reason) { + // Log the client rejected + logEvent(String8::format("REJECT device %s client for package %s (PID %d), reason: (%s)", + cameraId, clientPackage, clientPid, reason)); +} + +void CameraService::logUserSwitch(int oldUserId, int newUserId) { + // Log the new and old users + logEvent(String8::format("USER_SWITCH from old user: %d , to new user: %d", oldUserId, + newUserId)); +} + +void CameraService::logDeviceRemoved(const char* cameraId, const char* reason) { + // Log the device removal + logEvent(String8::format("REMOVE device %s, reason: (%s)", cameraId, reason)); +} + +void CameraService::logDeviceAdded(const char* cameraId, const char* reason) { + // Log the device removal + logEvent(String8::format("ADD device %s, reason: (%s)", cameraId, reason)); +} + +void CameraService::logClientDied(int clientPid, const char* reason) { + // Log the device removal + logEvent(String8::format("DIED client(s) with PID %d, reason: (%s)", clientPid, reason)); } status_t CameraService::onTransact(uint32_t code, const Parcel& data, Parcel* reply, @@ -1911,7 +1977,7 @@ static bool tryLock(Mutex& mutex) } status_t CameraService::dump(int fd, const Vector<String16>& args) { - String8 result; + String8 result("Dump of the Camera Service:\n"); if (checkCallingPermission(String16("android.permission.DUMP")) == false) { result.appendFormat("Permission Denial: " "can't dump CameraService from pid=%d, uid=%d\n", @@ -1957,12 +2023,15 @@ status_t CameraService::dump(int fd, const Vector<String16>& args) { result = String8("Prior client events (most recent at top):\n"); - for (const auto& msg : mEventLog) { - result.appendFormat("%s\n", msg.string()); - } + { + Mutex::Autolock l(mLogLock); + for (const auto& msg : mEventLog) { + result.appendFormat("%s\n", msg.string()); + } - if (mEventLog.size() == DEFAULT_EVICTION_LOG_LENGTH) { - result.append("...\n"); + if (mEventLog.size() == DEFAULT_EVENT_LOG_LENGTH) { + result.append("...\n"); + } } write(fd, result.string(), result.size()); @@ -2094,10 +2163,12 @@ void CameraService::handleTorchClientBinderDied(const wp<IBinder> &who) { /*virtual*/void CameraService::binderDied(const wp<IBinder> &who) { /** - * While tempting to promote the wp<IBinder> into a sp, - * it's actually not supported by the binder driver + * While tempting to promote the wp<IBinder> into a sp, it's actually not supported by the + * binder driver */ + logClientDied(getCallingPid(), String8("Binder died unexpectedly")); + // check torch client handleTorchClientBinderDied(who); diff --git a/services/camera/libcameraservice/CameraService.h b/services/camera/libcameraservice/CameraService.h index ca1c504..91c7d59 100644 --- a/services/camera/libcameraservice/CameraService.h +++ b/services/camera/libcameraservice/CameraService.h @@ -90,8 +90,11 @@ public: // 3 second busy timeout when other clients are connecting static const nsecs_t DEFAULT_CONNECT_TIMEOUT_NS = 3000000000; + // 1 second busy timeout when other clients are disconnecting + static const nsecs_t DEFAULT_DISCONNECT_TIMEOUT_NS = 1000000000; + // Default number of messages to store in eviction log - static const size_t DEFAULT_EVICTION_LOG_LENGTH = 50; + static const size_t DEFAULT_EVENT_LOG_LENGTH = 100; enum { // Default last user id @@ -492,6 +495,7 @@ private: // Circular buffer for storing event logging for dumps RingBuffer<String8> mEventLog; + Mutex mLogLock; // UID of last user. int mLastUserId; @@ -546,14 +550,45 @@ private: void doUserSwitch(int newUserId); /** - * Add a event log message that a client has been disconnected. + * Add an event log message. + */ + void logEvent(const char* event); + + /** + * Add an event log message that a client has been disconnected. + */ + void logDisconnected(const char* cameraId, int clientPid, const char* clientPackage); + + /** + * Add an event log message that a client has been connected. + */ + void logConnected(const char* cameraId, int clientPid, const char* clientPackage); + + /** + * Add an event log message that a client's connect attempt has been rejected. */ - void logDisconnected(const String8& cameraId, int clientPid, const String8& clientPackage); + void logRejected(const char* cameraId, int clientPid, const char* clientPackage, + const char* reason); /** - * Add a event log message that a client has been connected. + * Add an event log message that the current device user has been switched. */ - void logConnected(const String8& cameraId, int clientPid, const String8& clientPackage); + void logUserSwitch(int oldUserId, int newUserId); + + /** + * Add an event log message that a device has been removed by the HAL + */ + void logDeviceRemoved(const char* cameraId, const char* reason); + + /** + * Add an event log message that a device has been added by the HAL + */ + void logDeviceAdded(const char* cameraId, const char* reason); + + /** + * Add an event log message that a client has unexpectedly died. + */ + void logClientDied(int clientPid, const char* reason); int mNumberOfCameras; @@ -714,9 +749,10 @@ status_t CameraService::connectHelper(const sp<CALLBACK>& cameraCb, const String String8 clientName8(clientPackageName); int clientPid = getCallingPid(); - ALOGI("CameraService::connect call E (PID %d \"%s\", camera ID %s) for HAL version %d and " + ALOGI("CameraService::connect call (PID %d \"%s\", camera ID %s) for HAL version %s and " "Camera API version %d", clientPid, clientName8.string(), cameraId.string(), - halVersion, static_cast<int>(effectiveApiLevel)); + (halVersion == -1) ? "default" : std::to_string(halVersion).c_str(), + static_cast<int>(effectiveApiLevel)); sp<CLIENT> client = nullptr; { @@ -734,7 +770,15 @@ status_t CameraService::connectHelper(const sp<CALLBACK>& cameraCb, const String if((ret = validateConnectLocked(cameraId, /*inout*/clientUid)) != NO_ERROR) { return ret; } - mLastUserId = multiuser_get_user_id(clientUid); + int userId = multiuser_get_user_id(clientUid); + + if (userId != mLastUserId && clientPid != getpid() ) { + // If no previous user ID had been set, set to the user of the caller. + logUserSwitch(mLastUserId, userId); + LOG_ALWAYS_FATAL_IF(mLastUserId != DEFAULT_LAST_USER_ID, + "Invalid state: Should never update user ID here unless was default"); + mLastUserId = userId; + } // Check the shim parameters after acquiring lock, if they have already been updated and // we were doing a shim update, return immediately diff --git a/services/camera/libcameraservice/api1/Camera2Client.cpp b/services/camera/libcameraservice/api1/Camera2Client.cpp index f53f425..05ede92 100644 --- a/services/camera/libcameraservice/api1/Camera2Client.cpp +++ b/services/camera/libcameraservice/api1/Camera2Client.cpp @@ -1710,6 +1710,40 @@ status_t Camera2Client::commandSetVideoBufferCountL(size_t count) { return mStreamingProcessor->setRecordingBufferCount(count); } +void Camera2Client::notifyError(ICameraDeviceCallbacks::CameraErrorCode errorCode, + const CaptureResultExtras& resultExtras) { + int32_t err = CAMERA_ERROR_UNKNOWN; + switch(errorCode) { + case ICameraDeviceCallbacks::ERROR_CAMERA_DISCONNECTED: + err = CAMERA_ERROR_RELEASED; + break; + case ICameraDeviceCallbacks::ERROR_CAMERA_DEVICE: + err = CAMERA_ERROR_UNKNOWN; + break; + case ICameraDeviceCallbacks::ERROR_CAMERA_SERVICE: + err = CAMERA_ERROR_SERVER_DIED; + break; + case ICameraDeviceCallbacks::ERROR_CAMERA_REQUEST: + case ICameraDeviceCallbacks::ERROR_CAMERA_RESULT: + case ICameraDeviceCallbacks::ERROR_CAMERA_BUFFER: + ALOGW("%s: Received recoverable error %d from HAL - ignoring, requestId %" PRId32, + __FUNCTION__, errorCode, resultExtras.requestId); + return; + default: + err = CAMERA_ERROR_UNKNOWN; + break; + } + + ALOGE("%s: Error condition %d reported by HAL, requestId %" PRId32, __FUNCTION__, errorCode, + resultExtras.requestId); + + SharedCameraCallbacks::Lock l(mSharedCameraCallbacks); + if (l.mRemoteCallback != nullptr) { + l.mRemoteCallback->notifyCallback(CAMERA_MSG_ERROR, err, 0); + } +} + + /** Device-related methods */ void Camera2Client::notifyAutoFocus(uint8_t newState, int triggerId) { ALOGV("%s: Autofocus state now %d, last trigger %d", diff --git a/services/camera/libcameraservice/api1/Camera2Client.h b/services/camera/libcameraservice/api1/Camera2Client.h index 5a8241f..a988037 100644 --- a/services/camera/libcameraservice/api1/Camera2Client.h +++ b/services/camera/libcameraservice/api1/Camera2Client.h @@ -77,6 +77,8 @@ public: virtual status_t setParameters(const String8& params); virtual String8 getParameters() const; virtual status_t sendCommand(int32_t cmd, int32_t arg1, int32_t arg2); + virtual void notifyError(ICameraDeviceCallbacks::CameraErrorCode errorCode, + const CaptureResultExtras& resultExtras); /** * Interface used by CameraService diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp index 8587e0e..bf1692d 100644 --- a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp +++ b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp @@ -65,6 +65,7 @@ CameraDeviceClient::CameraDeviceClient(const sp<CameraService>& cameraService, int servicePid) : Camera2ClientBase(cameraService, remoteCallback, clientPackageName, cameraId, cameraFacing, clientPid, clientUid, servicePid), + mInputStream(), mRequestIdCounter(0) { ATRACE_CALL(); @@ -127,6 +128,7 @@ status_t CameraDeviceClient::submitRequestList(List<sp<CaptureRequest> > request List<const CameraMetadata> metadataRequestList; int32_t requestId = mRequestIdCounter; uint32_t loopCounter = 0; + bool isReprocess = false; for (List<sp<CaptureRequest> >::iterator it = requests.begin(); it != requests.end(); ++it) { sp<CaptureRequest> request = *it; @@ -134,6 +136,18 @@ status_t CameraDeviceClient::submitRequestList(List<sp<CaptureRequest> > request ALOGE("%s: Camera %d: Sent null request.", __FUNCTION__, mCameraId); return BAD_VALUE; + } else if (it == requests.begin()) { + isReprocess = request->mIsReprocess; + if (isReprocess && !mInputStream.configured) { + ALOGE("%s: Camera %d: no input stream is configured."); + return BAD_VALUE; + } else if (isReprocess && streaming) { + ALOGE("%s: Camera %d: streaming reprocess requests not supported."); + return BAD_VALUE; + } + } else if (isReprocess != request->mIsReprocess) { + ALOGE("%s: Camera %d: Sent regular and reprocess requests."); + return BAD_VALUE; } CameraMetadata metadata(request->mMetadata); @@ -182,6 +196,10 @@ status_t CameraDeviceClient::submitRequestList(List<sp<CaptureRequest> > request metadata.update(ANDROID_REQUEST_OUTPUT_STREAMS, &outputStreamIds[0], outputStreamIds.size()); + if (isReprocess) { + metadata.update(ANDROID_REQUEST_INPUT_STREAMS, &mInputStream.id, 1); + } + metadata.update(ANDROID_REQUEST_ID, &requestId, /*size*/1); loopCounter++; // loopCounter starts from 1 ALOGV("%s: Camera %d: Creating request with ID %d (%d of %zu)", @@ -260,8 +278,8 @@ status_t CameraDeviceClient::beginConfigure() { } status_t CameraDeviceClient::endConfigure() { - ALOGV("%s: ending configure (%zu streams)", - __FUNCTION__, mStreamMap.size()); + ALOGV("%s: ending configure (%d input stream, %zu output streams)", + __FUNCTION__, mInputStream.configured ? 1 : 0, mStreamMap.size()); status_t res; if ( (res = checkPid(__FUNCTION__) ) != OK) return res; @@ -284,19 +302,25 @@ status_t CameraDeviceClient::deleteStream(int streamId) { if (!mDevice.get()) return DEAD_OBJECT; - // Guard against trying to delete non-created streams + bool isInput = false; ssize_t index = NAME_NOT_FOUND; - for (size_t i = 0; i < mStreamMap.size(); ++i) { - if (streamId == mStreamMap.valueAt(i)) { - index = i; - break; + + if (mInputStream.configured && mInputStream.id == streamId) { + isInput = true; + } else { + // Guard against trying to delete non-created streams + for (size_t i = 0; i < mStreamMap.size(); ++i) { + if (streamId == mStreamMap.valueAt(i)) { + index = i; + break; + } } - } - if (index == NAME_NOT_FOUND) { - ALOGW("%s: Camera %d: Invalid stream ID (%d) specified, no stream " - "created yet", __FUNCTION__, mCameraId, streamId); - return BAD_VALUE; + if (index == NAME_NOT_FOUND) { + ALOGW("%s: Camera %d: Invalid stream ID (%d) specified, no stream " + "created yet", __FUNCTION__, mCameraId, streamId); + return BAD_VALUE; + } } // Also returns BAD_VALUE if stream ID was not valid @@ -307,8 +331,11 @@ status_t CameraDeviceClient::deleteStream(int streamId) { " already checked and the stream ID (%d) should be valid.", __FUNCTION__, mCameraId, streamId); } else if (res == OK) { - mStreamMap.removeItemsAt(index); - + if (isInput) { + mInputStream.configured = false; + } else { + mStreamMap.removeItemsAt(index); + } } return res; @@ -450,6 +477,58 @@ status_t CameraDeviceClient::createStream(const OutputConfiguration &outputConfi } +status_t CameraDeviceClient::createInputStream(int width, int height, + int format) { + + ATRACE_CALL(); + ALOGV("%s (w = %d, h = %d, f = 0x%x)", __FUNCTION__, width, height, format); + + status_t res; + if ( (res = checkPid(__FUNCTION__) ) != OK) return res; + + Mutex::Autolock icl(mBinderSerializationLock); + if (!mDevice.get()) return DEAD_OBJECT; + + if (mInputStream.configured) { + ALOGE("%s: Camera %d: Already has an input stream " + " configuration. (ID %zd)", __FUNCTION__, mCameraId, + mInputStream.id); + return ALREADY_EXISTS; + } + + int streamId = -1; + res = mDevice->createInputStream(width, height, format, &streamId); + if (res == OK) { + mInputStream.configured = true; + mInputStream.width = width; + mInputStream.height = height; + mInputStream.format = format; + mInputStream.id = streamId; + + ALOGV("%s: Camera %d: Successfully created a new input stream ID %d", + __FUNCTION__, mCameraId, streamId); + + return streamId; + } + + return res; +} + +status_t CameraDeviceClient::getInputBufferProducer( + /*out*/sp<IGraphicBufferProducer> *producer) { + status_t res; + if ( (res = checkPid(__FUNCTION__) ) != OK) return res; + + if (producer == NULL) { + return BAD_VALUE; + } + + Mutex::Autolock icl(mBinderSerializationLock); + if (!mDevice.get()) return DEAD_OBJECT; + + return mDevice->getInputBufferProducer(producer); +} + bool CameraDeviceClient::roundBufferDimensionNearest(int32_t width, int32_t height, int32_t format, android_dataspace dataSpace, const CameraMetadata& info, /*out*/int32_t* outWidth, /*out*/int32_t* outHeight) { @@ -592,6 +671,42 @@ status_t CameraDeviceClient::flush(int64_t* lastFrameNumber) { return mDevice->flush(lastFrameNumber); } +status_t CameraDeviceClient::prepare(int streamId) { + ATRACE_CALL(); + ALOGV("%s", __FUNCTION__); + + status_t res = OK; + if ( (res = checkPid(__FUNCTION__) ) != OK) return res; + + Mutex::Autolock icl(mBinderSerializationLock); + + // Guard against trying to prepare non-created streams + ssize_t index = NAME_NOT_FOUND; + for (size_t i = 0; i < mStreamMap.size(); ++i) { + if (streamId == mStreamMap.valueAt(i)) { + index = i; + break; + } + } + + if (index == NAME_NOT_FOUND) { + ALOGW("%s: Camera %d: Invalid stream ID (%d) specified, no stream " + "created yet", __FUNCTION__, mCameraId, streamId); + return BAD_VALUE; + } + + // Also returns BAD_VALUE if stream ID was not valid + res = mDevice->prepare(streamId); + + if (res == BAD_VALUE) { + ALOGE("%s: Camera %d: Unexpected BAD_VALUE when preparing stream, but we" + " already checked and the stream ID (%d) should be valid.", + __FUNCTION__, mCameraId, streamId); + } + + return res; +} + status_t CameraDeviceClient::dump(int fd, const Vector<String16>& args) { String8 result; result.appendFormat("CameraDeviceClient[%d] (%p) dump:\n", @@ -602,13 +717,19 @@ status_t CameraDeviceClient::dump(int fd, const Vector<String16>& args) { result.append(" State:\n"); result.appendFormat(" Request ID counter: %d\n", mRequestIdCounter); + if (mInputStream.configured) { + result.appendFormat(" Current input stream ID: %d\n", + mInputStream.id); + } else { + result.append(" No input stream configured.\n"); + } if (!mStreamMap.isEmpty()) { - result.append(" Current stream IDs:\n"); + result.append(" Current output stream IDs:\n"); for (size_t i = 0; i < mStreamMap.size(); i++) { result.appendFormat(" Stream %d\n", mStreamMap.valueAt(i)); } } else { - result.append(" No streams configured.\n"); + result.append(" No output streams configured.\n"); } write(fd, result.string(), result.size()); // TODO: print dynamic/request section from most recent requests @@ -645,6 +766,14 @@ void CameraDeviceClient::notifyShutter(const CaptureResultExtras& resultExtras, } } +void CameraDeviceClient::notifyPrepared(int streamId) { + // Thread safe. Don't bother locking. + sp<ICameraDeviceCallbacks> remoteCb = getRemoteCallback(); + if (remoteCb != 0) { + remoteCb->onPrepared(streamId); + } +} + void CameraDeviceClient::detachDevice() { if (mDevice == 0) return; diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.h b/services/camera/libcameraservice/api2/CameraDeviceClient.h index a3dbb90..b8d8bea 100644 --- a/services/camera/libcameraservice/api2/CameraDeviceClient.h +++ b/services/camera/libcameraservice/api2/CameraDeviceClient.h @@ -86,6 +86,13 @@ public: virtual status_t createStream(const OutputConfiguration &outputConfiguration); + // Create an input stream of width, height, and format. + virtual status_t createInputStream(int width, int height, int format); + + // Get the buffer producer of the input stream + virtual status_t getInputBufferProducer( + /*out*/sp<IGraphicBufferProducer> *producer); + // Create a request object from a template. virtual status_t createDefaultRequest(int templateId, /*out*/ @@ -102,6 +109,9 @@ public: virtual status_t flush(/*out*/ int64_t* lastFrameNumber = NULL); + // Prepare stream by preallocating its buffers + virtual status_t prepare(int streamId); + /** * Interface used by CameraService */ @@ -128,6 +138,7 @@ public: virtual void notifyError(ICameraDeviceCallbacks::CameraErrorCode errorCode, const CaptureResultExtras& resultExtras); virtual void notifyShutter(const CaptureResultExtras& resultExtras, nsecs_t timestamp); + virtual void notifyPrepared(int streamId); /** * Interface used by independent components of CameraDeviceClient. @@ -161,10 +172,18 @@ private: android_dataspace dataSpace, const CameraMetadata& info, /*out*/int32_t* outWidth, /*out*/int32_t* outHeight); - // IGraphicsBufferProducer binder -> Stream ID + // IGraphicsBufferProducer binder -> Stream ID for output streams KeyedVector<sp<IBinder>, int> mStreamMap; - // Stream ID + struct InputStreamConfiguration { + bool configured; + int32_t width; + int32_t height; + int32_t format; + int32_t id; + } mInputStream; + + // Request ID Vector<int> mStreamingRequestList; int32_t mRequestIdCounter; diff --git a/services/camera/libcameraservice/common/Camera2ClientBase.cpp b/services/camera/libcameraservice/common/Camera2ClientBase.cpp index c0c2314..ba0b264 100644 --- a/services/camera/libcameraservice/common/Camera2ClientBase.cpp +++ b/services/camera/libcameraservice/common/Camera2ClientBase.cpp @@ -280,6 +280,14 @@ void Camera2ClientBase<TClientBase>::notifyAutoWhitebalance(uint8_t newState, } template <typename TClientBase> +void Camera2ClientBase<TClientBase>::notifyPrepared(int streamId) { + (void)streamId; + + ALOGV("%s: Stream %d now prepared", + __FUNCTION__, streamId); +} + +template <typename TClientBase> int Camera2ClientBase<TClientBase>::getCameraId() const { return TClientBase::mCameraId; } diff --git a/services/camera/libcameraservice/common/Camera2ClientBase.h b/services/camera/libcameraservice/common/Camera2ClientBase.h index 168ea0a..f1cacdf 100644 --- a/services/camera/libcameraservice/common/Camera2ClientBase.h +++ b/services/camera/libcameraservice/common/Camera2ClientBase.h @@ -72,7 +72,7 @@ public: virtual void notifyAutoExposure(uint8_t newState, int triggerId); virtual void notifyAutoWhitebalance(uint8_t newState, int triggerId); - + virtual void notifyPrepared(int streamId); int getCameraId() const; const sp<CameraDeviceBase>& diff --git a/services/camera/libcameraservice/common/CameraDeviceBase.h b/services/camera/libcameraservice/common/CameraDeviceBase.h index fe55b9e..f02fc32 100644 --- a/services/camera/libcameraservice/common/CameraDeviceBase.h +++ b/services/camera/libcameraservice/common/CameraDeviceBase.h @@ -30,6 +30,7 @@ #include "camera/CameraMetadata.h" #include "camera/CaptureResult.h" #include "common/CameraModule.h" +#include "gui/IGraphicBufferProducer.h" namespace android { @@ -110,6 +111,14 @@ class CameraDeviceBase : public virtual RefBase { android_dataspace dataSpace, camera3_stream_rotation_t rotation, int *id) = 0; /** + * Create an input stream of width, height, and format. + * + * Return value is the stream ID if non-negative and an error if negative. + */ + virtual status_t createInputStream(uint32_t width, uint32_t height, + int32_t format, /*out*/ int32_t *id) = 0; + + /** * Create an input reprocess stream that uses buffers from an existing * output stream. */ @@ -150,6 +159,10 @@ class CameraDeviceBase : public virtual RefBase { */ virtual status_t configureStreams() = 0; + // get the buffer producer of the input stream + virtual status_t getInputBufferProducer( + sp<IGraphicBufferProducer> *producer) = 0; + /** * Create a metadata buffer with fields that the HAL device believes are * best for the given use case @@ -186,6 +199,7 @@ class CameraDeviceBase : public virtual RefBase { virtual void notifyIdle() = 0; virtual void notifyShutter(const CaptureResultExtras &resultExtras, nsecs_t timestamp) = 0; + virtual void notifyPrepared(int streamId) = 0; // Required only for API1 virtual void notifyAutoFocus(uint8_t newState, int triggerId) = 0; @@ -268,6 +282,12 @@ class CameraDeviceBase : public virtual RefBase { virtual status_t flush(int64_t *lastFrameNumber = NULL) = 0; /** + * Prepare stream by preallocating buffers for it asynchronously. + * Calls notifyPrepared() once allocation is complete. + */ + virtual status_t prepare(int streamId) = 0; + + /** * Get the HAL device version. */ virtual uint32_t getDeviceVersion() = 0; diff --git a/services/camera/libcameraservice/common/CameraModule.cpp b/services/camera/libcameraservice/common/CameraModule.cpp index e5b12ae..b861d71 100644 --- a/services/camera/libcameraservice/common/CameraModule.cpp +++ b/services/camera/libcameraservice/common/CameraModule.cpp @@ -78,6 +78,12 @@ int CameraModule::getCameraInfo(int cameraId, struct camera_info *info) { if (ret != 0) { return ret; } + int deviceVersion = cameraInfo.device_version; + if (deviceVersion < CAMERA_DEVICE_API_VERSION_2_0) { + // static_camera_characteristics is invalid + *info = rawInfo; + return ret; + } CameraMetadata m; m = rawInfo.static_camera_characteristics; deriveCameraCharacteristicsKeys(rawInfo.device_version, m); diff --git a/services/camera/libcameraservice/device2/Camera2Device.cpp b/services/camera/libcameraservice/device2/Camera2Device.cpp index 878986b..f6645f3 100644 --- a/services/camera/libcameraservice/device2/Camera2Device.cpp +++ b/services/camera/libcameraservice/device2/Camera2Device.cpp @@ -618,6 +618,12 @@ status_t Camera2Device::flush(int64_t* /*lastFrameNumber*/) { return waitUntilDrained(); } +status_t Camera2Device::prepare(int streamId) { + ATRACE_CALL(); + ALOGE("%s: Camera %d: unimplemented", __FUNCTION__, mId); + return NO_INIT; +} + uint32_t Camera2Device::getDeviceVersion() { ATRACE_CALL(); return mDeviceVersion; @@ -1581,4 +1587,18 @@ int Camera2Device::ReprocessStreamAdapter::release_buffer( return OK; } +// camera 2 devices don't support reprocessing +status_t Camera2Device::createInputStream( + uint32_t width, uint32_t height, int format, int *id) { + ALOGE("%s: camera 2 devices don't support reprocessing", __FUNCTION__); + return INVALID_OPERATION; +} + +// camera 2 devices don't support reprocessing +status_t Camera2Device::getInputBufferProducer( + sp<IGraphicBufferProducer> *producer) { + ALOGE("%s: camera 2 devices don't support reprocessing", __FUNCTION__); + return INVALID_OPERATION; +} + }; // namespace android diff --git a/services/camera/libcameraservice/device2/Camera2Device.h b/services/camera/libcameraservice/device2/Camera2Device.h index 9b32fa6..fd1240a 100644 --- a/services/camera/libcameraservice/device2/Camera2Device.h +++ b/services/camera/libcameraservice/device2/Camera2Device.h @@ -59,6 +59,8 @@ class Camera2Device: public CameraDeviceBase { virtual status_t createStream(sp<ANativeWindow> consumer, uint32_t width, uint32_t height, int format, android_dataspace dataSpace, camera3_stream_rotation_t rotation, int *id); + virtual status_t createInputStream( + uint32_t width, uint32_t height, int format, int *id); virtual status_t createReprocessStreamFromStream(int outputId, int *id); virtual status_t getStreamInfo(int id, uint32_t *width, uint32_t *height, uint32_t *format); @@ -67,6 +69,8 @@ class Camera2Device: public CameraDeviceBase { virtual status_t deleteReprocessStream(int id); // No-op on HAL2 devices virtual status_t configureStreams(); + virtual status_t getInputBufferProducer( + sp<IGraphicBufferProducer> *producer); virtual status_t createDefaultRequest(int templateId, CameraMetadata *request); virtual status_t waitUntilDrained(); virtual status_t setNotifyCallback(NotificationListener *listener); @@ -80,6 +84,9 @@ class Camera2Device: public CameraDeviceBase { buffer_handle_t *buffer, wp<BufferReleasedListener> listener); // Flush implemented as just a wait virtual status_t flush(int64_t *lastFrameNumber = NULL); + // Prepare is a no-op + virtual status_t prepare(int streamId); + virtual uint32_t getDeviceVersion(); virtual ssize_t getJpegBufferSize(uint32_t width, uint32_t height) const; diff --git a/services/camera/libcameraservice/device3/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp index 8236788..d2c2482 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.cpp +++ b/services/camera/libcameraservice/device3/Camera3Device.cpp @@ -62,6 +62,7 @@ Camera3Device::Camera3Device(int id): mUsePartialResult(false), mNumPartialResults(1), mNextResultFrameNumber(0), + mNextReprocessResultFrameNumber(0), mNextShutterFrameNumber(0), mListener(NULL) { @@ -174,6 +175,8 @@ status_t Camera3Device::initialize(CameraModule *module) return res; } + mPreparerThread = new PreparerThread(); + /** Everything is good to go */ mDeviceVersion = device->common.version; @@ -201,6 +204,17 @@ status_t Camera3Device::initialize(CameraModule *module) } } + camera_metadata_entry configs = + mDeviceInfo.find(ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS); + for (uint32_t i = 0; i < configs.count; i += 4) { + if (configs.data.i32[i] == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED && + configs.data.i32[i + 3] == + ANDROID_SCALER_AVAILABLE_STREAM_CONFIGURATIONS_INPUT) { + mSupportedOpaqueInputSizes.add(Size(configs.data.i32[i + 1], + configs.data.i32[i + 2])); + } + } + return OK; } @@ -1019,6 +1033,20 @@ status_t Camera3Device::configureStreams() { return configureStreamsLocked(); } +status_t Camera3Device::getInputBufferProducer( + sp<IGraphicBufferProducer> *producer) { + Mutex::Autolock il(mInterfaceLock); + Mutex::Autolock l(mLock); + + if (producer == NULL) { + return BAD_VALUE; + } else if (mInputStream == NULL) { + return INVALID_OPERATION; + } + + return mInputStream->getInputBufferProducer(producer); +} + status_t Camera3Device::createDefaultRequest(int templateId, CameraMetadata *request) { ATRACE_CALL(); @@ -1054,9 +1082,9 @@ status_t Camera3Device::createDefaultRequest(int templateId, mHal3Device, templateId); ATRACE_END(); if (rawRequest == NULL) { - SET_ERR_L("HAL is unable to construct default settings for template %d", - templateId); - return DEAD_OBJECT; + ALOGI("%s: template %d is not supported on this camera device", + __FUNCTION__, templateId); + return BAD_VALUE; } *request = rawRequest; mRequestTemplateCache[templateId] = rawRequest; @@ -1164,7 +1192,8 @@ status_t Camera3Device::setNotifyCallback(NotificationListener *listener) { ALOGW("%s: Replacing old callback listener", __FUNCTION__); } mListener = listener; - mRequestThread->setNotifyCallback(listener); + mRequestThread->setNotificationListener(listener); + mPreparerThread->setNotificationListener(listener); return OK; } @@ -1310,6 +1339,34 @@ status_t Camera3Device::flush(int64_t *frameNumber) { return res; } +status_t Camera3Device::prepare(int streamId) { + ATRACE_CALL(); + ALOGV("%s: Camera %d: Preparing stream %d", __FUNCTION__, mId, streamId); + + sp<Camera3StreamInterface> stream; + ssize_t outputStreamIdx = mOutputStreams.indexOfKey(streamId); + if (outputStreamIdx == NAME_NOT_FOUND) { + CLOGE("Stream %d does not exist", streamId); + return BAD_VALUE; + } + + stream = mOutputStreams.editValueAt(outputStreamIdx); + + if (stream->isUnpreparable() || stream->hasOutstandingBuffers() ) { + ALOGE("%s: Camera %d: Stream %d has already been a request target", + __FUNCTION__, mId, streamId); + return BAD_VALUE; + } + + if (mRequestThread->isStreamPending(stream)) { + ALOGE("%s: Camera %d: Stream %d is already a target in a pending request", + __FUNCTION__, mId, streamId); + return BAD_VALUE; + } + + return mPreparerThread->prepare(stream); +} + uint32_t Camera3Device::getDeviceVersion() { ATRACE_CALL(); Mutex::Autolock il(mInterfaceLock); @@ -1383,6 +1440,11 @@ sp<Camera3Device::CaptureRequest> Camera3Device::createCaptureRequest( return NULL; } } + // Check if stream is being prepared + if (mInputStream->isPreparing()) { + CLOGE("Request references an input stream that's being prepared!"); + return NULL; + } newRequest->mInputStream = mInputStream; newRequest->mSettings.erase(ANDROID_REQUEST_INPUT_STREAMS); @@ -1415,6 +1477,11 @@ sp<Camera3Device::CaptureRequest> Camera3Device::createCaptureRequest( return NULL; } } + // Check if stream is being prepared + if (stream->isPreparing()) { + CLOGE("Request references an output stream that's being prepared!"); + return NULL; + } newRequest->mOutputStreams.push(stream); } @@ -1423,6 +1490,17 @@ sp<Camera3Device::CaptureRequest> Camera3Device::createCaptureRequest( return newRequest; } +bool Camera3Device::isOpaqueInputSizeSupported(uint32_t width, uint32_t height) { + for (uint32_t i = 0; i < mSupportedOpaqueInputSizes.size(); i++) { + Size size = mSupportedOpaqueInputSizes[i]; + if (size.width == width && size.height == height) { + return true; + } + } + + return false; +} + status_t Camera3Device::configureStreamsLocked() { ATRACE_CALL(); status_t res; @@ -1879,7 +1957,6 @@ bool Camera3Device::insert3AResult(CameraMetadata& result, int32_t tag, return true; } - void Camera3Device::returnOutputBuffers( const camera3_stream_buffer_t *outputBuffers, size_t numBuffers, nsecs_t timestamp) { @@ -1947,20 +2024,31 @@ void Camera3Device::removeInFlightRequestIfReadyLocked(int idx) { void Camera3Device::sendCaptureResult(CameraMetadata &pendingMetadata, CaptureResultExtras &resultExtras, CameraMetadata &collectedPartialResult, - uint32_t frameNumber) { + uint32_t frameNumber, + bool reprocess) { if (pendingMetadata.isEmpty()) return; Mutex::Autolock l(mOutputLock); // TODO: need to track errors for tighter bounds on expected frame number - if (frameNumber < mNextResultFrameNumber) { - SET_ERR("Out-of-order capture result metadata submitted! " + if (reprocess) { + if (frameNumber < mNextReprocessResultFrameNumber) { + SET_ERR("Out-of-order reprocess capture result metadata submitted! " "(got frame number %d, expecting %d)", - frameNumber, mNextResultFrameNumber); - return; + frameNumber, mNextReprocessResultFrameNumber); + return; + } + mNextReprocessResultFrameNumber = frameNumber + 1; + } else { + if (frameNumber < mNextResultFrameNumber) { + SET_ERR("Out-of-order capture result metadata submitted! " + "(got frame number %d, expecting %d)", + frameNumber, mNextResultFrameNumber); + return; + } + mNextResultFrameNumber = frameNumber + 1; } - mNextResultFrameNumber = frameNumber + 1; CaptureResult captureResult; captureResult.mResultExtras = resultExtras; @@ -2170,7 +2258,7 @@ void Camera3Device::processCaptureResult(const camera3_capture_result *result) { CameraMetadata metadata; metadata = result->result; sendCaptureResult(metadata, request.resultExtras, - collectedPartialResult, frameNumber); + collectedPartialResult, frameNumber, hasInputBufferInRequest); } } @@ -2332,7 +2420,8 @@ void Camera3Device::notifyShutter(const camera3_shutter_msg_t &msg, // send pending result and buffers sendCaptureResult(r.pendingMetadata, r.resultExtras, - r.partialResult.collectedResult, msg.frame_number); + r.partialResult.collectedResult, msg.frame_number, + r.hasInputBuffer); returnOutputBuffers(r.pendingOutputBuffers.array(), r.pendingOutputBuffers.size(), r.shutterTimestamp); r.pendingOutputBuffers.clear(); @@ -2367,7 +2456,7 @@ CameraMetadata Camera3Device::getLatestRequestLocked() { Camera3Device::RequestThread::RequestThread(wp<Camera3Device> parent, sp<StatusTracker> statusTracker, camera3_device_t *hal3Device) : - Thread(false), + Thread(/*canCallJava*/false), mParent(parent), mStatusTracker(statusTracker), mHal3Device(hal3Device), @@ -2383,7 +2472,7 @@ Camera3Device::RequestThread::RequestThread(wp<Camera3Device> parent, mStatusId = statusTracker->addComponent(); } -void Camera3Device::RequestThread::setNotifyCallback( +void Camera3Device::RequestThread::setNotificationListener( NotificationListener *listener) { Mutex::Autolock l(mRequestLock); mListener = listener; @@ -2669,7 +2758,6 @@ bool Camera3Device::RequestThread::threadLoop() { // Fill in buffers if (nextRequest->mInputStream != NULL) { - request.input_buffer = &inputBuffer; res = nextRequest->mInputStream->getInputBuffer(&inputBuffer); if (res != OK) { // Can't get input buffer from gralloc queue - this could be due to @@ -2686,6 +2774,7 @@ bool Camera3Device::RequestThread::threadLoop() { cleanUpFailedRequest(request, nextRequest, outputBuffers); return true; } + request.input_buffer = &inputBuffer; totalNumBuffers += 1; } else { request.input_buffer = NULL; @@ -2797,6 +2886,26 @@ CameraMetadata Camera3Device::RequestThread::getLatestRequest() const { return mLatestRequest; } +bool Camera3Device::RequestThread::isStreamPending( + sp<Camera3StreamInterface>& stream) { + Mutex::Autolock l(mRequestLock); + + for (const auto& request : mRequestQueue) { + for (const auto& s : request->mOutputStreams) { + if (stream == s) return true; + } + if (stream == request->mInputStream) return true; + } + + for (const auto& request : mRepeatingRequests) { + for (const auto& s : request->mOutputStreams) { + if (stream == s) return true; + } + if (stream == request->mInputStream) return true; + } + + return false; +} void Camera3Device::RequestThread::cleanUpFailedRequest( camera3_capture_request_t &request, @@ -3144,6 +3253,138 @@ status_t Camera3Device::RequestThread::addDummyTriggerIds( return OK; } +/** + * PreparerThread inner class methods + */ + +Camera3Device::PreparerThread::PreparerThread() : + Thread(/*canCallJava*/false), mActive(false), mCancelNow(false) { +} + +Camera3Device::PreparerThread::~PreparerThread() { + Thread::requestExitAndWait(); + if (mCurrentStream != nullptr) { + mCurrentStream->cancelPrepare(); + ATRACE_ASYNC_END("stream prepare", mCurrentStream->getId()); + mCurrentStream.clear(); + } + clear(); +} + +status_t Camera3Device::PreparerThread::prepare(sp<Camera3StreamInterface>& stream) { + status_t res; + + Mutex::Autolock l(mLock); + + res = stream->startPrepare(); + if (res == OK) { + // No preparation needed, fire listener right off + ALOGV("%s: Stream %d already prepared", __FUNCTION__, stream->getId()); + if (mListener) { + mListener->notifyPrepared(stream->getId()); + } + return OK; + } else if (res != NOT_ENOUGH_DATA) { + return res; + } + + // Need to prepare, start up thread if necessary + if (!mActive) { + // mRunning will change to false before the thread fully shuts down, so wait to be sure it + // isn't running + Thread::requestExitAndWait(); + res = Thread::run("C3PrepThread", PRIORITY_BACKGROUND); + if (res != OK) { + ALOGE("%s: Unable to start preparer stream: %d (%s)", __FUNCTION__, res, strerror(-res)); + if (mListener) { + mListener->notifyPrepared(stream->getId()); + } + return res; + } + mCancelNow = false; + mActive = true; + ALOGV("%s: Preparer stream started", __FUNCTION__); + } + + // queue up the work + mPendingStreams.push_back(stream); + ALOGV("%s: Stream %d queued for preparing", __FUNCTION__, stream->getId()); + + return OK; +} + +status_t Camera3Device::PreparerThread::clear() { + status_t res; + + Mutex::Autolock l(mLock); + + for (const auto& stream : mPendingStreams) { + stream->cancelPrepare(); + } + mPendingStreams.clear(); + mCancelNow = true; + + return OK; +} + +void Camera3Device::PreparerThread::setNotificationListener(NotificationListener *listener) { + Mutex::Autolock l(mLock); + mListener = listener; +} + +bool Camera3Device::PreparerThread::threadLoop() { + status_t res; + { + Mutex::Autolock l(mLock); + if (mCurrentStream == nullptr) { + // End thread if done with work + if (mPendingStreams.empty()) { + ALOGV("%s: Preparer stream out of work", __FUNCTION__); + // threadLoop _must not_ re-acquire mLock after it sets mActive to false; would + // cause deadlock with prepare()'s requestExitAndWait triggered by !mActive. + mActive = false; + return false; + } + + // Get next stream to prepare + auto it = mPendingStreams.begin(); + mCurrentStream = *it; + mPendingStreams.erase(it); + ATRACE_ASYNC_BEGIN("stream prepare", mCurrentStream->getId()); + ALOGV("%s: Preparing stream %d", __FUNCTION__, mCurrentStream->getId()); + } else if (mCancelNow) { + mCurrentStream->cancelPrepare(); + ATRACE_ASYNC_END("stream prepare", mCurrentStream->getId()); + ALOGV("%s: Cancelling stream %d prepare", __FUNCTION__, mCurrentStream->getId()); + mCurrentStream.clear(); + mCancelNow = false; + return true; + } + } + + res = mCurrentStream->prepareNextBuffer(); + if (res == NOT_ENOUGH_DATA) return true; + if (res != OK) { + // Something bad happened; try to recover by cancelling prepare and + // signalling listener anyway + ALOGE("%s: Stream %d returned error %d (%s) during prepare", __FUNCTION__, + mCurrentStream->getId(), res, strerror(-res)); + mCurrentStream->cancelPrepare(); + } + + // This stream has finished, notify listener + Mutex::Autolock l(mLock); + if (mListener) { + ALOGV("%s: Stream %d prepare done, signaling listener", __FUNCTION__, + mCurrentStream->getId()); + mListener->notifyPrepared(mCurrentStream->getId()); + } + + ATRACE_ASYNC_END("stream prepare", mCurrentStream->getId()); + mCurrentStream.clear(); + + return true; +} /** * Static callback forwarding methods from HAL to instance diff --git a/services/camera/libcameraservice/device3/Camera3Device.h b/services/camera/libcameraservice/device3/Camera3Device.h index a77548d..4fbcb2e 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.h +++ b/services/camera/libcameraservice/device3/Camera3Device.h @@ -116,6 +116,8 @@ class Camera3Device : virtual status_t deleteReprocessStream(int id); virtual status_t configureStreams(); + virtual status_t getInputBufferProducer( + sp<IGraphicBufferProducer> *producer); virtual status_t createDefaultRequest(int templateId, CameraMetadata *request); @@ -136,6 +138,8 @@ class Camera3Device : virtual status_t flush(int64_t *lastFrameNumber = NULL); + virtual status_t prepare(int streamId); + virtual uint32_t getDeviceVersion(); virtual ssize_t getJpegBufferSize(uint32_t width, uint32_t height) const; @@ -179,6 +183,14 @@ class Camera3Device : uint32_t mDeviceVersion; + struct Size { + uint32_t width; + uint32_t height; + Size(uint32_t w = 0, uint32_t h = 0) : width(w), height(h){} + }; + // Map from format to size. + Vector<Size> mSupportedOpaqueInputSizes; + enum Status { STATUS_ERROR, STATUS_UNINITIALIZED, @@ -324,11 +336,11 @@ class Camera3Device : */ bool tryLockSpinRightRound(Mutex& lock); - struct Size { - int width; - int height; - Size(int w, int h) : width(w), height(h){} - }; + /** + * Helper function to determine if an input size for implementation defined + * format is supported. + */ + bool isOpaqueInputSizeSupported(uint32_t width, uint32_t height); /** * Helper function to get the largest Jpeg resolution (in area) @@ -364,7 +376,7 @@ class Camera3Device : sp<camera3::StatusTracker> statusTracker, camera3_device_t *hal3Device); - void setNotifyCallback(NotificationListener *listener); + void setNotificationListener(NotificationListener *listener); /** * Call after stream (re)-configuration is completed. @@ -428,6 +440,12 @@ class Camera3Device : */ CameraMetadata getLatestRequest() const; + /** + * Returns true if the stream is a target of any queued or repeating + * capture request + */ + bool isStreamPending(sp<camera3::Camera3StreamInterface>& stream); + protected: virtual bool threadLoop(); @@ -549,7 +567,6 @@ class Camera3Device : Vector<camera3_stream_buffer_t> pendingOutputBuffers; - // Fields used by the partial result only struct PartialResultInFlight { // Set by process_capture_result once 3A has been sent to clients @@ -600,7 +617,8 @@ class Camera3Device : resultExtras(extras), hasInputBuffer(hasInput){ } -}; + }; + // Map from frame number to the in-flight request state typedef KeyedVector<uint32_t, InFlightRequest> InFlightMap; @@ -632,6 +650,45 @@ class Camera3Device : sp<camera3::StatusTracker> mStatusTracker; /** + * Thread for preparing streams + */ + class PreparerThread : private Thread, public virtual RefBase { + public: + PreparerThread(); + ~PreparerThread(); + + void setNotificationListener(NotificationListener *listener); + + /** + * Queue up a stream to be prepared. Streams are processed by + * a background thread in FIFO order + */ + status_t prepare(sp<camera3::Camera3StreamInterface>& stream); + + /** + * Cancel all current and pending stream preparation + */ + status_t clear(); + + private: + Mutex mLock; + + virtual bool threadLoop(); + + // Guarded by mLock + + NotificationListener *mListener; + List<sp<camera3::Camera3StreamInterface> > mPendingStreams; + bool mActive; + bool mCancelNow; + + // Only accessed by threadLoop and the destructor + + sp<camera3::Camera3StreamInterface> mCurrentStream; + }; + sp<PreparerThread> mPreparerThread; + + /** * Output result queue and current HAL device 3A state */ @@ -639,8 +696,10 @@ class Camera3Device : Mutex mOutputLock; /**** Scope for mOutputLock ****/ - + // the minimal frame number of the next non-reprocess result uint32_t mNextResultFrameNumber; + // the minimal frame number of the next reprocess result + uint32_t mNextReprocessResultFrameNumber; uint32_t mNextShutterFrameNumber; List<CaptureResult> mResultQueue; Condition mResultSignal; @@ -669,7 +728,8 @@ class Camera3Device : // partial results, and the frame number to the result queue. void sendCaptureResult(CameraMetadata &pendingMetadata, CaptureResultExtras &resultExtras, - CameraMetadata &collectedPartialResult, uint32_t frameNumber); + CameraMetadata &collectedPartialResult, uint32_t frameNumber, + bool reprocess); /**** Scope for mInFlightLock ****/ diff --git a/services/camera/libcameraservice/device3/Camera3DummyStream.cpp b/services/camera/libcameraservice/device3/Camera3DummyStream.cpp index 01edfff..ecb8ac8 100644 --- a/services/camera/libcameraservice/device3/Camera3DummyStream.cpp +++ b/services/camera/libcameraservice/device3/Camera3DummyStream.cpp @@ -87,7 +87,7 @@ status_t Camera3DummyStream::disconnectLocked() { return OK; } -status_t Camera3DummyStream::getEndpointUsage(uint32_t *usage) { +status_t Camera3DummyStream::getEndpointUsage(uint32_t *usage) const { *usage = DUMMY_USAGE; return OK; } diff --git a/services/camera/libcameraservice/device3/Camera3DummyStream.h b/services/camera/libcameraservice/device3/Camera3DummyStream.h index d023c57..3a3dbf4 100644 --- a/services/camera/libcameraservice/device3/Camera3DummyStream.h +++ b/services/camera/libcameraservice/device3/Camera3DummyStream.h @@ -89,7 +89,7 @@ class Camera3DummyStream : virtual status_t configureQueueLocked(); - virtual status_t getEndpointUsage(uint32_t *usage); + virtual status_t getEndpointUsage(uint32_t *usage) const; }; // class Camera3DummyStream diff --git a/services/camera/libcameraservice/device3/Camera3IOStreamBase.cpp b/services/camera/libcameraservice/device3/Camera3IOStreamBase.cpp index 8696413..23b1c45 100644 --- a/services/camera/libcameraservice/device3/Camera3IOStreamBase.cpp +++ b/services/camera/libcameraservice/device3/Camera3IOStreamBase.cpp @@ -67,13 +67,18 @@ bool Camera3IOStreamBase::hasOutstandingBuffersLocked() const { void Camera3IOStreamBase::dump(int fd, const Vector<String16> &args) const { (void) args; String8 lines; + + uint32_t consumerUsage = 0; + status_t res = getEndpointUsage(&consumerUsage); + if (res != OK) consumerUsage = 0; + lines.appendFormat(" State: %d\n", mState); - lines.appendFormat(" Dims: %d x %d, format 0x%x\n", + lines.appendFormat(" Dims: %d x %d, format 0x%x, dataspace 0x%x\n", camera3_stream::width, camera3_stream::height, - camera3_stream::format); + camera3_stream::format, camera3_stream::data_space); lines.appendFormat(" Max size: %zu\n", mMaxSize); - lines.appendFormat(" Usage: %d, max HAL buffers: %d\n", - camera3_stream::usage, camera3_stream::max_buffers); + lines.appendFormat(" Combined usage: %d, max HAL buffers: %d\n", + camera3_stream::usage | consumerUsage, camera3_stream::max_buffers); lines.appendFormat(" Frames produced: %d, last timestamp: %" PRId64 " ns\n", mFrameCount, mLastTimestamp); lines.appendFormat(" Total buffers: %zu, currently dequeued: %zu\n", @@ -156,13 +161,11 @@ void Camera3IOStreamBase::handoutBufferLocked(camera3_stream_buffer &buffer, // Inform tracker about becoming busy if (mHandoutTotalBufferCount == 0 && mState != STATE_IN_CONFIG && - mState != STATE_IN_RECONFIG) { + mState != STATE_IN_RECONFIG && mState != STATE_PREPARING) { /** * Avoid a spurious IDLE->ACTIVE->IDLE transition when using buffers * before/after register_stream_buffers during initial configuration - * or re-configuration. - * - * TODO: IN_CONFIG and IN_RECONFIG checks only make sense for <HAL3.2 + * or re-configuration, or during prepare pre-allocation */ sp<StatusTracker> statusTracker = mStatusTracker.promote(); if (statusTracker != 0) { @@ -177,9 +180,11 @@ void Camera3IOStreamBase::handoutBufferLocked(camera3_stream_buffer &buffer, } status_t Camera3IOStreamBase::getBufferPreconditionCheckLocked() const { - // Allow dequeue during IN_[RE]CONFIG for registration + // Allow dequeue during IN_[RE]CONFIG for registration, in + // PREPARING for pre-allocation if (mState != STATE_CONFIGURED && - mState != STATE_IN_CONFIG && mState != STATE_IN_RECONFIG) { + mState != STATE_IN_CONFIG && mState != STATE_IN_RECONFIG && + mState != STATE_PREPARING) { ALOGE("%s: Stream %d: Can't get buffers in unconfigured state %d", __FUNCTION__, mId, mState); return INVALID_OPERATION; @@ -240,13 +245,11 @@ status_t Camera3IOStreamBase::returnAnyBufferLocked( mHandoutTotalBufferCount--; if (mHandoutTotalBufferCount == 0 && mState != STATE_IN_CONFIG && - mState != STATE_IN_RECONFIG) { + mState != STATE_IN_RECONFIG && mState != STATE_PREPARING) { /** * Avoid a spurious IDLE->ACTIVE->IDLE transition when using buffers * before/after register_stream_buffers during initial configuration - * or re-configuration. - * - * TODO: IN_CONFIG and IN_RECONFIG checks only make sense for <HAL3.2 + * or re-configuration, or during prepare pre-allocation */ ALOGV("%s: Stream %d: All buffers returned; now idle", __FUNCTION__, mId); diff --git a/services/camera/libcameraservice/device3/Camera3IOStreamBase.h b/services/camera/libcameraservice/device3/Camera3IOStreamBase.h index abcf2b1..f5727e8 100644 --- a/services/camera/libcameraservice/device3/Camera3IOStreamBase.h +++ b/services/camera/libcameraservice/device3/Camera3IOStreamBase.h @@ -84,7 +84,7 @@ class Camera3IOStreamBase : virtual size_t getHandoutInputBufferCountLocked(); - virtual status_t getEndpointUsage(uint32_t *usage) = 0; + virtual status_t getEndpointUsage(uint32_t *usage) const = 0; status_t getBufferPreconditionCheckLocked() const; status_t returnBufferPreconditionCheckLocked() const; diff --git a/services/camera/libcameraservice/device3/Camera3InputStream.cpp b/services/camera/libcameraservice/device3/Camera3InputStream.cpp index 6bf671e..84c5754 100644 --- a/services/camera/libcameraservice/device3/Camera3InputStream.cpp +++ b/services/camera/libcameraservice/device3/Camera3InputStream.cpp @@ -65,8 +65,8 @@ status_t Camera3InputStream::getInputBufferLocked( assert(mConsumer != 0); BufferItem bufferItem; - res = mConsumer->acquireBuffer(&bufferItem, /*waitForFence*/false); + res = mConsumer->acquireBuffer(&bufferItem, /*waitForFence*/false); if (res != OK) { ALOGE("%s: Stream %d: Can't acquire next output buffer: %s (%d)", __FUNCTION__, mId, strerror(-res), res); @@ -162,6 +162,21 @@ status_t Camera3InputStream::returnInputBufferLocked( return returnAnyBufferLocked(buffer, /*timestamp*/0, /*output*/false); } +status_t Camera3InputStream::getInputBufferProducerLocked( + sp<IGraphicBufferProducer> *producer) { + ATRACE_CALL(); + + if (producer == NULL) { + return BAD_VALUE; + } else if (mProducer == NULL) { + ALOGE("%s: No input stream is configured"); + return INVALID_OPERATION; + } + + *producer = mProducer; + return OK; +} + status_t Camera3InputStream::disconnectLocked() { status_t res; @@ -212,10 +227,17 @@ status_t Camera3InputStream::configureQueueLocked() { res = producer->query(NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBuffers); if (res != OK || minUndequeuedBuffers < 0) { ALOGE("%s: Stream %d: Could not query min undequeued buffers (error %d, bufCount %d)", - __FUNCTION__, mId, res, minUndequeuedBuffers); + __FUNCTION__, mId, res, minUndequeuedBuffers); return res; } size_t minBufs = static_cast<size_t>(minUndequeuedBuffers); + + if (camera3_stream::max_buffers == 0) { + ALOGE("%s: %d: HAL sets max_buffer to 0. Must be at least 1.", + __FUNCTION__, __LINE__); + return INVALID_OPERATION; + } + /* * We promise never to 'acquire' more than camera3_stream::max_buffers * at any one time. @@ -232,6 +254,8 @@ status_t Camera3InputStream::configureQueueLocked() { mConsumer = new BufferItemConsumer(consumer, camera3_stream::usage, mTotalBufferCount); mConsumer->setName(String8::format("Camera3-InputStream-%d", mId)); + + mProducer = producer; } res = mConsumer->setDefaultBufferSize(camera3_stream::width, @@ -251,7 +275,7 @@ status_t Camera3InputStream::configureQueueLocked() { return OK; } -status_t Camera3InputStream::getEndpointUsage(uint32_t *usage) { +status_t Camera3InputStream::getEndpointUsage(uint32_t *usage) const { // Per HAL3 spec, input streams have 0 for their initial usage field. *usage = 0; return OK; diff --git a/services/camera/libcameraservice/device3/Camera3InputStream.h b/services/camera/libcameraservice/device3/Camera3InputStream.h index fd17f4f..9f3de10 100644 --- a/services/camera/libcameraservice/device3/Camera3InputStream.h +++ b/services/camera/libcameraservice/device3/Camera3InputStream.h @@ -49,6 +49,7 @@ class Camera3InputStream : public Camera3IOStreamBase { private: sp<BufferItemConsumer> mConsumer; + sp<IGraphicBufferProducer> mProducer; Vector<BufferItem> mBuffersInFlight; /** @@ -68,11 +69,13 @@ class Camera3InputStream : public Camera3IOStreamBase { virtual status_t getInputBufferLocked(camera3_stream_buffer *buffer); virtual status_t returnInputBufferLocked( const camera3_stream_buffer &buffer); + virtual status_t getInputBufferProducerLocked( + sp<IGraphicBufferProducer> *producer); virtual status_t disconnectLocked(); virtual status_t configureQueueLocked(); - virtual status_t getEndpointUsage(uint32_t *usage); + virtual status_t getEndpointUsage(uint32_t *usage) const; }; // class Camera3InputStream diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp index 0c739e9..7a0331b 100644 --- a/services/camera/libcameraservice/device3/Camera3OutputStream.cpp +++ b/services/camera/libcameraservice/device3/Camera3OutputStream.cpp @@ -209,6 +209,13 @@ status_t Camera3OutputStream::returnBufferCheckedLocked( } } mLock.lock(); + + // Once a valid buffer has been returned to the queue, can no longer + // dequeue all buffers for preallocation. + if (buffer.status != CAMERA3_BUFFER_STATUS_ERROR) { + mStreamUnpreparable = true; + } + if (res != OK) { close(anwReleaseFence); } @@ -390,14 +397,28 @@ status_t Camera3OutputStream::disconnectLocked() { return OK; } -status_t Camera3OutputStream::getEndpointUsage(uint32_t *usage) { +status_t Camera3OutputStream::getEndpointUsage(uint32_t *usage) const { status_t res; int32_t u = 0; res = mConsumer->query(mConsumer.get(), NATIVE_WINDOW_CONSUMER_USAGE_BITS, &u); - *usage = u; + // If an opaque output stream's endpoint is ImageReader, add + // GRALLOC_USAGE_HW_CAMERA_ZSL to the usage so HAL knows it will be used + // for the ZSL use case. + // Assume it's for ImageReader if the consumer usage doesn't have any of these bits set: + // 1. GRALLOC_USAGE_HW_TEXTURE + // 2. GRALLOC_USAGE_HW_RENDER + // 3. GRALLOC_USAGE_HW_COMPOSER + // 4. GRALLOC_USAGE_HW_VIDEO_ENCODER + if (camera3_stream::format == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED && + (u & (GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_HW_RENDER | GRALLOC_USAGE_HW_COMPOSER | + GRALLOC_USAGE_HW_VIDEO_ENCODER)) == 0) { + u |= GRALLOC_USAGE_HW_CAMERA_ZSL; + } + + *usage = u; return res; } diff --git a/services/camera/libcameraservice/device3/Camera3OutputStream.h b/services/camera/libcameraservice/device3/Camera3OutputStream.h index 12b2ebb..513b695 100644 --- a/services/camera/libcameraservice/device3/Camera3OutputStream.h +++ b/services/camera/libcameraservice/device3/Camera3OutputStream.h @@ -99,7 +99,7 @@ class Camera3OutputStream : virtual status_t configureQueueLocked(); - virtual status_t getEndpointUsage(uint32_t *usage); + virtual status_t getEndpointUsage(uint32_t *usage) const; }; // class Camera3OutputStream diff --git a/services/camera/libcameraservice/device3/Camera3Stream.cpp b/services/camera/libcameraservice/device3/Camera3Stream.cpp index 4acbce3..3821da1 100644 --- a/services/camera/libcameraservice/device3/Camera3Stream.cpp +++ b/services/camera/libcameraservice/device3/Camera3Stream.cpp @@ -194,6 +194,11 @@ status_t Camera3Stream::finishConfiguration(camera3_device *hal3Device) { return OK; } + // Reset prepared state, since buffer config has changed, and existing + // allocations are no longer valid + mPrepared = false; + mStreamUnpreparable = false; + status_t res; res = configureQueueLocked(); if (res != OK) { @@ -244,6 +249,125 @@ status_t Camera3Stream::cancelConfiguration() { return OK; } +bool Camera3Stream::isUnpreparable() { + ATRACE_CALL(); + + Mutex::Autolock l(mLock); + return mStreamUnpreparable; +} + +status_t Camera3Stream::startPrepare() { + ATRACE_CALL(); + + Mutex::Autolock l(mLock); + status_t res = OK; + + // This function should be only called when the stream is configured already. + if (mState != STATE_CONFIGURED) { + ALOGE("%s: Stream %d: Can't prepare stream if stream is not in CONFIGURED " + "state %d", __FUNCTION__, mId, mState); + return INVALID_OPERATION; + } + + // This function can't be called if the stream has already received filled + // buffers + if (mStreamUnpreparable) { + ALOGE("%s: Stream %d: Can't prepare stream that's already in use", + __FUNCTION__, mId); + return INVALID_OPERATION; + } + + if (getHandoutOutputBufferCountLocked() > 0) { + ALOGE("%s: Stream %d: Can't prepare stream that has outstanding buffers", + __FUNCTION__, mId); + return INVALID_OPERATION; + } + + if (mPrepared) return OK; + + size_t bufferCount = getBufferCountLocked(); + + mPreparedBuffers.insertAt(camera3_stream_buffer_t(), /*index*/0, bufferCount); + mPreparedBufferIdx = 0; + + mState = STATE_PREPARING; + + return NOT_ENOUGH_DATA; +} + +bool Camera3Stream::isPreparing() const { + Mutex::Autolock l(mLock); + return mState == STATE_PREPARING; +} + +status_t Camera3Stream::prepareNextBuffer() { + ATRACE_CALL(); + + Mutex::Autolock l(mLock); + status_t res = OK; + + // This function should be only called when the stream is preparing + if (mState != STATE_PREPARING) { + ALOGE("%s: Stream %d: Can't prepare buffer if stream is not in PREPARING " + "state %d", __FUNCTION__, mId, mState); + return INVALID_OPERATION; + } + + // Get next buffer - this may allocate, and take a while for large buffers + res = getBufferLocked( &mPreparedBuffers.editItemAt(mPreparedBufferIdx) ); + if (res != OK) { + ALOGE("%s: Stream %d: Unable to allocate buffer %d during preparation", + __FUNCTION__, mId, mPreparedBufferIdx); + return NO_INIT; + } + + mPreparedBufferIdx++; + + // Check if we still have buffers left to allocate + if (mPreparedBufferIdx < mPreparedBuffers.size()) { + return NOT_ENOUGH_DATA; + } + + // Done with prepare - mark stream as such, and return all buffers + // via cancelPrepare + mPrepared = true; + + return cancelPrepareLocked(); +} + +status_t Camera3Stream::cancelPrepare() { + ATRACE_CALL(); + + Mutex::Autolock l(mLock); + + return cancelPrepareLocked(); +} + +status_t Camera3Stream::cancelPrepareLocked() { + status_t res = OK; + + // This function should be only called when the stream is mid-preparing. + if (mState != STATE_PREPARING) { + ALOGE("%s: Stream %d: Can't cancel prepare stream if stream is not in " + "PREPARING state %d", __FUNCTION__, mId, mState); + return INVALID_OPERATION; + } + + // Return all valid buffers to stream, in ERROR state to indicate + // they weren't filled. + for (size_t i = 0; i < mPreparedBufferIdx; i++) { + mPreparedBuffers.editItemAt(i).release_fence = -1; + mPreparedBuffers.editItemAt(i).status = CAMERA3_BUFFER_STATUS_ERROR; + returnBufferLocked(mPreparedBuffers[i], 0); + } + mPreparedBuffers.clear(); + mPreparedBufferIdx = 0; + + mState = STATE_CONFIGURED; + + return res; +} + status_t Camera3Stream::getBuffer(camera3_stream_buffer *buffer) { ATRACE_CALL(); Mutex::Autolock l(mLock); @@ -346,6 +470,13 @@ status_t Camera3Stream::returnInputBuffer(const camera3_stream_buffer &buffer) { return res; } +status_t Camera3Stream::getInputBufferProducer(sp<IGraphicBufferProducer> *producer) { + ATRACE_CALL(); + Mutex::Autolock l(mLock); + + return getInputBufferProducerLocked(producer); +} + void Camera3Stream::fireBufferListenersLocked( const camera3_stream_buffer& /*buffer*/, bool acquired, bool output) { List<wp<Camera3StreamBufferListener> >::iterator it, end; @@ -420,15 +551,13 @@ status_t Camera3Stream::registerBuffersLocked(camera3_device *hal3Device) { ALOGE("%s: register_stream_buffers is deprecated in HAL3.2; " "must be set to NULL in camera3_device::ops", __FUNCTION__); return INVALID_OPERATION; - } else { - ALOGD("%s: Skipping NULL check for deprecated register_stream_buffers", __FUNCTION__); } return OK; - } else { - ALOGV("%s: register_stream_buffers using deprecated code path", __FUNCTION__); } + ALOGV("%s: register_stream_buffers using deprecated code path", __FUNCTION__); + status_t res; size_t bufferCount = getBufferCountLocked(); @@ -484,6 +613,8 @@ status_t Camera3Stream::registerBuffersLocked(camera3_device *hal3Device) { returnBufferLocked(streamBuffers[i], 0); } + mPrepared = true; + return res; } @@ -505,6 +636,10 @@ status_t Camera3Stream::returnInputBufferLocked( ALOGE("%s: This type of stream does not support input", __FUNCTION__); return INVALID_OPERATION; } +status_t Camera3Stream::getInputBufferProducerLocked(sp<IGraphicBufferProducer> *producer) { + ALOGE("%s: This type of stream does not support input", __FUNCTION__); + return INVALID_OPERATION; +} void Camera3Stream::addBufferListener( wp<Camera3StreamBufferListener> listener) { diff --git a/services/camera/libcameraservice/device3/Camera3Stream.h b/services/camera/libcameraservice/device3/Camera3Stream.h index aba27fe..0543c66 100644 --- a/services/camera/libcameraservice/device3/Camera3Stream.h +++ b/services/camera/libcameraservice/device3/Camera3Stream.h @@ -57,8 +57,15 @@ namespace camera3 { * re-registering buffers with HAL. * * STATE_CONFIGURED: Stream is configured, and has registered buffers with the - * HAL. The stream's getBuffer/returnBuffer work. The priv pointer may still be - * modified. + * HAL (if necessary). The stream's getBuffer/returnBuffer work. The priv + * pointer may still be modified. + * + * STATE_PREPARING: The stream's buffers are being pre-allocated for use. On + * older HALs, this is done as part of configuration, but in newer HALs + * buffers may be allocated at time of first use. But some use cases require + * buffer allocation upfront, to minmize disruption due to lengthy allocation + * duration. In this state, only prepareNextBuffer() and cancelPrepare() + * may be called. * * Transition table: * @@ -82,6 +89,12 @@ namespace camera3 { * STATE_CONFIGURED => STATE_CONSTRUCTED: * When disconnect() is called after making sure stream is idle with * waitUntilIdle(). + * STATE_CONFIGURED => STATE_PREPARING: + * When startPrepare is called before the stream has a buffer + * queued back into it for the first time. + * STATE_PREPARING => STATE_CONFIGURED: + * When sufficient prepareNextBuffer calls have been made to allocate + * all stream buffers, or cancelPrepare is called. * * Status Tracking: * Each stream is tracked by StatusTracker as a separate component, @@ -167,6 +180,73 @@ class Camera3Stream : status_t cancelConfiguration(); /** + * Determine whether the stream has already become in-use (has received + * a valid filled buffer), which determines if a stream can still have + * prepareNextBuffer called on it. + */ + bool isUnpreparable(); + + /** + * Start stream preparation. May only be called in the CONFIGURED state, + * when no valid buffers have yet been returned to this stream. + * + * If no prepartion is necessary, returns OK and does not transition to + * PREPARING state. Otherwise, returns NOT_ENOUGH_DATA and transitions + * to PREPARING. + * + * This call performs no allocation, so is quick to call. + * + * Returns: + * OK if no more buffers need to be preallocated + * NOT_ENOUGH_DATA if calls to prepareNextBuffer are needed to finish + * buffer pre-allocation, and transitions to the PREPARING state. + * NO_INIT in case of a serious error from the HAL device + * INVALID_OPERATION if called when not in CONFIGURED state, or a + * valid buffer has already been returned to this stream. + */ + status_t startPrepare(); + + /** + * Check if the stream is mid-preparing. + */ + bool isPreparing() const; + + /** + * Continue stream buffer preparation by allocating the next + * buffer for this stream. May only be called in the PREPARED state. + * + * Returns OK and transitions to the CONFIGURED state if all buffers + * are allocated after the call concludes. Otherwise returns NOT_ENOUGH_DATA. + * + * This call allocates one buffer, which may take several milliseconds for + * large buffers. + * + * Returns: + * OK if no more buffers need to be preallocated, and transitions + * to the CONFIGURED state. + * NOT_ENOUGH_DATA if more calls to prepareNextBuffer are needed to finish + * buffer pre-allocation. + * NO_INIT in case of a serious error from the HAL device + * INVALID_OPERATION if called when not in CONFIGURED state, or a + * valid buffer has already been returned to this stream. + */ + status_t prepareNextBuffer(); + + /** + * Cancel stream preparation early. In case allocation needs to be + * stopped, this method transitions the stream back to the CONFIGURED state. + * Buffers that have been allocated with prepareNextBuffer remain that way, + * but a later use of prepareNextBuffer will require just as many + * calls as if the earlier prepare attempt had not existed. + * + * Returns: + * OK if cancellation succeeded, and transitions to the CONFIGURED state + * INVALID_OPERATION if not in the PREPARING state + * NO_INIT in case of a serious error from the HAL device + */ + status_t cancelPrepare(); + + /** * Fill in the camera3_stream_buffer with the next valid buffer for this * stream, to hand over to the HAL. * @@ -205,6 +285,10 @@ class Camera3Stream : */ status_t returnInputBuffer(const camera3_stream_buffer &buffer); + // get the buffer producer of the input buffer queue. + // only apply to input streams. + status_t getInputBufferProducer(sp<IGraphicBufferProducer> *producer); + /** * Whether any of the stream's buffers are currently in use by the HAL, * including buffers that have been returned but not yet had their @@ -259,7 +343,8 @@ class Camera3Stream : STATE_CONSTRUCTED, STATE_IN_CONFIG, STATE_IN_RECONFIG, - STATE_CONFIGURED + STATE_CONFIGURED, + STATE_PREPARING } mState; mutable Mutex mLock; @@ -285,6 +370,9 @@ class Camera3Stream : virtual status_t returnInputBufferLocked( const camera3_stream_buffer &buffer); virtual bool hasOutstandingBuffersLocked() const = 0; + // Get the buffer producer of the input buffer queue. Only apply to input streams. + virtual status_t getInputBufferProducerLocked(sp<IGraphicBufferProducer> *producer); + // Can return -ENOTCONN when we are already disconnected (not an error) virtual status_t disconnectLocked() = 0; @@ -305,13 +393,17 @@ class Camera3Stream : // Get the usage flags for the other endpoint, or return // INVALID_OPERATION if they cannot be obtained. - virtual status_t getEndpointUsage(uint32_t *usage) = 0; + virtual status_t getEndpointUsage(uint32_t *usage) const = 0; // Tracking for idle state wp<StatusTracker> mStatusTracker; // Status tracker component ID int mStatusId; + // Tracking for stream prepare - whether this stream can still have + // prepareNextBuffer called on it. + bool mStreamUnpreparable; + private: uint32_t oldUsage; uint32_t oldMaxBuffers; @@ -326,6 +418,18 @@ class Camera3Stream : bool acquired, bool output); List<wp<Camera3StreamBufferListener> > mBufferListenerList; + status_t cancelPrepareLocked(); + + // Tracking for PREPARING state + + // State of buffer preallocation. Only true if either prepareNextBuffer + // has been called sufficient number of times, or stream configuration + // had to register buffers with the HAL + bool mPrepared; + + Vector<camera3_stream_buffer_t> mPreparedBuffers; + size_t mPreparedBufferIdx; + }; // class Camera3Stream }; // namespace camera3 diff --git a/services/camera/libcameraservice/device3/Camera3StreamInterface.h b/services/camera/libcameraservice/device3/Camera3StreamInterface.h index da989cd..d177b57 100644 --- a/services/camera/libcameraservice/device3/Camera3StreamInterface.h +++ b/services/camera/libcameraservice/device3/Camera3StreamInterface.h @@ -89,6 +89,68 @@ class Camera3StreamInterface : public virtual RefBase { virtual status_t cancelConfiguration() = 0; /** + * Determine whether the stream has already become in-use (has received + * a valid filled buffer), which determines if a stream can still have + * prepareNextBuffer called on it. + */ + virtual bool isUnpreparable() = 0; + + /** + * Start stream preparation. May only be called in the CONFIGURED state, + * when no valid buffers have yet been returned to this stream. + * + * If no prepartion is necessary, returns OK and does not transition to + * PREPARING state. Otherwise, returns NOT_ENOUGH_DATA and transitions + * to PREPARING. + * + * Returns: + * OK if no more buffers need to be preallocated + * NOT_ENOUGH_DATA if calls to prepareNextBuffer are needed to finish + * buffer pre-allocation, and transitions to the PREPARING state. + * NO_INIT in case of a serious error from the HAL device + * INVALID_OPERATION if called when not in CONFIGURED state, or a + * valid buffer has already been returned to this stream. + */ + virtual status_t startPrepare() = 0; + + /** + * Check if the stream is mid-preparing. + */ + virtual bool isPreparing() const = 0; + + /** + * Continue stream buffer preparation by allocating the next + * buffer for this stream. May only be called in the PREPARED state. + * + * Returns OK and transitions to the CONFIGURED state if all buffers + * are allocated after the call concludes. Otherwise returns NOT_ENOUGH_DATA. + * + * Returns: + * OK if no more buffers need to be preallocated, and transitions + * to the CONFIGURED state. + * NOT_ENOUGH_DATA if more calls to prepareNextBuffer are needed to finish + * buffer pre-allocation. + * NO_INIT in case of a serious error from the HAL device + * INVALID_OPERATION if called when not in CONFIGURED state, or a + * valid buffer has already been returned to this stream. + */ + virtual status_t prepareNextBuffer() = 0; + + /** + * Cancel stream preparation early. In case allocation needs to be + * stopped, this method transitions the stream back to the CONFIGURED state. + * Buffers that have been allocated with prepareNextBuffer remain that way, + * but a later use of prepareNextBuffer will require just as many + * calls as if the earlier prepare attempt had not existed. + * + * Returns: + * OK if cancellation succeeded, and transitions to the CONFIGURED state + * INVALID_OPERATION if not in the PREPARING state + * NO_INIT in case of a serious error from the HAL device + */ + virtual status_t cancelPrepare() = 0; + + /** * Fill in the camera3_stream_buffer with the next valid buffer for this * stream, to hand over to the HAL. * @@ -128,6 +190,13 @@ class Camera3StreamInterface : public virtual RefBase { virtual status_t returnInputBuffer(const camera3_stream_buffer &buffer) = 0; /** + * Get the buffer producer of the input buffer queue. + * + * This method only applies to input streams. + */ + virtual status_t getInputBufferProducer(sp<IGraphicBufferProducer> *producer) = 0; + + /** * Whether any of the stream's buffers are currently in use by the HAL, * including buffers that have been returned but not yet had their * release fence signaled. diff --git a/services/camera/libcameraservice/utils/ClientManager.h b/services/camera/libcameraservice/utils/ClientManager.h index ad5486d..aa40a2d 100644 --- a/services/camera/libcameraservice/utils/ClientManager.h +++ b/services/camera/libcameraservice/utils/ClientManager.h @@ -17,7 +17,9 @@ #ifndef ANDROID_SERVICE_UTILS_EVICTION_POLICY_MANAGER_H #define ANDROID_SERVICE_UTILS_EVICTION_POLICY_MANAGER_H +#include <utils/Condition.h> #include <utils/Mutex.h> +#include <utils/Timers.h> #include <algorithm> #include <utility> @@ -263,6 +265,16 @@ public: */ std::shared_ptr<ClientDescriptor<KEY, VALUE>> get(const KEY& key) const; + /** + * Block until the given client is no longer in the active clients list, or the timeout + * occurred. + * + * Returns NO_ERROR if this succeeded, -ETIMEDOUT on a timeout, or a negative error code on + * failure. + */ + status_t waitUntilRemoved(const std::shared_ptr<ClientDescriptor<KEY, VALUE>> client, + nsecs_t timeout) const; + protected: ~ClientManager(); @@ -284,6 +296,7 @@ private: int64_t getCurrentCostLocked() const; mutable Mutex mLock; + mutable Condition mRemovedCondition; int32_t mMaxCost; // LRU ordered, most recent at end std::vector<std::shared_ptr<ClientDescriptor<KEY, VALUE>>> mClients; @@ -430,6 +443,7 @@ std::vector<std::shared_ptr<ClientDescriptor<KEY, VALUE>>> ClientManager<KEY, VA }), mClients.end()); mClients.push_back(client); + mRemovedCondition.broadcast(); return evicted; } @@ -487,6 +501,7 @@ template<class KEY, class VALUE> void ClientManager<KEY, VALUE>::removeAll() { Mutex::Autolock lock(mLock); mClients.clear(); + mRemovedCondition.broadcast(); } template<class KEY, class VALUE> @@ -505,6 +520,39 @@ std::shared_ptr<ClientDescriptor<KEY, VALUE>> ClientManager<KEY, VALUE>::remove( return false; }), mClients.end()); + mRemovedCondition.broadcast(); + return ret; +} + +template<class KEY, class VALUE> +status_t ClientManager<KEY, VALUE>::waitUntilRemoved( + const std::shared_ptr<ClientDescriptor<KEY, VALUE>> client, + nsecs_t timeout) const { + status_t ret = NO_ERROR; + Mutex::Autolock lock(mLock); + + bool isRemoved = false; + + // Figure out what time in the future we should hit the timeout + nsecs_t failTime = systemTime(SYSTEM_TIME_MONOTONIC) + timeout; + + while (!isRemoved) { + isRemoved = true; + for (const auto& i : mClients) { + if (i == client) { + isRemoved = false; + } + } + + if (!isRemoved) { + ret = mRemovedCondition.waitRelative(mLock, timeout); + if (ret != NO_ERROR) { + break; + } + timeout = failTime - systemTime(SYSTEM_TIME_MONOTONIC); + } + } + return ret; } @@ -520,6 +568,7 @@ void ClientManager<KEY, VALUE>::remove( } return false; }), mClients.end()); + mRemovedCondition.broadcast(); } template<class KEY, class VALUE> diff --git a/services/mediaresourcemanager/ResourceManagerService.cpp b/services/mediaresourcemanager/ResourceManagerService.cpp index 7296d47..75a69ed 100644 --- a/services/mediaresourcemanager/ResourceManagerService.cpp +++ b/services/mediaresourcemanager/ResourceManagerService.cpp @@ -126,6 +126,7 @@ void ResourceManagerService::addResource( Mutex::Autolock lock(mLock); ResourceInfos& infos = getResourceInfosForEdit(pid, mMap); ResourceInfo& info = getResourceInfoForEdit(clientId, client, infos); + // TODO: do the merge instead of append. info.resources.appendVector(resources); } @@ -197,19 +198,58 @@ bool ResourceManagerService::reclaimResource( } } } + + if (clients.size() == 0) { + // if we are here, run the third pass to free one codec with the same type. + for (size_t i = 0; i < resources.size(); ++i) { + String8 type = resources[i].mType; + if (type == kResourceSecureCodec || type == kResourceNonSecureCodec) { + sp<IResourceManagerClient> client; + if (!getLowestPriorityBiggestClient_l(callingPid, type, &client)) { + return false; + } + clients.push_back(client); + } + } + } } if (clients.size() == 0) { return false; } + sp<IResourceManagerClient> failedClient; for (size_t i = 0; i < clients.size(); ++i) { ALOGV("reclaimResource from client %p", clients[i].get()); if (!clients[i]->reclaimResource()) { - return false; + failedClient = clients[i]; + break; } } - return true; + + { + Mutex::Autolock lock(mLock); + bool found = false; + for (size_t i = 0; i < mMap.size(); ++i) { + ResourceInfos &infos = mMap.editValueAt(i); + for (size_t j = 0; j < infos.size();) { + if (infos[j].client == failedClient) { + j = infos.removeAt(j); + found = true; + } else { + ++j; + } + } + if (found) { + break; + } + } + if (!found) { + ALOGV("didn't find failed client"); + } + } + + return (failedClient == NULL); } bool ResourceManagerService::getAllClients_l( diff --git a/services/mediaresourcemanager/test/ResourceManagerService_test.cpp b/services/mediaresourcemanager/test/ResourceManagerService_test.cpp index b73e1bc..48d1395 100644 --- a/services/mediaresourcemanager/test/ResourceManagerService_test.cpp +++ b/services/mediaresourcemanager/test/ResourceManagerService_test.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015 The Android Open Source Project + * Copyright 2015 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -118,6 +118,20 @@ protected: client3->reset(); } + // test set up + // --------------------------------------------------------------------------------- + // pid priority client type number + // --------------------------------------------------------------------------------- + // kTestPid1(30) 30 mTestClient1 secure codec 1 + // graphic memory 200 + // graphic memory 200 + // --------------------------------------------------------------------------------- + // kTestPid2(20) 20 mTestClient2 non-secure codec 1 + // graphic memory 300 + // ------------------------------------------- + // mTestClient3 secure codec 1 + // graphic memory 100 + // --------------------------------------------------------------------------------- void addResource() { // kTestPid1 mTestClient1 Vector<MediaResource> resources1; @@ -202,10 +216,12 @@ protected: int lowPriorityPid = 100; EXPECT_FALSE(mService->getAllClients_l(lowPriorityPid, type, &clients)); int midPriorityPid = 25; - EXPECT_FALSE(mService->getAllClients_l(lowPriorityPid, type, &clients)); + // some higher priority process (e.g. kTestPid2) owns the resource, so getAllClients_l + // will fail. + EXPECT_FALSE(mService->getAllClients_l(midPriorityPid, type, &clients)); int highPriorityPid = 10; - EXPECT_TRUE(mService->getAllClients_l(10, unknowType, &clients)); - EXPECT_TRUE(mService->getAllClients_l(10, type, &clients)); + EXPECT_TRUE(mService->getAllClients_l(highPriorityPid, unknowType, &clients)); + EXPECT_TRUE(mService->getAllClients_l(highPriorityPid, type, &clients)); EXPECT_EQ(2u, clients.size()); EXPECT_EQ(mTestClient3, clients[0]); @@ -308,6 +324,30 @@ protected: // nothing left EXPECT_FALSE(mService->reclaimResource(10, resources)); } + + // ### secure codecs can coexist and secure codec can coexist with non-secure codec ### + { + addResource(); + mService->mSupportsMultipleSecureCodecs = true; + mService->mSupportsSecureWithNonSecureCodec = true; + + Vector<MediaResource> resources; + resources.push_back(MediaResource(String8(kResourceSecureCodec), 1)); + + EXPECT_TRUE(mService->reclaimResource(10, resources)); + // secure codec from lowest process got reclaimed + verifyClients(true, false, false); + + // call again should reclaim another secure codec from lowest process + EXPECT_TRUE(mService->reclaimResource(10, resources)); + verifyClients(false, false, true); + + // nothing left + EXPECT_FALSE(mService->reclaimResource(10, resources)); + + // clean up client 2 which still has non secure codec left + mService->removeResource((int64_t) mTestClient2.get()); + } } void testReclaimResourceNonSecure() { @@ -360,6 +400,26 @@ protected: // nothing left EXPECT_FALSE(mService->reclaimResource(10, resources)); } + + // ### secure codec can coexist with non-secure codec ### + { + addResource(); + mService->mSupportsSecureWithNonSecureCodec = true; + + Vector<MediaResource> resources; + resources.push_back(MediaResource(String8(kResourceNonSecureCodec), 1)); + + EXPECT_TRUE(mService->reclaimResource(10, resources)); + // one non secure codec from lowest process got reclaimed + verifyClients(false, true, false); + + // nothing left + EXPECT_FALSE(mService->reclaimResource(10, resources)); + + // clean up client 1 and 3 which still have secure codec left + mService->removeResource((int64_t) mTestClient1.get()); + mService->removeResource((int64_t) mTestClient3.get()); + } } void testGetLowestPriorityBiggestClient() { |