From da7581b7b61b84f15e8d671c86fd117c322b009e Mon Sep 17 00:00:00 2001 From: Eric Laurent Date: Fri, 2 Jul 2010 08:12:41 -0700 Subject: Added Visualizer effect. The visualizer enables application to retrieve part of the currently playing audio for visualization purpose. It is not an audio recording interface and only returns partial and low quality audio content as a waveform or a frequency representation (FFT). Removed temporary hack made in MediaPlayer for animated wall papers based on audio visualization (snoop() method. This commit also includes a change in AudioEffect class: - the enable()/disable() methods have been replaced bya more standard setEnabled() method. - some fixes in javadoc Change-Id: Id092a1340e9e38dae68646ade7be054e3a36980e --- media/libmedia/Android.mk | 3 +- media/libmedia/AudioEffect.cpp | 36 ++-- media/libmedia/IMediaPlayerService.cpp | 17 +- media/libmedia/Visualizer.cpp | 330 +++++++++++++++++++++++++++++++++ media/libmedia/mediaplayer.cpp | 57 ------ 5 files changed, 347 insertions(+), 96 deletions(-) create mode 100644 media/libmedia/Visualizer.cpp (limited to 'media/libmedia') diff --git a/media/libmedia/Android.mk b/media/libmedia/Android.mk index de9e51d..977e6be 100644 --- a/media/libmedia/Android.mk +++ b/media/libmedia/Android.mk @@ -30,7 +30,8 @@ LOCAL_SRC_FILES:= \ MediaProfiles.cpp \ IEffect.cpp \ IEffectClient.cpp \ - AudioEffect.cpp + AudioEffect.cpp \ + Visualizer.cpp LOCAL_SHARED_LIBRARIES := \ libui libcutils libutils libbinder libsonivox libicuuc libexpat libsurfaceflinger_client libcamera_client diff --git a/media/libmedia/AudioEffect.cpp b/media/libmedia/AudioEffect.cpp index 4afa2dc..783249d 100644 --- a/media/libmedia/AudioEffect.cpp +++ b/media/libmedia/AudioEffect.cpp @@ -171,7 +171,7 @@ AudioEffect::~AudioEffect() LOGV("Destructor %p", this); if (mStatus == NO_ERROR || mStatus == ALREADY_EXISTS) { - disable(); + setEnabled(false); if (mIEffect != NULL) { mIEffect->disconnect(); mIEffect->asBinder()->unlinkToDeath(mIEffectClient); @@ -196,36 +196,28 @@ effect_descriptor_t AudioEffect::descriptor() const return mDescriptor; } -bool AudioEffect::isEnabled() const +bool AudioEffect::getEnabled() const { return (mEnabled != 0); } -status_t AudioEffect::enable() +status_t AudioEffect::setEnabled(bool enabled) { if (mStatus != NO_ERROR) { return INVALID_OPERATION; } - LOGV("enable %p", this); - if (android_atomic_or(1, &mEnabled) == 0) { - return mIEffect->enable(); - } - - return INVALID_OPERATION; -} - -status_t AudioEffect::disable() -{ - if (mStatus != NO_ERROR) { - return INVALID_OPERATION; - } - LOGV("disable %p", this); - - if (android_atomic_and(~1, &mEnabled) == 1) { - return mIEffect->disable(); + if (enabled) { + LOGV("enable %p", this); + if (android_atomic_or(1, &mEnabled) == 0) { + return mIEffect->enable(); + } + } else { + LOGV("disable %p", this); + if (android_atomic_and(~1, &mEnabled) == 1) { + return mIEffect->disable(); + } } - return INVALID_OPERATION; } @@ -349,7 +341,7 @@ void AudioEffect::controlStatusChanged(bool controlGranted) void AudioEffect::enableStatusChanged(bool enabled) { - LOGV("enableStatusChanged %p enabled %d", this, enabled); + LOGV("enableStatusChanged %p enabled %d mCbf %p", this, enabled, mCbf); if (mStatus == ALREADY_EXISTS) { mEnabled = enabled; if (mCbf) { diff --git a/media/libmedia/IMediaPlayerService.cpp b/media/libmedia/IMediaPlayerService.cpp index 1ae222e..4abfa75 100644 --- a/media/libmedia/IMediaPlayerService.cpp +++ b/media/libmedia/IMediaPlayerService.cpp @@ -35,8 +35,7 @@ enum { DECODE_FD, CREATE_MEDIA_RECORDER, CREATE_METADATA_RETRIEVER, - GET_OMX, - SNOOP + GET_OMX }; class BpMediaPlayerService: public BpInterface @@ -134,14 +133,6 @@ public: return interface_cast(reply.readStrongBinder()); } - virtual sp snoop() - { - Parcel data, reply; - data.writeInterfaceToken(IMediaPlayerService::getInterfaceDescriptor()); - remote()->transact(SNOOP, data, &reply); - return interface_cast(reply.readStrongBinder()); - } - virtual sp getOMX() { Parcel data, reply; data.writeInterfaceToken(IMediaPlayerService::getInterfaceDescriptor()); @@ -221,12 +212,6 @@ status_t BnMediaPlayerService::onTransact( reply->writeStrongBinder(player->asBinder()); return NO_ERROR; } break; - case SNOOP: { - CHECK_INTERFACE(IMediaPlayerService, data, reply); - sp snooped_audio = snoop(); - reply->writeStrongBinder(snooped_audio->asBinder()); - return NO_ERROR; - } break; case CREATE_MEDIA_RECORDER: { CHECK_INTERFACE(IMediaPlayerService, data, reply); pid_t pid = data.readInt32(); diff --git a/media/libmedia/Visualizer.cpp b/media/libmedia/Visualizer.cpp new file mode 100644 index 0000000..47e96e5 --- /dev/null +++ b/media/libmedia/Visualizer.cpp @@ -0,0 +1,330 @@ +/* +** +** Copyright 2010, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + + +//#define LOG_NDEBUG 0 +#define LOG_TAG "Visualizer" +#include + +#include +#include +#include + +#include + +extern "C" { +#define FLOATING_POINT 1 +#include "fftwrap.h" +} + +namespace android { + +// --------------------------------------------------------------------------- + +Visualizer::Visualizer (int32_t priority, + effect_callback_t cbf, + void* user, + int sessionId) + : AudioEffect(SL_IID_VISUALIZATION, NULL, priority, cbf, user, sessionId), + mCaptureRate(CAPTURE_RATE_DEF), + mCaptureSize(CAPTURE_SIZE_DEF), + mSampleRate(44100000), + mCaptureCallBack(NULL), + mCaptureCbkUser(NULL) +{ + initCaptureSize(); + if (mCaptureSize != 0) { + mFftTable = spx_fft_init(mCaptureSize); + } else { + mFftTable = NULL; + } +} + +Visualizer::~Visualizer() +{ + if (mFftTable != NULL) { + spx_fft_destroy(mFftTable); + } +} + +status_t Visualizer::setEnabled(bool enabled) +{ + Mutex::Autolock _l(mLock); + + sp t = mCaptureThread; + if (t != 0) { + if (enabled) { + if (t->exitPending()) { + if (t->requestExitAndWait() == WOULD_BLOCK) { + LOGE("Visualizer::enable() called from thread"); + return INVALID_OPERATION; + } + } + } + t->mLock.lock(); + } + + status_t status = AudioEffect::setEnabled(enabled); + + if (status == NO_ERROR) { + if (t != 0) { + if (enabled) { + t->run("AudioTrackThread"); + } else { + t->requestExit(); + } + } + } + + if (t != 0) { + t->mLock.unlock(); + } + + return status; +} + +status_t Visualizer::setCaptureCallBack(capture_cbk_t cbk, void* user, uint32_t flags, uint32_t rate) +{ + if (rate > CAPTURE_RATE_MAX) { + return BAD_VALUE; + } + Mutex::Autolock _l(mLock); + + if (mEnabled) { + return INVALID_OPERATION; + } + + sp t = mCaptureThread; + if (t != 0) { + t->mLock.lock(); + } + mCaptureThread.clear(); + mCaptureCallBack = cbk; + mCaptureCbkUser = user; + mCaptureFlags = flags; + mCaptureRate = rate; + + if (t != 0) { + t->mLock.unlock(); + } + + if (cbk != NULL) { + mCaptureThread = new CaptureThread(*this, rate, ((flags & CAPTURE_CALL_JAVA) != 0)); + if (mCaptureThread == 0) { + LOGE("Could not create callback thread"); + return NO_INIT; + } + } + LOGV("setCaptureCallBack() rate: %d thread %p flags 0x%08x", + rate, mCaptureThread.get(), mCaptureFlags); + return NO_ERROR; +} + +status_t Visualizer::setCaptureSize(uint32_t size) +{ + if (size > VISUALIZER_CAPTURE_SIZE_MAX || + size < VISUALIZER_CAPTURE_SIZE_MIN || + AudioSystem::popCount(size) != 1) { + return BAD_VALUE; + } + + Mutex::Autolock _l(mLock); + if (mEnabled) { + return INVALID_OPERATION; + } + + uint32_t buf32[sizeof(effect_param_t) / sizeof(uint32_t) + 2]; + effect_param_t *p = (effect_param_t *)buf32; + + p->psize = sizeof(uint32_t); + p->vsize = sizeof(uint32_t); + *(int32_t *)p->data = VISU_PARAM_CAPTURE_SIZE; + *((int32_t *)p->data + 1)= size; + status_t status = setParameter(p); + + LOGV("setCaptureSize size %d status %d p->status %d", size, status, p->status); + + if (status == NO_ERROR) { + status = p->status; + } + if (status == NO_ERROR) { + mCaptureSize = size; + if (mFftTable != NULL) { + spx_fft_destroy(mFftTable); + } + mFftTable = spx_fft_init(mCaptureSize); + LOGV("setCaptureSize size %d mFftTable %p", mCaptureSize, mFftTable); + } + + return status; +} + +status_t Visualizer::getWaveForm(uint8_t *waveform) +{ + if (waveform == NULL) { + return BAD_VALUE; + } + if (mCaptureSize == 0) { + return NO_INIT; + } + + status_t status = NO_ERROR; + if (mEnabled) { + int32_t replySize = mCaptureSize; + status_t status = command(VISU_CMD_CAPTURE, 0, NULL, &replySize, waveform); + if (replySize == 0) { + status = NOT_ENOUGH_DATA; + } + } else { + memset(waveform, 0x80, mCaptureSize); + } + return status; +} + +status_t Visualizer::getFft(uint8_t *fft) +{ + if (fft == NULL) { + return BAD_VALUE; + } + if (mCaptureSize == 0) { + return NO_INIT; + } + + status_t status = NO_ERROR; + if (mEnabled) { + uint8_t buf[mCaptureSize]; + status_t status = getWaveForm(buf); + if (status == NO_ERROR) { + status = doFft(fft, buf); + } + } else { + memset(fft, 0, mCaptureSize); + } + return status; +} + +status_t Visualizer::doFft(uint8_t *fft, uint8_t *waveform) +{ + if (mFftTable == NULL) { + return NO_INIT; + } + + float fsrc[mCaptureSize]; + for (uint32_t i = 0; i < mCaptureSize; i++) { + fsrc[i] = (int16_t)(waveform[i] ^ 0x80) << 8; + } + float fdst[mCaptureSize]; + spx_fft_float(mFftTable, fsrc, fdst); + for (uint32_t i = 0; i < mCaptureSize; i++) { + fft[i] = (uint8_t)((int32_t)fdst[i] >> 8); + } + return NO_ERROR; +} + +void Visualizer::periodicCapture() +{ + Mutex::Autolock _l(mLock); + LOGV("periodicCapture() %p mCaptureCallBack %p mCaptureFlags 0x%08x", + this, mCaptureCallBack, mCaptureFlags); + if (mCaptureCallBack != NULL && + (mCaptureFlags & (CAPTURE_WAVEFORM|CAPTURE_FFT)) && + mCaptureSize != 0) { + uint8_t waveform[mCaptureSize]; + status_t status = getWaveForm(waveform); + if (status != NO_ERROR) { + return; + } + uint8_t fft[mCaptureSize]; + if (mCaptureFlags & CAPTURE_FFT) { + status = doFft(fft, waveform); + } + if (status != NO_ERROR) { + return; + } + uint8_t *wavePtr = NULL; + uint8_t *fftPtr = NULL; + uint32_t waveSize = 0; + uint32_t fftSize = 0; + if (mCaptureFlags & CAPTURE_WAVEFORM) { + wavePtr = waveform; + waveSize = mCaptureSize; + } + if (mCaptureFlags & CAPTURE_FFT) { + fftPtr = fft; + fftSize = mCaptureSize; + } + mCaptureCallBack(mCaptureCbkUser, waveSize, wavePtr, fftSize, fftPtr, mSampleRate); + } +} + +uint32_t Visualizer::initCaptureSize() +{ + uint32_t buf32[sizeof(effect_param_t) / sizeof(uint32_t) + 2]; + effect_param_t *p = (effect_param_t *)buf32; + + p->psize = sizeof(uint32_t); + p->vsize = sizeof(uint32_t); + *(int32_t *)p->data = VISU_PARAM_CAPTURE_SIZE; + status_t status = getParameter(p); + + if (status == NO_ERROR) { + status = p->status; + } + + uint32_t size = 0; + if (status == NO_ERROR) { + size = *((int32_t *)p->data + 1); + } + mCaptureSize = size; + + LOGV("initCaptureSize size %d status %d", mCaptureSize, status); + + return size; +} + +//------------------------------------------------------------------------- + +Visualizer::CaptureThread::CaptureThread(Visualizer& receiver, uint32_t captureRate, bool bCanCallJava) + : Thread(bCanCallJava), mReceiver(receiver) +{ + mSleepTimeUs = 1000000000 / captureRate; + LOGV("CaptureThread cstor %p captureRate %d mSleepTimeUs %d", this, captureRate, mSleepTimeUs); +} + +bool Visualizer::CaptureThread::threadLoop() +{ + LOGV("CaptureThread %p enter", this); + while (!exitPending()) + { + usleep(mSleepTimeUs); + mReceiver.periodicCapture(); + } + LOGV("CaptureThread %p exiting", this); + return false; +} + +status_t Visualizer::CaptureThread::readyToRun() +{ + return NO_ERROR; +} + +void Visualizer::CaptureThread::onFirstRef() +{ +} + +}; // namespace android + diff --git a/media/libmedia/mediaplayer.cpp b/media/libmedia/mediaplayer.cpp index d5a3c13..b43f75f 100644 --- a/media/libmedia/mediaplayer.cpp +++ b/media/libmedia/mediaplayer.cpp @@ -658,61 +658,4 @@ void MediaPlayer::died() } -extern "C" { -#define FLOATING_POINT 1 -#include "fftwrap.h" -} - -static void *ffttable = NULL; - -// peeks at the audio data and fills 'data' with the requested kind -// (currently kind=0 returns mono 16 bit PCM data, and kind=1 returns -// 256 point FFT data). Return value is number of samples returned, -// which may be 0. -/*static*/ int MediaPlayer::snoop(short* data, int len, int kind) { - - sp p; - const sp& service = getMediaPlayerService(); - if (service != 0) { - // Take a peek at the waveform. The returned data consists of 16 bit mono PCM data. - p = service->snoop(); - - if (p == NULL) { - return 0; - } - - if (kind == 0) { // return waveform data - int plen = p->size(); - len *= 2; // number of shorts -> number of bytes - short *src = (short*) p->pointer(); - if (plen > len) { - plen = len; - } - memcpy(data, src, plen); - return plen / sizeof(short); // return number of samples - } else if (kind == 1) { - // TODO: use a more efficient FFT - // Right now this uses the speex library, which is compiled to do a float FFT - if (!ffttable) ffttable = spx_fft_init(512); - short *usrc = (short*) p->pointer(); - float fsrc[512]; - for (int i=0;i<512;i++) - fsrc[i] = usrc[i]; - float fdst[512]; - spx_fft_float(ffttable, fsrc, fdst); - if (len > 512) { - len = 512; - } - len /= 2; // only half the output data is valid - for (int i=0; i < len; i++) - data[i] = fdst[i]; - return len; - } - - } else { - LOGE("Unable to locate media service"); - } - return 0; -} - }; // namespace android -- cgit v1.1