diff options
43 files changed, 3625 insertions, 248 deletions
diff --git a/cmds/stagefright/Android.mk b/cmds/stagefright/Android.mk index 561ce02..78c89bb 100644 --- a/cmds/stagefright/Android.mk +++ b/cmds/stagefright/Android.mk @@ -169,6 +169,44 @@ include $(BUILD_EXECUTABLE) include $(CLEAR_VARS) +LOCAL_NDK_STL_VARIANT := stlport_static + +LOCAL_SRC_FILES:= \ + filters/argbtorgba.rs \ + filters/nightvision.rs \ + filters/saturation.rs \ + mediafilter.cpp \ + +LOCAL_SHARED_LIBRARIES := \ + libstagefright liblog libutils libbinder libstagefright_foundation \ + libmedia libgui libcutils libui + +LOCAL_C_INCLUDES:= \ + $(TOP)/bionic \ + $(TOP)/external/stlport/stlport \ + $(TOP)/frameworks/av/media/libstagefright \ + $(TOP)/frameworks/native/include/media/openmax \ + $(TOP)/frameworks/rs/cpp \ + $(TOP)/frameworks/rs \ + +intermediates := $(call intermediates-dir-for,STATIC_LIBRARIES,libRS,TARGET,) +LOCAL_C_INCLUDES += $(intermediates) + +LOCAL_STATIC_LIBRARIES:= \ + libstagefright_mediafilter + +LOCAL_CFLAGS += -Wno-multichar + +LOCAL_MODULE_TAGS := optional + +LOCAL_MODULE:= mediafilter + +include $(BUILD_EXECUTABLE) + +################################################################################ + +include $(CLEAR_VARS) + LOCAL_SRC_FILES:= \ muxer.cpp \ diff --git a/cmds/stagefright/codec.cpp b/cmds/stagefright/codec.cpp index fd02bcc..d987250 100644 --- a/cmds/stagefright/codec.cpp +++ b/cmds/stagefright/codec.cpp @@ -45,9 +45,10 @@ static void usage(const char *me) { fprintf(stderr, "usage: %s [-a] use audio\n" "\t\t[-v] use video\n" "\t\t[-p] playback\n" - "\t\t[-S] allocate buffers from a surface\n", + "\t\t[-S] allocate buffers from a surface\n" + "\t\t[-R] render output to surface (enables -S)\n" + "\t\t[-T] use render timestamps (enables -R)\n", me); - exit(1); } @@ -71,7 +72,9 @@ static int decode( const char *path, bool useAudio, bool useVideo, - const android::sp<android::Surface> &surface) { + const android::sp<android::Surface> &surface, + bool renderSurface, + bool useTimestamp) { using namespace android; static int64_t kTimeout = 500ll; @@ -136,6 +139,7 @@ static int decode( CHECK(!stateByTrack.isEmpty()); int64_t startTimeUs = ALooper::GetNowUs(); + int64_t startTimeRender = -1; for (size_t i = 0; i < stateByTrack.size(); ++i) { CodecState *state = &stateByTrack.editValueAt(i); @@ -260,7 +264,23 @@ static int decode( ++state->mNumBuffersDecoded; state->mNumBytesDecoded += size; - err = state->mCodec->releaseOutputBuffer(index); + if (surface == NULL || !renderSurface) { + err = state->mCodec->releaseOutputBuffer(index); + } else if (useTimestamp) { + if (startTimeRender == -1) { + // begin rendering 2 vsyncs (~33ms) after first decode + startTimeRender = + systemTime(SYSTEM_TIME_MONOTONIC) + 33000000 + - (presentationTimeUs * 1000); + } + presentationTimeUs = + (presentationTimeUs * 1000) + startTimeRender; + err = state->mCodec->renderOutputBufferAndRelease( + index, presentationTimeUs); + } else { + err = state->mCodec->renderOutputBufferAndRelease(index); + } + CHECK_EQ(err, (status_t)OK); if (flags & MediaCodec::BUFFER_FLAG_EOS) { @@ -320,34 +340,42 @@ int main(int argc, char **argv) { bool useVideo = false; bool playback = false; bool useSurface = false; + bool renderSurface = false; + bool useTimestamp = false; int res; - while ((res = getopt(argc, argv, "havpSD")) >= 0) { + while ((res = getopt(argc, argv, "havpSDRT")) >= 0) { switch (res) { case 'a': { useAudio = true; break; } - case 'v': { useVideo = true; break; } - case 'p': { playback = true; break; } - + case 'T': + { + useTimestamp = true; + } + // fall through + case 'R': + { + renderSurface = true; + } + // fall through case 'S': { useSurface = true; break; } - case '?': case 'h': default: @@ -422,7 +450,8 @@ int main(int argc, char **argv) { player->stop(); player->reset(); } else { - decode(looper, argv[0], useAudio, useVideo, surface); + decode(looper, argv[0], useAudio, useVideo, surface, renderSurface, + useTimestamp); } if (playback || (useSurface && useVideo)) { diff --git a/cmds/stagefright/filters/argbtorgba.rs b/cmds/stagefright/filters/argbtorgba.rs new file mode 100644 index 0000000..229ff8c --- /dev/null +++ b/cmds/stagefright/filters/argbtorgba.rs @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma version(1) +#pragma rs java_package_name(com.android.rs.cppbasic) +#pragma rs_fp_relaxed + +void root(const uchar4 *v_in, uchar4 *v_out) { + v_out->x = v_in->y; + v_out->y = v_in->z; + v_out->z = v_in->w; + v_out->w = v_in->x; +}
\ No newline at end of file diff --git a/cmds/stagefright/filters/nightvision.rs b/cmds/stagefright/filters/nightvision.rs new file mode 100644 index 0000000..f61413c --- /dev/null +++ b/cmds/stagefright/filters/nightvision.rs @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma version(1) +#pragma rs java_package_name(com.android.rs.cppbasic) +#pragma rs_fp_relaxed + +const static float3 gMonoMult = {0.299f, 0.587f, 0.114f}; +const static float3 gNightVisionMult = {0.5f, 1.f, 0.5f}; + +// calculates luminance of pixel, then biases color balance toward green +void root(const uchar4 *v_in, uchar4 *v_out) { + v_out->x = v_in->x; // don't modify A + + // get RGB, scale 0-255 uchar to 0-1.0 float + float3 rgb = {v_in->y * 0.003921569f, v_in->z * 0.003921569f, + v_in->w * 0.003921569f}; + + // apply filter + float3 result = dot(rgb, gMonoMult) * gNightVisionMult; + + v_out->y = (uchar)clamp((result.r * 255.f + 0.5f), 0.f, 255.f); + v_out->z = (uchar)clamp((result.g * 255.f + 0.5f), 0.f, 255.f); + v_out->w = (uchar)clamp((result.b * 255.f + 0.5f), 0.f, 255.f); +} diff --git a/cmds/stagefright/filters/saturation.rs b/cmds/stagefright/filters/saturation.rs new file mode 100644 index 0000000..1de9dd8 --- /dev/null +++ b/cmds/stagefright/filters/saturation.rs @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma version(1) +#pragma rs java_package_name(com.android.rs.cppbasic) +#pragma rs_fp_relaxed + +const static float3 gMonoMult = {0.299f, 0.587f, 0.114f}; + +// global variables (parameters accessible to application code) +float gSaturation = 1.0f; + +void root(const uchar4 *v_in, uchar4 *v_out) { + v_out->x = v_in->x; // don't modify A + + // get RGB, scale 0-255 uchar to 0-1.0 float + float3 rgb = {v_in->y * 0.003921569f, v_in->z * 0.003921569f, + v_in->w * 0.003921569f}; + + // apply saturation filter + float3 result = dot(rgb, gMonoMult); + result = mix(result, rgb, gSaturation); + + v_out->y = (uchar)clamp((result.r * 255.f + 0.5f), 0.f, 255.f); + v_out->z = (uchar)clamp((result.g * 255.f + 0.5f), 0.f, 255.f); + v_out->w = (uchar)clamp((result.b * 255.f + 0.5f), 0.f, 255.f); +} diff --git a/cmds/stagefright/mediafilter.cpp b/cmds/stagefright/mediafilter.cpp new file mode 100644 index 0000000..f77b38b --- /dev/null +++ b/cmds/stagefright/mediafilter.cpp @@ -0,0 +1,785 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "mediafilterTest" + +#include <inttypes.h> + +#include <binder/ProcessState.h> +#include <filters/ColorConvert.h> +#include <gui/ISurfaceComposer.h> +#include <gui/SurfaceComposerClient.h> +#include <gui/Surface.h> +#include <media/ICrypto.h> +#include <media/IMediaHTTPService.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/DataSource.h> +#include <media/stagefright/MediaCodec.h> +#include <media/stagefright/NuMediaExtractor.h> +#include <media/stagefright/RenderScriptWrapper.h> +#include <OMX_IVCommon.h> +#include <ui/DisplayInfo.h> + +#include "RenderScript.h" +#include "ScriptC_argbtorgba.h" +#include "ScriptC_nightvision.h" +#include "ScriptC_saturation.h" + +// test parameters +static const bool kTestFlush = true; // Note: true will drop 1 out of +static const int kFlushAfterFrames = 25; // kFlushAfterFrames output frames +static const int64_t kTimeout = 500ll; + +// built-in filter parameters +static const int32_t kInvert = false; // ZeroFilter param +static const float kBlurRadius = 15.0f; // IntrinsicBlurFilter param +static const float kSaturation = 0.0f; // SaturationFilter param + +static void usage(const char *me) { + fprintf(stderr, "usage: [flags] %s\n" + "\t[-b] use IntrinsicBlurFilter\n" + "\t[-c] use argb to rgba conversion RSFilter\n" + "\t[-n] use night vision RSFilter\n" + "\t[-r] use saturation RSFilter\n" + "\t[-s] use SaturationFilter\n" + "\t[-z] use ZeroFilter (copy filter)\n" + "\t[-R] render output to surface (enables -S)\n" + "\t[-S] allocate buffers from a surface\n" + "\t[-T] use render timestamps (enables -R)\n", + me); + exit(1); +} + +namespace android { + +struct SaturationRSFilter : RenderScriptWrapper::RSFilterCallback { + void init(RSC::sp<RSC::RS> context) { + mScript = new ScriptC_saturation(context); + mScript->set_gSaturation(3.f); + } + + virtual status_t processBuffers( + RSC::Allocation *inBuffer, RSC::Allocation *outBuffer) { + mScript->forEach_root(inBuffer, outBuffer); + + return OK; + } + + status_t handleSetParameters(const sp<AMessage> &msg) { + return OK; + } + +private: + RSC::sp<ScriptC_saturation> mScript; +}; + +struct NightVisionRSFilter : RenderScriptWrapper::RSFilterCallback { + void init(RSC::sp<RSC::RS> context) { + mScript = new ScriptC_nightvision(context); + } + + virtual status_t processBuffers( + RSC::Allocation *inBuffer, RSC::Allocation *outBuffer) { + mScript->forEach_root(inBuffer, outBuffer); + + return OK; + } + + status_t handleSetParameters(const sp<AMessage> &msg) { + return OK; + } + +private: + RSC::sp<ScriptC_nightvision> mScript; +}; + +struct ARGBToRGBARSFilter : RenderScriptWrapper::RSFilterCallback { + void init(RSC::sp<RSC::RS> context) { + mScript = new ScriptC_argbtorgba(context); + } + + virtual status_t processBuffers( + RSC::Allocation *inBuffer, RSC::Allocation *outBuffer) { + mScript->forEach_root(inBuffer, outBuffer); + + return OK; + } + + status_t handleSetParameters(const sp<AMessage> &msg) { + return OK; + } + +private: + RSC::sp<ScriptC_argbtorgba> mScript; +}; + +struct CodecState { + sp<MediaCodec> mCodec; + Vector<sp<ABuffer> > mInBuffers; + Vector<sp<ABuffer> > mOutBuffers; + bool mSignalledInputEOS; + bool mSawOutputEOS; + int64_t mNumBuffersDecoded; +}; + +struct DecodedFrame { + size_t index; + size_t offset; + size_t size; + int64_t presentationTimeUs; + uint32_t flags; +}; + +enum FilterType { + FILTERTYPE_ZERO, + FILTERTYPE_INTRINSIC_BLUR, + FILTERTYPE_SATURATION, + FILTERTYPE_RS_SATURATION, + FILTERTYPE_RS_NIGHT_VISION, + FILTERTYPE_RS_ARGB_TO_RGBA, +}; + +size_t inputFramesSinceFlush = 0; +void tryCopyDecodedBuffer( + List<DecodedFrame> *decodedFrameIndices, + CodecState *filterState, + CodecState *vidState) { + if (decodedFrameIndices->empty()) { + return; + } + + size_t filterIndex; + status_t err = filterState->mCodec->dequeueInputBuffer( + &filterIndex, kTimeout); + if (err != OK) { + return; + } + + ++inputFramesSinceFlush; + + DecodedFrame frame = *decodedFrameIndices->begin(); + + // only consume a buffer if we are not going to flush, since we expect + // the dequeue -> flush -> queue operation to cause an error and + // not produce an output frame + if (!kTestFlush || inputFramesSinceFlush < kFlushAfterFrames) { + decodedFrameIndices->erase(decodedFrameIndices->begin()); + } + size_t outIndex = frame.index; + + const sp<ABuffer> &srcBuffer = + vidState->mOutBuffers.itemAt(outIndex); + const sp<ABuffer> &destBuffer = + filterState->mInBuffers.itemAt(filterIndex); + + sp<AMessage> srcFormat, destFormat; + vidState->mCodec->getOutputFormat(&srcFormat); + filterState->mCodec->getInputFormat(&destFormat); + + int32_t srcWidth, srcHeight, srcStride, srcSliceHeight; + int32_t srcColorFormat, destColorFormat; + int32_t destWidth, destHeight, destStride, destSliceHeight; + CHECK(srcFormat->findInt32("stride", &srcStride) + && srcFormat->findInt32("slice-height", &srcSliceHeight) + && srcFormat->findInt32("width", &srcWidth) + && srcFormat->findInt32("height", & srcHeight) + && srcFormat->findInt32("color-format", &srcColorFormat)); + CHECK(destFormat->findInt32("stride", &destStride) + && destFormat->findInt32("slice-height", &destSliceHeight) + && destFormat->findInt32("width", &destWidth) + && destFormat->findInt32("height", & destHeight) + && destFormat->findInt32("color-format", &destColorFormat)); + + CHECK(srcWidth <= destStride && srcHeight <= destSliceHeight); + + convertYUV420spToARGB( + srcBuffer->data(), + srcBuffer->data() + srcStride * srcSliceHeight, + srcWidth, + srcHeight, + destBuffer->data()); + + // copy timestamp + int64_t timeUs; + CHECK(srcBuffer->meta()->findInt64("timeUs", &timeUs)); + destBuffer->meta()->setInt64("timeUs", timeUs); + + if (kTestFlush && inputFramesSinceFlush >= kFlushAfterFrames) { + inputFramesSinceFlush = 0; + + // check that queueing a buffer that was dequeued before flush + // fails with expected error EACCES + filterState->mCodec->flush(); + + err = filterState->mCodec->queueInputBuffer( + filterIndex, 0 /* offset */, destBuffer->size(), + timeUs, frame.flags); + + if (err == OK) { + ALOGE("FAIL: queue after flush returned OK"); + } else if (err != -EACCES) { + ALOGE("queueInputBuffer after flush returned %d, " + "expected -EACCES (-13)", err); + } + } else { + err = filterState->mCodec->queueInputBuffer( + filterIndex, 0 /* offset */, destBuffer->size(), + timeUs, frame.flags); + CHECK(err == OK); + + err = vidState->mCodec->releaseOutputBuffer(outIndex); + CHECK(err == OK); + } +} + +size_t outputFramesSinceFlush = 0; +void tryDrainOutputBuffer( + CodecState *filterState, + const sp<Surface> &surface, bool renderSurface, + bool useTimestamp, int64_t *startTimeRender) { + size_t index; + size_t offset; + size_t size; + int64_t presentationTimeUs; + uint32_t flags; + status_t err = filterState->mCodec->dequeueOutputBuffer( + &index, &offset, &size, &presentationTimeUs, &flags, + kTimeout); + + if (err != OK) { + return; + } + + ++outputFramesSinceFlush; + + if (kTestFlush && outputFramesSinceFlush >= kFlushAfterFrames) { + filterState->mCodec->flush(); + } + + if (surface == NULL || !renderSurface) { + err = filterState->mCodec->releaseOutputBuffer(index); + } else if (useTimestamp) { + if (*startTimeRender == -1) { + // begin rendering 2 vsyncs after first decode + *startTimeRender = systemTime(SYSTEM_TIME_MONOTONIC) + + 33000000 - (presentationTimeUs * 1000); + } + presentationTimeUs = + (presentationTimeUs * 1000) + *startTimeRender; + err = filterState->mCodec->renderOutputBufferAndRelease( + index, presentationTimeUs); + } else { + err = filterState->mCodec->renderOutputBufferAndRelease(index); + } + + if (kTestFlush && outputFramesSinceFlush >= kFlushAfterFrames) { + outputFramesSinceFlush = 0; + + // releasing the buffer dequeued before flush should cause an error + // if so, the frame will also be skipped in output stream + if (err == OK) { + ALOGE("FAIL: release after flush returned OK"); + } else if (err != -EACCES) { + ALOGE("releaseOutputBuffer after flush returned %d, " + "expected -EACCES (-13)", err); + } + } else { + CHECK(err == OK); + } + + if (flags & MediaCodec::BUFFER_FLAG_EOS) { + ALOGV("reached EOS on output."); + filterState->mSawOutputEOS = true; + } +} + +static int decode( + const sp<ALooper> &looper, + const char *path, + const sp<Surface> &surface, + bool renderSurface, + bool useTimestamp, + FilterType filterType) { + + static int64_t kTimeout = 500ll; + + sp<NuMediaExtractor> extractor = new NuMediaExtractor; + if (extractor->setDataSource(NULL /* httpService */, path) != OK) { + fprintf(stderr, "unable to instantiate extractor.\n"); + return 1; + } + + KeyedVector<size_t, CodecState> stateByTrack; + + CodecState *vidState = NULL; + for (size_t i = 0; i < extractor->countTracks(); ++i) { + sp<AMessage> format; + status_t err = extractor->getTrackFormat(i, &format); + CHECK(err == OK); + + AString mime; + CHECK(format->findString("mime", &mime)); + bool isVideo = !strncasecmp(mime.c_str(), "video/", 6); + if (!isVideo) { + continue; + } + + ALOGV("selecting track %zu", i); + + err = extractor->selectTrack(i); + CHECK(err == OK); + + CodecState *state = + &stateByTrack.editValueAt(stateByTrack.add(i, CodecState())); + + vidState = state; + + state->mNumBuffersDecoded = 0; + + state->mCodec = MediaCodec::CreateByType( + looper, mime.c_str(), false /* encoder */); + + CHECK(state->mCodec != NULL); + + err = state->mCodec->configure( + format, NULL /* surface */, NULL /* crypto */, 0 /* flags */); + + CHECK(err == OK); + + state->mSignalledInputEOS = false; + state->mSawOutputEOS = false; + + break; + } + CHECK(!stateByTrack.isEmpty()); + CHECK(vidState != NULL); + sp<AMessage> vidFormat; + vidState->mCodec->getOutputFormat(&vidFormat); + + // set filter to use ARGB8888 + vidFormat->setInt32("color-format", OMX_COLOR_Format32bitARGB8888); + // set app cache directory path + vidFormat->setString("cacheDir", "/system/bin"); + + // create RenderScript context for RSFilters + RSC::sp<RSC::RS> context = new RSC::RS(); + context->init("/system/bin"); + + sp<RenderScriptWrapper::RSFilterCallback> rsFilter; + + // create renderscript wrapper for RSFilters + sp<RenderScriptWrapper> rsWrapper = new RenderScriptWrapper; + rsWrapper->mContext = context.get(); + + CodecState *filterState = new CodecState(); + filterState->mNumBuffersDecoded = 0; + + sp<AMessage> params = new AMessage(); + + switch (filterType) { + case FILTERTYPE_ZERO: + { + filterState->mCodec = MediaCodec::CreateByComponentName( + looper, "android.filter.zerofilter"); + params->setInt32("invert", kInvert); + break; + } + case FILTERTYPE_INTRINSIC_BLUR: + { + filterState->mCodec = MediaCodec::CreateByComponentName( + looper, "android.filter.intrinsicblur"); + params->setFloat("blur-radius", kBlurRadius); + break; + } + case FILTERTYPE_SATURATION: + { + filterState->mCodec = MediaCodec::CreateByComponentName( + looper, "android.filter.saturation"); + params->setFloat("saturation", kSaturation); + break; + } + case FILTERTYPE_RS_SATURATION: + { + SaturationRSFilter *satFilter = new SaturationRSFilter; + satFilter->init(context); + rsFilter = satFilter; + rsWrapper->mCallback = rsFilter; + vidFormat->setObject("rs-wrapper", rsWrapper); + + filterState->mCodec = MediaCodec::CreateByComponentName( + looper, "android.filter.RenderScript"); + break; + } + case FILTERTYPE_RS_NIGHT_VISION: + { + NightVisionRSFilter *nightVisionFilter = new NightVisionRSFilter; + nightVisionFilter->init(context); + rsFilter = nightVisionFilter; + rsWrapper->mCallback = rsFilter; + vidFormat->setObject("rs-wrapper", rsWrapper); + + filterState->mCodec = MediaCodec::CreateByComponentName( + looper, "android.filter.RenderScript"); + break; + } + case FILTERTYPE_RS_ARGB_TO_RGBA: + { + ARGBToRGBARSFilter *argbToRgbaFilter = new ARGBToRGBARSFilter; + argbToRgbaFilter->init(context); + rsFilter = argbToRgbaFilter; + rsWrapper->mCallback = rsFilter; + vidFormat->setObject("rs-wrapper", rsWrapper); + + filterState->mCodec = MediaCodec::CreateByComponentName( + looper, "android.filter.RenderScript"); + break; + } + default: + { + LOG_ALWAYS_FATAL("mediacodec.cpp error: unrecognized FilterType"); + break; + } + } + CHECK(filterState->mCodec != NULL); + + status_t err = filterState->mCodec->configure( + vidFormat /* format */, surface, NULL /* crypto */, 0 /* flags */); + CHECK(err == OK); + + filterState->mSignalledInputEOS = false; + filterState->mSawOutputEOS = false; + + int64_t startTimeUs = ALooper::GetNowUs(); + int64_t startTimeRender = -1; + + for (size_t i = 0; i < stateByTrack.size(); ++i) { + CodecState *state = &stateByTrack.editValueAt(i); + + sp<MediaCodec> codec = state->mCodec; + + CHECK_EQ((status_t)OK, codec->start()); + + CHECK_EQ((status_t)OK, codec->getInputBuffers(&state->mInBuffers)); + CHECK_EQ((status_t)OK, codec->getOutputBuffers(&state->mOutBuffers)); + + ALOGV("got %zu input and %zu output buffers", + state->mInBuffers.size(), state->mOutBuffers.size()); + } + + CHECK_EQ((status_t)OK, filterState->mCodec->setParameters(params)); + + if (kTestFlush) { + status_t flushErr = filterState->mCodec->flush(); + if (flushErr == OK) { + ALOGE("FAIL: Flush before start returned OK"); + } else { + ALOGV("Flush before start returned status %d, usually ENOSYS (-38)", + flushErr); + } + } + + CHECK_EQ((status_t)OK, filterState->mCodec->start()); + CHECK_EQ((status_t)OK, filterState->mCodec->getInputBuffers( + &filterState->mInBuffers)); + CHECK_EQ((status_t)OK, filterState->mCodec->getOutputBuffers( + &filterState->mOutBuffers)); + + if (kTestFlush) { + status_t flushErr = filterState->mCodec->flush(); + if (flushErr != OK) { + ALOGE("FAIL: Flush after start returned %d, expect OK (0)", + flushErr); + } else { + ALOGV("Flush immediately after start OK"); + } + } + + List<DecodedFrame> decodedFrameIndices; + + // loop until decoder reaches EOS + bool sawInputEOS = false; + bool sawOutputEOSOnAllTracks = false; + while (!sawOutputEOSOnAllTracks) { + if (!sawInputEOS) { + size_t trackIndex; + status_t err = extractor->getSampleTrackIndex(&trackIndex); + + if (err != OK) { + ALOGV("saw input eos"); + sawInputEOS = true; + } else { + CodecState *state = &stateByTrack.editValueFor(trackIndex); + + size_t index; + err = state->mCodec->dequeueInputBuffer(&index, kTimeout); + + if (err == OK) { + ALOGV("filling input buffer %zu", index); + + const sp<ABuffer> &buffer = state->mInBuffers.itemAt(index); + + err = extractor->readSampleData(buffer); + CHECK(err == OK); + + int64_t timeUs; + err = extractor->getSampleTime(&timeUs); + CHECK(err == OK); + + uint32_t bufferFlags = 0; + + err = state->mCodec->queueInputBuffer( + index, 0 /* offset */, buffer->size(), + timeUs, bufferFlags); + + CHECK(err == OK); + + extractor->advance(); + } else { + CHECK_EQ(err, -EAGAIN); + } + } + } else { + for (size_t i = 0; i < stateByTrack.size(); ++i) { + CodecState *state = &stateByTrack.editValueAt(i); + + if (!state->mSignalledInputEOS) { + size_t index; + status_t err = + state->mCodec->dequeueInputBuffer(&index, kTimeout); + + if (err == OK) { + ALOGV("signalling input EOS on track %zu", i); + + err = state->mCodec->queueInputBuffer( + index, 0 /* offset */, 0 /* size */, + 0ll /* timeUs */, MediaCodec::BUFFER_FLAG_EOS); + + CHECK(err == OK); + + state->mSignalledInputEOS = true; + } else { + CHECK_EQ(err, -EAGAIN); + } + } + } + } + + sawOutputEOSOnAllTracks = true; + for (size_t i = 0; i < stateByTrack.size(); ++i) { + CodecState *state = &stateByTrack.editValueAt(i); + + if (state->mSawOutputEOS) { + continue; + } else { + sawOutputEOSOnAllTracks = false; + } + + DecodedFrame frame; + status_t err = state->mCodec->dequeueOutputBuffer( + &frame.index, &frame.offset, &frame.size, + &frame.presentationTimeUs, &frame.flags, kTimeout); + + if (err == OK) { + ALOGV("draining decoded buffer %zu, time = %lld us", + frame.index, frame.presentationTimeUs); + + ++(state->mNumBuffersDecoded); + + decodedFrameIndices.push_back(frame); + + if (frame.flags & MediaCodec::BUFFER_FLAG_EOS) { + ALOGV("reached EOS on decoder output."); + state->mSawOutputEOS = true; + } + + } else if (err == INFO_OUTPUT_BUFFERS_CHANGED) { + ALOGV("INFO_OUTPUT_BUFFERS_CHANGED"); + CHECK_EQ((status_t)OK, state->mCodec->getOutputBuffers( + &state->mOutBuffers)); + + ALOGV("got %zu output buffers", state->mOutBuffers.size()); + } else if (err == INFO_FORMAT_CHANGED) { + sp<AMessage> format; + CHECK_EQ((status_t)OK, state->mCodec->getOutputFormat(&format)); + + ALOGV("INFO_FORMAT_CHANGED: %s", + format->debugString().c_str()); + } else { + CHECK_EQ(err, -EAGAIN); + } + + tryCopyDecodedBuffer(&decodedFrameIndices, filterState, vidState); + + tryDrainOutputBuffer( + filterState, surface, renderSurface, + useTimestamp, &startTimeRender); + } + } + + // after EOS on decoder, let filter reach EOS + while (!filterState->mSawOutputEOS) { + tryCopyDecodedBuffer(&decodedFrameIndices, filterState, vidState); + + tryDrainOutputBuffer( + filterState, surface, renderSurface, + useTimestamp, &startTimeRender); + } + + int64_t elapsedTimeUs = ALooper::GetNowUs() - startTimeUs; + + for (size_t i = 0; i < stateByTrack.size(); ++i) { + CodecState *state = &stateByTrack.editValueAt(i); + + CHECK_EQ((status_t)OK, state->mCodec->release()); + + printf("track %zu: %" PRId64 " frames decoded and filtered, " + "%.2f fps.\n", i, state->mNumBuffersDecoded, + state->mNumBuffersDecoded * 1E6 / elapsedTimeUs); + } + + return 0; +} + +} // namespace android + +int main(int argc, char **argv) { + using namespace android; + + const char *me = argv[0]; + + bool useSurface = false; + bool renderSurface = false; + bool useTimestamp = false; + FilterType filterType = FILTERTYPE_ZERO; + + int res; + while ((res = getopt(argc, argv, "bcnrszTRSh")) >= 0) { + switch (res) { + case 'b': + { + filterType = FILTERTYPE_INTRINSIC_BLUR; + break; + } + case 'c': + { + filterType = FILTERTYPE_RS_ARGB_TO_RGBA; + break; + } + case 'n': + { + filterType = FILTERTYPE_RS_NIGHT_VISION; + break; + } + case 'r': + { + filterType = FILTERTYPE_RS_SATURATION; + break; + } + case 's': + { + filterType = FILTERTYPE_SATURATION; + break; + } + case 'z': + { + filterType = FILTERTYPE_ZERO; + break; + } + case 'T': + { + useTimestamp = true; + } + // fall through + case 'R': + { + renderSurface = true; + } + // fall through + case 'S': + { + useSurface = true; + break; + } + case '?': + case 'h': + default: + { + usage(me); + break; + } + } + } + + argc -= optind; + argv += optind; + + if (argc != 1) { + usage(me); + } + + ProcessState::self()->startThreadPool(); + + DataSource::RegisterDefaultSniffers(); + + android::sp<ALooper> looper = new ALooper; + looper->start(); + + android::sp<SurfaceComposerClient> composerClient; + android::sp<SurfaceControl> control; + android::sp<Surface> surface; + + if (useSurface) { + composerClient = new SurfaceComposerClient; + CHECK_EQ((status_t)OK, composerClient->initCheck()); + + android::sp<IBinder> display(SurfaceComposerClient::getBuiltInDisplay( + ISurfaceComposer::eDisplayIdMain)); + DisplayInfo info; + SurfaceComposerClient::getDisplayInfo(display, &info); + ssize_t displayWidth = info.w; + ssize_t displayHeight = info.h; + + ALOGV("display is %zd x %zd", displayWidth, displayHeight); + + control = composerClient->createSurface( + String8("A Surface"), displayWidth, displayHeight, + PIXEL_FORMAT_RGBA_8888, 0); + + CHECK(control != NULL); + CHECK(control->isValid()); + + SurfaceComposerClient::openGlobalTransaction(); + CHECK_EQ((status_t)OK, control->setLayer(INT_MAX)); + CHECK_EQ((status_t)OK, control->show()); + SurfaceComposerClient::closeGlobalTransaction(); + + surface = control->getSurface(); + CHECK(surface != NULL); + } + + decode(looper, argv[0], surface, renderSurface, useTimestamp, filterType); + + if (useSurface) { + composerClient->dispose(); + } + + looper->stop(); + + return 0; +} diff --git a/include/media/EffectsFactoryApi.h b/include/media/EffectsFactoryApi.h index b1ed7b0..64a3212 100644 --- a/include/media/EffectsFactoryApi.h +++ b/include/media/EffectsFactoryApi.h @@ -171,6 +171,8 @@ int EffectGetDescriptor(const effect_uuid_t *pEffectUuid, effect_descriptor_t *p //////////////////////////////////////////////////////////////////////////////// int EffectIsNullUuid(const effect_uuid_t *pEffectUuid); +int EffectDumpEffects(int fd); + #if __cplusplus } // extern "C" #endif diff --git a/include/media/stagefright/MediaFilter.h b/include/media/stagefright/MediaFilter.h new file mode 100644 index 0000000..7b3f700 --- /dev/null +++ b/include/media/stagefright/MediaFilter.h @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MEDIA_FILTER_H_ +#define MEDIA_FILTER_H_ + +#include <media/stagefright/CodecBase.h> + +namespace android { + +struct ABuffer; +struct GraphicBufferListener; +struct MemoryDealer; +struct SimpleFilter; + +struct MediaFilter : public CodecBase { + MediaFilter(); + + virtual void setNotificationMessage(const sp<AMessage> &msg); + + virtual void initiateAllocateComponent(const sp<AMessage> &msg); + virtual void initiateConfigureComponent(const sp<AMessage> &msg); + virtual void initiateCreateInputSurface(); + virtual void initiateStart(); + virtual void initiateShutdown(bool keepComponentAllocated = false); + + virtual void signalFlush(); + virtual void signalResume(); + + virtual void signalRequestIDRFrame(); + virtual void signalSetParameters(const sp<AMessage> &msg); + virtual void signalEndOfInputStream(); + + virtual void onMessageReceived(const sp<AMessage> &msg); + + struct PortDescription : public CodecBase::PortDescription { + virtual size_t countBuffers(); + virtual IOMX::buffer_id bufferIDAt(size_t index) const; + virtual sp<ABuffer> bufferAt(size_t index) const; + + protected: + PortDescription(); + + private: + friend struct MediaFilter; + + Vector<IOMX::buffer_id> mBufferIDs; + Vector<sp<ABuffer> > mBuffers; + + void addBuffer(IOMX::buffer_id id, const sp<ABuffer> &buffer); + + DISALLOW_EVIL_CONSTRUCTORS(PortDescription); + }; + +protected: + virtual ~MediaFilter(); + +private: + struct BufferInfo { + enum Status { + OWNED_BY_US, + OWNED_BY_UPSTREAM, + }; + + IOMX::buffer_id mBufferID; + int32_t mGeneration; + int32_t mOutputFlags; + Status mStatus; + + sp<ABuffer> mData; + }; + + enum State { + UNINITIALIZED, + INITIALIZED, + CONFIGURED, + STARTED, + }; + + enum { + kWhatInputBufferFilled = 'inpF', + kWhatOutputBufferDrained = 'outD', + kWhatShutdown = 'shut', + kWhatFlush = 'flus', + kWhatResume = 'resm', + kWhatAllocateComponent = 'allo', + kWhatConfigureComponent = 'conf', + kWhatCreateInputSurface = 'cisf', + kWhatSignalEndOfInputStream = 'eois', + kWhatStart = 'star', + kWhatSetParameters = 'setP', + kWhatProcessBuffers = 'proc', + }; + + enum { + kPortIndexInput = 0, + kPortIndexOutput = 1 + }; + + // member variables + AString mComponentName; + State mState; + status_t mInputEOSResult; + int32_t mWidth, mHeight; + int32_t mStride, mSliceHeight; + int32_t mColorFormatIn, mColorFormatOut; + size_t mMaxInputSize, mMaxOutputSize; + int32_t mGeneration; + sp<AMessage> mNotify; + sp<AMessage> mInputFormat; + sp<AMessage> mOutputFormat; + + sp<MemoryDealer> mDealer[2]; + Vector<BufferInfo> mBuffers[2]; + Vector<BufferInfo*> mAvailableInputBuffers; + Vector<BufferInfo*> mAvailableOutputBuffers; + bool mPortEOS[2]; + + sp<SimpleFilter> mFilter; + sp<GraphicBufferListener> mGraphicBufferListener; + + // helper functions + void signalProcessBuffers(); + void signalError(status_t error); + + status_t allocateBuffersOnPort(OMX_U32 portIndex); + BufferInfo *findBufferByID( + uint32_t portIndex, IOMX::buffer_id bufferID, + ssize_t *index = NULL); + void postFillThisBuffer(BufferInfo *info); + void postDrainThisBuffer(BufferInfo *info); + void postEOS(); + void sendFormatChange(); + void requestFillEmptyInput(); + void processBuffers(); + + void onAllocateComponent(const sp<AMessage> &msg); + void onConfigureComponent(const sp<AMessage> &msg); + void onStart(); + void onInputBufferFilled(const sp<AMessage> &msg); + void onOutputBufferDrained(const sp<AMessage> &msg); + void onShutdown(const sp<AMessage> &msg); + void onFlush(); + void onSetParameters(const sp<AMessage> &msg); + void onCreateInputSurface(); + void onInputFrameAvailable(); + void onSignalEndOfInputStream(); + + DISALLOW_EVIL_CONSTRUCTORS(MediaFilter); +}; + +} // namespace android + +#endif // MEDIA_FILTER_H_ diff --git a/include/media/stagefright/RenderScriptWrapper.h b/include/media/stagefright/RenderScriptWrapper.h new file mode 100644 index 0000000..b42649e --- /dev/null +++ b/include/media/stagefright/RenderScriptWrapper.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RENDERSCRIPT_WRAPPER_H_ +#define RENDERSCRIPT_WRAPPER_H_ + +#include <RenderScript.h> + +namespace android { + +struct RenderScriptWrapper : public RefBase { +public: + struct RSFilterCallback : public RefBase { + public: + // called by RSFilter to process each input buffer + virtual status_t processBuffers( + RSC::Allocation* inBuffer, + RSC::Allocation* outBuffer) = 0; + + virtual status_t handleSetParameters(const sp<AMessage> &msg) = 0; + }; + + sp<RSFilterCallback> mCallback; + RSC::sp<RSC::RS> mContext; +}; + +} // namespace android + +#endif // RENDERSCRIPT_WRAPPER_H_ diff --git a/media/libeffects/factory/EffectsFactory.c b/media/libeffects/factory/EffectsFactory.c index 6d30d64..c310fe2 100644 --- a/media/libeffects/factory/EffectsFactory.c +++ b/media/libeffects/factory/EffectsFactory.c @@ -28,6 +28,7 @@ static list_elem_t *gEffectList; // list of effect_entry_t: all currently created effects static list_elem_t *gLibraryList; // list of lib_entry_t: all currently loaded libraries +static list_elem_t *gSkippedEffects; // list of effects skipped because of duplicate uuid // list of effect_descriptor and list of sub effects : all currently loaded // It does not contain effects without sub effects. static list_sub_elem_t *gSubEffectList; @@ -63,10 +64,10 @@ static int findEffect(const effect_uuid_t *type, lib_entry_t **lib, effect_descriptor_t **desc); // To search a subeffect in the gSubEffectList -int findSubEffect(const effect_uuid_t *uuid, +static int findSubEffect(const effect_uuid_t *uuid, lib_entry_t **lib, effect_descriptor_t **desc); -static void dumpEffectDescriptor(effect_descriptor_t *desc, char *str, size_t len); +static void dumpEffectDescriptor(effect_descriptor_t *desc, char *str, size_t len, int indent); static int stringToUuid(const char *str, effect_uuid_t *uuid); static int uuidToString(const effect_uuid_t *uuid, char *str, size_t maxLen); @@ -237,8 +238,8 @@ int EffectQueryEffect(uint32_t index, effect_descriptor_t *pDescriptor) } #if (LOG_NDEBUG == 0) - char str[256]; - dumpEffectDescriptor(pDescriptor, str, 256); + char str[512]; + dumpEffectDescriptor(pDescriptor, str, sizeof(str), 0 /* indent */); ALOGV("EffectQueryEffect() desc:%s", str); #endif pthread_mutex_unlock(&gLibLock); @@ -503,15 +504,31 @@ int loadLibrary(cnode *root, const char *name) audio_effect_library_t *desc; list_elem_t *e; lib_entry_t *l; + char path[PATH_MAX]; + char *str; + size_t len; node = config_find(root, PATH_TAG); if (node == NULL) { return -EINVAL; } + // audio_effects.conf always specifies 32 bit lib path: convert to 64 bit path if needed + strlcpy(path, node->value, PATH_MAX); +#ifdef __LP64__ + str = strstr(path, "/lib/"); + if (str == NULL) + return -EINVAL; + len = str - path; + path[len] = '\0'; + strlcat(path, "/lib64/", PATH_MAX); + strlcat(path, node->value + len + strlen("/lib/"), PATH_MAX); +#endif + if (strlen(path) >= PATH_MAX - 1) + return -EINVAL; - hdl = dlopen(node->value, RTLD_NOW); + hdl = dlopen(path, RTLD_NOW); if (hdl == NULL) { - ALOGW("loadLibrary() failed to open %s", node->value); + ALOGW("loadLibrary() failed to open %s", path); goto error; } @@ -535,7 +552,7 @@ int loadLibrary(cnode *root, const char *name) // add entry for library in gLibraryList l = malloc(sizeof(lib_entry_t)); l->name = strndup(name, PATH_MAX); - l->path = strndup(node->value, PATH_MAX); + l->path = strndup(path, PATH_MAX); l->handle = hdl; l->desc = desc; l->effects = NULL; @@ -547,7 +564,7 @@ int loadLibrary(cnode *root, const char *name) e->next = gLibraryList; gLibraryList = e; pthread_mutex_unlock(&gLibLock); - ALOGV("getLibrary() linked library %p for path %s", l, node->value); + ALOGV("getLibrary() linked library %p for path %s", l, path); return 0; @@ -595,8 +612,8 @@ int addSubEffect(cnode *root) return -EINVAL; } #if (LOG_NDEBUG==0) - char s[256]; - dumpEffectDescriptor(d, s, 256); + char s[512]; + dumpEffectDescriptor(d, s, sizeof(s), 0 /* indent */); ALOGV("addSubEffect() read descriptor %p:%s",d, s); #endif if (EFFECT_API_VERSION_MAJOR(d->apiVersion) != @@ -660,6 +677,13 @@ int loadEffect(cnode *root) ALOGW("loadEffect() invalid uuid %s", node->value); return -EINVAL; } + lib_entry_t *tmp; + bool skip = false; + if (findEffect(NULL, &uuid, &tmp, NULL) == 0) { + ALOGW("skipping duplicate uuid %s %s", node->value, + node->next ? "and its sub-effects" : ""); + skip = true; + } d = malloc(sizeof(effect_descriptor_t)); if (l->desc->get_descriptor(&uuid, d) != 0) { @@ -670,8 +694,8 @@ int loadEffect(cnode *root) return -EINVAL; } #if (LOG_NDEBUG==0) - char s[256]; - dumpEffectDescriptor(d, s, 256); + char s[512]; + dumpEffectDescriptor(d, s, sizeof(s), 0 /* indent */); ALOGV("loadEffect() read descriptor %p:%s",d, s); #endif if (EFFECT_API_VERSION_MAJOR(d->apiVersion) != @@ -682,8 +706,14 @@ int loadEffect(cnode *root) } e = malloc(sizeof(list_elem_t)); e->object = d; - e->next = l->effects; - l->effects = e; + if (skip) { + e->next = gSkippedEffects; + gSkippedEffects = e; + return -EINVAL; + } else { + e->next = l->effects; + l->effects = e; + } // After the UUID node in the config_tree, if node->next is valid, // that would be sub effect node. @@ -876,22 +906,30 @@ int findEffect(const effect_uuid_t *type, return ret; } -void dumpEffectDescriptor(effect_descriptor_t *desc, char *str, size_t len) { +void dumpEffectDescriptor(effect_descriptor_t *desc, char *str, size_t len, int indent) { char s[256]; + char ss[256]; + char idt[indent + 1]; - snprintf(str, len, "\nEffect Descriptor %p:\n", desc); - strncat(str, "- TYPE: ", len); - uuidToString(&desc->uuid, s, 256); - snprintf(str, len, "- UUID: %s\n", s); - uuidToString(&desc->type, s, 256); - snprintf(str, len, "- TYPE: %s\n", s); - sprintf(s, "- apiVersion: %08X\n- flags: %08X\n", - desc->apiVersion, desc->flags); - strncat(str, s, len); - sprintf(s, "- name: %s\n", desc->name); - strncat(str, s, len); - sprintf(s, "- implementor: %s\n", desc->implementor); - strncat(str, s, len); + memset(idt, ' ', indent); + idt[indent] = 0; + + str[0] = 0; + + snprintf(s, sizeof(s), "%s%s / %s\n", idt, desc->name, desc->implementor); + strlcat(str, s, len); + + uuidToString(&desc->uuid, s, sizeof(s)); + snprintf(ss, sizeof(ss), "%s UUID: %s\n", idt, s); + strlcat(str, ss, len); + + uuidToString(&desc->type, s, sizeof(s)); + snprintf(ss, sizeof(ss), "%s TYPE: %s\n", idt, s); + strlcat(str, ss, len); + + sprintf(s, "%s apiVersion: %08X\n%s flags: %08X\n", idt, + desc->apiVersion, idt, desc->flags); + strlcat(str, s, len); } int stringToUuid(const char *str, effect_uuid_t *uuid) @@ -934,3 +972,40 @@ int uuidToString(const effect_uuid_t *uuid, char *str, size_t maxLen) return 0; } +int EffectDumpEffects(int fd) { + char s[512]; + list_elem_t *e = gLibraryList; + lib_entry_t *l = NULL; + effect_descriptor_t *d = NULL; + int found = 0; + int ret = 0; + + while (e) { + l = (lib_entry_t *)e->object; + list_elem_t *efx = l->effects; + dprintf(fd, "Library %s\n", l->name); + if (!efx) { + dprintf(fd, " (no effects)\n"); + } + while (efx) { + d = (effect_descriptor_t *)efx->object; + dumpEffectDescriptor(d, s, sizeof(s), 2); + dprintf(fd, "%s", s); + efx = efx->next; + } + e = e->next; + } + + e = gSkippedEffects; + if (e) { + dprintf(fd, "Skipped effects\n"); + while(e) { + d = (effect_descriptor_t *)e->object; + dumpEffectDescriptor(d, s, sizeof(s), 2 /* indent */); + dprintf(fd, "%s", s); + e = e->next; + } + } + return ret; +} + diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp index e09567a..de16ec9 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp @@ -344,6 +344,14 @@ status_t NuPlayerDriver::seekTo(int msec) { case STATE_PREPARED: case STATE_STOPPED_AND_PREPARED: { + int curpos = 0; + if (mPositionUs > 0) { + curpos = (mPositionUs + 500ll) / 1000; + } + if (curpos == msec) { + // nothing to do, and doing something anyway could result in deadlock (b/15323063) + break; + } mStartupSeekTimeUs = seekTimeUs; // pretend that the seek completed. It will actually happen when starting playback. // TODO: actually perform the seek here, so the player is ready to go at the new diff --git a/media/libnbaio/MonoPipeReader.cpp b/media/libnbaio/MonoPipeReader.cpp index de82229..e4d3ed8 100644 --- a/media/libnbaio/MonoPipeReader.cpp +++ b/media/libnbaio/MonoPipeReader.cpp @@ -39,7 +39,7 @@ ssize_t MonoPipeReader::availableToRead() return NEGOTIATE; } ssize_t ret = android_atomic_acquire_load(&mPipe->mRear) - mPipe->mFront; - ALOG_ASSERT((0 <= ret) && (ret <= mMaxFrames)); + ALOG_ASSERT((0 <= ret) && ((size_t) ret <= mPipe->mMaxFrames)); return ret; } diff --git a/media/libstagefright/Android.mk b/media/libstagefright/Android.mk index 193f8a7..7c4f92a 100644 --- a/media/libstagefright/Android.mk +++ b/media/libstagefright/Android.mk @@ -103,6 +103,7 @@ LOCAL_STATIC_LIBRARIES := \ libstagefright_color_conversion \ libstagefright_aacenc \ libstagefright_matroska \ + libstagefright_mediafilter \ libstagefright_webm \ libstagefright_timedtext \ libvpx \ @@ -110,7 +111,8 @@ LOCAL_STATIC_LIBRARIES := \ libstagefright_mpeg2ts \ libstagefright_id3 \ libFLAC \ - libmedia_helper + libmedia_helper \ + libRScpp_static LOCAL_SHARED_LIBRARIES += \ libstagefright_enc_common \ diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp index 11069e4..3622eb7 100644 --- a/media/libstagefright/MediaCodec.cpp +++ b/media/libstagefright/MediaCodec.cpp @@ -36,6 +36,7 @@ #include <media/stagefright/MediaCodecList.h> #include <media/stagefright/MediaDefs.h> #include <media/stagefright/MediaErrors.h> +#include <media/stagefright/MediaFilter.h> #include <media/stagefright/MetaData.h> #include <media/stagefright/NativeWindowWrapper.h> #include <private/android_filesystem_config.h> @@ -189,7 +190,16 @@ status_t MediaCodec::init(const AString &name, bool nameIsType, bool encoder) { // quickly, violating the OpenMAX specs, until that is remedied // we need to invest in an extra looper to free the main event // queue. - mCodec = new ACodec; + + if (nameIsType || !strncasecmp(name.c_str(), "omx.", 4)) { + mCodec = new ACodec; + } else if (!nameIsType + && !strncasecmp(name.c_str(), "android.filter.", 15)) { + mCodec = new MediaFilter; + } else { + return NAME_NOT_FOUND; + } + bool needDedicatedLooper = false; if (nameIsType && !strncasecmp(name.c_str(), "video/", 6)) { needDedicatedLooper = true; diff --git a/media/libstagefright/colorconversion/SoftwareRenderer.cpp b/media/libstagefright/colorconversion/SoftwareRenderer.cpp index 1899b40..6474906 100644 --- a/media/libstagefright/colorconversion/SoftwareRenderer.cpp +++ b/media/libstagefright/colorconversion/SoftwareRenderer.cpp @@ -98,32 +98,48 @@ void SoftwareRenderer::resetFormatIfChanged(const sp<AMessage> &format) { mCropWidth = mCropRight - mCropLeft + 1; mCropHeight = mCropBottom - mCropTop + 1; - int halFormat; - size_t bufWidth, bufHeight; - - switch (mColorFormat) { - case OMX_COLOR_FormatYUV420Planar: - case OMX_TI_COLOR_FormatYUV420PackedSemiPlanar: - { - if (!runningInEmulator()) { + // by default convert everything to RGB565 + int halFormat = HAL_PIXEL_FORMAT_RGB_565; + size_t bufWidth = mCropWidth; + size_t bufHeight = mCropHeight; + + // hardware has YUV12 and RGBA8888 support, so convert known formats + if (!runningInEmulator()) { + switch (mColorFormat) { + case OMX_COLOR_FormatYUV420Planar: + case OMX_TI_COLOR_FormatYUV420PackedSemiPlanar: + { halFormat = HAL_PIXEL_FORMAT_YV12; bufWidth = (mCropWidth + 1) & ~1; bufHeight = (mCropHeight + 1) & ~1; break; } - - // fall through. + case OMX_COLOR_Format24bitRGB888: + { + halFormat = HAL_PIXEL_FORMAT_RGB_888; + bufWidth = (mCropWidth + 1) & ~1; + bufHeight = (mCropHeight + 1) & ~1; + break; + } + case OMX_COLOR_Format32bitARGB8888: + case OMX_COLOR_Format32BitRGBA8888: + { + halFormat = HAL_PIXEL_FORMAT_RGBA_8888; + bufWidth = (mCropWidth + 1) & ~1; + bufHeight = (mCropHeight + 1) & ~1; + break; + } + default: + { + break; + } } + } - default: - halFormat = HAL_PIXEL_FORMAT_RGB_565; - bufWidth = mCropWidth; - bufHeight = mCropHeight; - - mConverter = new ColorConverter( - mColorFormat, OMX_COLOR_Format16bitRGB565); - CHECK(mConverter->isValid()); - break; + if (halFormat == HAL_PIXEL_FORMAT_RGB_565) { + mConverter = new ColorConverter( + mColorFormat, OMX_COLOR_Format16bitRGB565); + CHECK(mConverter->isValid()); } CHECK(mNativeWindow != NULL); @@ -200,6 +216,8 @@ void SoftwareRenderer::render( CHECK_EQ(0, mapper.lock( buf->handle, GRALLOC_USAGE_SW_WRITE_OFTEN, bounds, &dst)); + // TODO move the other conversions also into ColorConverter, and + // fix cropping issues (when mCropLeft/Top != 0 or mWidth != mCropWidth) if (mConverter) { mConverter->convert( data, @@ -210,7 +228,8 @@ void SoftwareRenderer::render( 0, 0, mCropWidth - 1, mCropHeight - 1); } else if (mColorFormat == OMX_COLOR_FormatYUV420Planar) { const uint8_t *src_y = (const uint8_t *)data; - const uint8_t *src_u = (const uint8_t *)data + mWidth * mHeight; + const uint8_t *src_u = + (const uint8_t *)data + mWidth * mHeight; const uint8_t *src_v = src_u + (mWidth / 2 * mHeight / 2); uint8_t *dst_y = (uint8_t *)dst; @@ -236,14 +255,10 @@ void SoftwareRenderer::render( dst_u += dst_c_stride; dst_v += dst_c_stride; } - } else { - CHECK_EQ(mColorFormat, OMX_TI_COLOR_FormatYUV420PackedSemiPlanar); - - const uint8_t *src_y = - (const uint8_t *)data; - - const uint8_t *src_uv = - (const uint8_t *)data + mWidth * (mHeight - mCropTop / 2); + } else if (mColorFormat == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar) { + const uint8_t *src_y = (const uint8_t *)data; + const uint8_t *src_uv = (const uint8_t *)data + + mWidth * (mHeight - mCropTop / 2); uint8_t *dst_y = (uint8_t *)dst; @@ -271,6 +286,40 @@ void SoftwareRenderer::render( dst_u += dst_c_stride; dst_v += dst_c_stride; } + } else if (mColorFormat == OMX_COLOR_Format24bitRGB888) { + uint8_t* srcPtr = (uint8_t*)data; + uint8_t* dstPtr = (uint8_t*)dst; + + for (size_t y = 0; y < (size_t)mCropHeight; ++y) { + memcpy(dstPtr, srcPtr, mCropWidth * 3); + srcPtr += mWidth * 3; + dstPtr += buf->stride * 3; + } + } else if (mColorFormat == OMX_COLOR_Format32bitARGB8888) { + uint8_t *srcPtr, *dstPtr; + + for (size_t y = 0; y < (size_t)mCropHeight; ++y) { + srcPtr = (uint8_t*)data + mWidth * 4 * y; + dstPtr = (uint8_t*)dst + buf->stride * 4 * y; + for (size_t x = 0; x < (size_t)mCropWidth; ++x) { + uint8_t a = *srcPtr++; + for (size_t i = 0; i < 3; ++i) { // copy RGB + *dstPtr++ = *srcPtr++; + } + *dstPtr++ = a; // alpha last (ARGB to RGBA) + } + } + } else if (mColorFormat == OMX_COLOR_Format32BitRGBA8888) { + uint8_t* srcPtr = (uint8_t*)data; + uint8_t* dstPtr = (uint8_t*)dst; + + for (size_t y = 0; y < (size_t)mCropHeight; ++y) { + memcpy(dstPtr, srcPtr, mCropWidth * 4); + srcPtr += mWidth * 4; + dstPtr += buf->stride * 4; + } + } else { + LOG_ALWAYS_FATAL("bad color format %#x", mColorFormat); } CHECK_EQ(0, mapper.unlock(buf->handle)); diff --git a/media/libstagefright/filters/Android.mk b/media/libstagefright/filters/Android.mk new file mode 100644 index 0000000..5e895e1 --- /dev/null +++ b/media/libstagefright/filters/Android.mk @@ -0,0 +1,32 @@ +LOCAL_PATH:= $(call my-dir) +include $(CLEAR_VARS) + +LOCAL_NDK_STL_VARIANT := stlport_static + +LOCAL_SRC_FILES := \ + ColorConvert.cpp \ + GraphicBufferListener.cpp \ + IntrinsicBlurFilter.cpp \ + MediaFilter.cpp \ + RSFilter.cpp \ + SaturationFilter.cpp \ + saturationARGB.rs \ + SimpleFilter.cpp \ + ZeroFilter.cpp + +LOCAL_C_INCLUDES := \ + $(TOP)/bionic \ + $(TOP)/bionic/libstdc++/include \ + $(TOP)/external/stlport/stlport \ + $(TOP)/frameworks/native/include/media/openmax \ + $(TOP)/frameworks/rs/cpp \ + $(TOP)/frameworks/rs \ + +intermediates := $(call intermediates-dir-for,STATIC_LIBRARIES,libRS,TARGET,) +LOCAL_C_INCLUDES += $(intermediates) + +LOCAL_CFLAGS += -Wno-multichar + +LOCAL_MODULE:= libstagefright_mediafilter + +include $(BUILD_STATIC_LIBRARY) diff --git a/media/libstagefright/filters/ColorConvert.cpp b/media/libstagefright/filters/ColorConvert.cpp new file mode 100644 index 0000000..a5039f9 --- /dev/null +++ b/media/libstagefright/filters/ColorConvert.cpp @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ColorConvert.h" + +#ifndef max +#define max(a,b) ((a) > (b) ? (a) : (b)) +#endif +#ifndef min +#define min(a,b) ((a) < (b) ? (a) : (b)) +#endif + +namespace android { + +void YUVToRGB( + int32_t y, int32_t u, int32_t v, + int32_t* r, int32_t* g, int32_t* b) { + y -= 16; + u -= 128; + v -= 128; + + *b = 1192 * y + 2066 * u; + *g = 1192 * y - 833 * v - 400 * u; + *r = 1192 * y + 1634 * v; + + *r = min(262143, max(0, *r)); + *g = min(262143, max(0, *g)); + *b = min(262143, max(0, *b)); + + *r >>= 10; + *g >>= 10; + *b >>= 10; +} + +void convertYUV420spToARGB( + uint8_t *pY, uint8_t *pUV, int32_t width, int32_t height, + uint8_t *dest) { + const int32_t bytes_per_pixel = 2; + + for (int32_t i = 0; i < height; i++) { + for (int32_t j = 0; j < width; j++) { + int32_t y = *(pY + i * width + j); + int32_t u = *(pUV + (i/2) * width + bytes_per_pixel * (j/2)); + int32_t v = *(pUV + (i/2) * width + bytes_per_pixel * (j/2) + 1); + + int32_t r, g, b; + YUVToRGB(y, u, v, &r, &g, &b); + + *dest++ = 0xFF; + *dest++ = r; + *dest++ = g; + *dest++ = b; + } + } +} + +void convertYUV420spToRGB888( + uint8_t *pY, uint8_t *pUV, int32_t width, int32_t height, + uint8_t *dest) { + const int32_t bytes_per_pixel = 2; + + for (int32_t i = 0; i < height; i++) { + for (int32_t j = 0; j < width; j++) { + int32_t y = *(pY + i * width + j); + int32_t u = *(pUV + (i/2) * width + bytes_per_pixel * (j/2)); + int32_t v = *(pUV + (i/2) * width + bytes_per_pixel * (j/2) + 1); + + int32_t r, g, b; + YUVToRGB(y, u, v, &r, &g, &b); + + *dest++ = r; + *dest++ = g; + *dest++ = b; + } + } +} + +// HACK - not even slightly optimized +// TODO: remove when RGBA support is added to SoftwareRenderer +void convertRGBAToARGB( + uint8_t *src, int32_t width, int32_t height, uint32_t stride, + uint8_t *dest) { + for (size_t i = 0; i < height; ++i) { + for (size_t j = 0; j < width; ++j) { + uint8_t r = *src++; + uint8_t g = *src++; + uint8_t b = *src++; + uint8_t a = *src++; + *dest++ = a; + *dest++ = r; + *dest++ = g; + *dest++ = b; + } + src += (stride - width) * 4; + } +} + +} // namespace android diff --git a/media/libstagefright/filters/ColorConvert.h b/media/libstagefright/filters/ColorConvert.h new file mode 100644 index 0000000..13faa02 --- /dev/null +++ b/media/libstagefright/filters/ColorConvert.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COLOR_CONVERT_H_ +#define COLOR_CONVERT_H_ + +#include <inttypes.h> + +namespace android { + +void YUVToRGB( + int32_t y, int32_t u, int32_t v, + int32_t* r, int32_t* g, int32_t* b); + +void convertYUV420spToARGB( + uint8_t *pY, uint8_t *pUV, int32_t width, int32_t height, + uint8_t *dest); + +void convertYUV420spToRGB888( + uint8_t *pY, uint8_t *pUV, int32_t width, int32_t height, + uint8_t *dest); + +// TODO: remove when RGBA support is added to SoftwareRenderer +void convertRGBAToARGB( + uint8_t *src, int32_t width, int32_t height, uint32_t stride, + uint8_t *dest); + +} // namespace android + +#endif // COLOR_CONVERT_H_ diff --git a/media/libstagefright/filters/GraphicBufferListener.cpp b/media/libstagefright/filters/GraphicBufferListener.cpp new file mode 100644 index 0000000..fa38192 --- /dev/null +++ b/media/libstagefright/filters/GraphicBufferListener.cpp @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "GraphicBufferListener" + +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/MediaErrors.h> + +#include "GraphicBufferListener.h" + +namespace android { + +status_t GraphicBufferListener::init( + const sp<AMessage> ¬ify, + size_t bufferWidth, size_t bufferHeight, size_t bufferCount) { + mNotify = notify; + + String8 name("GraphicBufferListener"); + BufferQueue::createBufferQueue(&mProducer, &mConsumer); + mConsumer->setConsumerName(name); + mConsumer->setDefaultBufferSize(bufferWidth, bufferHeight); + mConsumer->setConsumerUsageBits(GRALLOC_USAGE_SW_READ_OFTEN); + + status_t err = mConsumer->setMaxAcquiredBufferCount(bufferCount); + if (err != NO_ERROR) { + ALOGE("Unable to set BQ max acquired buffer count to %u: %d", + bufferCount, err); + return err; + } + + wp<BufferQueue::ConsumerListener> listener = + static_cast<BufferQueue::ConsumerListener*>(this); + sp<BufferQueue::ProxyConsumerListener> proxy = + new BufferQueue::ProxyConsumerListener(listener); + + err = mConsumer->consumerConnect(proxy, false); + if (err != NO_ERROR) { + ALOGE("Error connecting to BufferQueue: %s (%d)", + strerror(-err), err); + return err; + } + + ALOGV("init() successful."); + + return OK; +} + +void GraphicBufferListener::onFrameAvailable(const BufferItem& /* item */) { + ALOGV("onFrameAvailable() called"); + + { + Mutex::Autolock autoLock(mMutex); + mNumFramesAvailable++; + } + + sp<AMessage> notify = mNotify->dup(); + mNotify->setWhat(kWhatFrameAvailable); + mNotify->post(); +} + +void GraphicBufferListener::onBuffersReleased() { + ALOGV("onBuffersReleased() called"); + // nothing to do +} + +void GraphicBufferListener::onSidebandStreamChanged() { + ALOGW("GraphicBufferListener cannot consume sideband streams."); + // nothing to do +} + +BufferQueue::BufferItem GraphicBufferListener::getBufferItem() { + BufferQueue::BufferItem item; + + { + Mutex::Autolock autoLock(mMutex); + if (mNumFramesAvailable <= 0) { + ALOGE("getBuffer() called with no frames available"); + return item; + } + mNumFramesAvailable--; + } + + status_t err = mConsumer->acquireBuffer(&item, 0); + if (err == BufferQueue::NO_BUFFER_AVAILABLE) { + // shouldn't happen, since we track num frames available + ALOGE("frame was not available"); + item.mBuf = -1; + return item; + } else if (err != OK) { + ALOGE("acquireBuffer returned err=%d", err); + item.mBuf = -1; + return item; + } + + // Wait for it to become available. + err = item.mFence->waitForever("GraphicBufferListener::getBufferItem"); + if (err != OK) { + ALOGW("failed to wait for buffer fence: %d", err); + // keep going + } + + // If this is the first time we're seeing this buffer, add it to our + // slot table. + if (item.mGraphicBuffer != NULL) { + ALOGV("setting mBufferSlot %d", item.mBuf); + mBufferSlot[item.mBuf] = item.mGraphicBuffer; + } + + return item; +} + +sp<GraphicBuffer> GraphicBufferListener::getBuffer( + BufferQueue::BufferItem item) { + sp<GraphicBuffer> buf; + if (item.mBuf < 0 || item.mBuf >= BufferQueue::NUM_BUFFER_SLOTS) { + ALOGE("getBuffer() received invalid BufferItem: mBuf==%d", item.mBuf); + return buf; + } + + buf = mBufferSlot[item.mBuf]; + CHECK(buf.get() != NULL); + + return buf; +} + +status_t GraphicBufferListener::releaseBuffer( + BufferQueue::BufferItem item) { + if (item.mBuf < 0 || item.mBuf >= BufferQueue::NUM_BUFFER_SLOTS) { + ALOGE("getBuffer() received invalid BufferItem: mBuf==%d", item.mBuf); + return ERROR_OUT_OF_RANGE; + } + + mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber, + EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE); + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/GraphicBufferListener.h b/media/libstagefright/filters/GraphicBufferListener.h new file mode 100644 index 0000000..b3e0ee3 --- /dev/null +++ b/media/libstagefright/filters/GraphicBufferListener.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GRAPHIC_BUFFER_LISTENER_H_ +#define GRAPHIC_BUFFER_LISTENER_H_ + +#include <gui/BufferQueue.h> + +namespace android { + +struct AMessage; + +struct GraphicBufferListener : public BufferQueue::ConsumerListener { +public: + GraphicBufferListener() {}; + + status_t init( + const sp<AMessage> ¬ify, + size_t bufferWidth, size_t bufferHeight, size_t bufferCount); + + virtual void onFrameAvailable(const BufferItem& item); + virtual void onBuffersReleased(); + virtual void onSidebandStreamChanged(); + + // Returns the handle to the producer side of the BufferQueue. Buffers + // queued on this will be received by GraphicBufferListener. + sp<IGraphicBufferProducer> getIGraphicBufferProducer() const { + return mProducer; + } + + BufferQueue::BufferItem getBufferItem(); + sp<GraphicBuffer> getBuffer(BufferQueue::BufferItem item); + status_t releaseBuffer(BufferQueue::BufferItem item); + + enum { + kWhatFrameAvailable = 'frav', + }; + +private: + sp<AMessage> mNotify; + size_t mNumFramesAvailable; + + mutable Mutex mMutex; + + // Our BufferQueue interfaces. mProducer is passed to the producer through + // getIGraphicBufferProducer, and mConsumer is used internally to retrieve + // the buffers queued by the producer. + sp<IGraphicBufferProducer> mProducer; + sp<IGraphicBufferConsumer> mConsumer; + + // Cache of GraphicBuffers from the buffer queue. + sp<GraphicBuffer> mBufferSlot[BufferQueue::NUM_BUFFER_SLOTS]; +}; + +} // namespace android + +#endif // GRAPHIC_BUFFER_LISTENER_H diff --git a/media/libstagefright/filters/IntrinsicBlurFilter.cpp b/media/libstagefright/filters/IntrinsicBlurFilter.cpp new file mode 100644 index 0000000..cbcf699 --- /dev/null +++ b/media/libstagefright/filters/IntrinsicBlurFilter.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "IntrinsicBlurFilter" + +#include <utils/Log.h> + +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include "IntrinsicBlurFilter.h" + +namespace android { + +status_t IntrinsicBlurFilter::configure(const sp<AMessage> &msg) { + status_t err = SimpleFilter::configure(msg); + if (err != OK) { + return err; + } + + if (!msg->findString("cacheDir", &mCacheDir)) { + ALOGE("Failed to find cache directory in config message."); + return NAME_NOT_FOUND; + } + + return OK; +} + +status_t IntrinsicBlurFilter::start() { + // TODO: use a single RS context object for entire application + mRS = new RSC::RS(); + + if (!mRS->init(mCacheDir.c_str())) { + ALOGE("Failed to initialize RenderScript context."); + return NO_INIT; + } + + // 32-bit elements for ARGB8888 + RSC::sp<const RSC::Element> e = RSC::Element::U8_4(mRS); + + RSC::Type::Builder tb(mRS, e); + tb.setX(mWidth); + tb.setY(mHeight); + RSC::sp<const RSC::Type> t = tb.create(); + + mAllocIn = RSC::Allocation::createTyped(mRS, t); + mAllocOut = RSC::Allocation::createTyped(mRS, t); + + mBlur = RSC::ScriptIntrinsicBlur::create(mRS, e); + mBlur->setRadius(mBlurRadius); + mBlur->setInput(mAllocIn); + + return OK; +} + +void IntrinsicBlurFilter::reset() { + mBlur.clear(); + mAllocOut.clear(); + mAllocIn.clear(); + mRS.clear(); +} + +status_t IntrinsicBlurFilter::setParameters(const sp<AMessage> &msg) { + sp<AMessage> params; + CHECK(msg->findMessage("params", ¶ms)); + + float blurRadius; + if (params->findFloat("blur-radius", &blurRadius)) { + mBlurRadius = blurRadius; + } + + return OK; +} + +status_t IntrinsicBlurFilter::processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer) { + mAllocIn->copy1DRangeFrom(0, mWidth * mHeight, srcBuffer->data()); + mBlur->forEach(mAllocOut); + mAllocOut->copy1DRangeTo(0, mWidth * mHeight, outBuffer->data()); + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/IntrinsicBlurFilter.h b/media/libstagefright/filters/IntrinsicBlurFilter.h new file mode 100644 index 0000000..4707ab7 --- /dev/null +++ b/media/libstagefright/filters/IntrinsicBlurFilter.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef INTRINSIC_BLUR_FILTER_H_ +#define INTRINSIC_BLUR_FILTER_H_ + +#include "RenderScript.h" +#include "SimpleFilter.h" + +namespace android { + +struct IntrinsicBlurFilter : public SimpleFilter { +public: + IntrinsicBlurFilter() : mBlurRadius(1.f) {}; + + virtual status_t configure(const sp<AMessage> &msg); + virtual status_t start(); + virtual void reset(); + virtual status_t setParameters(const sp<AMessage> &msg); + virtual status_t processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer); + +protected: + virtual ~IntrinsicBlurFilter() {}; + +private: + AString mCacheDir; + RSC::sp<RSC::RS> mRS; + RSC::sp<RSC::Allocation> mAllocIn; + RSC::sp<RSC::Allocation> mAllocOut; + RSC::sp<RSC::ScriptIntrinsicBlur> mBlur; + float mBlurRadius; +}; + +} // namespace android + +#endif // INTRINSIC_BLUR_FILTER_H_ diff --git a/media/libstagefright/filters/MediaFilter.cpp b/media/libstagefright/filters/MediaFilter.cpp new file mode 100644 index 0000000..c5289b6 --- /dev/null +++ b/media/libstagefright/filters/MediaFilter.cpp @@ -0,0 +1,816 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "MediaFilter" + +#include <inttypes.h> +#include <utils/Trace.h> + +#include <binder/MemoryDealer.h> + +#include <media/stagefright/BufferProducerWrapper.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include <media/stagefright/MediaDefs.h> +#include <media/stagefright/MediaErrors.h> +#include <media/stagefright/MediaFilter.h> + +#include "ColorConvert.h" +#include "GraphicBufferListener.h" +#include "IntrinsicBlurFilter.h" +#include "RSFilter.h" +#include "SaturationFilter.h" +#include "ZeroFilter.h" + +namespace android { + +// parameter: number of input and output buffers +static const size_t kBufferCountActual = 4; + +MediaFilter::MediaFilter() + : mState(UNINITIALIZED), + mGeneration(0), + mGraphicBufferListener(NULL) { +} + +MediaFilter::~MediaFilter() { +} + +//////////////////// PUBLIC FUNCTIONS ////////////////////////////////////////// + +void MediaFilter::setNotificationMessage(const sp<AMessage> &msg) { + mNotify = msg; +} + +void MediaFilter::initiateAllocateComponent(const sp<AMessage> &msg) { + msg->setWhat(kWhatAllocateComponent); + msg->setTarget(id()); + msg->post(); +} + +void MediaFilter::initiateConfigureComponent(const sp<AMessage> &msg) { + msg->setWhat(kWhatConfigureComponent); + msg->setTarget(id()); + msg->post(); +} + +void MediaFilter::initiateCreateInputSurface() { + (new AMessage(kWhatCreateInputSurface, id()))->post(); +} + +void MediaFilter::initiateStart() { + (new AMessage(kWhatStart, id()))->post(); +} + +void MediaFilter::initiateShutdown(bool keepComponentAllocated) { + sp<AMessage> msg = new AMessage(kWhatShutdown, id()); + msg->setInt32("keepComponentAllocated", keepComponentAllocated); + msg->post(); +} + +void MediaFilter::signalFlush() { + (new AMessage(kWhatFlush, id()))->post(); +} + +void MediaFilter::signalResume() { + (new AMessage(kWhatResume, id()))->post(); +} + +// nothing to do +void MediaFilter::signalRequestIDRFrame() { + return; +} + +void MediaFilter::signalSetParameters(const sp<AMessage> ¶ms) { + sp<AMessage> msg = new AMessage(kWhatSetParameters, id()); + msg->setMessage("params", params); + msg->post(); +} + +void MediaFilter::signalEndOfInputStream() { + (new AMessage(kWhatSignalEndOfInputStream, id()))->post(); +} + +void MediaFilter::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatAllocateComponent: + { + onAllocateComponent(msg); + break; + } + case kWhatConfigureComponent: + { + onConfigureComponent(msg); + break; + } + case kWhatStart: + { + onStart(); + break; + } + case kWhatProcessBuffers: + { + processBuffers(); + break; + } + case kWhatInputBufferFilled: + { + onInputBufferFilled(msg); + break; + } + case kWhatOutputBufferDrained: + { + onOutputBufferDrained(msg); + break; + } + case kWhatShutdown: + { + onShutdown(msg); + break; + } + case kWhatFlush: + { + onFlush(); + break; + } + case kWhatResume: + { + // nothing to do + break; + } + case kWhatSetParameters: + { + onSetParameters(msg); + break; + } + case kWhatCreateInputSurface: + { + onCreateInputSurface(); + break; + } + case GraphicBufferListener::kWhatFrameAvailable: + { + onInputFrameAvailable(); + break; + } + case kWhatSignalEndOfInputStream: + { + onSignalEndOfInputStream(); + break; + } + default: + { + ALOGE("Message not handled:\n%s", msg->debugString().c_str()); + break; + } + } +} + +//////////////////// PORT DESCRIPTION ////////////////////////////////////////// + +MediaFilter::PortDescription::PortDescription() { +} + +void MediaFilter::PortDescription::addBuffer( + IOMX::buffer_id id, const sp<ABuffer> &buffer) { + mBufferIDs.push_back(id); + mBuffers.push_back(buffer); +} + +size_t MediaFilter::PortDescription::countBuffers() { + return mBufferIDs.size(); +} + +IOMX::buffer_id MediaFilter::PortDescription::bufferIDAt(size_t index) const { + return mBufferIDs.itemAt(index); +} + +sp<ABuffer> MediaFilter::PortDescription::bufferAt(size_t index) const { + return mBuffers.itemAt(index); +} + +//////////////////// HELPER FUNCTIONS ////////////////////////////////////////// + +void MediaFilter::signalProcessBuffers() { + (new AMessage(kWhatProcessBuffers, id()))->post(); +} + +void MediaFilter::signalError(status_t error) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatError); + notify->setInt32("err", error); + notify->post(); +} + +status_t MediaFilter::allocateBuffersOnPort(OMX_U32 portIndex) { + CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); + const bool isInput = portIndex == kPortIndexInput; + const size_t bufferSize = isInput ? mMaxInputSize : mMaxOutputSize; + + CHECK(mDealer[portIndex] == NULL); + CHECK(mBuffers[portIndex].isEmpty()); + + ALOGV("Allocating %zu buffers of size %zu on %s port", + kBufferCountActual, bufferSize, + isInput ? "input" : "output"); + + size_t totalSize = kBufferCountActual * bufferSize; + + mDealer[portIndex] = new MemoryDealer(totalSize, "MediaFilter"); + + for (size_t i = 0; i < kBufferCountActual; ++i) { + sp<IMemory> mem = mDealer[portIndex]->allocate(bufferSize); + CHECK(mem.get() != NULL); + + BufferInfo info; + info.mStatus = BufferInfo::OWNED_BY_US; + info.mBufferID = i; + info.mGeneration = mGeneration; + info.mOutputFlags = 0; + info.mData = new ABuffer(mem->pointer(), bufferSize); + info.mData->meta()->setInt64("timeUs", 0); + + mBuffers[portIndex].push_back(info); + + if (!isInput) { + mAvailableOutputBuffers.push( + &mBuffers[portIndex].editItemAt(i)); + } + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatBuffersAllocated); + + notify->setInt32("portIndex", portIndex); + + sp<PortDescription> desc = new PortDescription; + + for (size_t i = 0; i < mBuffers[portIndex].size(); ++i) { + const BufferInfo &info = mBuffers[portIndex][i]; + + desc->addBuffer(info.mBufferID, info.mData); + } + + notify->setObject("portDesc", desc); + notify->post(); + + return OK; +} + +MediaFilter::BufferInfo* MediaFilter::findBufferByID( + uint32_t portIndex, IOMX::buffer_id bufferID, + ssize_t *index) { + for (size_t i = 0; i < mBuffers[portIndex].size(); ++i) { + BufferInfo *info = &mBuffers[portIndex].editItemAt(i); + + if (info->mBufferID == bufferID) { + if (index != NULL) { + *index = i; + } + return info; + } + } + + TRESPASS(); + + return NULL; +} + +void MediaFilter::postFillThisBuffer(BufferInfo *info) { + ALOGV("postFillThisBuffer on buffer %d", info->mBufferID); + if (mPortEOS[kPortIndexInput]) { + return; + } + + CHECK_EQ((int)info->mStatus, (int)BufferInfo::OWNED_BY_US); + + info->mGeneration = mGeneration; + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatFillThisBuffer); + notify->setInt32("buffer-id", info->mBufferID); + + info->mData->meta()->clear(); + notify->setBuffer("buffer", info->mData); + + sp<AMessage> reply = new AMessage(kWhatInputBufferFilled, id()); + reply->setInt32("buffer-id", info->mBufferID); + + notify->setMessage("reply", reply); + + info->mStatus = BufferInfo::OWNED_BY_UPSTREAM; + notify->post(); +} + +void MediaFilter::postDrainThisBuffer(BufferInfo *info) { + CHECK_EQ((int)info->mStatus, (int)BufferInfo::OWNED_BY_US); + + info->mGeneration = mGeneration; + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatDrainThisBuffer); + notify->setInt32("buffer-id", info->mBufferID); + notify->setInt32("flags", info->mOutputFlags); + notify->setBuffer("buffer", info->mData); + + sp<AMessage> reply = new AMessage(kWhatOutputBufferDrained, id()); + reply->setInt32("buffer-id", info->mBufferID); + + notify->setMessage("reply", reply); + + notify->post(); + + info->mStatus = BufferInfo::OWNED_BY_UPSTREAM; +} + +void MediaFilter::postEOS() { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatEOS); + notify->setInt32("err", ERROR_END_OF_STREAM); + notify->post(); + + ALOGV("Sent kWhatEOS."); +} + +void MediaFilter::sendFormatChange() { + sp<AMessage> notify = mNotify->dup(); + + notify->setInt32("what", kWhatOutputFormatChanged); + + AString mime; + CHECK(mOutputFormat->findString("mime", &mime)); + notify->setString("mime", mime.c_str()); + + notify->setInt32("stride", mStride); + notify->setInt32("slice-height", mSliceHeight); + notify->setInt32("color-format", mColorFormatOut); + notify->setRect("crop", 0, 0, mStride - 1, mSliceHeight - 1); + notify->setInt32("width", mWidth); + notify->setInt32("height", mHeight); + + notify->post(); +} + +void MediaFilter::requestFillEmptyInput() { + if (mPortEOS[kPortIndexInput]) { + return; + } + + for (size_t i = 0; i < mBuffers[kPortIndexInput].size(); ++i) { + BufferInfo *info = &mBuffers[kPortIndexInput].editItemAt(i); + + if (info->mStatus == BufferInfo::OWNED_BY_US) { + postFillThisBuffer(info); + } + } +} + +void MediaFilter::processBuffers() { + if (mAvailableInputBuffers.empty() || mAvailableOutputBuffers.empty()) { + ALOGV("Skipping process (buffers unavailable)"); + return; + } + + if (mPortEOS[kPortIndexOutput]) { + // TODO notify caller of queueInput error when it is supported + // in MediaCodec + ALOGW("Tried to process a buffer after EOS."); + return; + } + + BufferInfo *inputInfo = mAvailableInputBuffers[0]; + mAvailableInputBuffers.removeAt(0); + BufferInfo *outputInfo = mAvailableOutputBuffers[0]; + mAvailableOutputBuffers.removeAt(0); + + status_t err; + err = mFilter->processBuffers(inputInfo->mData, outputInfo->mData); + if (err != (status_t)OK) { + outputInfo->mData->meta()->setInt32("err", err); + } + + int64_t timeUs; + CHECK(inputInfo->mData->meta()->findInt64("timeUs", &timeUs)); + outputInfo->mData->meta()->setInt64("timeUs", timeUs); + outputInfo->mOutputFlags = 0; + int32_t eos = 0; + if (inputInfo->mData->meta()->findInt32("eos", &eos) && eos != 0) { + outputInfo->mOutputFlags |= OMX_BUFFERFLAG_EOS; + mPortEOS[kPortIndexOutput] = true; + outputInfo->mData->meta()->setInt32("eos", eos); + postEOS(); + ALOGV("Output stream saw EOS."); + } + + ALOGV("Processed input buffer %u [%zu], output buffer %u [%zu]", + inputInfo->mBufferID, inputInfo->mData->size(), + outputInfo->mBufferID, outputInfo->mData->size()); + + if (mGraphicBufferListener != NULL) { + delete inputInfo; + } else { + postFillThisBuffer(inputInfo); + } + postDrainThisBuffer(outputInfo); + + // prevent any corner case where buffers could get stuck in queue + signalProcessBuffers(); +} + +void MediaFilter::onAllocateComponent(const sp<AMessage> &msg) { + CHECK_EQ(mState, UNINITIALIZED); + + CHECK(msg->findString("componentName", &mComponentName)); + const char* name = mComponentName.c_str(); + if (!strcasecmp(name, "android.filter.zerofilter")) { + mFilter = new ZeroFilter; + } else if (!strcasecmp(name, "android.filter.saturation")) { + mFilter = new SaturationFilter; + } else if (!strcasecmp(name, "android.filter.intrinsicblur")) { + mFilter = new IntrinsicBlurFilter; + } else if (!strcasecmp(name, "android.filter.RenderScript")) { + mFilter = new RSFilter; + } else { + ALOGE("Unrecognized filter name: %s", name); + signalError(NAME_NOT_FOUND); + return; + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatComponentAllocated); + // HACK - need "OMX.google" to use MediaCodec's software renderer + notify->setString("componentName", "OMX.google.MediaFilter"); + notify->post(); + mState = INITIALIZED; + ALOGV("Handled kWhatAllocateComponent."); +} + +void MediaFilter::onConfigureComponent(const sp<AMessage> &msg) { + // TODO: generalize to allow audio filters as well as video + + CHECK_EQ(mState, INITIALIZED); + + // get params - at least mime, width & height + AString mime; + CHECK(msg->findString("mime", &mime)); + if (strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_RAW)) { + ALOGE("Bad mime: %s", mime.c_str()); + signalError(BAD_VALUE); + return; + } + + CHECK(msg->findInt32("width", &mWidth)); + CHECK(msg->findInt32("height", &mHeight)); + if (!msg->findInt32("stride", &mStride)) { + mStride = mWidth; + } + if (!msg->findInt32("slice-height", &mSliceHeight)) { + mSliceHeight = mHeight; + } + + mMaxInputSize = mWidth * mHeight * 4; // room for ARGB8888 + int32_t maxInputSize; + if (msg->findInt32("max-input-size", &maxInputSize) + && (size_t)maxInputSize > mMaxInputSize) { + mMaxInputSize = maxInputSize; + } + + if (!msg->findInt32("color-format", &mColorFormatIn)) { + // default to OMX_COLOR_Format32bitARGB8888 + mColorFormatIn = OMX_COLOR_Format32bitARGB8888; + msg->setInt32("color-format", mColorFormatIn); + } + mColorFormatOut = mColorFormatIn; + + mMaxOutputSize = mWidth * mHeight * 4; // room for ARGB8888 + + AString cacheDir; + if (!msg->findString("cacheDir", &cacheDir)) { + ALOGE("Failed to find cache directory in config message."); + signalError(NAME_NOT_FOUND); + return; + } + + status_t err; + err = mFilter->configure(msg); + if (err != (status_t)OK) { + ALOGE("Failed to configure filter component, err %d", err); + signalError(err); + return; + } + + mInputFormat = new AMessage(); + mInputFormat->setString("mime", mime.c_str()); + mInputFormat->setInt32("stride", mStride); + mInputFormat->setInt32("slice-height", mSliceHeight); + mInputFormat->setInt32("color-format", mColorFormatIn); + mInputFormat->setRect("crop", 0, 0, mStride, mSliceHeight); + mInputFormat->setInt32("width", mWidth); + mInputFormat->setInt32("height", mHeight); + + mOutputFormat = new AMessage(); + mOutputFormat->setString("mime", mime.c_str()); + mOutputFormat->setInt32("stride", mStride); + mOutputFormat->setInt32("slice-height", mSliceHeight); + mOutputFormat->setInt32("color-format", mColorFormatOut); + mOutputFormat->setRect("crop", 0, 0, mStride, mSliceHeight); + mOutputFormat->setInt32("width", mWidth); + mOutputFormat->setInt32("height", mHeight); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatComponentConfigured); + notify->setString("componentName", "MediaFilter"); + notify->setMessage("input-format", mInputFormat); + notify->setMessage("output-format", mOutputFormat); + notify->post(); + mState = CONFIGURED; + ALOGV("Handled kWhatConfigureComponent."); + + sendFormatChange(); +} + +void MediaFilter::onStart() { + CHECK_EQ(mState, CONFIGURED); + + allocateBuffersOnPort(kPortIndexInput); + + allocateBuffersOnPort(kPortIndexOutput); + + status_t err = mFilter->start(); + if (err != (status_t)OK) { + ALOGE("Failed to start filter component, err %d", err); + signalError(err); + return; + } + + mPortEOS[kPortIndexInput] = false; + mPortEOS[kPortIndexOutput] = false; + mInputEOSResult = OK; + mState = STARTED; + + requestFillEmptyInput(); + ALOGV("Handled kWhatStart."); +} + +void MediaFilter::onInputBufferFilled(const sp<AMessage> &msg) { + IOMX::buffer_id bufferID; + CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID)); + BufferInfo *info = findBufferByID(kPortIndexInput, bufferID); + + if (mState != STARTED) { + // we're not running, so we'll just keep that buffer... + info->mStatus = BufferInfo::OWNED_BY_US; + return; + } + + if (info->mGeneration != mGeneration) { + ALOGV("Caught a stale input buffer [ID %d]", bufferID); + // buffer is stale (taken before a flush/shutdown) - repost it + CHECK_EQ(info->mStatus, BufferInfo::OWNED_BY_US); + postFillThisBuffer(info); + return; + } + + CHECK_EQ(info->mStatus, BufferInfo::OWNED_BY_UPSTREAM); + info->mStatus = BufferInfo::OWNED_BY_US; + + sp<ABuffer> buffer; + int32_t err = OK; + bool eos = false; + + if (!msg->findBuffer("buffer", &buffer)) { + // these are unfilled buffers returned by client + CHECK(msg->findInt32("err", &err)); + + if (err == OK) { + // buffers with no errors are returned on MediaCodec.flush + ALOGV("saw unfilled buffer (MediaCodec.flush)"); + postFillThisBuffer(info); + return; + } else { + ALOGV("saw error %d instead of an input buffer", err); + eos = true; + } + + buffer.clear(); + } + + int32_t isCSD; + if (buffer != NULL && buffer->meta()->findInt32("csd", &isCSD) + && isCSD != 0) { + // ignore codec-specific data buffers + ALOGW("MediaFilter received a codec-specific data buffer"); + postFillThisBuffer(info); + return; + } + + int32_t tmp; + if (buffer != NULL && buffer->meta()->findInt32("eos", &tmp) && tmp) { + eos = true; + err = ERROR_END_OF_STREAM; + } + + mAvailableInputBuffers.push_back(info); + processBuffers(); + + if (eos) { + mPortEOS[kPortIndexInput] = true; + mInputEOSResult = err; + } + + ALOGV("Handled kWhatInputBufferFilled. [ID %u]", bufferID); +} + +void MediaFilter::onOutputBufferDrained(const sp<AMessage> &msg) { + IOMX::buffer_id bufferID; + CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID)); + BufferInfo *info = findBufferByID(kPortIndexOutput, bufferID); + + if (mState != STARTED) { + // we're not running, so we'll just keep that buffer... + info->mStatus = BufferInfo::OWNED_BY_US; + return; + } + + if (info->mGeneration != mGeneration) { + ALOGV("Caught a stale output buffer [ID %d]", bufferID); + // buffer is stale (taken before a flush/shutdown) - keep it + CHECK_EQ(info->mStatus, BufferInfo::OWNED_BY_US); + return; + } + + CHECK_EQ(info->mStatus, BufferInfo::OWNED_BY_UPSTREAM); + info->mStatus = BufferInfo::OWNED_BY_US; + + mAvailableOutputBuffers.push_back(info); + + processBuffers(); + + ALOGV("Handled kWhatOutputBufferDrained. [ID %u]", + bufferID); +} + +void MediaFilter::onShutdown(const sp<AMessage> &msg) { + mGeneration++; + + if (mState != UNINITIALIZED) { + mFilter->reset(); + } + + int32_t keepComponentAllocated; + CHECK(msg->findInt32("keepComponentAllocated", &keepComponentAllocated)); + if (!keepComponentAllocated || mState == UNINITIALIZED) { + mState = UNINITIALIZED; + } else { + mState = INITIALIZED; + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatShutdownCompleted); + notify->post(); +} + +void MediaFilter::onFlush() { + mGeneration++; + + mAvailableInputBuffers.clear(); + for (size_t i = 0; i < mBuffers[kPortIndexInput].size(); ++i) { + BufferInfo *info = &mBuffers[kPortIndexInput].editItemAt(i); + info->mStatus = BufferInfo::OWNED_BY_US; + } + mAvailableOutputBuffers.clear(); + for (size_t i = 0; i < mBuffers[kPortIndexOutput].size(); ++i) { + BufferInfo *info = &mBuffers[kPortIndexOutput].editItemAt(i); + info->mStatus = BufferInfo::OWNED_BY_US; + mAvailableOutputBuffers.push_back(info); + } + + mPortEOS[kPortIndexInput] = false; + mPortEOS[kPortIndexOutput] = false; + mInputEOSResult = OK; + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatFlushCompleted); + notify->post(); + ALOGV("Posted kWhatFlushCompleted"); + + // MediaCodec returns all input buffers after flush, so in + // onInputBufferFilled we call postFillThisBuffer on them +} + +void MediaFilter::onSetParameters(const sp<AMessage> &msg) { + CHECK(mState != STARTED); + + status_t err = mFilter->setParameters(msg); + if (err != (status_t)OK) { + ALOGE("setParameters returned err %d", err); + } +} + +void MediaFilter::onCreateInputSurface() { + CHECK(mState == CONFIGURED); + + mGraphicBufferListener = new GraphicBufferListener; + + sp<AMessage> notify = new AMessage(); + notify->setTarget(id()); + status_t err = mGraphicBufferListener->init( + notify, mStride, mSliceHeight, kBufferCountActual); + + if (err != OK) { + ALOGE("Failed to init mGraphicBufferListener: %d", err); + signalError(err); + return; + } + + sp<AMessage> reply = mNotify->dup(); + reply->setInt32("what", CodecBase::kWhatInputSurfaceCreated); + reply->setObject( + "input-surface", + new BufferProducerWrapper( + mGraphicBufferListener->getIGraphicBufferProducer())); + reply->post(); +} + +void MediaFilter::onInputFrameAvailable() { + BufferQueue::BufferItem item = mGraphicBufferListener->getBufferItem(); + sp<GraphicBuffer> buf = mGraphicBufferListener->getBuffer(item); + + // get pointer to graphic buffer + void* bufPtr; + buf->lock(GraphicBuffer::USAGE_SW_READ_OFTEN, &bufPtr); + + // HACK - there is no OMX_COLOR_FORMATTYPE value for RGBA, so the format + // conversion is hardcoded until we add this. + // TODO: check input format and convert only if necessary + // copy RGBA graphic buffer into temporary ARGB input buffer + BufferInfo *inputInfo = new BufferInfo; + inputInfo->mData = new ABuffer(buf->getWidth() * buf->getHeight() * 4); + ALOGV("Copying surface data into temp buffer."); + convertRGBAToARGB( + (uint8_t*)bufPtr, buf->getWidth(), buf->getHeight(), + buf->getStride(), inputInfo->mData->data()); + inputInfo->mBufferID = item.mBuf; + inputInfo->mGeneration = mGeneration; + inputInfo->mOutputFlags = 0; + inputInfo->mStatus = BufferInfo::OWNED_BY_US; + inputInfo->mData->meta()->setInt64("timeUs", item.mTimestamp / 1000); + + mAvailableInputBuffers.push_back(inputInfo); + + mGraphicBufferListener->releaseBuffer(item); + + signalProcessBuffers(); +} + +void MediaFilter::onSignalEndOfInputStream() { + // if using input surface, need to send an EOS output buffer + if (mGraphicBufferListener != NULL) { + Vector<BufferInfo> *outputBufs = &mBuffers[kPortIndexOutput]; + BufferInfo* eosBuf; + bool foundBuf = false; + for (size_t i = 0; i < kBufferCountActual; i++) { + eosBuf = &outputBufs->editItemAt(i); + if (eosBuf->mStatus == BufferInfo::OWNED_BY_US) { + foundBuf = true; + break; + } + } + + if (!foundBuf) { + ALOGE("onSignalEndOfInputStream failed to find an output buffer"); + return; + } + + eosBuf->mOutputFlags = OMX_BUFFERFLAG_EOS; + eosBuf->mGeneration = mGeneration; + eosBuf->mData->setRange(0, 0); + postDrainThisBuffer(eosBuf); + ALOGV("Posted EOS on output buffer %zu", eosBuf->mBufferID); + } + + mPortEOS[kPortIndexOutput] = true; + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatSignaledInputEOS); + notify->post(); + + ALOGV("Output stream saw EOS."); +} + +} // namespace android diff --git a/media/libstagefright/filters/RSFilter.cpp b/media/libstagefright/filters/RSFilter.cpp new file mode 100644 index 0000000..b569945 --- /dev/null +++ b/media/libstagefright/filters/RSFilter.cpp @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "RSFilter" + +#include <utils/Log.h> + +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include "RSFilter.h" + +namespace android { + +RSFilter::RSFilter() { + +} + +RSFilter::~RSFilter() { + +} + +status_t RSFilter::configure(const sp<AMessage> &msg) { + status_t err = SimpleFilter::configure(msg); + if (err != OK) { + return err; + } + + if (!msg->findString("cacheDir", &mCacheDir)) { + ALOGE("Failed to find cache directory in config message."); + return NAME_NOT_FOUND; + } + + sp<RenderScriptWrapper> wrapper; + if (!msg->findObject("rs-wrapper", (sp<RefBase>*)&wrapper)) { + ALOGE("Failed to find RenderScriptWrapper in config message."); + return NAME_NOT_FOUND; + } + + mRS = wrapper->mContext; + mCallback = wrapper->mCallback; + + return OK; +} + +status_t RSFilter::start() { + // 32-bit elements for ARGB8888 + RSC::sp<const RSC::Element> e = RSC::Element::U8_4(mRS); + + RSC::Type::Builder tb(mRS, e); + tb.setX(mWidth); + tb.setY(mHeight); + RSC::sp<const RSC::Type> t = tb.create(); + + mAllocIn = RSC::Allocation::createTyped(mRS, t); + mAllocOut = RSC::Allocation::createTyped(mRS, t); + + return OK; +} + +void RSFilter::reset() { + mCallback.clear(); + mAllocOut.clear(); + mAllocIn.clear(); + mRS.clear(); +} + +status_t RSFilter::setParameters(const sp<AMessage> &msg) { + return mCallback->handleSetParameters(msg); +} + +status_t RSFilter::processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer) { + mAllocIn->copy1DRangeFrom(0, mWidth * mHeight, srcBuffer->data()); + mCallback->processBuffers(mAllocIn.get(), mAllocOut.get()); + mAllocOut->copy1DRangeTo(0, mWidth * mHeight, outBuffer->data()); + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/RSFilter.h b/media/libstagefright/filters/RSFilter.h new file mode 100644 index 0000000..c5b5074 --- /dev/null +++ b/media/libstagefright/filters/RSFilter.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RS_FILTER_H_ +#define RS_FILTER_H_ + +#include <media/stagefright/RenderScriptWrapper.h> +#include <RenderScript.h> + +#include "SimpleFilter.h" + +namespace android { + +struct AString; + +struct RSFilter : public SimpleFilter { +public: + RSFilter(); + + virtual status_t configure(const sp<AMessage> &msg); + virtual status_t start(); + virtual void reset(); + virtual status_t setParameters(const sp<AMessage> &msg); + virtual status_t processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer); + +protected: + virtual ~RSFilter(); + +private: + AString mCacheDir; + sp<RenderScriptWrapper::RSFilterCallback> mCallback; + RSC::sp<RSC::RS> mRS; + RSC::sp<RSC::Allocation> mAllocIn; + RSC::sp<RSC::Allocation> mAllocOut; +}; + +} // namespace android + +#endif // RS_FILTER_H_ diff --git a/media/libstagefright/filters/SaturationFilter.cpp b/media/libstagefright/filters/SaturationFilter.cpp new file mode 100644 index 0000000..ba5f75a --- /dev/null +++ b/media/libstagefright/filters/SaturationFilter.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "SaturationFilter" + +#include <utils/Log.h> + +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include "SaturationFilter.h" + +namespace android { + +status_t SaturationFilter::configure(const sp<AMessage> &msg) { + status_t err = SimpleFilter::configure(msg); + if (err != OK) { + return err; + } + + if (!msg->findString("cacheDir", &mCacheDir)) { + ALOGE("Failed to find cache directory in config message."); + return NAME_NOT_FOUND; + } + + return OK; +} + +status_t SaturationFilter::start() { + // TODO: use a single RS context object for entire application + mRS = new RSC::RS(); + + if (!mRS->init(mCacheDir.c_str())) { + ALOGE("Failed to initialize RenderScript context."); + return NO_INIT; + } + + // 32-bit elements for ARGB8888 + RSC::sp<const RSC::Element> e = RSC::Element::U8_4(mRS); + + RSC::Type::Builder tb(mRS, e); + tb.setX(mWidth); + tb.setY(mHeight); + RSC::sp<const RSC::Type> t = tb.create(); + + mAllocIn = RSC::Allocation::createTyped(mRS, t); + mAllocOut = RSC::Allocation::createTyped(mRS, t); + + mScript = new ScriptC_saturationARGB(mRS); + + mScript->set_gSaturation(mSaturation); + + return OK; +} + +void SaturationFilter::reset() { + mScript.clear(); + mAllocOut.clear(); + mAllocIn.clear(); + mRS.clear(); +} + +status_t SaturationFilter::setParameters(const sp<AMessage> &msg) { + sp<AMessage> params; + CHECK(msg->findMessage("params", ¶ms)); + + float saturation; + if (params->findFloat("saturation", &saturation)) { + mSaturation = saturation; + } + + return OK; +} + +status_t SaturationFilter::processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer) { + mAllocIn->copy1DRangeFrom(0, mWidth * mHeight, srcBuffer->data()); + mScript->forEach_root(mAllocIn, mAllocOut); + mAllocOut->copy1DRangeTo(0, mWidth * mHeight, outBuffer->data()); + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/SaturationFilter.h b/media/libstagefright/filters/SaturationFilter.h new file mode 100644 index 0000000..0545021 --- /dev/null +++ b/media/libstagefright/filters/SaturationFilter.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SATURATION_FILTER_H_ +#define SATURATION_FILTER_H_ + +#include <RenderScript.h> + +#include "ScriptC_saturationARGB.h" +#include "SimpleFilter.h" + +namespace android { + +struct SaturationFilter : public SimpleFilter { +public: + SaturationFilter() : mSaturation(1.f) {}; + + virtual status_t configure(const sp<AMessage> &msg); + virtual status_t start(); + virtual void reset(); + virtual status_t setParameters(const sp<AMessage> &msg); + virtual status_t processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer); + +protected: + virtual ~SaturationFilter() {}; + +private: + AString mCacheDir; + RSC::sp<RSC::RS> mRS; + RSC::sp<RSC::Allocation> mAllocIn; + RSC::sp<RSC::Allocation> mAllocOut; + RSC::sp<ScriptC_saturationARGB> mScript; + float mSaturation; +}; + +} // namespace android + +#endif // SATURATION_FILTER_H_ diff --git a/media/libstagefright/filters/SimpleFilter.cpp b/media/libstagefright/filters/SimpleFilter.cpp new file mode 100644 index 0000000..6c1ca2c --- /dev/null +++ b/media/libstagefright/filters/SimpleFilter.cpp @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include "SimpleFilter.h" + +namespace android { + +status_t SimpleFilter::configure(const sp<AMessage> &msg) { + CHECK(msg->findInt32("width", &mWidth)); + CHECK(msg->findInt32("height", &mHeight)); + if (!msg->findInt32("stride", &mStride)) { + mStride = mWidth; + } + if (!msg->findInt32("slice-height", &mSliceHeight)) { + mSliceHeight = mHeight; + } + CHECK(msg->findInt32("color-format", &mColorFormatIn)); + mColorFormatOut = mColorFormatIn; + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/SimpleFilter.h b/media/libstagefright/filters/SimpleFilter.h new file mode 100644 index 0000000..4cd37ef --- /dev/null +++ b/media/libstagefright/filters/SimpleFilter.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SIMPLE_FILTER_H_ +#define SIMPLE_FILTER_H_ + +#include <stdint.h> +#include <utils/Errors.h> +#include <utils/RefBase.h> + +struct ABuffer; +struct AMessage; + +namespace android { + +struct SimpleFilter : public RefBase { +public: + SimpleFilter() : mWidth(0), mHeight(0), mStride(0), mSliceHeight(0), + mColorFormatIn(0), mColorFormatOut(0) {}; + + virtual status_t configure(const sp<AMessage> &msg); + + virtual status_t start() = 0; + virtual void reset() = 0; + virtual status_t setParameters(const sp<AMessage> &msg) = 0; + virtual status_t processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer) = 0; + +protected: + int32_t mWidth, mHeight; + int32_t mStride, mSliceHeight; + int32_t mColorFormatIn, mColorFormatOut; + + virtual ~SimpleFilter() {}; +}; + +} // namespace android + +#endif // SIMPLE_FILTER_H_ diff --git a/media/libstagefright/filters/ZeroFilter.cpp b/media/libstagefright/filters/ZeroFilter.cpp new file mode 100644 index 0000000..3f1243c --- /dev/null +++ b/media/libstagefright/filters/ZeroFilter.cpp @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "ZeroFilter" + +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include "ZeroFilter.h" + +namespace android { + +status_t ZeroFilter::setParameters(const sp<AMessage> &msg) { + sp<AMessage> params; + CHECK(msg->findMessage("params", ¶ms)); + + int32_t invert; + if (params->findInt32("invert", &invert)) { + mInvertData = (invert != 0); + } + + return OK; +} + +status_t ZeroFilter::processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer) { + // assuming identical input & output buffers, since we're a copy filter + if (mInvertData) { + uint32_t* src = (uint32_t*)srcBuffer->data(); + uint32_t* dest = (uint32_t*)outBuffer->data(); + for (size_t i = 0; i < srcBuffer->size() / 4; ++i) { + *(dest++) = *(src++) ^ 0xFFFFFFFF; + } + } else { + memcpy(outBuffer->data(), srcBuffer->data(), srcBuffer->size()); + } + outBuffer->setRange(0, srcBuffer->size()); + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/ZeroFilter.h b/media/libstagefright/filters/ZeroFilter.h new file mode 100644 index 0000000..bd34dfb --- /dev/null +++ b/media/libstagefright/filters/ZeroFilter.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZERO_FILTER_H_ +#define ZERO_FILTER_H_ + +#include "SimpleFilter.h" + +namespace android { + +struct ZeroFilter : public SimpleFilter { +public: + ZeroFilter() : mInvertData(false) {}; + + virtual status_t start() { return OK; }; + virtual void reset() {}; + virtual status_t setParameters(const sp<AMessage> &msg); + virtual status_t processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer); + +protected: + virtual ~ZeroFilter() {}; + +private: + bool mInvertData; +}; + +} // namespace android + +#endif // ZERO_FILTER_H_ diff --git a/media/libstagefright/filters/saturation.rs b/media/libstagefright/filters/saturation.rs new file mode 100644 index 0000000..2c867ac --- /dev/null +++ b/media/libstagefright/filters/saturation.rs @@ -0,0 +1,40 @@ +// Sample script for RGB888 support (compare to saturationARGB.rs) +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma version(1) +#pragma rs java_package_name(com.android.rs.cppbasic) +#pragma rs_fp_relaxed + +const static float3 gMonoMult = {0.299f, 0.587f, 0.114f}; + +// global variables (parameters accessible to application code) +float gSaturation = 1.0f; + +void root(const uchar3 *v_in, uchar3 *v_out) { + // scale 0-255 uchar to 0-1.0 float + float3 in = {v_in->r * 0.003921569f, v_in->g * 0.003921569f, + v_in->b * 0.003921569f}; + + // apply saturation filter + float3 result = dot(in, gMonoMult); + result = mix(result, in, gSaturation); + + // convert to uchar, copied from rsPackColorTo8888 + v_out->x = (uchar)clamp((result.r * 255.f + 0.5f), 0.f, 255.f); + v_out->y = (uchar)clamp((result.g * 255.f + 0.5f), 0.f, 255.f); + v_out->z = (uchar)clamp((result.b * 255.f + 0.5f), 0.f, 255.f); +} diff --git a/media/libstagefright/filters/saturationARGB.rs b/media/libstagefright/filters/saturationARGB.rs new file mode 100644 index 0000000..1de9dd8 --- /dev/null +++ b/media/libstagefright/filters/saturationARGB.rs @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma version(1) +#pragma rs java_package_name(com.android.rs.cppbasic) +#pragma rs_fp_relaxed + +const static float3 gMonoMult = {0.299f, 0.587f, 0.114f}; + +// global variables (parameters accessible to application code) +float gSaturation = 1.0f; + +void root(const uchar4 *v_in, uchar4 *v_out) { + v_out->x = v_in->x; // don't modify A + + // get RGB, scale 0-255 uchar to 0-1.0 float + float3 rgb = {v_in->y * 0.003921569f, v_in->z * 0.003921569f, + v_in->w * 0.003921569f}; + + // apply saturation filter + float3 result = dot(rgb, gMonoMult); + result = mix(result, rgb, gSaturation); + + v_out->y = (uchar)clamp((result.r * 255.f + 0.5f), 0.f, 255.f); + v_out->z = (uchar)clamp((result.g * 255.f + 0.5f), 0.f, 255.f); + v_out->w = (uchar)clamp((result.b * 255.f + 0.5f), 0.f, 255.f); +} diff --git a/media/libstagefright/mpeg2ts/ESQueue.cpp b/media/libstagefright/mpeg2ts/ESQueue.cpp index ef1cd3d..3c8f03e 100644 --- a/media/libstagefright/mpeg2ts/ESQueue.cpp +++ b/media/libstagefright/mpeg2ts/ESQueue.cpp @@ -604,8 +604,6 @@ sp<ABuffer> ElementaryStreamQueue::dequeueAccessUnitAAC() { // having to interpolate. // The final AAC frame may well extend into the next RangeInfo but // that's ok. - // TODO: the logic commented above is skipped because codec cannot take - // arbitrary sized input buffers; size_t offset = 0; while (offset < info.mLength) { if (offset + 7 > mBuffer->size()) { @@ -670,12 +668,9 @@ sp<ABuffer> ElementaryStreamQueue::dequeueAccessUnitAAC() { size_t headerSize = protection_absent ? 7 : 9; offset += aac_frame_length; - // TODO: move back to concatenation when codec can support arbitrary input buffers. - // For now only queue a single buffer - break; } - int64_t timeUs = fetchTimestampAAC(offset); + int64_t timeUs = fetchTimestamp(offset); sp<ABuffer> accessUnit = new ABuffer(offset); memcpy(accessUnit->data(), mBuffer->data(), offset); @@ -722,45 +717,6 @@ int64_t ElementaryStreamQueue::fetchTimestamp(size_t size) { return timeUs; } -// TODO: avoid interpolating timestamps once codec supports arbitrary sized input buffers -int64_t ElementaryStreamQueue::fetchTimestampAAC(size_t size) { - int64_t timeUs = -1; - bool first = true; - - size_t samplesize = size; - while (size > 0) { - CHECK(!mRangeInfos.empty()); - - RangeInfo *info = &*mRangeInfos.begin(); - - if (first) { - timeUs = info->mTimestampUs; - first = false; - } - - if (info->mLength > size) { - int32_t sampleRate; - CHECK(mFormat->findInt32(kKeySampleRate, &sampleRate)); - info->mLength -= size; - size_t numSamples = 1024 * size / samplesize; - info->mTimestampUs += numSamples * 1000000ll / sampleRate; - size = 0; - } else { - size -= info->mLength; - - mRangeInfos.erase(mRangeInfos.begin()); - info = NULL; - } - - } - - if (timeUs == 0ll) { - ALOGV("Returning 0 timestamp"); - } - - return timeUs; -} - struct NALPosition { size_t nalOffset; size_t nalSize; diff --git a/media/libstagefright/mpeg2ts/ESQueue.h b/media/libstagefright/mpeg2ts/ESQueue.h index eb4b1c9..45b4624 100644 --- a/media/libstagefright/mpeg2ts/ESQueue.h +++ b/media/libstagefright/mpeg2ts/ESQueue.h @@ -77,7 +77,6 @@ private: // consume a logical (compressed) access unit of size "size", // returns its timestamp in us (or -1 if no time information). int64_t fetchTimestamp(size_t size); - int64_t fetchTimestampAAC(size_t size); DISALLOW_EVIL_CONSTRUCTORS(ElementaryStreamQueue); }; diff --git a/services/audioflinger/AudioFlinger.cpp b/services/audioflinger/AudioFlinger.cpp index fed85e9..a17a49e 100644 --- a/services/audioflinger/AudioFlinger.cpp +++ b/services/audioflinger/AudioFlinger.cpp @@ -401,6 +401,9 @@ status_t AudioFlinger::dump(int fd, const Vector<String16>& args) String8 result(kClientLockedString); write(fd, result.string(), result.size()); } + + EffectDumpEffects(fd); + dumpClients(fd, args); if (clientLocked) { mClientLock.unlock(); diff --git a/services/audioflinger/AudioMixer.cpp b/services/audioflinger/AudioMixer.cpp index fd28ea1..0d4b358 100644 --- a/services/audioflinger/AudioMixer.cpp +++ b/services/audioflinger/AudioMixer.cpp @@ -430,6 +430,10 @@ void AudioMixer::setLog(NBLog::Writer *log) mState.mLog = log; } +static inline audio_format_t selectMixerInFormat(audio_format_t inputFormat __unused) { + return kUseFloat && kUseNewMixer ? AUDIO_FORMAT_PCM_FLOAT : AUDIO_FORMAT_PCM_16_BIT; +} + int AudioMixer::getTrackName(audio_channel_mask_t channelMask, audio_format_t format, int sessionId) { @@ -492,24 +496,23 @@ int AudioMixer::getTrackName(audio_channel_mask_t channelMask, t->mInputBufferProvider = NULL; t->mReformatBufferProvider = NULL; t->downmixerBufferProvider = NULL; + t->mPostDownmixReformatBufferProvider = NULL; t->mMixerFormat = AUDIO_FORMAT_PCM_16_BIT; t->mFormat = format; - t->mMixerInFormat = kUseFloat && kUseNewMixer - ? AUDIO_FORMAT_PCM_FLOAT : AUDIO_FORMAT_PCM_16_BIT; + t->mMixerInFormat = selectMixerInFormat(format); + t->mDownmixRequiresFormat = AUDIO_FORMAT_INVALID; // no format required t->mMixerChannelMask = audio_channel_mask_from_representation_and_bits( AUDIO_CHANNEL_REPRESENTATION_POSITION, AUDIO_CHANNEL_OUT_STEREO); t->mMixerChannelCount = audio_channel_count_from_out_mask(t->mMixerChannelMask); // Check the downmixing (or upmixing) requirements. - status_t status = initTrackDownmix(t, n); + status_t status = t->prepareForDownmix(); if (status != OK) { ALOGE("AudioMixer::getTrackName invalid channelMask (%#x)", channelMask); return -1; } - // initTrackDownmix() may change the input format requirement. - // If you desire floating point input to the mixer, it may change - // to integer because the downmixer requires integer to process. + // prepareForDownmix() may change mDownmixRequiresFormat ALOGVV("mMixerFormat:%#x mMixerInFormat:%#x\n", t->mMixerFormat, t->mMixerInFormat); - prepareTrackForReformat(t, n); + t->prepareForReformat(); mTrackNames |= 1 << n; return TRACK0 + n; } @@ -526,7 +529,7 @@ void AudioMixer::invalidateState(uint32_t mask) } // Called when channel masks have changed for a track name -// TODO: Fix Downmixbufferprofider not to (possibly) change mixer input format, +// TODO: Fix DownmixerBufferProvider not to (possibly) change mixer input format, // which will simplify this logic. bool AudioMixer::setChannelMasks(int name, audio_channel_mask_t trackChannelMask, audio_channel_mask_t mixerChannelMask) { @@ -551,21 +554,18 @@ bool AudioMixer::setChannelMasks(int name, // channel masks have changed, does this track need a downmixer? // update to try using our desired format (if we aren't already using it) - const audio_format_t prevMixerInFormat = track.mMixerInFormat; - track.mMixerInFormat = kUseFloat && kUseNewMixer - ? AUDIO_FORMAT_PCM_FLOAT : AUDIO_FORMAT_PCM_16_BIT; - const status_t status = initTrackDownmix(&mState.tracks[name], name); + const audio_format_t prevDownmixerFormat = track.mDownmixRequiresFormat; + const status_t status = mState.tracks[name].prepareForDownmix(); ALOGE_IF(status != OK, - "initTrackDownmix error %d, track channel mask %#x, mixer channel mask %#x", + "prepareForDownmix error %d, track channel mask %#x, mixer channel mask %#x", status, track.channelMask, track.mMixerChannelMask); - const bool mixerInFormatChanged = prevMixerInFormat != track.mMixerInFormat; - if (mixerInFormatChanged) { - prepareTrackForReformat(&track, name); // because of downmixer, track format may change! + if (prevDownmixerFormat != track.mDownmixRequiresFormat) { + track.prepareForReformat(); // because of downmixer, track format may change! } - if (track.resampler && (mixerInFormatChanged || mixerChannelCountChanged)) { - // resampler input format or channels may have changed. + if (track.resampler && mixerChannelCountChanged) { + // resampler channels may have changed. const uint32_t resetToSampleRate = track.sampleRate; delete track.resampler; track.resampler = NULL; @@ -576,99 +576,122 @@ bool AudioMixer::setChannelMasks(int name, return true; } -status_t AudioMixer::initTrackDownmix(track_t* pTrack, int trackName) -{ - // Only remix (upmix or downmix) if the track and mixer/device channel masks - // are not the same and not handled internally, as mono -> stereo currently is. - if (pTrack->channelMask != pTrack->mMixerChannelMask - && !(pTrack->channelMask == AUDIO_CHANNEL_OUT_MONO - && pTrack->mMixerChannelMask == AUDIO_CHANNEL_OUT_STEREO)) { - return prepareTrackForDownmix(pTrack, trackName); - } - // no remix necessary - unprepareTrackForDownmix(pTrack, trackName); - return NO_ERROR; -} - -void AudioMixer::unprepareTrackForDownmix(track_t* pTrack, int trackName __unused) { - ALOGV("AudioMixer::unprepareTrackForDownmix(%d)", trackName); +void AudioMixer::track_t::unprepareForDownmix() { + ALOGV("AudioMixer::unprepareForDownmix(%p)", this); - if (pTrack->downmixerBufferProvider != NULL) { + mDownmixRequiresFormat = AUDIO_FORMAT_INVALID; + if (downmixerBufferProvider != NULL) { // this track had previously been configured with a downmixer, delete it ALOGV(" deleting old downmixer"); - delete pTrack->downmixerBufferProvider; - pTrack->downmixerBufferProvider = NULL; - reconfigureBufferProviders(pTrack); + delete downmixerBufferProvider; + downmixerBufferProvider = NULL; + reconfigureBufferProviders(); } else { ALOGV(" nothing to do, no downmixer to delete"); } } -status_t AudioMixer::prepareTrackForDownmix(track_t* pTrack, int trackName) +status_t AudioMixer::track_t::prepareForDownmix() { - ALOGV("AudioMixer::prepareTrackForDownmix(%d) with mask 0x%x", trackName, pTrack->channelMask); + ALOGV("AudioMixer::prepareForDownmix(%p) with mask 0x%x", + this, channelMask); // discard the previous downmixer if there was one - unprepareTrackForDownmix(pTrack, trackName); + unprepareForDownmix(); + // Only remix (upmix or downmix) if the track and mixer/device channel masks + // are not the same and not handled internally, as mono -> stereo currently is. + if (channelMask == mMixerChannelMask + || (channelMask == AUDIO_CHANNEL_OUT_MONO + && mMixerChannelMask == AUDIO_CHANNEL_OUT_STEREO)) { + return NO_ERROR; + } if (DownmixerBufferProvider::isMultichannelCapable()) { - DownmixerBufferProvider* pDbp = new DownmixerBufferProvider(pTrack->channelMask, - pTrack->mMixerChannelMask, - AUDIO_FORMAT_PCM_16_BIT /* TODO: use pTrack->mMixerInFormat, now only PCM 16 */, - pTrack->sampleRate, pTrack->sessionId, kCopyBufferFrameCount); + DownmixerBufferProvider* pDbp = new DownmixerBufferProvider(channelMask, + mMixerChannelMask, + AUDIO_FORMAT_PCM_16_BIT /* TODO: use mMixerInFormat, now only PCM 16 */, + sampleRate, sessionId, kCopyBufferFrameCount); if (pDbp->isValid()) { // if constructor completed properly - pTrack->mMixerInFormat = AUDIO_FORMAT_PCM_16_BIT; // PCM 16 bit required for downmix - pTrack->downmixerBufferProvider = pDbp; - reconfigureBufferProviders(pTrack); + mDownmixRequiresFormat = AUDIO_FORMAT_PCM_16_BIT; // PCM 16 bit required for downmix + downmixerBufferProvider = pDbp; + reconfigureBufferProviders(); return NO_ERROR; } delete pDbp; } // Effect downmixer does not accept the channel conversion. Let's use our remixer. - RemixBufferProvider* pRbp = new RemixBufferProvider(pTrack->channelMask, - pTrack->mMixerChannelMask, pTrack->mMixerInFormat, kCopyBufferFrameCount); + RemixBufferProvider* pRbp = new RemixBufferProvider(channelMask, + mMixerChannelMask, mMixerInFormat, kCopyBufferFrameCount); // Remix always finds a conversion whereas Downmixer effect above may fail. - pTrack->downmixerBufferProvider = pRbp; - reconfigureBufferProviders(pTrack); + downmixerBufferProvider = pRbp; + reconfigureBufferProviders(); return NO_ERROR; } -void AudioMixer::unprepareTrackForReformat(track_t* pTrack, int trackName __unused) { - ALOGV("AudioMixer::unprepareTrackForReformat(%d)", trackName); - if (pTrack->mReformatBufferProvider != NULL) { - delete pTrack->mReformatBufferProvider; - pTrack->mReformatBufferProvider = NULL; - reconfigureBufferProviders(pTrack); +void AudioMixer::track_t::unprepareForReformat() { + ALOGV("AudioMixer::unprepareForReformat(%p)", this); + bool requiresReconfigure = false; + if (mReformatBufferProvider != NULL) { + delete mReformatBufferProvider; + mReformatBufferProvider = NULL; + requiresReconfigure = true; + } + if (mPostDownmixReformatBufferProvider != NULL) { + delete mPostDownmixReformatBufferProvider; + mPostDownmixReformatBufferProvider = NULL; + requiresReconfigure = true; + } + if (requiresReconfigure) { + reconfigureBufferProviders(); } } -status_t AudioMixer::prepareTrackForReformat(track_t* pTrack, int trackName) +status_t AudioMixer::track_t::prepareForReformat() { - ALOGV("AudioMixer::prepareTrackForReformat(%d) with format %#x", trackName, pTrack->mFormat); - // discard the previous reformatter if there was one - unprepareTrackForReformat(pTrack, trackName); - // only configure reformatter if needed - if (pTrack->mFormat != pTrack->mMixerInFormat) { - pTrack->mReformatBufferProvider = new ReformatBufferProvider( - audio_channel_count_from_out_mask(pTrack->channelMask), - pTrack->mFormat, pTrack->mMixerInFormat, + ALOGV("AudioMixer::prepareForReformat(%p) with format %#x", this, mFormat); + // discard previous reformatters + unprepareForReformat(); + // only configure reformatters as needed + const audio_format_t targetFormat = mDownmixRequiresFormat != AUDIO_FORMAT_INVALID + ? mDownmixRequiresFormat : mMixerInFormat; + bool requiresReconfigure = false; + if (mFormat != targetFormat) { + mReformatBufferProvider = new ReformatBufferProvider( + audio_channel_count_from_out_mask(channelMask), + mFormat, + targetFormat, kCopyBufferFrameCount); - reconfigureBufferProviders(pTrack); + requiresReconfigure = true; + } + if (targetFormat != mMixerInFormat) { + mPostDownmixReformatBufferProvider = new ReformatBufferProvider( + audio_channel_count_from_out_mask(mMixerChannelMask), + targetFormat, + mMixerInFormat, + kCopyBufferFrameCount); + requiresReconfigure = true; + } + if (requiresReconfigure) { + reconfigureBufferProviders(); } return NO_ERROR; } -void AudioMixer::reconfigureBufferProviders(track_t* pTrack) +void AudioMixer::track_t::reconfigureBufferProviders() { - pTrack->bufferProvider = pTrack->mInputBufferProvider; - if (pTrack->mReformatBufferProvider) { - pTrack->mReformatBufferProvider->setBufferProvider(pTrack->bufferProvider); - pTrack->bufferProvider = pTrack->mReformatBufferProvider; + bufferProvider = mInputBufferProvider; + if (mReformatBufferProvider) { + mReformatBufferProvider->setBufferProvider(bufferProvider); + bufferProvider = mReformatBufferProvider; + } + if (downmixerBufferProvider) { + downmixerBufferProvider->setBufferProvider(bufferProvider); + bufferProvider = downmixerBufferProvider; } - if (pTrack->downmixerBufferProvider) { - pTrack->downmixerBufferProvider->setBufferProvider(pTrack->bufferProvider); - pTrack->bufferProvider = pTrack->downmixerBufferProvider; + if (mPostDownmixReformatBufferProvider) { + mPostDownmixReformatBufferProvider->setBufferProvider(bufferProvider); + bufferProvider = mPostDownmixReformatBufferProvider; } } @@ -687,9 +710,9 @@ void AudioMixer::deleteTrackName(int name) delete track.resampler; track.resampler = NULL; // delete the downmixer - unprepareTrackForDownmix(&mState.tracks[name], name); + mState.tracks[name].unprepareForDownmix(); // delete the reformatter - unprepareTrackForReformat(&mState.tracks[name], name); + mState.tracks[name].unprepareForReformat(); mTrackNames &= ~(1<<name); } @@ -828,7 +851,7 @@ void AudioMixer::setParameter(int name, int target, int param, void *value) ALOG_ASSERT(audio_is_linear_pcm(format), "Invalid format %#x", format); track.mFormat = format; ALOGV("setParameter(TRACK, FORMAT, %#x)", format); - prepareTrackForReformat(&track, name); + track.prepareForReformat(); invalidateState(1 << name); } } break; @@ -1032,10 +1055,13 @@ void AudioMixer::setBufferProvider(int name, AudioBufferProvider* bufferProvider if (mState.tracks[name].mReformatBufferProvider != NULL) { mState.tracks[name].mReformatBufferProvider->reset(); } else if (mState.tracks[name].downmixerBufferProvider != NULL) { + mState.tracks[name].downmixerBufferProvider->reset(); + } else if (mState.tracks[name].mPostDownmixReformatBufferProvider != NULL) { + mState.tracks[name].mPostDownmixReformatBufferProvider->reset(); } mState.tracks[name].mInputBufferProvider = bufferProvider; - reconfigureBufferProviders(&mState.tracks[name]); + mState.tracks[name].reconfigureBufferProviders(); } diff --git a/services/audioflinger/AudioMixer.h b/services/audioflinger/AudioMixer.h index f4f142b..c2ff985 100644 --- a/services/audioflinger/AudioMixer.h +++ b/services/audioflinger/AudioMixer.h @@ -205,17 +205,34 @@ private: int32_t* auxBuffer; // 16-byte boundary + + /* Buffer providers are constructed to translate the track input data as needed. + * + * 1) mInputBufferProvider: The AudioTrack buffer provider. + * 2) mReformatBufferProvider: If not NULL, performs the audio reformat to + * match either mMixerInFormat or mDownmixRequiresFormat, if the downmixer + * requires reformat. For example, it may convert floating point input to + * PCM_16_bit if that's required by the downmixer. + * 3) downmixerBufferProvider: If not NULL, performs the channel remixing to match + * the number of channels required by the mixer sink. + * 4) mPostDownmixReformatBufferProvider: If not NULL, performs reformatting from + * the downmixer requirements to the mixer engine input requirements. + */ AudioBufferProvider* mInputBufferProvider; // externally provided buffer provider. CopyBufferProvider* mReformatBufferProvider; // provider wrapper for reformatting. CopyBufferProvider* downmixerBufferProvider; // wrapper for channel conversion. + CopyBufferProvider* mPostDownmixReformatBufferProvider; + // 16-byte boundary int32_t sessionId; - // 16-byte boundary audio_format_t mMixerFormat; // output mix format: AUDIO_FORMAT_PCM_(FLOAT|16_BIT) audio_format_t mFormat; // input track format audio_format_t mMixerInFormat; // mix internal format AUDIO_FORMAT_PCM_(FLOAT|16_BIT) // each track must be converted to this format. + audio_format_t mDownmixRequiresFormat; // required downmixer format + // AUDIO_FORMAT_PCM_16_BIT if 16 bit necessary + // AUDIO_FORMAT_INVALID if no required format float mVolume[MAX_NUM_VOLUMES]; // floating point set volume float mPrevVolume[MAX_NUM_VOLUMES]; // floating point previous volume @@ -225,7 +242,6 @@ private: float mPrevAuxLevel; // floating point prev aux level float mAuxInc; // floating point aux increment - // 16-byte boundary audio_channel_mask_t mMixerChannelMask; uint32_t mMixerChannelCount; @@ -236,6 +252,12 @@ private: void adjustVolumeRamp(bool aux, bool useFloat = false); size_t getUnreleasedFrames() const { return resampler != NULL ? resampler->getUnreleasedFrames() : 0; }; + + status_t prepareForDownmix(); + void unprepareForDownmix(); + status_t prepareForReformat(); + void unprepareForReformat(); + void reconfigureBufferProviders(); }; typedef void (*process_hook_t)(state_t* state, int64_t pts); @@ -382,14 +404,6 @@ private: bool setChannelMasks(int name, audio_channel_mask_t trackChannelMask, audio_channel_mask_t mixerChannelMask); - // TODO: remove unused trackName/trackNum from functions below. - static status_t initTrackDownmix(track_t* pTrack, int trackName); - static status_t prepareTrackForDownmix(track_t* pTrack, int trackNum); - static void unprepareTrackForDownmix(track_t* pTrack, int trackName); - static status_t prepareTrackForReformat(track_t* pTrack, int trackNum); - static void unprepareTrackForReformat(track_t* pTrack, int trackName); - static void reconfigureBufferProviders(track_t* pTrack); - static void track__genericResample(track_t* t, int32_t* out, size_t numFrames, int32_t* temp, int32_t* aux); static void track__nop(track_t* t, int32_t* out, size_t numFrames, int32_t* temp, int32_t* aux); diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp index a75f784..aef1844 100644 --- a/services/audioflinger/Threads.cpp +++ b/services/audioflinger/Threads.cpp @@ -314,6 +314,64 @@ void CpuStats::sample(const String8 &title // ThreadBase // ---------------------------------------------------------------------------- +// static +const char *AudioFlinger::ThreadBase::threadTypeToString(AudioFlinger::ThreadBase::type_t type) +{ + switch (type) { + case MIXER: + return "MIXER"; + case DIRECT: + return "DIRECT"; + case DUPLICATING: + return "DUPLICATING"; + case RECORD: + return "RECORD"; + case OFFLOAD: + return "OFFLOAD"; + default: + return "unknown"; + } +} + +static String8 outputFlagsToString(audio_output_flags_t flags) +{ + static const struct mapping { + audio_output_flags_t mFlag; + const char * mString; + } mappings[] = { + AUDIO_OUTPUT_FLAG_DIRECT, "DIRECT", + AUDIO_OUTPUT_FLAG_PRIMARY, "PRIMARY", + AUDIO_OUTPUT_FLAG_FAST, "FAST", + AUDIO_OUTPUT_FLAG_DEEP_BUFFER, "DEEP_BUFFER", + AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD, "COMPRESS_OFFLOAAD", + AUDIO_OUTPUT_FLAG_NON_BLOCKING, "NON_BLOCKING", + AUDIO_OUTPUT_FLAG_HW_AV_SYNC, "HW_AV_SYNC", + AUDIO_OUTPUT_FLAG_NONE, "NONE", // must be last + }; + String8 result; + audio_output_flags_t allFlags = AUDIO_OUTPUT_FLAG_NONE; + const mapping *entry; + for (entry = mappings; entry->mFlag != AUDIO_OUTPUT_FLAG_NONE; entry++) { + allFlags = (audio_output_flags_t) (allFlags | entry->mFlag); + if (flags & entry->mFlag) { + if (!result.isEmpty()) { + result.append("|"); + } + result.append(entry->mString); + } + } + if (flags & ~allFlags) { + if (!result.isEmpty()) { + result.append("|"); + } + result.appendFormat("0x%X", flags & ~allFlags); + } + if (result.isEmpty()) { + result.append(entry->mString); + } + return result; +} + AudioFlinger::ThreadBase::ThreadBase(const sp<AudioFlinger>& audioFlinger, audio_io_handle_t id, audio_devices_t outDevice, audio_devices_t inDevice, type_t type) : Thread(false /*canCallJava*/), @@ -577,20 +635,21 @@ void AudioFlinger::ThreadBase::dumpBase(int fd, const Vector<String16>& args __u bool locked = AudioFlinger::dumpTryLock(mLock); if (!locked) { - dprintf(fd, "thread %p maybe dead locked\n", this); + dprintf(fd, "thread %p may be deadlocked\n", this); } dprintf(fd, " I/O handle: %d\n", mId); dprintf(fd, " TID: %d\n", getTid()); dprintf(fd, " Standby: %s\n", mStandby ? "yes" : "no"); - dprintf(fd, " Sample rate: %u\n", mSampleRate); + dprintf(fd, " Sample rate: %u Hz\n", mSampleRate); dprintf(fd, " HAL frame count: %zu\n", mFrameCount); + dprintf(fd, " HAL format: 0x%x (%s)\n", mHALFormat, formatToString(mHALFormat)); dprintf(fd, " HAL buffer size: %u bytes\n", mBufferSize); - dprintf(fd, " Channel Count: %u\n", mChannelCount); - dprintf(fd, " Channel Mask: 0x%08x (%s)\n", mChannelMask, + dprintf(fd, " Channel count: %u\n", mChannelCount); + dprintf(fd, " Channel mask: 0x%08x (%s)\n", mChannelMask, channelMaskToString(mChannelMask, mType != RECORD).string()); - dprintf(fd, " Format: 0x%x (%s)\n", mHALFormat, formatToString(mHALFormat)); - dprintf(fd, " Frame size: %zu\n", mFrameSize); + dprintf(fd, " Format: 0x%x (%s)\n", mFormat, formatToString(mFormat)); + dprintf(fd, " Frame size: %zu bytes\n", mFrameSize); dprintf(fd, " Pending config events:"); size_t numConfig = mConfigEvents.size(); if (numConfig) { @@ -1317,7 +1376,7 @@ void AudioFlinger::PlaybackThread::dumpTracks(int fd, const Vector<String16>& ar void AudioFlinger::PlaybackThread::dumpInternals(int fd, const Vector<String16>& args) { - dprintf(fd, "\nOutput thread %p:\n", this); + dprintf(fd, "\nOutput thread %p type %d (%s):\n", this, type(), threadTypeToString(type())); dprintf(fd, " Normal frame count: %zu\n", mNormalFrameCount); dprintf(fd, " Last write occurred (msecs): %llu\n", ns2ms(systemTime() - mLastWriteTime)); dprintf(fd, " Total writes: %d\n", mNumWrites); @@ -1328,6 +1387,10 @@ void AudioFlinger::PlaybackThread::dumpInternals(int fd, const Vector<String16>& dprintf(fd, " Mixer buffer: %p\n", mMixerBuffer); dprintf(fd, " Effect buffer: %p\n", mEffectBuffer); dprintf(fd, " Fast track availMask=%#x\n", mFastTrackAvailMask); + AudioStreamOut *output = mOutput; + audio_output_flags_t flags = output != NULL ? output->flags : AUDIO_OUTPUT_FLAG_NONE; + String8 flagsAsString = outputFlagsToString(flags); + dprintf(fd, " AudioStreamOut: %p flags %#x (%s)\n", output, flags, flagsAsString.string()); dumpBase(fd, args); } @@ -2122,6 +2185,7 @@ ssize_t AudioFlinger::PlaybackThread::threadLoop_write() } else { bytesWritten = framesWritten; } + mLatchDValid = false; status_t status = mNormalSink->getTimestamp(mLatchD.mTimestamp); if (status == NO_ERROR) { size_t totalFramesWritten = mNormalSink->framesWritten(); @@ -2619,7 +2683,9 @@ bool AudioFlinger::PlaybackThread::threadLoop() } } else { + ATRACE_BEGIN("sleep"); usleep(sleepTime); + ATRACE_END(); } } @@ -2819,6 +2885,7 @@ AudioFlinger::MixerThread::MixerThread(const sp<AudioFlinger>& audioFlinger, Aud NBAIO_Format format = mOutputSink->format(); NBAIO_Format origformat = format; // adjust format to match that of the Fast Mixer + ALOGV("format changed from %d to %d", format.mFormat, fastMixerFormat); format.mFormat = fastMixerFormat; format.mFrameSize = audio_bytes_per_sample(format.mFormat) * format.mChannelCount; @@ -3382,6 +3449,23 @@ AudioFlinger::PlaybackThread::mixer_state AudioFlinger::MixerThread::prepareTrac } size_t framesReady = track->framesReady(); + if (ATRACE_ENABLED()) { + // I wish we had formatted trace names + char traceName[16]; + strcpy(traceName, "nRdy"); + int name = track->name(); + if (AudioMixer::TRACK0 <= name && + name < (int) (AudioMixer::TRACK0 + AudioMixer::MAX_NUM_TRACKS)) { + name -= AudioMixer::TRACK0; + traceName[4] = (name / 10) + '0'; + traceName[5] = (name % 10) + '0'; + } else { + traceName[4] = '?'; + traceName[5] = '?'; + } + traceName[6] = '\0'; + ATRACE_INT(traceName, framesReady); + } if ((framesReady >= minFrames) && track->isReady() && !track->isPaused() && !track->isTerminated()) { @@ -5032,7 +5116,9 @@ reacquire_wakelock: // sleep with mutex unlocked if (sleepUs > 0) { + ATRACE_BEGIN("sleep"); usleep(sleepUs); + ATRACE_END(); sleepUs = 0; } diff --git a/services/audioflinger/Threads.h b/services/audioflinger/Threads.h index bb9aa18..bcdb0e5 100644 --- a/services/audioflinger/Threads.h +++ b/services/audioflinger/Threads.h @@ -32,6 +32,8 @@ public: OFFLOAD // Thread class is OffloadThread }; + static const char *threadTypeToString(type_t type); + ThreadBase(const sp<AudioFlinger>& audioFlinger, audio_io_handle_t id, audio_devices_t outDevice, audio_devices_t inDevice, type_t type); virtual ~ThreadBase(); @@ -406,6 +408,7 @@ protected: audio_channel_mask_t mChannelMask; uint32_t mChannelCount; size_t mFrameSize; + // not HAL frame size, this is for output sink (to pipe to fast mixer) audio_format_t mFormat; // Source format for Recording and // Sink format for Playback. // Sink format may be different than diff --git a/services/audioflinger/Tracks.cpp b/services/audioflinger/Tracks.cpp index 037c73b..6ce2d42 100644 --- a/services/audioflinger/Tracks.cpp +++ b/services/audioflinger/Tracks.cpp @@ -860,6 +860,7 @@ void AudioFlinger::PlaybackThread::Track::reset() if (mState == FLUSHED) { mState = IDLE; } + mPreviousValid = false; } } diff --git a/services/audioflinger/tests/mixer_to_wav_tests.sh b/services/audioflinger/tests/mixer_to_wav_tests.sh index 9b39e77..e60e6d5 100755 --- a/services/audioflinger/tests/mixer_to_wav_tests.sh +++ b/services/audioflinger/tests/mixer_to_wav_tests.sh @@ -63,8 +63,18 @@ function createwav() { # process__genericResampling # track__Resample / track__genericResample adb shell test-mixer $1 -s 48000 \ + -o /sdcard/tm48000grif.wav \ + sine:2,4000,7520 chirp:2,9200 sine:1,3000,18000 \ + sine:f,6,6000,19000 chirp:i,4,30000 + adb pull /sdcard/tm48000grif.wav $2 + +# Test: +# process__genericResampling +# track__Resample / track__genericResample + adb shell test-mixer $1 -s 48000 \ -o /sdcard/tm48000gr.wav \ - sine:2,4000,7520 chirp:2,9200 sine:1,3000,18000 + sine:2,4000,7520 chirp:2,9200 sine:1,3000,18000 \ + sine:6,6000,19000 adb pull /sdcard/tm48000gr.wav $2 # Test: diff --git a/services/audioflinger/tests/test-mixer.cpp b/services/audioflinger/tests/test-mixer.cpp index 9a4fad6..8da6245 100644 --- a/services/audioflinger/tests/test-mixer.cpp +++ b/services/audioflinger/tests/test-mixer.cpp @@ -39,7 +39,7 @@ static void usage(const char* name) { fprintf(stderr, "Usage: %s [-f] [-m] [-c channels]" " [-s sample-rate] [-o <output-file>] [-a <aux-buffer-file>] [-P csv]" " (<input-file> | <command>)+\n", name); - fprintf(stderr, " -f enable floating point input track\n"); + fprintf(stderr, " -f enable floating point input track by default\n"); fprintf(stderr, " -m enable floating point mixer output\n"); fprintf(stderr, " -c number of mixer output channels\n"); fprintf(stderr, " -s mixer sample-rate\n"); @@ -47,8 +47,8 @@ static void usage(const char* name) { fprintf(stderr, " -a <aux-buffer-file>\n"); fprintf(stderr, " -P # frames provided per call to resample() in CSV format\n"); fprintf(stderr, " <input-file> is a WAV file\n"); - fprintf(stderr, " <command> can be 'sine:<channels>,<frequency>,<samplerate>'\n"); - fprintf(stderr, " 'chirp:<channels>,<samplerate>'\n"); + fprintf(stderr, " <command> can be 'sine:[(i|f),]<channels>,<frequency>,<samplerate>'\n"); + fprintf(stderr, " 'chirp:[(i|f),]<channels>,<samplerate>'\n"); } static int writeFile(const char *filename, const void *buffer, @@ -78,6 +78,18 @@ static int writeFile(const char *filename, const void *buffer, return EXIT_SUCCESS; } +const char *parseFormat(const char *s, bool *useFloat) { + if (!strncmp(s, "f,", 2)) { + *useFloat = true; + return s + 2; + } + if (!strncmp(s, "i,", 2)) { + *useFloat = false; + return s + 2; + } + return s; +} + int main(int argc, char* argv[]) { const char* const progname = argv[0]; bool useInputFloat = false; @@ -88,8 +100,9 @@ int main(int argc, char* argv[]) { std::vector<int> Pvalues; const char* outputFilename = NULL; const char* auxFilename = NULL; - std::vector<int32_t> Names; - std::vector<SignalProvider> Providers; + std::vector<int32_t> names; + std::vector<SignalProvider> providers; + std::vector<audio_format_t> formats; for (int ch; (ch = getopt(argc, argv, "fmc:s:o:a:P:")) != -1;) { switch (ch) { @@ -138,54 +151,65 @@ int main(int argc, char* argv[]) { size_t outputFrames = 0; // create providers for each track - Providers.resize(argc); + names.resize(argc); + providers.resize(argc); + formats.resize(argc); for (int i = 0; i < argc; ++i) { static const char chirp[] = "chirp:"; static const char sine[] = "sine:"; static const double kSeconds = 1; + bool useFloat = useInputFloat; if (!strncmp(argv[i], chirp, strlen(chirp))) { std::vector<int> v; + const char *s = parseFormat(argv[i] + strlen(chirp), &useFloat); - parseCSV(argv[i] + strlen(chirp), v); + parseCSV(s, v); if (v.size() == 2) { printf("creating chirp(%d %d)\n", v[0], v[1]); - if (useInputFloat) { - Providers[i].setChirp<float>(v[0], 0, v[1]/2, v[1], kSeconds); + if (useFloat) { + providers[i].setChirp<float>(v[0], 0, v[1]/2, v[1], kSeconds); + formats[i] = AUDIO_FORMAT_PCM_FLOAT; } else { - Providers[i].setChirp<int16_t>(v[0], 0, v[1]/2, v[1], kSeconds); + providers[i].setChirp<int16_t>(v[0], 0, v[1]/2, v[1], kSeconds); + formats[i] = AUDIO_FORMAT_PCM_16_BIT; } - Providers[i].setIncr(Pvalues); + providers[i].setIncr(Pvalues); } else { fprintf(stderr, "malformed input '%s'\n", argv[i]); } } else if (!strncmp(argv[i], sine, strlen(sine))) { std::vector<int> v; + const char *s = parseFormat(argv[i] + strlen(sine), &useFloat); - parseCSV(argv[i] + strlen(sine), v); + parseCSV(s, v); if (v.size() == 3) { printf("creating sine(%d %d %d)\n", v[0], v[1], v[2]); - if (useInputFloat) { - Providers[i].setSine<float>(v[0], v[1], v[2], kSeconds); + if (useFloat) { + providers[i].setSine<float>(v[0], v[1], v[2], kSeconds); + formats[i] = AUDIO_FORMAT_PCM_FLOAT; } else { - Providers[i].setSine<int16_t>(v[0], v[1], v[2], kSeconds); + providers[i].setSine<int16_t>(v[0], v[1], v[2], kSeconds); + formats[i] = AUDIO_FORMAT_PCM_16_BIT; } - Providers[i].setIncr(Pvalues); + providers[i].setIncr(Pvalues); } else { fprintf(stderr, "malformed input '%s'\n", argv[i]); } } else { printf("creating filename(%s)\n", argv[i]); if (useInputFloat) { - Providers[i].setFile<float>(argv[i]); + providers[i].setFile<float>(argv[i]); + formats[i] = AUDIO_FORMAT_PCM_FLOAT; } else { - Providers[i].setFile<short>(argv[i]); + providers[i].setFile<short>(argv[i]); + formats[i] = AUDIO_FORMAT_PCM_16_BIT; } - Providers[i].setIncr(Pvalues); + providers[i].setIncr(Pvalues); } // calculate the number of output frames - size_t nframes = (int64_t) Providers[i].getNumFrames() * outputSampleRate - / Providers[i].getSampleRate(); + size_t nframes = (int64_t) providers[i].getNumFrames() * outputSampleRate + / providers[i].getSampleRate(); if (i == 0 || outputFrames > nframes) { // choose minimum for outputFrames outputFrames = nframes; } @@ -213,22 +237,20 @@ int main(int argc, char* argv[]) { // create the mixer. const size_t mixerFrameCount = 320; // typical numbers may range from 240 or 960 AudioMixer *mixer = new AudioMixer(mixerFrameCount, outputSampleRate); - audio_format_t inputFormat = useInputFloat - ? AUDIO_FORMAT_PCM_FLOAT : AUDIO_FORMAT_PCM_16_BIT; audio_format_t mixerFormat = useMixerFloat ? AUDIO_FORMAT_PCM_FLOAT : AUDIO_FORMAT_PCM_16_BIT; - float f = AudioMixer::UNITY_GAIN_FLOAT / Providers.size(); // normalize volume by # tracks + float f = AudioMixer::UNITY_GAIN_FLOAT / providers.size(); // normalize volume by # tracks static float f0; // zero // set up the tracks. - for (size_t i = 0; i < Providers.size(); ++i) { - //printf("track %d out of %d\n", i, Providers.size()); - uint32_t channelMask = audio_channel_out_mask_from_count(Providers[i].getNumChannels()); + for (size_t i = 0; i < providers.size(); ++i) { + //printf("track %d out of %d\n", i, providers.size()); + uint32_t channelMask = audio_channel_out_mask_from_count(providers[i].getNumChannels()); int32_t name = mixer->getTrackName(channelMask, - inputFormat, AUDIO_SESSION_OUTPUT_MIX); + formats[i], AUDIO_SESSION_OUTPUT_MIX); ALOG_ASSERT(name >= 0); - Names.push_back(name); - mixer->setBufferProvider(name, &Providers[i]); + names[i] = name; + mixer->setBufferProvider(name, &providers[i]); mixer->setParameter(name, AudioMixer::TRACK, AudioMixer::MAIN_BUFFER, (void *)outputAddr); mixer->setParameter( @@ -240,7 +262,7 @@ int main(int argc, char* argv[]) { name, AudioMixer::TRACK, AudioMixer::FORMAT, - (void *)(uintptr_t)inputFormat); + (void *)(uintptr_t)formats[i]); mixer->setParameter( name, AudioMixer::TRACK, @@ -255,7 +277,7 @@ int main(int argc, char* argv[]) { name, AudioMixer::RESAMPLE, AudioMixer::SAMPLE_RATE, - (void *)(uintptr_t)Providers[i].getSampleRate()); + (void *)(uintptr_t)providers[i].getSampleRate()); if (useRamp) { mixer->setParameter(name, AudioMixer::VOLUME, AudioMixer::VOLUME0, &f0); mixer->setParameter(name, AudioMixer::VOLUME, AudioMixer::VOLUME1, &f0); @@ -277,11 +299,11 @@ int main(int argc, char* argv[]) { // pump the mixer to process data. size_t i; for (i = 0; i < outputFrames - mixerFrameCount; i += mixerFrameCount) { - for (size_t j = 0; j < Names.size(); ++j) { - mixer->setParameter(Names[j], AudioMixer::TRACK, AudioMixer::MAIN_BUFFER, + for (size_t j = 0; j < names.size(); ++j) { + mixer->setParameter(names[j], AudioMixer::TRACK, AudioMixer::MAIN_BUFFER, (char *) outputAddr + i * outputFrameSize); if (auxFilename) { - mixer->setParameter(Names[j], AudioMixer::TRACK, AudioMixer::AUX_BUFFER, + mixer->setParameter(names[j], AudioMixer::TRACK, AudioMixer::AUX_BUFFER, (char *) auxAddr + i * auxFrameSize); } } |