diff options
34 files changed, 3246 insertions, 45 deletions
diff --git a/cmds/stagefright/Android.mk b/cmds/stagefright/Android.mk index 561ce02..78c89bb 100644 --- a/cmds/stagefright/Android.mk +++ b/cmds/stagefright/Android.mk @@ -169,6 +169,44 @@ include $(BUILD_EXECUTABLE) include $(CLEAR_VARS) +LOCAL_NDK_STL_VARIANT := stlport_static + +LOCAL_SRC_FILES:= \ + filters/argbtorgba.rs \ + filters/nightvision.rs \ + filters/saturation.rs \ + mediafilter.cpp \ + +LOCAL_SHARED_LIBRARIES := \ + libstagefright liblog libutils libbinder libstagefright_foundation \ + libmedia libgui libcutils libui + +LOCAL_C_INCLUDES:= \ + $(TOP)/bionic \ + $(TOP)/external/stlport/stlport \ + $(TOP)/frameworks/av/media/libstagefright \ + $(TOP)/frameworks/native/include/media/openmax \ + $(TOP)/frameworks/rs/cpp \ + $(TOP)/frameworks/rs \ + +intermediates := $(call intermediates-dir-for,STATIC_LIBRARIES,libRS,TARGET,) +LOCAL_C_INCLUDES += $(intermediates) + +LOCAL_STATIC_LIBRARIES:= \ + libstagefright_mediafilter + +LOCAL_CFLAGS += -Wno-multichar + +LOCAL_MODULE_TAGS := optional + +LOCAL_MODULE:= mediafilter + +include $(BUILD_EXECUTABLE) + +################################################################################ + +include $(CLEAR_VARS) + LOCAL_SRC_FILES:= \ muxer.cpp \ diff --git a/cmds/stagefright/codec.cpp b/cmds/stagefright/codec.cpp index fd02bcc..d987250 100644 --- a/cmds/stagefright/codec.cpp +++ b/cmds/stagefright/codec.cpp @@ -45,9 +45,10 @@ static void usage(const char *me) { fprintf(stderr, "usage: %s [-a] use audio\n" "\t\t[-v] use video\n" "\t\t[-p] playback\n" - "\t\t[-S] allocate buffers from a surface\n", + "\t\t[-S] allocate buffers from a surface\n" + "\t\t[-R] render output to surface (enables -S)\n" + "\t\t[-T] use render timestamps (enables -R)\n", me); - exit(1); } @@ -71,7 +72,9 @@ static int decode( const char *path, bool useAudio, bool useVideo, - const android::sp<android::Surface> &surface) { + const android::sp<android::Surface> &surface, + bool renderSurface, + bool useTimestamp) { using namespace android; static int64_t kTimeout = 500ll; @@ -136,6 +139,7 @@ static int decode( CHECK(!stateByTrack.isEmpty()); int64_t startTimeUs = ALooper::GetNowUs(); + int64_t startTimeRender = -1; for (size_t i = 0; i < stateByTrack.size(); ++i) { CodecState *state = &stateByTrack.editValueAt(i); @@ -260,7 +264,23 @@ static int decode( ++state->mNumBuffersDecoded; state->mNumBytesDecoded += size; - err = state->mCodec->releaseOutputBuffer(index); + if (surface == NULL || !renderSurface) { + err = state->mCodec->releaseOutputBuffer(index); + } else if (useTimestamp) { + if (startTimeRender == -1) { + // begin rendering 2 vsyncs (~33ms) after first decode + startTimeRender = + systemTime(SYSTEM_TIME_MONOTONIC) + 33000000 + - (presentationTimeUs * 1000); + } + presentationTimeUs = + (presentationTimeUs * 1000) + startTimeRender; + err = state->mCodec->renderOutputBufferAndRelease( + index, presentationTimeUs); + } else { + err = state->mCodec->renderOutputBufferAndRelease(index); + } + CHECK_EQ(err, (status_t)OK); if (flags & MediaCodec::BUFFER_FLAG_EOS) { @@ -320,34 +340,42 @@ int main(int argc, char **argv) { bool useVideo = false; bool playback = false; bool useSurface = false; + bool renderSurface = false; + bool useTimestamp = false; int res; - while ((res = getopt(argc, argv, "havpSD")) >= 0) { + while ((res = getopt(argc, argv, "havpSDRT")) >= 0) { switch (res) { case 'a': { useAudio = true; break; } - case 'v': { useVideo = true; break; } - case 'p': { playback = true; break; } - + case 'T': + { + useTimestamp = true; + } + // fall through + case 'R': + { + renderSurface = true; + } + // fall through case 'S': { useSurface = true; break; } - case '?': case 'h': default: @@ -422,7 +450,8 @@ int main(int argc, char **argv) { player->stop(); player->reset(); } else { - decode(looper, argv[0], useAudio, useVideo, surface); + decode(looper, argv[0], useAudio, useVideo, surface, renderSurface, + useTimestamp); } if (playback || (useSurface && useVideo)) { diff --git a/cmds/stagefright/filters/argbtorgba.rs b/cmds/stagefright/filters/argbtorgba.rs new file mode 100644 index 0000000..229ff8c --- /dev/null +++ b/cmds/stagefright/filters/argbtorgba.rs @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma version(1) +#pragma rs java_package_name(com.android.rs.cppbasic) +#pragma rs_fp_relaxed + +void root(const uchar4 *v_in, uchar4 *v_out) { + v_out->x = v_in->y; + v_out->y = v_in->z; + v_out->z = v_in->w; + v_out->w = v_in->x; +}
\ No newline at end of file diff --git a/cmds/stagefright/filters/nightvision.rs b/cmds/stagefright/filters/nightvision.rs new file mode 100644 index 0000000..f61413c --- /dev/null +++ b/cmds/stagefright/filters/nightvision.rs @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma version(1) +#pragma rs java_package_name(com.android.rs.cppbasic) +#pragma rs_fp_relaxed + +const static float3 gMonoMult = {0.299f, 0.587f, 0.114f}; +const static float3 gNightVisionMult = {0.5f, 1.f, 0.5f}; + +// calculates luminance of pixel, then biases color balance toward green +void root(const uchar4 *v_in, uchar4 *v_out) { + v_out->x = v_in->x; // don't modify A + + // get RGB, scale 0-255 uchar to 0-1.0 float + float3 rgb = {v_in->y * 0.003921569f, v_in->z * 0.003921569f, + v_in->w * 0.003921569f}; + + // apply filter + float3 result = dot(rgb, gMonoMult) * gNightVisionMult; + + v_out->y = (uchar)clamp((result.r * 255.f + 0.5f), 0.f, 255.f); + v_out->z = (uchar)clamp((result.g * 255.f + 0.5f), 0.f, 255.f); + v_out->w = (uchar)clamp((result.b * 255.f + 0.5f), 0.f, 255.f); +} diff --git a/cmds/stagefright/filters/saturation.rs b/cmds/stagefright/filters/saturation.rs new file mode 100644 index 0000000..1de9dd8 --- /dev/null +++ b/cmds/stagefright/filters/saturation.rs @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma version(1) +#pragma rs java_package_name(com.android.rs.cppbasic) +#pragma rs_fp_relaxed + +const static float3 gMonoMult = {0.299f, 0.587f, 0.114f}; + +// global variables (parameters accessible to application code) +float gSaturation = 1.0f; + +void root(const uchar4 *v_in, uchar4 *v_out) { + v_out->x = v_in->x; // don't modify A + + // get RGB, scale 0-255 uchar to 0-1.0 float + float3 rgb = {v_in->y * 0.003921569f, v_in->z * 0.003921569f, + v_in->w * 0.003921569f}; + + // apply saturation filter + float3 result = dot(rgb, gMonoMult); + result = mix(result, rgb, gSaturation); + + v_out->y = (uchar)clamp((result.r * 255.f + 0.5f), 0.f, 255.f); + v_out->z = (uchar)clamp((result.g * 255.f + 0.5f), 0.f, 255.f); + v_out->w = (uchar)clamp((result.b * 255.f + 0.5f), 0.f, 255.f); +} diff --git a/cmds/stagefright/mediafilter.cpp b/cmds/stagefright/mediafilter.cpp new file mode 100644 index 0000000..f77b38b --- /dev/null +++ b/cmds/stagefright/mediafilter.cpp @@ -0,0 +1,785 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "mediafilterTest" + +#include <inttypes.h> + +#include <binder/ProcessState.h> +#include <filters/ColorConvert.h> +#include <gui/ISurfaceComposer.h> +#include <gui/SurfaceComposerClient.h> +#include <gui/Surface.h> +#include <media/ICrypto.h> +#include <media/IMediaHTTPService.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/DataSource.h> +#include <media/stagefright/MediaCodec.h> +#include <media/stagefright/NuMediaExtractor.h> +#include <media/stagefright/RenderScriptWrapper.h> +#include <OMX_IVCommon.h> +#include <ui/DisplayInfo.h> + +#include "RenderScript.h" +#include "ScriptC_argbtorgba.h" +#include "ScriptC_nightvision.h" +#include "ScriptC_saturation.h" + +// test parameters +static const bool kTestFlush = true; // Note: true will drop 1 out of +static const int kFlushAfterFrames = 25; // kFlushAfterFrames output frames +static const int64_t kTimeout = 500ll; + +// built-in filter parameters +static const int32_t kInvert = false; // ZeroFilter param +static const float kBlurRadius = 15.0f; // IntrinsicBlurFilter param +static const float kSaturation = 0.0f; // SaturationFilter param + +static void usage(const char *me) { + fprintf(stderr, "usage: [flags] %s\n" + "\t[-b] use IntrinsicBlurFilter\n" + "\t[-c] use argb to rgba conversion RSFilter\n" + "\t[-n] use night vision RSFilter\n" + "\t[-r] use saturation RSFilter\n" + "\t[-s] use SaturationFilter\n" + "\t[-z] use ZeroFilter (copy filter)\n" + "\t[-R] render output to surface (enables -S)\n" + "\t[-S] allocate buffers from a surface\n" + "\t[-T] use render timestamps (enables -R)\n", + me); + exit(1); +} + +namespace android { + +struct SaturationRSFilter : RenderScriptWrapper::RSFilterCallback { + void init(RSC::sp<RSC::RS> context) { + mScript = new ScriptC_saturation(context); + mScript->set_gSaturation(3.f); + } + + virtual status_t processBuffers( + RSC::Allocation *inBuffer, RSC::Allocation *outBuffer) { + mScript->forEach_root(inBuffer, outBuffer); + + return OK; + } + + status_t handleSetParameters(const sp<AMessage> &msg) { + return OK; + } + +private: + RSC::sp<ScriptC_saturation> mScript; +}; + +struct NightVisionRSFilter : RenderScriptWrapper::RSFilterCallback { + void init(RSC::sp<RSC::RS> context) { + mScript = new ScriptC_nightvision(context); + } + + virtual status_t processBuffers( + RSC::Allocation *inBuffer, RSC::Allocation *outBuffer) { + mScript->forEach_root(inBuffer, outBuffer); + + return OK; + } + + status_t handleSetParameters(const sp<AMessage> &msg) { + return OK; + } + +private: + RSC::sp<ScriptC_nightvision> mScript; +}; + +struct ARGBToRGBARSFilter : RenderScriptWrapper::RSFilterCallback { + void init(RSC::sp<RSC::RS> context) { + mScript = new ScriptC_argbtorgba(context); + } + + virtual status_t processBuffers( + RSC::Allocation *inBuffer, RSC::Allocation *outBuffer) { + mScript->forEach_root(inBuffer, outBuffer); + + return OK; + } + + status_t handleSetParameters(const sp<AMessage> &msg) { + return OK; + } + +private: + RSC::sp<ScriptC_argbtorgba> mScript; +}; + +struct CodecState { + sp<MediaCodec> mCodec; + Vector<sp<ABuffer> > mInBuffers; + Vector<sp<ABuffer> > mOutBuffers; + bool mSignalledInputEOS; + bool mSawOutputEOS; + int64_t mNumBuffersDecoded; +}; + +struct DecodedFrame { + size_t index; + size_t offset; + size_t size; + int64_t presentationTimeUs; + uint32_t flags; +}; + +enum FilterType { + FILTERTYPE_ZERO, + FILTERTYPE_INTRINSIC_BLUR, + FILTERTYPE_SATURATION, + FILTERTYPE_RS_SATURATION, + FILTERTYPE_RS_NIGHT_VISION, + FILTERTYPE_RS_ARGB_TO_RGBA, +}; + +size_t inputFramesSinceFlush = 0; +void tryCopyDecodedBuffer( + List<DecodedFrame> *decodedFrameIndices, + CodecState *filterState, + CodecState *vidState) { + if (decodedFrameIndices->empty()) { + return; + } + + size_t filterIndex; + status_t err = filterState->mCodec->dequeueInputBuffer( + &filterIndex, kTimeout); + if (err != OK) { + return; + } + + ++inputFramesSinceFlush; + + DecodedFrame frame = *decodedFrameIndices->begin(); + + // only consume a buffer if we are not going to flush, since we expect + // the dequeue -> flush -> queue operation to cause an error and + // not produce an output frame + if (!kTestFlush || inputFramesSinceFlush < kFlushAfterFrames) { + decodedFrameIndices->erase(decodedFrameIndices->begin()); + } + size_t outIndex = frame.index; + + const sp<ABuffer> &srcBuffer = + vidState->mOutBuffers.itemAt(outIndex); + const sp<ABuffer> &destBuffer = + filterState->mInBuffers.itemAt(filterIndex); + + sp<AMessage> srcFormat, destFormat; + vidState->mCodec->getOutputFormat(&srcFormat); + filterState->mCodec->getInputFormat(&destFormat); + + int32_t srcWidth, srcHeight, srcStride, srcSliceHeight; + int32_t srcColorFormat, destColorFormat; + int32_t destWidth, destHeight, destStride, destSliceHeight; + CHECK(srcFormat->findInt32("stride", &srcStride) + && srcFormat->findInt32("slice-height", &srcSliceHeight) + && srcFormat->findInt32("width", &srcWidth) + && srcFormat->findInt32("height", & srcHeight) + && srcFormat->findInt32("color-format", &srcColorFormat)); + CHECK(destFormat->findInt32("stride", &destStride) + && destFormat->findInt32("slice-height", &destSliceHeight) + && destFormat->findInt32("width", &destWidth) + && destFormat->findInt32("height", & destHeight) + && destFormat->findInt32("color-format", &destColorFormat)); + + CHECK(srcWidth <= destStride && srcHeight <= destSliceHeight); + + convertYUV420spToARGB( + srcBuffer->data(), + srcBuffer->data() + srcStride * srcSliceHeight, + srcWidth, + srcHeight, + destBuffer->data()); + + // copy timestamp + int64_t timeUs; + CHECK(srcBuffer->meta()->findInt64("timeUs", &timeUs)); + destBuffer->meta()->setInt64("timeUs", timeUs); + + if (kTestFlush && inputFramesSinceFlush >= kFlushAfterFrames) { + inputFramesSinceFlush = 0; + + // check that queueing a buffer that was dequeued before flush + // fails with expected error EACCES + filterState->mCodec->flush(); + + err = filterState->mCodec->queueInputBuffer( + filterIndex, 0 /* offset */, destBuffer->size(), + timeUs, frame.flags); + + if (err == OK) { + ALOGE("FAIL: queue after flush returned OK"); + } else if (err != -EACCES) { + ALOGE("queueInputBuffer after flush returned %d, " + "expected -EACCES (-13)", err); + } + } else { + err = filterState->mCodec->queueInputBuffer( + filterIndex, 0 /* offset */, destBuffer->size(), + timeUs, frame.flags); + CHECK(err == OK); + + err = vidState->mCodec->releaseOutputBuffer(outIndex); + CHECK(err == OK); + } +} + +size_t outputFramesSinceFlush = 0; +void tryDrainOutputBuffer( + CodecState *filterState, + const sp<Surface> &surface, bool renderSurface, + bool useTimestamp, int64_t *startTimeRender) { + size_t index; + size_t offset; + size_t size; + int64_t presentationTimeUs; + uint32_t flags; + status_t err = filterState->mCodec->dequeueOutputBuffer( + &index, &offset, &size, &presentationTimeUs, &flags, + kTimeout); + + if (err != OK) { + return; + } + + ++outputFramesSinceFlush; + + if (kTestFlush && outputFramesSinceFlush >= kFlushAfterFrames) { + filterState->mCodec->flush(); + } + + if (surface == NULL || !renderSurface) { + err = filterState->mCodec->releaseOutputBuffer(index); + } else if (useTimestamp) { + if (*startTimeRender == -1) { + // begin rendering 2 vsyncs after first decode + *startTimeRender = systemTime(SYSTEM_TIME_MONOTONIC) + + 33000000 - (presentationTimeUs * 1000); + } + presentationTimeUs = + (presentationTimeUs * 1000) + *startTimeRender; + err = filterState->mCodec->renderOutputBufferAndRelease( + index, presentationTimeUs); + } else { + err = filterState->mCodec->renderOutputBufferAndRelease(index); + } + + if (kTestFlush && outputFramesSinceFlush >= kFlushAfterFrames) { + outputFramesSinceFlush = 0; + + // releasing the buffer dequeued before flush should cause an error + // if so, the frame will also be skipped in output stream + if (err == OK) { + ALOGE("FAIL: release after flush returned OK"); + } else if (err != -EACCES) { + ALOGE("releaseOutputBuffer after flush returned %d, " + "expected -EACCES (-13)", err); + } + } else { + CHECK(err == OK); + } + + if (flags & MediaCodec::BUFFER_FLAG_EOS) { + ALOGV("reached EOS on output."); + filterState->mSawOutputEOS = true; + } +} + +static int decode( + const sp<ALooper> &looper, + const char *path, + const sp<Surface> &surface, + bool renderSurface, + bool useTimestamp, + FilterType filterType) { + + static int64_t kTimeout = 500ll; + + sp<NuMediaExtractor> extractor = new NuMediaExtractor; + if (extractor->setDataSource(NULL /* httpService */, path) != OK) { + fprintf(stderr, "unable to instantiate extractor.\n"); + return 1; + } + + KeyedVector<size_t, CodecState> stateByTrack; + + CodecState *vidState = NULL; + for (size_t i = 0; i < extractor->countTracks(); ++i) { + sp<AMessage> format; + status_t err = extractor->getTrackFormat(i, &format); + CHECK(err == OK); + + AString mime; + CHECK(format->findString("mime", &mime)); + bool isVideo = !strncasecmp(mime.c_str(), "video/", 6); + if (!isVideo) { + continue; + } + + ALOGV("selecting track %zu", i); + + err = extractor->selectTrack(i); + CHECK(err == OK); + + CodecState *state = + &stateByTrack.editValueAt(stateByTrack.add(i, CodecState())); + + vidState = state; + + state->mNumBuffersDecoded = 0; + + state->mCodec = MediaCodec::CreateByType( + looper, mime.c_str(), false /* encoder */); + + CHECK(state->mCodec != NULL); + + err = state->mCodec->configure( + format, NULL /* surface */, NULL /* crypto */, 0 /* flags */); + + CHECK(err == OK); + + state->mSignalledInputEOS = false; + state->mSawOutputEOS = false; + + break; + } + CHECK(!stateByTrack.isEmpty()); + CHECK(vidState != NULL); + sp<AMessage> vidFormat; + vidState->mCodec->getOutputFormat(&vidFormat); + + // set filter to use ARGB8888 + vidFormat->setInt32("color-format", OMX_COLOR_Format32bitARGB8888); + // set app cache directory path + vidFormat->setString("cacheDir", "/system/bin"); + + // create RenderScript context for RSFilters + RSC::sp<RSC::RS> context = new RSC::RS(); + context->init("/system/bin"); + + sp<RenderScriptWrapper::RSFilterCallback> rsFilter; + + // create renderscript wrapper for RSFilters + sp<RenderScriptWrapper> rsWrapper = new RenderScriptWrapper; + rsWrapper->mContext = context.get(); + + CodecState *filterState = new CodecState(); + filterState->mNumBuffersDecoded = 0; + + sp<AMessage> params = new AMessage(); + + switch (filterType) { + case FILTERTYPE_ZERO: + { + filterState->mCodec = MediaCodec::CreateByComponentName( + looper, "android.filter.zerofilter"); + params->setInt32("invert", kInvert); + break; + } + case FILTERTYPE_INTRINSIC_BLUR: + { + filterState->mCodec = MediaCodec::CreateByComponentName( + looper, "android.filter.intrinsicblur"); + params->setFloat("blur-radius", kBlurRadius); + break; + } + case FILTERTYPE_SATURATION: + { + filterState->mCodec = MediaCodec::CreateByComponentName( + looper, "android.filter.saturation"); + params->setFloat("saturation", kSaturation); + break; + } + case FILTERTYPE_RS_SATURATION: + { + SaturationRSFilter *satFilter = new SaturationRSFilter; + satFilter->init(context); + rsFilter = satFilter; + rsWrapper->mCallback = rsFilter; + vidFormat->setObject("rs-wrapper", rsWrapper); + + filterState->mCodec = MediaCodec::CreateByComponentName( + looper, "android.filter.RenderScript"); + break; + } + case FILTERTYPE_RS_NIGHT_VISION: + { + NightVisionRSFilter *nightVisionFilter = new NightVisionRSFilter; + nightVisionFilter->init(context); + rsFilter = nightVisionFilter; + rsWrapper->mCallback = rsFilter; + vidFormat->setObject("rs-wrapper", rsWrapper); + + filterState->mCodec = MediaCodec::CreateByComponentName( + looper, "android.filter.RenderScript"); + break; + } + case FILTERTYPE_RS_ARGB_TO_RGBA: + { + ARGBToRGBARSFilter *argbToRgbaFilter = new ARGBToRGBARSFilter; + argbToRgbaFilter->init(context); + rsFilter = argbToRgbaFilter; + rsWrapper->mCallback = rsFilter; + vidFormat->setObject("rs-wrapper", rsWrapper); + + filterState->mCodec = MediaCodec::CreateByComponentName( + looper, "android.filter.RenderScript"); + break; + } + default: + { + LOG_ALWAYS_FATAL("mediacodec.cpp error: unrecognized FilterType"); + break; + } + } + CHECK(filterState->mCodec != NULL); + + status_t err = filterState->mCodec->configure( + vidFormat /* format */, surface, NULL /* crypto */, 0 /* flags */); + CHECK(err == OK); + + filterState->mSignalledInputEOS = false; + filterState->mSawOutputEOS = false; + + int64_t startTimeUs = ALooper::GetNowUs(); + int64_t startTimeRender = -1; + + for (size_t i = 0; i < stateByTrack.size(); ++i) { + CodecState *state = &stateByTrack.editValueAt(i); + + sp<MediaCodec> codec = state->mCodec; + + CHECK_EQ((status_t)OK, codec->start()); + + CHECK_EQ((status_t)OK, codec->getInputBuffers(&state->mInBuffers)); + CHECK_EQ((status_t)OK, codec->getOutputBuffers(&state->mOutBuffers)); + + ALOGV("got %zu input and %zu output buffers", + state->mInBuffers.size(), state->mOutBuffers.size()); + } + + CHECK_EQ((status_t)OK, filterState->mCodec->setParameters(params)); + + if (kTestFlush) { + status_t flushErr = filterState->mCodec->flush(); + if (flushErr == OK) { + ALOGE("FAIL: Flush before start returned OK"); + } else { + ALOGV("Flush before start returned status %d, usually ENOSYS (-38)", + flushErr); + } + } + + CHECK_EQ((status_t)OK, filterState->mCodec->start()); + CHECK_EQ((status_t)OK, filterState->mCodec->getInputBuffers( + &filterState->mInBuffers)); + CHECK_EQ((status_t)OK, filterState->mCodec->getOutputBuffers( + &filterState->mOutBuffers)); + + if (kTestFlush) { + status_t flushErr = filterState->mCodec->flush(); + if (flushErr != OK) { + ALOGE("FAIL: Flush after start returned %d, expect OK (0)", + flushErr); + } else { + ALOGV("Flush immediately after start OK"); + } + } + + List<DecodedFrame> decodedFrameIndices; + + // loop until decoder reaches EOS + bool sawInputEOS = false; + bool sawOutputEOSOnAllTracks = false; + while (!sawOutputEOSOnAllTracks) { + if (!sawInputEOS) { + size_t trackIndex; + status_t err = extractor->getSampleTrackIndex(&trackIndex); + + if (err != OK) { + ALOGV("saw input eos"); + sawInputEOS = true; + } else { + CodecState *state = &stateByTrack.editValueFor(trackIndex); + + size_t index; + err = state->mCodec->dequeueInputBuffer(&index, kTimeout); + + if (err == OK) { + ALOGV("filling input buffer %zu", index); + + const sp<ABuffer> &buffer = state->mInBuffers.itemAt(index); + + err = extractor->readSampleData(buffer); + CHECK(err == OK); + + int64_t timeUs; + err = extractor->getSampleTime(&timeUs); + CHECK(err == OK); + + uint32_t bufferFlags = 0; + + err = state->mCodec->queueInputBuffer( + index, 0 /* offset */, buffer->size(), + timeUs, bufferFlags); + + CHECK(err == OK); + + extractor->advance(); + } else { + CHECK_EQ(err, -EAGAIN); + } + } + } else { + for (size_t i = 0; i < stateByTrack.size(); ++i) { + CodecState *state = &stateByTrack.editValueAt(i); + + if (!state->mSignalledInputEOS) { + size_t index; + status_t err = + state->mCodec->dequeueInputBuffer(&index, kTimeout); + + if (err == OK) { + ALOGV("signalling input EOS on track %zu", i); + + err = state->mCodec->queueInputBuffer( + index, 0 /* offset */, 0 /* size */, + 0ll /* timeUs */, MediaCodec::BUFFER_FLAG_EOS); + + CHECK(err == OK); + + state->mSignalledInputEOS = true; + } else { + CHECK_EQ(err, -EAGAIN); + } + } + } + } + + sawOutputEOSOnAllTracks = true; + for (size_t i = 0; i < stateByTrack.size(); ++i) { + CodecState *state = &stateByTrack.editValueAt(i); + + if (state->mSawOutputEOS) { + continue; + } else { + sawOutputEOSOnAllTracks = false; + } + + DecodedFrame frame; + status_t err = state->mCodec->dequeueOutputBuffer( + &frame.index, &frame.offset, &frame.size, + &frame.presentationTimeUs, &frame.flags, kTimeout); + + if (err == OK) { + ALOGV("draining decoded buffer %zu, time = %lld us", + frame.index, frame.presentationTimeUs); + + ++(state->mNumBuffersDecoded); + + decodedFrameIndices.push_back(frame); + + if (frame.flags & MediaCodec::BUFFER_FLAG_EOS) { + ALOGV("reached EOS on decoder output."); + state->mSawOutputEOS = true; + } + + } else if (err == INFO_OUTPUT_BUFFERS_CHANGED) { + ALOGV("INFO_OUTPUT_BUFFERS_CHANGED"); + CHECK_EQ((status_t)OK, state->mCodec->getOutputBuffers( + &state->mOutBuffers)); + + ALOGV("got %zu output buffers", state->mOutBuffers.size()); + } else if (err == INFO_FORMAT_CHANGED) { + sp<AMessage> format; + CHECK_EQ((status_t)OK, state->mCodec->getOutputFormat(&format)); + + ALOGV("INFO_FORMAT_CHANGED: %s", + format->debugString().c_str()); + } else { + CHECK_EQ(err, -EAGAIN); + } + + tryCopyDecodedBuffer(&decodedFrameIndices, filterState, vidState); + + tryDrainOutputBuffer( + filterState, surface, renderSurface, + useTimestamp, &startTimeRender); + } + } + + // after EOS on decoder, let filter reach EOS + while (!filterState->mSawOutputEOS) { + tryCopyDecodedBuffer(&decodedFrameIndices, filterState, vidState); + + tryDrainOutputBuffer( + filterState, surface, renderSurface, + useTimestamp, &startTimeRender); + } + + int64_t elapsedTimeUs = ALooper::GetNowUs() - startTimeUs; + + for (size_t i = 0; i < stateByTrack.size(); ++i) { + CodecState *state = &stateByTrack.editValueAt(i); + + CHECK_EQ((status_t)OK, state->mCodec->release()); + + printf("track %zu: %" PRId64 " frames decoded and filtered, " + "%.2f fps.\n", i, state->mNumBuffersDecoded, + state->mNumBuffersDecoded * 1E6 / elapsedTimeUs); + } + + return 0; +} + +} // namespace android + +int main(int argc, char **argv) { + using namespace android; + + const char *me = argv[0]; + + bool useSurface = false; + bool renderSurface = false; + bool useTimestamp = false; + FilterType filterType = FILTERTYPE_ZERO; + + int res; + while ((res = getopt(argc, argv, "bcnrszTRSh")) >= 0) { + switch (res) { + case 'b': + { + filterType = FILTERTYPE_INTRINSIC_BLUR; + break; + } + case 'c': + { + filterType = FILTERTYPE_RS_ARGB_TO_RGBA; + break; + } + case 'n': + { + filterType = FILTERTYPE_RS_NIGHT_VISION; + break; + } + case 'r': + { + filterType = FILTERTYPE_RS_SATURATION; + break; + } + case 's': + { + filterType = FILTERTYPE_SATURATION; + break; + } + case 'z': + { + filterType = FILTERTYPE_ZERO; + break; + } + case 'T': + { + useTimestamp = true; + } + // fall through + case 'R': + { + renderSurface = true; + } + // fall through + case 'S': + { + useSurface = true; + break; + } + case '?': + case 'h': + default: + { + usage(me); + break; + } + } + } + + argc -= optind; + argv += optind; + + if (argc != 1) { + usage(me); + } + + ProcessState::self()->startThreadPool(); + + DataSource::RegisterDefaultSniffers(); + + android::sp<ALooper> looper = new ALooper; + looper->start(); + + android::sp<SurfaceComposerClient> composerClient; + android::sp<SurfaceControl> control; + android::sp<Surface> surface; + + if (useSurface) { + composerClient = new SurfaceComposerClient; + CHECK_EQ((status_t)OK, composerClient->initCheck()); + + android::sp<IBinder> display(SurfaceComposerClient::getBuiltInDisplay( + ISurfaceComposer::eDisplayIdMain)); + DisplayInfo info; + SurfaceComposerClient::getDisplayInfo(display, &info); + ssize_t displayWidth = info.w; + ssize_t displayHeight = info.h; + + ALOGV("display is %zd x %zd", displayWidth, displayHeight); + + control = composerClient->createSurface( + String8("A Surface"), displayWidth, displayHeight, + PIXEL_FORMAT_RGBA_8888, 0); + + CHECK(control != NULL); + CHECK(control->isValid()); + + SurfaceComposerClient::openGlobalTransaction(); + CHECK_EQ((status_t)OK, control->setLayer(INT_MAX)); + CHECK_EQ((status_t)OK, control->show()); + SurfaceComposerClient::closeGlobalTransaction(); + + surface = control->getSurface(); + CHECK(surface != NULL); + } + + decode(looper, argv[0], surface, renderSurface, useTimestamp, filterType); + + if (useSurface) { + composerClient->dispose(); + } + + looper->stop(); + + return 0; +} diff --git a/include/media/stagefright/MediaFilter.h b/include/media/stagefright/MediaFilter.h new file mode 100644 index 0000000..7b3f700 --- /dev/null +++ b/include/media/stagefright/MediaFilter.h @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MEDIA_FILTER_H_ +#define MEDIA_FILTER_H_ + +#include <media/stagefright/CodecBase.h> + +namespace android { + +struct ABuffer; +struct GraphicBufferListener; +struct MemoryDealer; +struct SimpleFilter; + +struct MediaFilter : public CodecBase { + MediaFilter(); + + virtual void setNotificationMessage(const sp<AMessage> &msg); + + virtual void initiateAllocateComponent(const sp<AMessage> &msg); + virtual void initiateConfigureComponent(const sp<AMessage> &msg); + virtual void initiateCreateInputSurface(); + virtual void initiateStart(); + virtual void initiateShutdown(bool keepComponentAllocated = false); + + virtual void signalFlush(); + virtual void signalResume(); + + virtual void signalRequestIDRFrame(); + virtual void signalSetParameters(const sp<AMessage> &msg); + virtual void signalEndOfInputStream(); + + virtual void onMessageReceived(const sp<AMessage> &msg); + + struct PortDescription : public CodecBase::PortDescription { + virtual size_t countBuffers(); + virtual IOMX::buffer_id bufferIDAt(size_t index) const; + virtual sp<ABuffer> bufferAt(size_t index) const; + + protected: + PortDescription(); + + private: + friend struct MediaFilter; + + Vector<IOMX::buffer_id> mBufferIDs; + Vector<sp<ABuffer> > mBuffers; + + void addBuffer(IOMX::buffer_id id, const sp<ABuffer> &buffer); + + DISALLOW_EVIL_CONSTRUCTORS(PortDescription); + }; + +protected: + virtual ~MediaFilter(); + +private: + struct BufferInfo { + enum Status { + OWNED_BY_US, + OWNED_BY_UPSTREAM, + }; + + IOMX::buffer_id mBufferID; + int32_t mGeneration; + int32_t mOutputFlags; + Status mStatus; + + sp<ABuffer> mData; + }; + + enum State { + UNINITIALIZED, + INITIALIZED, + CONFIGURED, + STARTED, + }; + + enum { + kWhatInputBufferFilled = 'inpF', + kWhatOutputBufferDrained = 'outD', + kWhatShutdown = 'shut', + kWhatFlush = 'flus', + kWhatResume = 'resm', + kWhatAllocateComponent = 'allo', + kWhatConfigureComponent = 'conf', + kWhatCreateInputSurface = 'cisf', + kWhatSignalEndOfInputStream = 'eois', + kWhatStart = 'star', + kWhatSetParameters = 'setP', + kWhatProcessBuffers = 'proc', + }; + + enum { + kPortIndexInput = 0, + kPortIndexOutput = 1 + }; + + // member variables + AString mComponentName; + State mState; + status_t mInputEOSResult; + int32_t mWidth, mHeight; + int32_t mStride, mSliceHeight; + int32_t mColorFormatIn, mColorFormatOut; + size_t mMaxInputSize, mMaxOutputSize; + int32_t mGeneration; + sp<AMessage> mNotify; + sp<AMessage> mInputFormat; + sp<AMessage> mOutputFormat; + + sp<MemoryDealer> mDealer[2]; + Vector<BufferInfo> mBuffers[2]; + Vector<BufferInfo*> mAvailableInputBuffers; + Vector<BufferInfo*> mAvailableOutputBuffers; + bool mPortEOS[2]; + + sp<SimpleFilter> mFilter; + sp<GraphicBufferListener> mGraphicBufferListener; + + // helper functions + void signalProcessBuffers(); + void signalError(status_t error); + + status_t allocateBuffersOnPort(OMX_U32 portIndex); + BufferInfo *findBufferByID( + uint32_t portIndex, IOMX::buffer_id bufferID, + ssize_t *index = NULL); + void postFillThisBuffer(BufferInfo *info); + void postDrainThisBuffer(BufferInfo *info); + void postEOS(); + void sendFormatChange(); + void requestFillEmptyInput(); + void processBuffers(); + + void onAllocateComponent(const sp<AMessage> &msg); + void onConfigureComponent(const sp<AMessage> &msg); + void onStart(); + void onInputBufferFilled(const sp<AMessage> &msg); + void onOutputBufferDrained(const sp<AMessage> &msg); + void onShutdown(const sp<AMessage> &msg); + void onFlush(); + void onSetParameters(const sp<AMessage> &msg); + void onCreateInputSurface(); + void onInputFrameAvailable(); + void onSignalEndOfInputStream(); + + DISALLOW_EVIL_CONSTRUCTORS(MediaFilter); +}; + +} // namespace android + +#endif // MEDIA_FILTER_H_ diff --git a/include/media/stagefright/RenderScriptWrapper.h b/include/media/stagefright/RenderScriptWrapper.h new file mode 100644 index 0000000..b42649e --- /dev/null +++ b/include/media/stagefright/RenderScriptWrapper.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RENDERSCRIPT_WRAPPER_H_ +#define RENDERSCRIPT_WRAPPER_H_ + +#include <RenderScript.h> + +namespace android { + +struct RenderScriptWrapper : public RefBase { +public: + struct RSFilterCallback : public RefBase { + public: + // called by RSFilter to process each input buffer + virtual status_t processBuffers( + RSC::Allocation* inBuffer, + RSC::Allocation* outBuffer) = 0; + + virtual status_t handleSetParameters(const sp<AMessage> &msg) = 0; + }; + + sp<RSFilterCallback> mCallback; + RSC::sp<RSC::RS> mContext; +}; + +} // namespace android + +#endif // RENDERSCRIPT_WRAPPER_H_ diff --git a/media/libeffects/factory/EffectsFactory.c b/media/libeffects/factory/EffectsFactory.c index 6d30d64..5bfe508 100644 --- a/media/libeffects/factory/EffectsFactory.c +++ b/media/libeffects/factory/EffectsFactory.c @@ -503,15 +503,31 @@ int loadLibrary(cnode *root, const char *name) audio_effect_library_t *desc; list_elem_t *e; lib_entry_t *l; + char path[PATH_MAX]; + char *str; + size_t len; node = config_find(root, PATH_TAG); if (node == NULL) { return -EINVAL; } + // audio_effects.conf always specifies 32 bit lib path: convert to 64 bit path if needed + strlcpy(path, node->value, PATH_MAX); +#ifdef __LP64__ + str = strstr(path, "/lib/"); + if (str == NULL) + return -EINVAL; + len = str - path; + path[len] = '\0'; + strlcat(path, "/lib64/", PATH_MAX); + strlcat(path, node->value + len + strlen("/lib/"), PATH_MAX); +#endif + if (strlen(path) >= PATH_MAX - 1) + return -EINVAL; - hdl = dlopen(node->value, RTLD_NOW); + hdl = dlopen(path, RTLD_NOW); if (hdl == NULL) { - ALOGW("loadLibrary() failed to open %s", node->value); + ALOGW("loadLibrary() failed to open %s", path); goto error; } @@ -535,7 +551,7 @@ int loadLibrary(cnode *root, const char *name) // add entry for library in gLibraryList l = malloc(sizeof(lib_entry_t)); l->name = strndup(name, PATH_MAX); - l->path = strndup(node->value, PATH_MAX); + l->path = strndup(path, PATH_MAX); l->handle = hdl; l->desc = desc; l->effects = NULL; @@ -547,7 +563,7 @@ int loadLibrary(cnode *root, const char *name) e->next = gLibraryList; gLibraryList = e; pthread_mutex_unlock(&gLibLock); - ALOGV("getLibrary() linked library %p for path %s", l, node->value); + ALOGV("getLibrary() linked library %p for path %s", l, path); return 0; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp index 7dd54c1..df19aef 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp @@ -263,6 +263,9 @@ status_t NuPlayerDriver::start() { case STATE_PAUSED: case STATE_STOPPED_AND_PREPARED: { + if (mAtEOS) { + mPlayer->seekToAsync(0); + } mPlayer->resume(); mPositionUs -= ALooper::GetNowUs() - mPauseStartedTimeUs; break; @@ -340,6 +343,14 @@ status_t NuPlayerDriver::seekTo(int msec) { switch (mState) { case STATE_PREPARED: { + int curpos = 0; + if (mPositionUs > 0) { + curpos = (mPositionUs + 500ll) / 1000; + } + if (curpos == msec) { + // nothing to do, and doing something anyway could result in deadlock (b/15323063) + break; + } mStartupSeekTimeUs = seekTimeUs; // pretend that the seek completed. It will actually happen when starting playback. // TODO: actually perform the seek here, so the player is ready to go at the new diff --git a/media/libnbaio/MonoPipeReader.cpp b/media/libnbaio/MonoPipeReader.cpp index de82229..e4d3ed8 100644 --- a/media/libnbaio/MonoPipeReader.cpp +++ b/media/libnbaio/MonoPipeReader.cpp @@ -39,7 +39,7 @@ ssize_t MonoPipeReader::availableToRead() return NEGOTIATE; } ssize_t ret = android_atomic_acquire_load(&mPipe->mRear) - mPipe->mFront; - ALOG_ASSERT((0 <= ret) && (ret <= mMaxFrames)); + ALOG_ASSERT((0 <= ret) && ((size_t) ret <= mPipe->mMaxFrames)); return ret; } diff --git a/media/libstagefright/Android.mk b/media/libstagefright/Android.mk index 193f8a7..7c4f92a 100644 --- a/media/libstagefright/Android.mk +++ b/media/libstagefright/Android.mk @@ -103,6 +103,7 @@ LOCAL_STATIC_LIBRARIES := \ libstagefright_color_conversion \ libstagefright_aacenc \ libstagefright_matroska \ + libstagefright_mediafilter \ libstagefright_webm \ libstagefright_timedtext \ libvpx \ @@ -110,7 +111,8 @@ LOCAL_STATIC_LIBRARIES := \ libstagefright_mpeg2ts \ libstagefright_id3 \ libFLAC \ - libmedia_helper + libmedia_helper \ + libRScpp_static LOCAL_SHARED_LIBRARIES += \ libstagefright_enc_common \ diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp index 0bfc6e4..9d6bb9a 100644 --- a/media/libstagefright/MediaCodec.cpp +++ b/media/libstagefright/MediaCodec.cpp @@ -36,6 +36,7 @@ #include <media/stagefright/MediaCodecList.h> #include <media/stagefright/MediaDefs.h> #include <media/stagefright/MediaErrors.h> +#include <media/stagefright/MediaFilter.h> #include <media/stagefright/MetaData.h> #include <media/stagefright/NativeWindowWrapper.h> #include <private/android_filesystem_config.h> @@ -189,7 +190,16 @@ status_t MediaCodec::init(const char *name, bool nameIsType, bool encoder) { // quickly, violating the OpenMAX specs, until that is remedied // we need to invest in an extra looper to free the main event // queue. - mCodec = new ACodec; + + if (nameIsType || !strncasecmp(name, "omx.", 4)) { + mCodec = new ACodec; + } else if (!nameIsType + && !strncasecmp(name, "android.filter.", 15)) { + mCodec = new MediaFilter; + } else { + return NAME_NOT_FOUND; + } + bool needDedicatedLooper = false; if (nameIsType && !strncasecmp(name, "video/", 6)) { needDedicatedLooper = true; diff --git a/media/libstagefright/colorconversion/SoftwareRenderer.cpp b/media/libstagefright/colorconversion/SoftwareRenderer.cpp index cc98da0..ac2b103 100644 --- a/media/libstagefright/colorconversion/SoftwareRenderer.cpp +++ b/media/libstagefright/colorconversion/SoftwareRenderer.cpp @@ -98,32 +98,48 @@ void SoftwareRenderer::resetFormatIfChanged(const sp<AMessage> &format) { mCropWidth = mCropRight - mCropLeft + 1; mCropHeight = mCropBottom - mCropTop + 1; - int halFormat; - size_t bufWidth, bufHeight; - - switch (mColorFormat) { - case OMX_COLOR_FormatYUV420Planar: - case OMX_TI_COLOR_FormatYUV420PackedSemiPlanar: - { - if (!runningInEmulator()) { + // by default convert everything to RGB565 + int halFormat = HAL_PIXEL_FORMAT_RGB_565; + size_t bufWidth = mCropWidth; + size_t bufHeight = mCropHeight; + + // hardware has YUV12 and RGBA8888 support, so convert known formats + if (!runningInEmulator()) { + switch (mColorFormat) { + case OMX_COLOR_FormatYUV420Planar: + case OMX_TI_COLOR_FormatYUV420PackedSemiPlanar: + { halFormat = HAL_PIXEL_FORMAT_YV12; bufWidth = (mCropWidth + 1) & ~1; bufHeight = (mCropHeight + 1) & ~1; break; } - - // fall through. + case OMX_COLOR_Format24bitRGB888: + { + halFormat = HAL_PIXEL_FORMAT_RGB_888; + bufWidth = (mCropWidth + 1) & ~1; + bufHeight = (mCropHeight + 1) & ~1; + break; + } + case OMX_COLOR_Format32bitARGB8888: + case OMX_COLOR_Format32BitRGBA8888: + { + halFormat = HAL_PIXEL_FORMAT_RGBA_8888; + bufWidth = (mCropWidth + 1) & ~1; + bufHeight = (mCropHeight + 1) & ~1; + break; + } + default: + { + break; + } } + } - default: - halFormat = HAL_PIXEL_FORMAT_RGB_565; - bufWidth = mCropWidth; - bufHeight = mCropHeight; - - mConverter = new ColorConverter( - mColorFormat, OMX_COLOR_Format16bitRGB565); - CHECK(mConverter->isValid()); - break; + if (halFormat == HAL_PIXEL_FORMAT_RGB_565) { + mConverter = new ColorConverter( + mColorFormat, OMX_COLOR_Format16bitRGB565); + CHECK(mConverter->isValid()); } CHECK(mNativeWindow != NULL); @@ -200,6 +216,8 @@ void SoftwareRenderer::render( CHECK_EQ(0, mapper.lock( buf->handle, GRALLOC_USAGE_SW_WRITE_OFTEN, bounds, &dst)); + // TODO move the other conversions also into ColorConverter, and + // fix cropping issues (when mCropLeft/Top != 0 or mWidth != mCropWidth) if (mConverter) { mConverter->convert( data, @@ -210,7 +228,8 @@ void SoftwareRenderer::render( 0, 0, mCropWidth - 1, mCropHeight - 1); } else if (mColorFormat == OMX_COLOR_FormatYUV420Planar) { const uint8_t *src_y = (const uint8_t *)data; - const uint8_t *src_u = (const uint8_t *)data + mWidth * mHeight; + const uint8_t *src_u = + (const uint8_t *)data + mWidth * mHeight; const uint8_t *src_v = src_u + (mWidth / 2 * mHeight / 2); uint8_t *dst_y = (uint8_t *)dst; @@ -236,14 +255,10 @@ void SoftwareRenderer::render( dst_u += dst_c_stride; dst_v += dst_c_stride; } - } else { - CHECK_EQ(mColorFormat, OMX_TI_COLOR_FormatYUV420PackedSemiPlanar); - - const uint8_t *src_y = - (const uint8_t *)data; - - const uint8_t *src_uv = - (const uint8_t *)data + mWidth * (mHeight - mCropTop / 2); + } else if (mColorFormat == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar) { + const uint8_t *src_y = (const uint8_t *)data; + const uint8_t *src_uv = (const uint8_t *)data + + mWidth * (mHeight - mCropTop / 2); uint8_t *dst_y = (uint8_t *)dst; @@ -271,6 +286,40 @@ void SoftwareRenderer::render( dst_u += dst_c_stride; dst_v += dst_c_stride; } + } else if (mColorFormat == OMX_COLOR_Format24bitRGB888) { + uint8_t* srcPtr = (uint8_t*)data; + uint8_t* dstPtr = (uint8_t*)dst; + + for (size_t y = 0; y < (size_t)mCropHeight; ++y) { + memcpy(dstPtr, srcPtr, mCropWidth * 3); + srcPtr += mWidth * 3; + dstPtr += buf->stride * 3; + } + } else if (mColorFormat == OMX_COLOR_Format32bitARGB8888) { + uint8_t *srcPtr, *dstPtr; + + for (size_t y = 0; y < (size_t)mCropHeight; ++y) { + srcPtr = (uint8_t*)data + mWidth * 4 * y; + dstPtr = (uint8_t*)dst + buf->stride * 4 * y; + for (size_t x = 0; x < (size_t)mCropWidth; ++x) { + uint8_t a = *srcPtr++; + for (size_t i = 0; i < 3; ++i) { // copy RGB + *dstPtr++ = *srcPtr++; + } + *dstPtr++ = a; // alpha last (ARGB to RGBA) + } + } + } else if (mColorFormat == OMX_COLOR_Format32BitRGBA8888) { + uint8_t* srcPtr = (uint8_t*)data; + uint8_t* dstPtr = (uint8_t*)dst; + + for (size_t y = 0; y < (size_t)mCropHeight; ++y) { + memcpy(dstPtr, srcPtr, mCropWidth * 4); + srcPtr += mWidth * 4; + dstPtr += buf->stride * 4; + } + } else { + LOG_ALWAYS_FATAL("bad color format %#x", mColorFormat); } CHECK_EQ(0, mapper.unlock(buf->handle)); diff --git a/media/libstagefright/filters/Android.mk b/media/libstagefright/filters/Android.mk new file mode 100644 index 0000000..5e895e1 --- /dev/null +++ b/media/libstagefright/filters/Android.mk @@ -0,0 +1,32 @@ +LOCAL_PATH:= $(call my-dir) +include $(CLEAR_VARS) + +LOCAL_NDK_STL_VARIANT := stlport_static + +LOCAL_SRC_FILES := \ + ColorConvert.cpp \ + GraphicBufferListener.cpp \ + IntrinsicBlurFilter.cpp \ + MediaFilter.cpp \ + RSFilter.cpp \ + SaturationFilter.cpp \ + saturationARGB.rs \ + SimpleFilter.cpp \ + ZeroFilter.cpp + +LOCAL_C_INCLUDES := \ + $(TOP)/bionic \ + $(TOP)/bionic/libstdc++/include \ + $(TOP)/external/stlport/stlport \ + $(TOP)/frameworks/native/include/media/openmax \ + $(TOP)/frameworks/rs/cpp \ + $(TOP)/frameworks/rs \ + +intermediates := $(call intermediates-dir-for,STATIC_LIBRARIES,libRS,TARGET,) +LOCAL_C_INCLUDES += $(intermediates) + +LOCAL_CFLAGS += -Wno-multichar + +LOCAL_MODULE:= libstagefright_mediafilter + +include $(BUILD_STATIC_LIBRARY) diff --git a/media/libstagefright/filters/ColorConvert.cpp b/media/libstagefright/filters/ColorConvert.cpp new file mode 100644 index 0000000..a5039f9 --- /dev/null +++ b/media/libstagefright/filters/ColorConvert.cpp @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ColorConvert.h" + +#ifndef max +#define max(a,b) ((a) > (b) ? (a) : (b)) +#endif +#ifndef min +#define min(a,b) ((a) < (b) ? (a) : (b)) +#endif + +namespace android { + +void YUVToRGB( + int32_t y, int32_t u, int32_t v, + int32_t* r, int32_t* g, int32_t* b) { + y -= 16; + u -= 128; + v -= 128; + + *b = 1192 * y + 2066 * u; + *g = 1192 * y - 833 * v - 400 * u; + *r = 1192 * y + 1634 * v; + + *r = min(262143, max(0, *r)); + *g = min(262143, max(0, *g)); + *b = min(262143, max(0, *b)); + + *r >>= 10; + *g >>= 10; + *b >>= 10; +} + +void convertYUV420spToARGB( + uint8_t *pY, uint8_t *pUV, int32_t width, int32_t height, + uint8_t *dest) { + const int32_t bytes_per_pixel = 2; + + for (int32_t i = 0; i < height; i++) { + for (int32_t j = 0; j < width; j++) { + int32_t y = *(pY + i * width + j); + int32_t u = *(pUV + (i/2) * width + bytes_per_pixel * (j/2)); + int32_t v = *(pUV + (i/2) * width + bytes_per_pixel * (j/2) + 1); + + int32_t r, g, b; + YUVToRGB(y, u, v, &r, &g, &b); + + *dest++ = 0xFF; + *dest++ = r; + *dest++ = g; + *dest++ = b; + } + } +} + +void convertYUV420spToRGB888( + uint8_t *pY, uint8_t *pUV, int32_t width, int32_t height, + uint8_t *dest) { + const int32_t bytes_per_pixel = 2; + + for (int32_t i = 0; i < height; i++) { + for (int32_t j = 0; j < width; j++) { + int32_t y = *(pY + i * width + j); + int32_t u = *(pUV + (i/2) * width + bytes_per_pixel * (j/2)); + int32_t v = *(pUV + (i/2) * width + bytes_per_pixel * (j/2) + 1); + + int32_t r, g, b; + YUVToRGB(y, u, v, &r, &g, &b); + + *dest++ = r; + *dest++ = g; + *dest++ = b; + } + } +} + +// HACK - not even slightly optimized +// TODO: remove when RGBA support is added to SoftwareRenderer +void convertRGBAToARGB( + uint8_t *src, int32_t width, int32_t height, uint32_t stride, + uint8_t *dest) { + for (size_t i = 0; i < height; ++i) { + for (size_t j = 0; j < width; ++j) { + uint8_t r = *src++; + uint8_t g = *src++; + uint8_t b = *src++; + uint8_t a = *src++; + *dest++ = a; + *dest++ = r; + *dest++ = g; + *dest++ = b; + } + src += (stride - width) * 4; + } +} + +} // namespace android diff --git a/media/libstagefright/filters/ColorConvert.h b/media/libstagefright/filters/ColorConvert.h new file mode 100644 index 0000000..13faa02 --- /dev/null +++ b/media/libstagefright/filters/ColorConvert.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COLOR_CONVERT_H_ +#define COLOR_CONVERT_H_ + +#include <inttypes.h> + +namespace android { + +void YUVToRGB( + int32_t y, int32_t u, int32_t v, + int32_t* r, int32_t* g, int32_t* b); + +void convertYUV420spToARGB( + uint8_t *pY, uint8_t *pUV, int32_t width, int32_t height, + uint8_t *dest); + +void convertYUV420spToRGB888( + uint8_t *pY, uint8_t *pUV, int32_t width, int32_t height, + uint8_t *dest); + +// TODO: remove when RGBA support is added to SoftwareRenderer +void convertRGBAToARGB( + uint8_t *src, int32_t width, int32_t height, uint32_t stride, + uint8_t *dest); + +} // namespace android + +#endif // COLOR_CONVERT_H_ diff --git a/media/libstagefright/filters/GraphicBufferListener.cpp b/media/libstagefright/filters/GraphicBufferListener.cpp new file mode 100644 index 0000000..e493137 --- /dev/null +++ b/media/libstagefright/filters/GraphicBufferListener.cpp @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "GraphicBufferListener" + +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/MediaErrors.h> + +#include "GraphicBufferListener.h" + +namespace android { + +status_t GraphicBufferListener::init( + const sp<AMessage> ¬ify, + size_t bufferWidth, size_t bufferHeight, size_t bufferCount) { + mNotify = notify; + + String8 name("GraphicBufferListener"); + BufferQueue::createBufferQueue(&mProducer, &mConsumer); + mConsumer->setConsumerName(name); + mConsumer->setDefaultBufferSize(bufferWidth, bufferHeight); + mConsumer->setConsumerUsageBits(GRALLOC_USAGE_SW_READ_OFTEN); + + status_t err = mConsumer->setMaxAcquiredBufferCount(bufferCount); + if (err != NO_ERROR) { + ALOGE("Unable to set BQ max acquired buffer count to %u: %d", + bufferCount, err); + return err; + } + + wp<BufferQueue::ConsumerListener> listener = + static_cast<BufferQueue::ConsumerListener*>(this); + sp<BufferQueue::ProxyConsumerListener> proxy = + new BufferQueue::ProxyConsumerListener(listener); + + err = mConsumer->consumerConnect(proxy, false); + if (err != NO_ERROR) { + ALOGE("Error connecting to BufferQueue: %s (%d)", + strerror(-err), err); + return err; + } + + ALOGV("init() successful."); + + return OK; +} + +void GraphicBufferListener::onFrameAvailable() { + ALOGV("onFrameAvailable() called"); + + { + Mutex::Autolock autoLock(mMutex); + mNumFramesAvailable++; + } + + sp<AMessage> notify = mNotify->dup(); + mNotify->setWhat(kWhatFrameAvailable); + mNotify->post(); +} + +void GraphicBufferListener::onBuffersReleased() { + ALOGV("onBuffersReleased() called"); + // nothing to do +} + +void GraphicBufferListener::onSidebandStreamChanged() { + ALOGW("GraphicBufferListener cannot consume sideband streams."); + // nothing to do +} + +BufferQueue::BufferItem GraphicBufferListener::getBufferItem() { + BufferQueue::BufferItem item; + + { + Mutex::Autolock autoLock(mMutex); + if (mNumFramesAvailable <= 0) { + ALOGE("getBuffer() called with no frames available"); + return item; + } + mNumFramesAvailable--; + } + + status_t err = mConsumer->acquireBuffer(&item, 0); + if (err == BufferQueue::NO_BUFFER_AVAILABLE) { + // shouldn't happen, since we track num frames available + ALOGE("frame was not available"); + item.mBuf = -1; + return item; + } else if (err != OK) { + ALOGE("acquireBuffer returned err=%d", err); + item.mBuf = -1; + return item; + } + + // Wait for it to become available. + err = item.mFence->waitForever("GraphicBufferListener::getBufferItem"); + if (err != OK) { + ALOGW("failed to wait for buffer fence: %d", err); + // keep going + } + + // If this is the first time we're seeing this buffer, add it to our + // slot table. + if (item.mGraphicBuffer != NULL) { + ALOGV("setting mBufferSlot %d", item.mBuf); + mBufferSlot[item.mBuf] = item.mGraphicBuffer; + } + + return item; +} + +sp<GraphicBuffer> GraphicBufferListener::getBuffer( + BufferQueue::BufferItem item) { + sp<GraphicBuffer> buf; + if (item.mBuf < 0 || item.mBuf >= BufferQueue::NUM_BUFFER_SLOTS) { + ALOGE("getBuffer() received invalid BufferItem: mBuf==%d", item.mBuf); + return buf; + } + + buf = mBufferSlot[item.mBuf]; + CHECK(buf.get() != NULL); + + return buf; +} + +status_t GraphicBufferListener::releaseBuffer( + BufferQueue::BufferItem item) { + if (item.mBuf < 0 || item.mBuf >= BufferQueue::NUM_BUFFER_SLOTS) { + ALOGE("getBuffer() received invalid BufferItem: mBuf==%d", item.mBuf); + return ERROR_OUT_OF_RANGE; + } + + mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber, + EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE); + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/GraphicBufferListener.h b/media/libstagefright/filters/GraphicBufferListener.h new file mode 100644 index 0000000..aefac0d --- /dev/null +++ b/media/libstagefright/filters/GraphicBufferListener.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GRAPHIC_BUFFER_LISTENER_H_ +#define GRAPHIC_BUFFER_LISTENER_H_ + +#include <gui/BufferQueue.h> + +namespace android { + +struct AMessage; + +struct GraphicBufferListener : public BufferQueue::ConsumerListener { +public: + GraphicBufferListener() {}; + + status_t init( + const sp<AMessage> ¬ify, + size_t bufferWidth, size_t bufferHeight, size_t bufferCount); + + virtual void onFrameAvailable(); + virtual void onBuffersReleased(); + virtual void onSidebandStreamChanged(); + + // Returns the handle to the producer side of the BufferQueue. Buffers + // queued on this will be received by GraphicBufferListener. + sp<IGraphicBufferProducer> getIGraphicBufferProducer() const { + return mProducer; + } + + BufferQueue::BufferItem getBufferItem(); + sp<GraphicBuffer> getBuffer(BufferQueue::BufferItem item); + status_t releaseBuffer(BufferQueue::BufferItem item); + + enum { + kWhatFrameAvailable = 'frav', + }; + +private: + sp<AMessage> mNotify; + size_t mNumFramesAvailable; + + mutable Mutex mMutex; + + // Our BufferQueue interfaces. mProducer is passed to the producer through + // getIGraphicBufferProducer, and mConsumer is used internally to retrieve + // the buffers queued by the producer. + sp<IGraphicBufferProducer> mProducer; + sp<IGraphicBufferConsumer> mConsumer; + + // Cache of GraphicBuffers from the buffer queue. + sp<GraphicBuffer> mBufferSlot[BufferQueue::NUM_BUFFER_SLOTS]; +}; + +} // namespace android + +#endif // GRAPHIC_BUFFER_LISTENER_H diff --git a/media/libstagefright/filters/IntrinsicBlurFilter.cpp b/media/libstagefright/filters/IntrinsicBlurFilter.cpp new file mode 100644 index 0000000..cbcf699 --- /dev/null +++ b/media/libstagefright/filters/IntrinsicBlurFilter.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "IntrinsicBlurFilter" + +#include <utils/Log.h> + +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include "IntrinsicBlurFilter.h" + +namespace android { + +status_t IntrinsicBlurFilter::configure(const sp<AMessage> &msg) { + status_t err = SimpleFilter::configure(msg); + if (err != OK) { + return err; + } + + if (!msg->findString("cacheDir", &mCacheDir)) { + ALOGE("Failed to find cache directory in config message."); + return NAME_NOT_FOUND; + } + + return OK; +} + +status_t IntrinsicBlurFilter::start() { + // TODO: use a single RS context object for entire application + mRS = new RSC::RS(); + + if (!mRS->init(mCacheDir.c_str())) { + ALOGE("Failed to initialize RenderScript context."); + return NO_INIT; + } + + // 32-bit elements for ARGB8888 + RSC::sp<const RSC::Element> e = RSC::Element::U8_4(mRS); + + RSC::Type::Builder tb(mRS, e); + tb.setX(mWidth); + tb.setY(mHeight); + RSC::sp<const RSC::Type> t = tb.create(); + + mAllocIn = RSC::Allocation::createTyped(mRS, t); + mAllocOut = RSC::Allocation::createTyped(mRS, t); + + mBlur = RSC::ScriptIntrinsicBlur::create(mRS, e); + mBlur->setRadius(mBlurRadius); + mBlur->setInput(mAllocIn); + + return OK; +} + +void IntrinsicBlurFilter::reset() { + mBlur.clear(); + mAllocOut.clear(); + mAllocIn.clear(); + mRS.clear(); +} + +status_t IntrinsicBlurFilter::setParameters(const sp<AMessage> &msg) { + sp<AMessage> params; + CHECK(msg->findMessage("params", ¶ms)); + + float blurRadius; + if (params->findFloat("blur-radius", &blurRadius)) { + mBlurRadius = blurRadius; + } + + return OK; +} + +status_t IntrinsicBlurFilter::processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer) { + mAllocIn->copy1DRangeFrom(0, mWidth * mHeight, srcBuffer->data()); + mBlur->forEach(mAllocOut); + mAllocOut->copy1DRangeTo(0, mWidth * mHeight, outBuffer->data()); + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/IntrinsicBlurFilter.h b/media/libstagefright/filters/IntrinsicBlurFilter.h new file mode 100644 index 0000000..4707ab7 --- /dev/null +++ b/media/libstagefright/filters/IntrinsicBlurFilter.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef INTRINSIC_BLUR_FILTER_H_ +#define INTRINSIC_BLUR_FILTER_H_ + +#include "RenderScript.h" +#include "SimpleFilter.h" + +namespace android { + +struct IntrinsicBlurFilter : public SimpleFilter { +public: + IntrinsicBlurFilter() : mBlurRadius(1.f) {}; + + virtual status_t configure(const sp<AMessage> &msg); + virtual status_t start(); + virtual void reset(); + virtual status_t setParameters(const sp<AMessage> &msg); + virtual status_t processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer); + +protected: + virtual ~IntrinsicBlurFilter() {}; + +private: + AString mCacheDir; + RSC::sp<RSC::RS> mRS; + RSC::sp<RSC::Allocation> mAllocIn; + RSC::sp<RSC::Allocation> mAllocOut; + RSC::sp<RSC::ScriptIntrinsicBlur> mBlur; + float mBlurRadius; +}; + +} // namespace android + +#endif // INTRINSIC_BLUR_FILTER_H_ diff --git a/media/libstagefright/filters/MediaFilter.cpp b/media/libstagefright/filters/MediaFilter.cpp new file mode 100644 index 0000000..c5289b6 --- /dev/null +++ b/media/libstagefright/filters/MediaFilter.cpp @@ -0,0 +1,816 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "MediaFilter" + +#include <inttypes.h> +#include <utils/Trace.h> + +#include <binder/MemoryDealer.h> + +#include <media/stagefright/BufferProducerWrapper.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include <media/stagefright/MediaDefs.h> +#include <media/stagefright/MediaErrors.h> +#include <media/stagefright/MediaFilter.h> + +#include "ColorConvert.h" +#include "GraphicBufferListener.h" +#include "IntrinsicBlurFilter.h" +#include "RSFilter.h" +#include "SaturationFilter.h" +#include "ZeroFilter.h" + +namespace android { + +// parameter: number of input and output buffers +static const size_t kBufferCountActual = 4; + +MediaFilter::MediaFilter() + : mState(UNINITIALIZED), + mGeneration(0), + mGraphicBufferListener(NULL) { +} + +MediaFilter::~MediaFilter() { +} + +//////////////////// PUBLIC FUNCTIONS ////////////////////////////////////////// + +void MediaFilter::setNotificationMessage(const sp<AMessage> &msg) { + mNotify = msg; +} + +void MediaFilter::initiateAllocateComponent(const sp<AMessage> &msg) { + msg->setWhat(kWhatAllocateComponent); + msg->setTarget(id()); + msg->post(); +} + +void MediaFilter::initiateConfigureComponent(const sp<AMessage> &msg) { + msg->setWhat(kWhatConfigureComponent); + msg->setTarget(id()); + msg->post(); +} + +void MediaFilter::initiateCreateInputSurface() { + (new AMessage(kWhatCreateInputSurface, id()))->post(); +} + +void MediaFilter::initiateStart() { + (new AMessage(kWhatStart, id()))->post(); +} + +void MediaFilter::initiateShutdown(bool keepComponentAllocated) { + sp<AMessage> msg = new AMessage(kWhatShutdown, id()); + msg->setInt32("keepComponentAllocated", keepComponentAllocated); + msg->post(); +} + +void MediaFilter::signalFlush() { + (new AMessage(kWhatFlush, id()))->post(); +} + +void MediaFilter::signalResume() { + (new AMessage(kWhatResume, id()))->post(); +} + +// nothing to do +void MediaFilter::signalRequestIDRFrame() { + return; +} + +void MediaFilter::signalSetParameters(const sp<AMessage> ¶ms) { + sp<AMessage> msg = new AMessage(kWhatSetParameters, id()); + msg->setMessage("params", params); + msg->post(); +} + +void MediaFilter::signalEndOfInputStream() { + (new AMessage(kWhatSignalEndOfInputStream, id()))->post(); +} + +void MediaFilter::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatAllocateComponent: + { + onAllocateComponent(msg); + break; + } + case kWhatConfigureComponent: + { + onConfigureComponent(msg); + break; + } + case kWhatStart: + { + onStart(); + break; + } + case kWhatProcessBuffers: + { + processBuffers(); + break; + } + case kWhatInputBufferFilled: + { + onInputBufferFilled(msg); + break; + } + case kWhatOutputBufferDrained: + { + onOutputBufferDrained(msg); + break; + } + case kWhatShutdown: + { + onShutdown(msg); + break; + } + case kWhatFlush: + { + onFlush(); + break; + } + case kWhatResume: + { + // nothing to do + break; + } + case kWhatSetParameters: + { + onSetParameters(msg); + break; + } + case kWhatCreateInputSurface: + { + onCreateInputSurface(); + break; + } + case GraphicBufferListener::kWhatFrameAvailable: + { + onInputFrameAvailable(); + break; + } + case kWhatSignalEndOfInputStream: + { + onSignalEndOfInputStream(); + break; + } + default: + { + ALOGE("Message not handled:\n%s", msg->debugString().c_str()); + break; + } + } +} + +//////////////////// PORT DESCRIPTION ////////////////////////////////////////// + +MediaFilter::PortDescription::PortDescription() { +} + +void MediaFilter::PortDescription::addBuffer( + IOMX::buffer_id id, const sp<ABuffer> &buffer) { + mBufferIDs.push_back(id); + mBuffers.push_back(buffer); +} + +size_t MediaFilter::PortDescription::countBuffers() { + return mBufferIDs.size(); +} + +IOMX::buffer_id MediaFilter::PortDescription::bufferIDAt(size_t index) const { + return mBufferIDs.itemAt(index); +} + +sp<ABuffer> MediaFilter::PortDescription::bufferAt(size_t index) const { + return mBuffers.itemAt(index); +} + +//////////////////// HELPER FUNCTIONS ////////////////////////////////////////// + +void MediaFilter::signalProcessBuffers() { + (new AMessage(kWhatProcessBuffers, id()))->post(); +} + +void MediaFilter::signalError(status_t error) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatError); + notify->setInt32("err", error); + notify->post(); +} + +status_t MediaFilter::allocateBuffersOnPort(OMX_U32 portIndex) { + CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); + const bool isInput = portIndex == kPortIndexInput; + const size_t bufferSize = isInput ? mMaxInputSize : mMaxOutputSize; + + CHECK(mDealer[portIndex] == NULL); + CHECK(mBuffers[portIndex].isEmpty()); + + ALOGV("Allocating %zu buffers of size %zu on %s port", + kBufferCountActual, bufferSize, + isInput ? "input" : "output"); + + size_t totalSize = kBufferCountActual * bufferSize; + + mDealer[portIndex] = new MemoryDealer(totalSize, "MediaFilter"); + + for (size_t i = 0; i < kBufferCountActual; ++i) { + sp<IMemory> mem = mDealer[portIndex]->allocate(bufferSize); + CHECK(mem.get() != NULL); + + BufferInfo info; + info.mStatus = BufferInfo::OWNED_BY_US; + info.mBufferID = i; + info.mGeneration = mGeneration; + info.mOutputFlags = 0; + info.mData = new ABuffer(mem->pointer(), bufferSize); + info.mData->meta()->setInt64("timeUs", 0); + + mBuffers[portIndex].push_back(info); + + if (!isInput) { + mAvailableOutputBuffers.push( + &mBuffers[portIndex].editItemAt(i)); + } + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatBuffersAllocated); + + notify->setInt32("portIndex", portIndex); + + sp<PortDescription> desc = new PortDescription; + + for (size_t i = 0; i < mBuffers[portIndex].size(); ++i) { + const BufferInfo &info = mBuffers[portIndex][i]; + + desc->addBuffer(info.mBufferID, info.mData); + } + + notify->setObject("portDesc", desc); + notify->post(); + + return OK; +} + +MediaFilter::BufferInfo* MediaFilter::findBufferByID( + uint32_t portIndex, IOMX::buffer_id bufferID, + ssize_t *index) { + for (size_t i = 0; i < mBuffers[portIndex].size(); ++i) { + BufferInfo *info = &mBuffers[portIndex].editItemAt(i); + + if (info->mBufferID == bufferID) { + if (index != NULL) { + *index = i; + } + return info; + } + } + + TRESPASS(); + + return NULL; +} + +void MediaFilter::postFillThisBuffer(BufferInfo *info) { + ALOGV("postFillThisBuffer on buffer %d", info->mBufferID); + if (mPortEOS[kPortIndexInput]) { + return; + } + + CHECK_EQ((int)info->mStatus, (int)BufferInfo::OWNED_BY_US); + + info->mGeneration = mGeneration; + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatFillThisBuffer); + notify->setInt32("buffer-id", info->mBufferID); + + info->mData->meta()->clear(); + notify->setBuffer("buffer", info->mData); + + sp<AMessage> reply = new AMessage(kWhatInputBufferFilled, id()); + reply->setInt32("buffer-id", info->mBufferID); + + notify->setMessage("reply", reply); + + info->mStatus = BufferInfo::OWNED_BY_UPSTREAM; + notify->post(); +} + +void MediaFilter::postDrainThisBuffer(BufferInfo *info) { + CHECK_EQ((int)info->mStatus, (int)BufferInfo::OWNED_BY_US); + + info->mGeneration = mGeneration; + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatDrainThisBuffer); + notify->setInt32("buffer-id", info->mBufferID); + notify->setInt32("flags", info->mOutputFlags); + notify->setBuffer("buffer", info->mData); + + sp<AMessage> reply = new AMessage(kWhatOutputBufferDrained, id()); + reply->setInt32("buffer-id", info->mBufferID); + + notify->setMessage("reply", reply); + + notify->post(); + + info->mStatus = BufferInfo::OWNED_BY_UPSTREAM; +} + +void MediaFilter::postEOS() { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatEOS); + notify->setInt32("err", ERROR_END_OF_STREAM); + notify->post(); + + ALOGV("Sent kWhatEOS."); +} + +void MediaFilter::sendFormatChange() { + sp<AMessage> notify = mNotify->dup(); + + notify->setInt32("what", kWhatOutputFormatChanged); + + AString mime; + CHECK(mOutputFormat->findString("mime", &mime)); + notify->setString("mime", mime.c_str()); + + notify->setInt32("stride", mStride); + notify->setInt32("slice-height", mSliceHeight); + notify->setInt32("color-format", mColorFormatOut); + notify->setRect("crop", 0, 0, mStride - 1, mSliceHeight - 1); + notify->setInt32("width", mWidth); + notify->setInt32("height", mHeight); + + notify->post(); +} + +void MediaFilter::requestFillEmptyInput() { + if (mPortEOS[kPortIndexInput]) { + return; + } + + for (size_t i = 0; i < mBuffers[kPortIndexInput].size(); ++i) { + BufferInfo *info = &mBuffers[kPortIndexInput].editItemAt(i); + + if (info->mStatus == BufferInfo::OWNED_BY_US) { + postFillThisBuffer(info); + } + } +} + +void MediaFilter::processBuffers() { + if (mAvailableInputBuffers.empty() || mAvailableOutputBuffers.empty()) { + ALOGV("Skipping process (buffers unavailable)"); + return; + } + + if (mPortEOS[kPortIndexOutput]) { + // TODO notify caller of queueInput error when it is supported + // in MediaCodec + ALOGW("Tried to process a buffer after EOS."); + return; + } + + BufferInfo *inputInfo = mAvailableInputBuffers[0]; + mAvailableInputBuffers.removeAt(0); + BufferInfo *outputInfo = mAvailableOutputBuffers[0]; + mAvailableOutputBuffers.removeAt(0); + + status_t err; + err = mFilter->processBuffers(inputInfo->mData, outputInfo->mData); + if (err != (status_t)OK) { + outputInfo->mData->meta()->setInt32("err", err); + } + + int64_t timeUs; + CHECK(inputInfo->mData->meta()->findInt64("timeUs", &timeUs)); + outputInfo->mData->meta()->setInt64("timeUs", timeUs); + outputInfo->mOutputFlags = 0; + int32_t eos = 0; + if (inputInfo->mData->meta()->findInt32("eos", &eos) && eos != 0) { + outputInfo->mOutputFlags |= OMX_BUFFERFLAG_EOS; + mPortEOS[kPortIndexOutput] = true; + outputInfo->mData->meta()->setInt32("eos", eos); + postEOS(); + ALOGV("Output stream saw EOS."); + } + + ALOGV("Processed input buffer %u [%zu], output buffer %u [%zu]", + inputInfo->mBufferID, inputInfo->mData->size(), + outputInfo->mBufferID, outputInfo->mData->size()); + + if (mGraphicBufferListener != NULL) { + delete inputInfo; + } else { + postFillThisBuffer(inputInfo); + } + postDrainThisBuffer(outputInfo); + + // prevent any corner case where buffers could get stuck in queue + signalProcessBuffers(); +} + +void MediaFilter::onAllocateComponent(const sp<AMessage> &msg) { + CHECK_EQ(mState, UNINITIALIZED); + + CHECK(msg->findString("componentName", &mComponentName)); + const char* name = mComponentName.c_str(); + if (!strcasecmp(name, "android.filter.zerofilter")) { + mFilter = new ZeroFilter; + } else if (!strcasecmp(name, "android.filter.saturation")) { + mFilter = new SaturationFilter; + } else if (!strcasecmp(name, "android.filter.intrinsicblur")) { + mFilter = new IntrinsicBlurFilter; + } else if (!strcasecmp(name, "android.filter.RenderScript")) { + mFilter = new RSFilter; + } else { + ALOGE("Unrecognized filter name: %s", name); + signalError(NAME_NOT_FOUND); + return; + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatComponentAllocated); + // HACK - need "OMX.google" to use MediaCodec's software renderer + notify->setString("componentName", "OMX.google.MediaFilter"); + notify->post(); + mState = INITIALIZED; + ALOGV("Handled kWhatAllocateComponent."); +} + +void MediaFilter::onConfigureComponent(const sp<AMessage> &msg) { + // TODO: generalize to allow audio filters as well as video + + CHECK_EQ(mState, INITIALIZED); + + // get params - at least mime, width & height + AString mime; + CHECK(msg->findString("mime", &mime)); + if (strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_RAW)) { + ALOGE("Bad mime: %s", mime.c_str()); + signalError(BAD_VALUE); + return; + } + + CHECK(msg->findInt32("width", &mWidth)); + CHECK(msg->findInt32("height", &mHeight)); + if (!msg->findInt32("stride", &mStride)) { + mStride = mWidth; + } + if (!msg->findInt32("slice-height", &mSliceHeight)) { + mSliceHeight = mHeight; + } + + mMaxInputSize = mWidth * mHeight * 4; // room for ARGB8888 + int32_t maxInputSize; + if (msg->findInt32("max-input-size", &maxInputSize) + && (size_t)maxInputSize > mMaxInputSize) { + mMaxInputSize = maxInputSize; + } + + if (!msg->findInt32("color-format", &mColorFormatIn)) { + // default to OMX_COLOR_Format32bitARGB8888 + mColorFormatIn = OMX_COLOR_Format32bitARGB8888; + msg->setInt32("color-format", mColorFormatIn); + } + mColorFormatOut = mColorFormatIn; + + mMaxOutputSize = mWidth * mHeight * 4; // room for ARGB8888 + + AString cacheDir; + if (!msg->findString("cacheDir", &cacheDir)) { + ALOGE("Failed to find cache directory in config message."); + signalError(NAME_NOT_FOUND); + return; + } + + status_t err; + err = mFilter->configure(msg); + if (err != (status_t)OK) { + ALOGE("Failed to configure filter component, err %d", err); + signalError(err); + return; + } + + mInputFormat = new AMessage(); + mInputFormat->setString("mime", mime.c_str()); + mInputFormat->setInt32("stride", mStride); + mInputFormat->setInt32("slice-height", mSliceHeight); + mInputFormat->setInt32("color-format", mColorFormatIn); + mInputFormat->setRect("crop", 0, 0, mStride, mSliceHeight); + mInputFormat->setInt32("width", mWidth); + mInputFormat->setInt32("height", mHeight); + + mOutputFormat = new AMessage(); + mOutputFormat->setString("mime", mime.c_str()); + mOutputFormat->setInt32("stride", mStride); + mOutputFormat->setInt32("slice-height", mSliceHeight); + mOutputFormat->setInt32("color-format", mColorFormatOut); + mOutputFormat->setRect("crop", 0, 0, mStride, mSliceHeight); + mOutputFormat->setInt32("width", mWidth); + mOutputFormat->setInt32("height", mHeight); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatComponentConfigured); + notify->setString("componentName", "MediaFilter"); + notify->setMessage("input-format", mInputFormat); + notify->setMessage("output-format", mOutputFormat); + notify->post(); + mState = CONFIGURED; + ALOGV("Handled kWhatConfigureComponent."); + + sendFormatChange(); +} + +void MediaFilter::onStart() { + CHECK_EQ(mState, CONFIGURED); + + allocateBuffersOnPort(kPortIndexInput); + + allocateBuffersOnPort(kPortIndexOutput); + + status_t err = mFilter->start(); + if (err != (status_t)OK) { + ALOGE("Failed to start filter component, err %d", err); + signalError(err); + return; + } + + mPortEOS[kPortIndexInput] = false; + mPortEOS[kPortIndexOutput] = false; + mInputEOSResult = OK; + mState = STARTED; + + requestFillEmptyInput(); + ALOGV("Handled kWhatStart."); +} + +void MediaFilter::onInputBufferFilled(const sp<AMessage> &msg) { + IOMX::buffer_id bufferID; + CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID)); + BufferInfo *info = findBufferByID(kPortIndexInput, bufferID); + + if (mState != STARTED) { + // we're not running, so we'll just keep that buffer... + info->mStatus = BufferInfo::OWNED_BY_US; + return; + } + + if (info->mGeneration != mGeneration) { + ALOGV("Caught a stale input buffer [ID %d]", bufferID); + // buffer is stale (taken before a flush/shutdown) - repost it + CHECK_EQ(info->mStatus, BufferInfo::OWNED_BY_US); + postFillThisBuffer(info); + return; + } + + CHECK_EQ(info->mStatus, BufferInfo::OWNED_BY_UPSTREAM); + info->mStatus = BufferInfo::OWNED_BY_US; + + sp<ABuffer> buffer; + int32_t err = OK; + bool eos = false; + + if (!msg->findBuffer("buffer", &buffer)) { + // these are unfilled buffers returned by client + CHECK(msg->findInt32("err", &err)); + + if (err == OK) { + // buffers with no errors are returned on MediaCodec.flush + ALOGV("saw unfilled buffer (MediaCodec.flush)"); + postFillThisBuffer(info); + return; + } else { + ALOGV("saw error %d instead of an input buffer", err); + eos = true; + } + + buffer.clear(); + } + + int32_t isCSD; + if (buffer != NULL && buffer->meta()->findInt32("csd", &isCSD) + && isCSD != 0) { + // ignore codec-specific data buffers + ALOGW("MediaFilter received a codec-specific data buffer"); + postFillThisBuffer(info); + return; + } + + int32_t tmp; + if (buffer != NULL && buffer->meta()->findInt32("eos", &tmp) && tmp) { + eos = true; + err = ERROR_END_OF_STREAM; + } + + mAvailableInputBuffers.push_back(info); + processBuffers(); + + if (eos) { + mPortEOS[kPortIndexInput] = true; + mInputEOSResult = err; + } + + ALOGV("Handled kWhatInputBufferFilled. [ID %u]", bufferID); +} + +void MediaFilter::onOutputBufferDrained(const sp<AMessage> &msg) { + IOMX::buffer_id bufferID; + CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID)); + BufferInfo *info = findBufferByID(kPortIndexOutput, bufferID); + + if (mState != STARTED) { + // we're not running, so we'll just keep that buffer... + info->mStatus = BufferInfo::OWNED_BY_US; + return; + } + + if (info->mGeneration != mGeneration) { + ALOGV("Caught a stale output buffer [ID %d]", bufferID); + // buffer is stale (taken before a flush/shutdown) - keep it + CHECK_EQ(info->mStatus, BufferInfo::OWNED_BY_US); + return; + } + + CHECK_EQ(info->mStatus, BufferInfo::OWNED_BY_UPSTREAM); + info->mStatus = BufferInfo::OWNED_BY_US; + + mAvailableOutputBuffers.push_back(info); + + processBuffers(); + + ALOGV("Handled kWhatOutputBufferDrained. [ID %u]", + bufferID); +} + +void MediaFilter::onShutdown(const sp<AMessage> &msg) { + mGeneration++; + + if (mState != UNINITIALIZED) { + mFilter->reset(); + } + + int32_t keepComponentAllocated; + CHECK(msg->findInt32("keepComponentAllocated", &keepComponentAllocated)); + if (!keepComponentAllocated || mState == UNINITIALIZED) { + mState = UNINITIALIZED; + } else { + mState = INITIALIZED; + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatShutdownCompleted); + notify->post(); +} + +void MediaFilter::onFlush() { + mGeneration++; + + mAvailableInputBuffers.clear(); + for (size_t i = 0; i < mBuffers[kPortIndexInput].size(); ++i) { + BufferInfo *info = &mBuffers[kPortIndexInput].editItemAt(i); + info->mStatus = BufferInfo::OWNED_BY_US; + } + mAvailableOutputBuffers.clear(); + for (size_t i = 0; i < mBuffers[kPortIndexOutput].size(); ++i) { + BufferInfo *info = &mBuffers[kPortIndexOutput].editItemAt(i); + info->mStatus = BufferInfo::OWNED_BY_US; + mAvailableOutputBuffers.push_back(info); + } + + mPortEOS[kPortIndexInput] = false; + mPortEOS[kPortIndexOutput] = false; + mInputEOSResult = OK; + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatFlushCompleted); + notify->post(); + ALOGV("Posted kWhatFlushCompleted"); + + // MediaCodec returns all input buffers after flush, so in + // onInputBufferFilled we call postFillThisBuffer on them +} + +void MediaFilter::onSetParameters(const sp<AMessage> &msg) { + CHECK(mState != STARTED); + + status_t err = mFilter->setParameters(msg); + if (err != (status_t)OK) { + ALOGE("setParameters returned err %d", err); + } +} + +void MediaFilter::onCreateInputSurface() { + CHECK(mState == CONFIGURED); + + mGraphicBufferListener = new GraphicBufferListener; + + sp<AMessage> notify = new AMessage(); + notify->setTarget(id()); + status_t err = mGraphicBufferListener->init( + notify, mStride, mSliceHeight, kBufferCountActual); + + if (err != OK) { + ALOGE("Failed to init mGraphicBufferListener: %d", err); + signalError(err); + return; + } + + sp<AMessage> reply = mNotify->dup(); + reply->setInt32("what", CodecBase::kWhatInputSurfaceCreated); + reply->setObject( + "input-surface", + new BufferProducerWrapper( + mGraphicBufferListener->getIGraphicBufferProducer())); + reply->post(); +} + +void MediaFilter::onInputFrameAvailable() { + BufferQueue::BufferItem item = mGraphicBufferListener->getBufferItem(); + sp<GraphicBuffer> buf = mGraphicBufferListener->getBuffer(item); + + // get pointer to graphic buffer + void* bufPtr; + buf->lock(GraphicBuffer::USAGE_SW_READ_OFTEN, &bufPtr); + + // HACK - there is no OMX_COLOR_FORMATTYPE value for RGBA, so the format + // conversion is hardcoded until we add this. + // TODO: check input format and convert only if necessary + // copy RGBA graphic buffer into temporary ARGB input buffer + BufferInfo *inputInfo = new BufferInfo; + inputInfo->mData = new ABuffer(buf->getWidth() * buf->getHeight() * 4); + ALOGV("Copying surface data into temp buffer."); + convertRGBAToARGB( + (uint8_t*)bufPtr, buf->getWidth(), buf->getHeight(), + buf->getStride(), inputInfo->mData->data()); + inputInfo->mBufferID = item.mBuf; + inputInfo->mGeneration = mGeneration; + inputInfo->mOutputFlags = 0; + inputInfo->mStatus = BufferInfo::OWNED_BY_US; + inputInfo->mData->meta()->setInt64("timeUs", item.mTimestamp / 1000); + + mAvailableInputBuffers.push_back(inputInfo); + + mGraphicBufferListener->releaseBuffer(item); + + signalProcessBuffers(); +} + +void MediaFilter::onSignalEndOfInputStream() { + // if using input surface, need to send an EOS output buffer + if (mGraphicBufferListener != NULL) { + Vector<BufferInfo> *outputBufs = &mBuffers[kPortIndexOutput]; + BufferInfo* eosBuf; + bool foundBuf = false; + for (size_t i = 0; i < kBufferCountActual; i++) { + eosBuf = &outputBufs->editItemAt(i); + if (eosBuf->mStatus == BufferInfo::OWNED_BY_US) { + foundBuf = true; + break; + } + } + + if (!foundBuf) { + ALOGE("onSignalEndOfInputStream failed to find an output buffer"); + return; + } + + eosBuf->mOutputFlags = OMX_BUFFERFLAG_EOS; + eosBuf->mGeneration = mGeneration; + eosBuf->mData->setRange(0, 0); + postDrainThisBuffer(eosBuf); + ALOGV("Posted EOS on output buffer %zu", eosBuf->mBufferID); + } + + mPortEOS[kPortIndexOutput] = true; + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatSignaledInputEOS); + notify->post(); + + ALOGV("Output stream saw EOS."); +} + +} // namespace android diff --git a/media/libstagefright/filters/RSFilter.cpp b/media/libstagefright/filters/RSFilter.cpp new file mode 100644 index 0000000..b569945 --- /dev/null +++ b/media/libstagefright/filters/RSFilter.cpp @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "RSFilter" + +#include <utils/Log.h> + +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include "RSFilter.h" + +namespace android { + +RSFilter::RSFilter() { + +} + +RSFilter::~RSFilter() { + +} + +status_t RSFilter::configure(const sp<AMessage> &msg) { + status_t err = SimpleFilter::configure(msg); + if (err != OK) { + return err; + } + + if (!msg->findString("cacheDir", &mCacheDir)) { + ALOGE("Failed to find cache directory in config message."); + return NAME_NOT_FOUND; + } + + sp<RenderScriptWrapper> wrapper; + if (!msg->findObject("rs-wrapper", (sp<RefBase>*)&wrapper)) { + ALOGE("Failed to find RenderScriptWrapper in config message."); + return NAME_NOT_FOUND; + } + + mRS = wrapper->mContext; + mCallback = wrapper->mCallback; + + return OK; +} + +status_t RSFilter::start() { + // 32-bit elements for ARGB8888 + RSC::sp<const RSC::Element> e = RSC::Element::U8_4(mRS); + + RSC::Type::Builder tb(mRS, e); + tb.setX(mWidth); + tb.setY(mHeight); + RSC::sp<const RSC::Type> t = tb.create(); + + mAllocIn = RSC::Allocation::createTyped(mRS, t); + mAllocOut = RSC::Allocation::createTyped(mRS, t); + + return OK; +} + +void RSFilter::reset() { + mCallback.clear(); + mAllocOut.clear(); + mAllocIn.clear(); + mRS.clear(); +} + +status_t RSFilter::setParameters(const sp<AMessage> &msg) { + return mCallback->handleSetParameters(msg); +} + +status_t RSFilter::processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer) { + mAllocIn->copy1DRangeFrom(0, mWidth * mHeight, srcBuffer->data()); + mCallback->processBuffers(mAllocIn.get(), mAllocOut.get()); + mAllocOut->copy1DRangeTo(0, mWidth * mHeight, outBuffer->data()); + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/RSFilter.h b/media/libstagefright/filters/RSFilter.h new file mode 100644 index 0000000..c5b5074 --- /dev/null +++ b/media/libstagefright/filters/RSFilter.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RS_FILTER_H_ +#define RS_FILTER_H_ + +#include <media/stagefright/RenderScriptWrapper.h> +#include <RenderScript.h> + +#include "SimpleFilter.h" + +namespace android { + +struct AString; + +struct RSFilter : public SimpleFilter { +public: + RSFilter(); + + virtual status_t configure(const sp<AMessage> &msg); + virtual status_t start(); + virtual void reset(); + virtual status_t setParameters(const sp<AMessage> &msg); + virtual status_t processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer); + +protected: + virtual ~RSFilter(); + +private: + AString mCacheDir; + sp<RenderScriptWrapper::RSFilterCallback> mCallback; + RSC::sp<RSC::RS> mRS; + RSC::sp<RSC::Allocation> mAllocIn; + RSC::sp<RSC::Allocation> mAllocOut; +}; + +} // namespace android + +#endif // RS_FILTER_H_ diff --git a/media/libstagefright/filters/SaturationFilter.cpp b/media/libstagefright/filters/SaturationFilter.cpp new file mode 100644 index 0000000..ba5f75a --- /dev/null +++ b/media/libstagefright/filters/SaturationFilter.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "SaturationFilter" + +#include <utils/Log.h> + +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include "SaturationFilter.h" + +namespace android { + +status_t SaturationFilter::configure(const sp<AMessage> &msg) { + status_t err = SimpleFilter::configure(msg); + if (err != OK) { + return err; + } + + if (!msg->findString("cacheDir", &mCacheDir)) { + ALOGE("Failed to find cache directory in config message."); + return NAME_NOT_FOUND; + } + + return OK; +} + +status_t SaturationFilter::start() { + // TODO: use a single RS context object for entire application + mRS = new RSC::RS(); + + if (!mRS->init(mCacheDir.c_str())) { + ALOGE("Failed to initialize RenderScript context."); + return NO_INIT; + } + + // 32-bit elements for ARGB8888 + RSC::sp<const RSC::Element> e = RSC::Element::U8_4(mRS); + + RSC::Type::Builder tb(mRS, e); + tb.setX(mWidth); + tb.setY(mHeight); + RSC::sp<const RSC::Type> t = tb.create(); + + mAllocIn = RSC::Allocation::createTyped(mRS, t); + mAllocOut = RSC::Allocation::createTyped(mRS, t); + + mScript = new ScriptC_saturationARGB(mRS); + + mScript->set_gSaturation(mSaturation); + + return OK; +} + +void SaturationFilter::reset() { + mScript.clear(); + mAllocOut.clear(); + mAllocIn.clear(); + mRS.clear(); +} + +status_t SaturationFilter::setParameters(const sp<AMessage> &msg) { + sp<AMessage> params; + CHECK(msg->findMessage("params", ¶ms)); + + float saturation; + if (params->findFloat("saturation", &saturation)) { + mSaturation = saturation; + } + + return OK; +} + +status_t SaturationFilter::processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer) { + mAllocIn->copy1DRangeFrom(0, mWidth * mHeight, srcBuffer->data()); + mScript->forEach_root(mAllocIn, mAllocOut); + mAllocOut->copy1DRangeTo(0, mWidth * mHeight, outBuffer->data()); + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/SaturationFilter.h b/media/libstagefright/filters/SaturationFilter.h new file mode 100644 index 0000000..0545021 --- /dev/null +++ b/media/libstagefright/filters/SaturationFilter.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SATURATION_FILTER_H_ +#define SATURATION_FILTER_H_ + +#include <RenderScript.h> + +#include "ScriptC_saturationARGB.h" +#include "SimpleFilter.h" + +namespace android { + +struct SaturationFilter : public SimpleFilter { +public: + SaturationFilter() : mSaturation(1.f) {}; + + virtual status_t configure(const sp<AMessage> &msg); + virtual status_t start(); + virtual void reset(); + virtual status_t setParameters(const sp<AMessage> &msg); + virtual status_t processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer); + +protected: + virtual ~SaturationFilter() {}; + +private: + AString mCacheDir; + RSC::sp<RSC::RS> mRS; + RSC::sp<RSC::Allocation> mAllocIn; + RSC::sp<RSC::Allocation> mAllocOut; + RSC::sp<ScriptC_saturationARGB> mScript; + float mSaturation; +}; + +} // namespace android + +#endif // SATURATION_FILTER_H_ diff --git a/media/libstagefright/filters/SimpleFilter.cpp b/media/libstagefright/filters/SimpleFilter.cpp new file mode 100644 index 0000000..6c1ca2c --- /dev/null +++ b/media/libstagefright/filters/SimpleFilter.cpp @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include "SimpleFilter.h" + +namespace android { + +status_t SimpleFilter::configure(const sp<AMessage> &msg) { + CHECK(msg->findInt32("width", &mWidth)); + CHECK(msg->findInt32("height", &mHeight)); + if (!msg->findInt32("stride", &mStride)) { + mStride = mWidth; + } + if (!msg->findInt32("slice-height", &mSliceHeight)) { + mSliceHeight = mHeight; + } + CHECK(msg->findInt32("color-format", &mColorFormatIn)); + mColorFormatOut = mColorFormatIn; + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/SimpleFilter.h b/media/libstagefright/filters/SimpleFilter.h new file mode 100644 index 0000000..4cd37ef --- /dev/null +++ b/media/libstagefright/filters/SimpleFilter.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SIMPLE_FILTER_H_ +#define SIMPLE_FILTER_H_ + +#include <stdint.h> +#include <utils/Errors.h> +#include <utils/RefBase.h> + +struct ABuffer; +struct AMessage; + +namespace android { + +struct SimpleFilter : public RefBase { +public: + SimpleFilter() : mWidth(0), mHeight(0), mStride(0), mSliceHeight(0), + mColorFormatIn(0), mColorFormatOut(0) {}; + + virtual status_t configure(const sp<AMessage> &msg); + + virtual status_t start() = 0; + virtual void reset() = 0; + virtual status_t setParameters(const sp<AMessage> &msg) = 0; + virtual status_t processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer) = 0; + +protected: + int32_t mWidth, mHeight; + int32_t mStride, mSliceHeight; + int32_t mColorFormatIn, mColorFormatOut; + + virtual ~SimpleFilter() {}; +}; + +} // namespace android + +#endif // SIMPLE_FILTER_H_ diff --git a/media/libstagefright/filters/ZeroFilter.cpp b/media/libstagefright/filters/ZeroFilter.cpp new file mode 100644 index 0000000..3f1243c --- /dev/null +++ b/media/libstagefright/filters/ZeroFilter.cpp @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "ZeroFilter" + +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include "ZeroFilter.h" + +namespace android { + +status_t ZeroFilter::setParameters(const sp<AMessage> &msg) { + sp<AMessage> params; + CHECK(msg->findMessage("params", ¶ms)); + + int32_t invert; + if (params->findInt32("invert", &invert)) { + mInvertData = (invert != 0); + } + + return OK; +} + +status_t ZeroFilter::processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer) { + // assuming identical input & output buffers, since we're a copy filter + if (mInvertData) { + uint32_t* src = (uint32_t*)srcBuffer->data(); + uint32_t* dest = (uint32_t*)outBuffer->data(); + for (size_t i = 0; i < srcBuffer->size() / 4; ++i) { + *(dest++) = *(src++) ^ 0xFFFFFFFF; + } + } else { + memcpy(outBuffer->data(), srcBuffer->data(), srcBuffer->size()); + } + outBuffer->setRange(0, srcBuffer->size()); + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/ZeroFilter.h b/media/libstagefright/filters/ZeroFilter.h new file mode 100644 index 0000000..bd34dfb --- /dev/null +++ b/media/libstagefright/filters/ZeroFilter.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZERO_FILTER_H_ +#define ZERO_FILTER_H_ + +#include "SimpleFilter.h" + +namespace android { + +struct ZeroFilter : public SimpleFilter { +public: + ZeroFilter() : mInvertData(false) {}; + + virtual status_t start() { return OK; }; + virtual void reset() {}; + virtual status_t setParameters(const sp<AMessage> &msg); + virtual status_t processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer); + +protected: + virtual ~ZeroFilter() {}; + +private: + bool mInvertData; +}; + +} // namespace android + +#endif // ZERO_FILTER_H_ diff --git a/media/libstagefright/filters/saturation.rs b/media/libstagefright/filters/saturation.rs new file mode 100644 index 0000000..2c867ac --- /dev/null +++ b/media/libstagefright/filters/saturation.rs @@ -0,0 +1,40 @@ +// Sample script for RGB888 support (compare to saturationARGB.rs) +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma version(1) +#pragma rs java_package_name(com.android.rs.cppbasic) +#pragma rs_fp_relaxed + +const static float3 gMonoMult = {0.299f, 0.587f, 0.114f}; + +// global variables (parameters accessible to application code) +float gSaturation = 1.0f; + +void root(const uchar3 *v_in, uchar3 *v_out) { + // scale 0-255 uchar to 0-1.0 float + float3 in = {v_in->r * 0.003921569f, v_in->g * 0.003921569f, + v_in->b * 0.003921569f}; + + // apply saturation filter + float3 result = dot(in, gMonoMult); + result = mix(result, in, gSaturation); + + // convert to uchar, copied from rsPackColorTo8888 + v_out->x = (uchar)clamp((result.r * 255.f + 0.5f), 0.f, 255.f); + v_out->y = (uchar)clamp((result.g * 255.f + 0.5f), 0.f, 255.f); + v_out->z = (uchar)clamp((result.b * 255.f + 0.5f), 0.f, 255.f); +} diff --git a/media/libstagefright/filters/saturationARGB.rs b/media/libstagefright/filters/saturationARGB.rs new file mode 100644 index 0000000..1de9dd8 --- /dev/null +++ b/media/libstagefright/filters/saturationARGB.rs @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma version(1) +#pragma rs java_package_name(com.android.rs.cppbasic) +#pragma rs_fp_relaxed + +const static float3 gMonoMult = {0.299f, 0.587f, 0.114f}; + +// global variables (parameters accessible to application code) +float gSaturation = 1.0f; + +void root(const uchar4 *v_in, uchar4 *v_out) { + v_out->x = v_in->x; // don't modify A + + // get RGB, scale 0-255 uchar to 0-1.0 float + float3 rgb = {v_in->y * 0.003921569f, v_in->z * 0.003921569f, + v_in->w * 0.003921569f}; + + // apply saturation filter + float3 result = dot(rgb, gMonoMult); + result = mix(result, rgb, gSaturation); + + v_out->y = (uchar)clamp((result.r * 255.f + 0.5f), 0.f, 255.f); + v_out->z = (uchar)clamp((result.g * 255.f + 0.5f), 0.f, 255.f); + v_out->w = (uchar)clamp((result.b * 255.f + 0.5f), 0.f, 255.f); +} diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp index 97b1753..618a719 100644 --- a/services/audioflinger/Threads.cpp +++ b/services/audioflinger/Threads.cpp @@ -2136,6 +2136,7 @@ ssize_t AudioFlinger::PlaybackThread::threadLoop_write() } else { bytesWritten = framesWritten; } + mLatchDValid = false; status_t status = mNormalSink->getTimestamp(mLatchD.mTimestamp); if (status == NO_ERROR) { size_t totalFramesWritten = mNormalSink->framesWritten(); diff --git a/services/audioflinger/Tracks.cpp b/services/audioflinger/Tracks.cpp index 6cbab04..c718b0c 100644 --- a/services/audioflinger/Tracks.cpp +++ b/services/audioflinger/Tracks.cpp @@ -853,6 +853,7 @@ void AudioFlinger::PlaybackThread::Track::reset() if (mState == FLUSHED) { mState = IDLE; } + mPreviousValid = false; } } |