stagefright: filter surface input, config cacheDir

Bug: 17203044
Change-Id: Ifb927429568fe68807143e8511065fea1a25b3a5

10 files changed, 384 insertions, 13 deletions

diff --git a/media/libstagefright/filters/Android.mk b/media/libstagefright/filters/Android.mk
index ce18827..08d349d 100644
--- a/media/libstagefright/filters/Android.mk
+++ b/media/libstagefright/filters/Android.mk
@@ -5,6 +5,7 @@ LOCAL_NDK_STL_VARIANT := stlport_static
 
 LOCAL_SRC_FILES := \
         ColorConvert.cpp          \
+        GraphicBufferListener.cpp \
         IntrinsicBlurFilter.cpp   \
         MediaFilter.cpp           \
         SaturationFilter.cpp      \
diff --git a/media/libstagefright/filters/ColorConvert.cpp b/media/libstagefright/filters/ColorConvert.cpp
index b2afdcc..a5039f9 100644
--- a/media/libstagefright/filters/ColorConvert.cpp
+++ b/media/libstagefright/filters/ColorConvert.cpp
@@ -88,4 +88,24 @@ void convertYUV420spToRGB888(
     }
 }
 
+// HACK - not even slightly optimized
+// TODO: remove when RGBA support is added to SoftwareRenderer
+void convertRGBAToARGB(
+        uint8_t *src, int32_t width, int32_t height, uint32_t stride,
+        uint8_t *dest) {
+    for (size_t i = 0; i < height; ++i) {
+        for (size_t j = 0; j < width; ++j) {
+            uint8_t r = *src++;
+            uint8_t g = *src++;
+            uint8_t b = *src++;
+            uint8_t a = *src++;
+            *dest++ = a;
+            *dest++ = r;
+            *dest++ = g;
+            *dest++ = b;
+        }
+        src += (stride - width) * 4;
+    }
+}
+
 }   // namespace android
diff --git a/media/libstagefright/filters/ColorConvert.h b/media/libstagefright/filters/ColorConvert.h
index 16519f1..13faa02 100644
--- a/media/libstagefright/filters/ColorConvert.h
+++ b/media/libstagefright/filters/ColorConvert.h
@@ -33,6 +33,11 @@ void convertYUV420spToRGB888(
         uint8_t *pY, uint8_t *pUV, int32_t width, int32_t height,
         uint8_t *dest);
 
+// TODO: remove when RGBA support is added to SoftwareRenderer
+void convertRGBAToARGB(
+        uint8_t *src, int32_t width, int32_t height, uint32_t stride,
+        uint8_t *dest);
+
 }   // namespace android
 
 #endif  // COLOR_CONVERT_H_
diff --git a/media/libstagefright/filters/GraphicBufferListener.cpp b/media/libstagefright/filters/GraphicBufferListener.cpp
new file mode 100644
index 0000000..e493137
--- /dev/null
+++ b/media/libstagefright/filters/GraphicBufferListener.cpp
@@ -0,0 +1,154 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "GraphicBufferListener"
+
+#include <media/stagefright/foundation/ADebug.h>
+#include <media/stagefright/foundation/AMessage.h>
+#include <media/stagefright/MediaErrors.h>
+
+#include "GraphicBufferListener.h"
+
+namespace android {
+
+status_t GraphicBufferListener::init(
+        const sp<AMessage> &notify,
+        size_t bufferWidth, size_t bufferHeight, size_t bufferCount) {
+    mNotify = notify;
+
+    String8 name("GraphicBufferListener");
+    BufferQueue::createBufferQueue(&mProducer, &mConsumer);
+    mConsumer->setConsumerName(name);
+    mConsumer->setDefaultBufferSize(bufferWidth, bufferHeight);
+    mConsumer->setConsumerUsageBits(GRALLOC_USAGE_SW_READ_OFTEN);
+
+    status_t err = mConsumer->setMaxAcquiredBufferCount(bufferCount);
+    if (err != NO_ERROR) {
+        ALOGE("Unable to set BQ max acquired buffer count to %u: %d",
+                bufferCount, err);
+        return err;
+    }
+
+    wp<BufferQueue::ConsumerListener> listener =
+        static_cast<BufferQueue::ConsumerListener*>(this);
+    sp<BufferQueue::ProxyConsumerListener> proxy =
+        new BufferQueue::ProxyConsumerListener(listener);
+
+    err = mConsumer->consumerConnect(proxy, false);
+    if (err != NO_ERROR) {
+        ALOGE("Error connecting to BufferQueue: %s (%d)",
+                strerror(-err), err);
+        return err;
+    }
+
+    ALOGV("init() successful.");
+
+    return OK;
+}
+
+void GraphicBufferListener::onFrameAvailable() {
+    ALOGV("onFrameAvailable() called");
+
+    {
+        Mutex::Autolock autoLock(mMutex);
+        mNumFramesAvailable++;
+    }
+
+    sp<AMessage> notify = mNotify->dup();
+    mNotify->setWhat(kWhatFrameAvailable);
+    mNotify->post();
+}
+
+void GraphicBufferListener::onBuffersReleased() {
+    ALOGV("onBuffersReleased() called");
+    // nothing to do
+}
+
+void GraphicBufferListener::onSidebandStreamChanged() {
+    ALOGW("GraphicBufferListener cannot consume sideband streams.");
+    // nothing to do
+}
+
+BufferQueue::BufferItem GraphicBufferListener::getBufferItem() {
+    BufferQueue::BufferItem item;
+
+    {
+        Mutex::Autolock autoLock(mMutex);
+        if (mNumFramesAvailable <= 0) {
+            ALOGE("getBuffer() called with no frames available");
+            return item;
+        }
+        mNumFramesAvailable--;
+    }
+
+    status_t err = mConsumer->acquireBuffer(&item, 0);
+    if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
+        // shouldn't happen, since we track num frames available
+        ALOGE("frame was not available");
+        item.mBuf = -1;
+        return item;
+    } else if (err != OK) {
+        ALOGE("acquireBuffer returned err=%d", err);
+        item.mBuf = -1;
+        return item;
+    }
+
+    // Wait for it to become available.
+    err = item.mFence->waitForever("GraphicBufferListener::getBufferItem");
+    if (err != OK) {
+        ALOGW("failed to wait for buffer fence: %d", err);
+        // keep going
+    }
+
+    // If this is the first time we're seeing this buffer, add it to our
+    // slot table.
+    if (item.mGraphicBuffer != NULL) {
+        ALOGV("setting mBufferSlot %d", item.mBuf);
+        mBufferSlot[item.mBuf] = item.mGraphicBuffer;
+    }
+
+    return item;
+}
+
+sp<GraphicBuffer> GraphicBufferListener::getBuffer(
+        BufferQueue::BufferItem item) {
+    sp<GraphicBuffer> buf;
+    if (item.mBuf < 0 || item.mBuf >= BufferQueue::NUM_BUFFER_SLOTS) {
+        ALOGE("getBuffer() received invalid BufferItem: mBuf==%d", item.mBuf);
+        return buf;
+    }
+
+    buf = mBufferSlot[item.mBuf];
+    CHECK(buf.get() != NULL);
+
+    return buf;
+}
+
+status_t GraphicBufferListener::releaseBuffer(
+        BufferQueue::BufferItem item) {
+    if (item.mBuf < 0 || item.mBuf >= BufferQueue::NUM_BUFFER_SLOTS) {
+        ALOGE("getBuffer() received invalid BufferItem: mBuf==%d", item.mBuf);
+        return ERROR_OUT_OF_RANGE;
+    }
+
+    mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber,
+            EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE);
+
+    return OK;
+}
+
+}   // namespace android
diff --git a/media/libstagefright/filters/GraphicBufferListener.h b/media/libstagefright/filters/GraphicBufferListener.h
new file mode 100644
index 0000000..aefac0d
--- /dev/null
+++ b/media/libstagefright/filters/GraphicBufferListener.h
@@ -0,0 +1,70 @@
+/*
+ * Copyright (C) 2014 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef GRAPHIC_BUFFER_LISTENER_H_
+#define GRAPHIC_BUFFER_LISTENER_H_
+
+#include <gui/BufferQueue.h>
+
+namespace android {
+
+struct AMessage;
+
+struct GraphicBufferListener : public BufferQueue::ConsumerListener {
+public:
+    GraphicBufferListener() {};
+
+    status_t init(
+            const sp<AMessage> &notify,
+            size_t bufferWidth, size_t bufferHeight, size_t bufferCount);
+
+    virtual void onFrameAvailable();
+    virtual void onBuffersReleased();
+    virtual void onSidebandStreamChanged();
+
+    // Returns the handle to the producer side of the BufferQueue.  Buffers
+    // queued on this will be received by GraphicBufferListener.
+    sp<IGraphicBufferProducer> getIGraphicBufferProducer() const {
+        return mProducer;
+    }
+
+    BufferQueue::BufferItem getBufferItem();
+    sp<GraphicBuffer> getBuffer(BufferQueue::BufferItem item);
+    status_t releaseBuffer(BufferQueue::BufferItem item);
+
+    enum {
+        kWhatFrameAvailable = 'frav',
+    };
+
+private:
+    sp<AMessage> mNotify;
+    size_t mNumFramesAvailable;
+
+    mutable Mutex mMutex;
+
+    // Our BufferQueue interfaces. mProducer is passed to the producer through
+    // getIGraphicBufferProducer, and mConsumer is used internally to retrieve
+    // the buffers queued by the producer.
+    sp<IGraphicBufferProducer> mProducer;
+    sp<IGraphicBufferConsumer> mConsumer;
+
+    // Cache of GraphicBuffers from the buffer queue.
+    sp<GraphicBuffer> mBufferSlot[BufferQueue::NUM_BUFFER_SLOTS];
+};
+
+}   // namespace android
+
+#endif  // GRAPHIC_BUFFER_LISTENER_H
diff --git a/media/libstagefright/filters/IntrinsicBlurFilter.cpp b/media/libstagefright/filters/IntrinsicBlurFilter.cpp
index cca97ef..2bae4d7 100644
--- a/media/libstagefright/filters/IntrinsicBlurFilter.cpp
+++ b/media/libstagefright/filters/IntrinsicBlurFilter.cpp
@@ -31,9 +31,7 @@ status_t IntrinsicBlurFilter::start() {
     // TODO: use a single RS context object for entire application
     mRS = new RSC::RS();
 
-    // only legitimate because this is a standalone executable
-    // TODO: do we need to dynamically determine the cache directory?
-    if (!mRS->init("/system/bin")) {
+    if (!mRS->init(mCacheDir.c_str())) {
         ALOGE("Failed to initialize RenderScript context.");
         return NO_INIT;
     }
diff --git a/media/libstagefright/filters/MediaFilter.cpp b/media/libstagefright/filters/MediaFilter.cpp
index 5a15ec5..3f9aa19 100644
--- a/media/libstagefright/filters/MediaFilter.cpp
+++ b/media/libstagefright/filters/MediaFilter.cpp
@@ -22,6 +22,7 @@
 
 #include <binder/MemoryDealer.h>
 
+#include <media/stagefright/BufferProducerWrapper.h>
 #include <media/stagefright/foundation/ABuffer.h>
 #include <media/stagefright/foundation/ADebug.h>
 #include <media/stagefright/foundation/AMessage.h>
@@ -30,6 +31,8 @@
 #include <media/stagefright/MediaErrors.h>
 #include <media/stagefright/MediaFilter.h>
 
+#include "ColorConvert.h"
+#include "GraphicBufferListener.h"
 #include "IntrinsicBlurFilter.h"
 #include "SaturationFilter.h"
 #include "ZeroFilter.h"
@@ -41,7 +44,8 @@ static const size_t kBufferCountActual = 4;
 
 MediaFilter::MediaFilter()
     : mState(UNINITIALIZED),
-      mGeneration(0) {
+      mGeneration(0),
+      mGraphicBufferListener(NULL) {
 }
 
 MediaFilter::~MediaFilter() {
@@ -154,6 +158,21 @@ void MediaFilter::onMessageReceived(const sp<AMessage> &msg) {
             onSetParameters(msg);
             break;
         }
+        case kWhatCreateInputSurface:
+        {
+            onCreateInputSurface();
+            break;
+        }
+        case GraphicBufferListener::kWhatFrameAvailable:
+        {
+            onInputFrameAvailable();
+            break;
+        }
+        case kWhatSignalEndOfInputStream:
+        {
+            onSignalEndOfInputStream();
+            break;
+        }
         default:
         {
             ALOGE("Message not handled:\n%s", msg->debugString().c_str());
@@ -400,7 +419,11 @@ void MediaFilter::processBuffers() {
                 inputInfo->mBufferID, inputInfo->mData->size(),
                 outputInfo->mBufferID, outputInfo->mData->size());
 
-    postFillThisBuffer(inputInfo);
+    if (mGraphicBufferListener != NULL) {
+        delete inputInfo;
+    } else {
+        postFillThisBuffer(inputInfo);
+    }
     postDrainThisBuffer(outputInfo);
 
     // prevent any corner case where buffers could get stuck in queue
@@ -468,11 +491,19 @@ void MediaFilter::onConfigureComponent(const sp<AMessage> &msg) {
         mColorFormatIn = OMX_COLOR_Format32bitARGB8888;
     }
     mColorFormatOut = mColorFormatIn;
+
     mMaxOutputSize = mWidth * mHeight * 4;  // room for ARGB8888
 
+    AString cacheDir;
+    if (!msg->findString("cacheDir", &cacheDir)) {
+        ALOGE("Failed to find cache directory in config message.");
+        signalError(NAME_NOT_FOUND);
+        return;
+    }
+
     status_t err;
     err = mFilter->configure(
-            mWidth, mHeight, mStride, mSliceHeight, mColorFormatIn);
+            mWidth, mHeight, mStride, mSliceHeight, mColorFormatIn, cacheDir);
     if (err != (status_t)OK) {
         ALOGE("Failed to configure filter component, err %d", err);
         signalError(err);
@@ -595,8 +626,7 @@ void MediaFilter::onInputBufferFilled(const sp<AMessage> &msg) {
         mInputEOSResult = err;
     }
 
-    ALOGV("Handled kWhatInputBufferFilled. [ID %u]",
-            bufferID);
+    ALOGV("Handled kWhatInputBufferFilled. [ID %u]", bufferID);
 }
 
 void MediaFilter::onOutputBufferDrained(const sp<AMessage> &msg) {
@@ -673,4 +703,94 @@ void MediaFilter::onSetParameters(const sp<AMessage> &msg) {
     }
 }
 
+void MediaFilter::onCreateInputSurface() {
+    CHECK(mState == CONFIGURED);
+
+    mGraphicBufferListener = new GraphicBufferListener;
+
+    sp<AMessage> notify = new AMessage();
+    notify->setTarget(id());
+    status_t err = mGraphicBufferListener->init(
+            notify, mStride, mSliceHeight, kBufferCountActual);
+
+    if (err != OK) {
+        ALOGE("Failed to init mGraphicBufferListener: %d", err);
+        signalError(err);
+        return;
+    }
+
+    sp<AMessage> reply = mNotify->dup();
+    reply->setInt32("what", CodecBase::kWhatInputSurfaceCreated);
+    reply->setObject(
+            "input-surface",
+            new BufferProducerWrapper(
+                    mGraphicBufferListener->getIGraphicBufferProducer()));
+    reply->post();
+}
+
+void MediaFilter::onInputFrameAvailable() {
+    BufferQueue::BufferItem item = mGraphicBufferListener->getBufferItem();
+    sp<GraphicBuffer> buf = mGraphicBufferListener->getBuffer(item);
+
+    // get pointer to graphic buffer
+    void* bufPtr;
+    buf->lock(GraphicBuffer::USAGE_SW_READ_OFTEN, &bufPtr);
+
+    // HACK - there is no OMX_COLOR_FORMATTYPE value for RGBA, so the format
+    // conversion is hardcoded until we add this.
+    // TODO: check input format and convert only if necessary
+    // copy RGBA graphic buffer into temporary ARGB input buffer
+    BufferInfo *inputInfo = new BufferInfo;
+    inputInfo->mData = new ABuffer(buf->getWidth() * buf->getHeight() * 4);
+    ALOGV("Copying surface data into temp buffer.");
+    convertRGBAToARGB(
+            (uint8_t*)bufPtr, buf->getWidth(), buf->getHeight(),
+            buf->getStride(), inputInfo->mData->data());
+    inputInfo->mBufferID = item.mBuf;
+    inputInfo->mGeneration = mGeneration;
+    inputInfo->mOutputFlags = 0;
+    inputInfo->mStatus = BufferInfo::OWNED_BY_US;
+    inputInfo->mData->meta()->setInt64("timeUs", item.mTimestamp / 1000);
+
+    mAvailableInputBuffers.push_back(inputInfo);
+
+    mGraphicBufferListener->releaseBuffer(item);
+
+    signalProcessBuffers();
+}
+
+void MediaFilter::onSignalEndOfInputStream() {
+    // if using input surface, need to send an EOS output buffer
+    if (mGraphicBufferListener != NULL) {
+        Vector<BufferInfo> *outputBufs = &mBuffers[kPortIndexOutput];
+        BufferInfo* eosBuf;
+        bool foundBuf = false;
+        for (size_t i = 0; i < kBufferCountActual; i++) {
+            eosBuf = &outputBufs->editItemAt(i);
+            if (eosBuf->mStatus == BufferInfo::OWNED_BY_US) {
+                foundBuf = true;
+                break;
+            }
+        }
+
+        if (!foundBuf) {
+            ALOGE("onSignalEndOfInputStream failed to find an output buffer");
+            return;
+        }
+
+        eosBuf->mOutputFlags = OMX_BUFFERFLAG_EOS;
+        eosBuf->mGeneration = mGeneration;
+        eosBuf->mData->setRange(0, 0);
+        postDrainThisBuffer(eosBuf);
+        ALOGV("Posted EOS on output buffer %zu", eosBuf->mBufferID);
+    }
+
+    mPortEOS[kPortIndexOutput] = true;
+    sp<AMessage> notify = mNotify->dup();
+    notify->setInt32("what", CodecBase::kWhatSignaledInputEOS);
+    notify->post();
+
+    ALOGV("Output stream saw EOS.");
+}
+
 }   // namespace android
diff --git a/media/libstagefright/filters/SaturationFilter.cpp b/media/libstagefright/filters/SaturationFilter.cpp
index 1963c20..24aa083 100644
--- a/media/libstagefright/filters/SaturationFilter.cpp
+++ b/media/libstagefright/filters/SaturationFilter.cpp
@@ -31,9 +31,7 @@ status_t SaturationFilter::start() {
     // TODO: use a single RS context object for entire application
     mRS = new RSC::RS();
 
-    // only legitimate because this is a standalone executable
-    // TODO: do we need to dynamically determine the cache directory?
-    if (!mRS->init("/system/bin")) {
+    if (!mRS->init(mCacheDir.c_str())) {
         ALOGE("Failed to initialize RenderScript context.");
         return NO_INIT;
     }
diff --git a/media/libstagefright/filters/SimpleFilter.h b/media/libstagefright/filters/SimpleFilter.h
index a99ca05..26be742 100644
--- a/media/libstagefright/filters/SimpleFilter.h
+++ b/media/libstagefright/filters/SimpleFilter.h
@@ -31,16 +31,19 @@ public:
     SimpleFilter() : mWidth(0), mHeight(0), mStride(0), mSliceHeight(0),
             mColorFormatIn(0), mColorFormatOut(0) {};
 
+    // TODO: change this to take the configure AMessage so that parameters
+    // aren't hardcoded for all filters
     virtual status_t configure(
             int32_t srcWidth, int32_t srcHeight,
             int32_t srcStride, int32_t srcSliceHeight,
-            int32_t srcColorFormat) {
+            int32_t srcColorFormat, AString cacheDir) {
         mWidth = srcWidth;
         mHeight = srcHeight;
         mStride = srcStride;
         mSliceHeight = srcSliceHeight;
         mColorFormatIn = srcColorFormat;
         mColorFormatOut = mColorFormatIn;
+        mCacheDir = cacheDir;
 
         return OK;
     }
@@ -55,6 +58,7 @@ protected:
     int32_t mWidth, mHeight;
     int32_t mStride, mSliceHeight;
     int32_t mColorFormatIn, mColorFormatOut;
+    AString mCacheDir;
 
     virtual ~SimpleFilter() {};
 };
diff --git a/media/libstagefright/filters/saturationARGB.rs b/media/libstagefright/filters/saturationARGB.rs
index 20cfab9..1de9dd8 100644
--- a/media/libstagefright/filters/saturationARGB.rs
+++ b/media/libstagefright/filters/saturationARGB.rs
@@ -24,6 +24,8 @@ const static float3 gMonoMult = {0.299f, 0.587f, 0.114f};
 float gSaturation = 1.0f;
 
 void root(const uchar4 *v_in, uchar4 *v_out) {
+    v_out->x = v_in->x; // don't modify A
+
     // get RGB, scale 0-255 uchar to 0-1.0 float
     float3 rgb = {v_in->y * 0.003921569f, v_in->z * 0.003921569f,
             v_in->w * 0.003921569f};
@@ -32,7 +34,6 @@ void root(const uchar4 *v_in, uchar4 *v_out) {
     float3 result = dot(rgb, gMonoMult);
     result = mix(result, rgb, gSaturation);
 
-    v_out->x = v_in->x; // don't modify A
     v_out->y = (uchar)clamp((result.r * 255.f + 0.5f), 0.f, 255.f);
     v_out->z = (uchar)clamp((result.g * 255.f + 0.5f), 0.f, 255.f);
     v_out->w = (uchar)clamp((result.b * 255.f + 0.5f), 0.f, 255.f);