Implement Surface input to MediaCodec.

Also, renamed a CHECK_INTERFACE macro that was clashing with the
Binder version.

Bug 7991062

Change-Id: If5e6ed0a06d9f67975497676e4b05abe3aa3d6c0

1 files changed, 441 insertions, 0 deletions

diff --git a/media/libstagefright/omx/GraphicBufferSource.cpp b/media/libstagefright/omx/GraphicBufferSource.cpp
new file mode 100644
index 0000000..f207954
--- /dev/null
+++ b/media/libstagefright/omx/GraphicBufferSource.cpp
@@ -0,0 +1,441 @@
+/*
+ * Copyright (C) 2013 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "GraphicBufferSource"
+#include <utils/Log.h>
+
+#include <GraphicBufferSource.h>
+
+#include <OMX_Core.h>
+#include <media/stagefright/foundation/ADebug.h>
+
+#include <MetadataBufferType.h>
+#include <ui/GraphicBuffer.h>
+
+namespace android {
+
+static const bool EXTRA_CHECK = true;
+
+
+GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance,
+        uint32_t bufferWidth, uint32_t bufferHeight) :
+    mInitCheck(UNKNOWN_ERROR),
+    mNodeInstance(nodeInstance),
+    mExecuting(false),
+    mNumFramesAvailable(0),
+    mEndOfStream(false),
+    mEndOfStreamSent(false) {
+
+    ALOGV("GraphicBufferSource w=%u h=%u", bufferWidth, bufferHeight);
+
+    if (bufferWidth == 0 || bufferHeight == 0) {
+        ALOGE("Invalid dimensions %dx%d", bufferWidth, bufferHeight);
+        mInitCheck = BAD_VALUE;
+        return;
+    }
+
+    mBufferQueue = new BufferQueue(true);
+    mBufferQueue->setDefaultBufferSize(bufferWidth, bufferHeight);
+    mBufferQueue->setSynchronousMode(true);
+    mBufferQueue->setConsumerUsageBits(GRALLOC_USAGE_HW_VIDEO_ENCODER |
+            GRALLOC_USAGE_HW_TEXTURE);
+
+    // Note that we can't create an sp<...>(this) in a ctor that will not keep a
+    // reference once the ctor ends, as that would cause the refcount of 'this'
+    // dropping to 0 at the end of the ctor.  Since all we need is a wp<...>
+    // that's what we create.
+    wp<BufferQueue::ConsumerListener> listener;
+    listener = static_cast<BufferQueue::ConsumerListener*>(this);
+
+    sp<BufferQueue::ConsumerListener> proxy;
+    proxy = new BufferQueue::ProxyConsumerListener(listener);
+
+    status_t err = mBufferQueue->consumerConnect(proxy);
+    if (err != NO_ERROR) {
+        ALOGE("Error connecting to BufferQueue: %s (%d)",
+                strerror(-err), err);
+        return;
+    }
+
+    mInitCheck = OK;
+}
+
+GraphicBufferSource::~GraphicBufferSource() {
+    ALOGV("~GraphicBufferSource");
+    status_t err = mBufferQueue->consumerDisconnect();
+    if (err != NO_ERROR) {
+        ALOGW("consumerDisconnect failed: %d", err);
+    }
+}
+
+void GraphicBufferSource::omxExecuting() {
+    Mutex::Autolock autoLock(mMutex);
+    ALOGV("--> executing; avail=%d, codec vec size=%zd",
+            mNumFramesAvailable, mCodecBuffers.size());
+    CHECK(!mExecuting);
+    mExecuting = true;
+
+    // Start by loading up as many buffers as possible.  We want to do this,
+    // rather than just submit the first buffer, to avoid a degenerate case:
+    // if all BQ buffers arrive before we start executing, and we only submit
+    // one here, the other BQ buffers will just sit until we get notified
+    // that the codec buffer has been released.  We'd then acquire and
+    // submit a single additional buffer, repeatedly, never using more than
+    // one codec buffer simultaneously.  (We could instead try to submit
+    // all BQ buffers whenever any codec buffer is freed, but if we get the
+    // initial conditions right that will never be useful.)
+    while (mNumFramesAvailable && isCodecBufferAvailable_l()) {
+        fillCodecBuffer_l();
+    }
+
+    ALOGV("done loading initial frames, avail=%d", mNumFramesAvailable);
+
+    // If EOS has already been signaled, and there are no more frames to
+    // submit, try to send EOS now as well.
+    if (mEndOfStream && mNumFramesAvailable == 0) {
+        submitEndOfInputStream_l();
+    }
+}
+
+void GraphicBufferSource::omxIdling(){
+    Mutex::Autolock autoLock(mMutex);
+    ALOGV("--> idling");
+    if (!mExecuting) {
+        // Transition from "loading" to "idling".  Nothing to do.
+        return;
+    }
+
+    ALOGV("Dropped down to idle, avail=%d eos=%d eosSent=%d",
+            mNumFramesAvailable, mEndOfStream, mEndOfStreamSent);
+
+    // Codec is no longer executing.  Discard all codec-related state.
+    mCodecBuffers.clear();
+    // TODO: scan mCodecBuffers to verify that all mGraphicBuffer entries
+    //       are null; complain if not
+
+    mExecuting = false;
+}
+
+void GraphicBufferSource::addCodecBuffer(OMX_BUFFERHEADERTYPE* header) {
+    Mutex::Autolock autoLock(mMutex);
+
+    if (mExecuting) {
+        // This should never happen -- buffers can only be allocated when
+        // transitioning from "loaded" to "idle".
+        ALOGE("addCodecBuffer: buffer added while executing");
+        return;
+    }
+
+    ALOGV("addCodecBuffer h=%p size=%lu p=%p",
+            header, header->nAllocLen, header->pBuffer);
+    CodecBuffer codecBuffer;
+    codecBuffer.mHeader = header;
+    mCodecBuffers.add(codecBuffer);
+}
+
+void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) {
+    Mutex::Autolock autoLock(mMutex);
+
+    CHECK(mExecuting);  // could this happen if app stop()s early?
+
+    int cbi = findMatchingCodecBuffer_l(header);
+    if (cbi < 0) {
+        // This should never happen.
+        ALOGE("codecBufferEmptied: buffer not recognized (h=%p)", header);
+        return;
+    }
+
+    ALOGV("codecBufferEmptied h=%p size=%lu filled=%lu p=%p",
+            header, header->nAllocLen, header->nFilledLen,
+            header->pBuffer);
+    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
+
+    // header->nFilledLen may not be the original value, so we can't compare
+    // that to zero to see of this was the EOS buffer.  Instead we just
+    // see if the GraphicBuffer reference was null, which should only ever
+    // happen for EOS.
+    if (codecBuffer.mGraphicBuffer == NULL) {
+        CHECK(mEndOfStream);
+        // No GraphicBuffer to deal with, no additional input or output is
+        // expected, so just return.
+        return;
+    }
+
+    if (EXTRA_CHECK) {
+        // Pull the graphic buffer handle back out of the buffer, and confirm
+        // that it matches expectations.
+        OMX_U8* data = header->pBuffer;
+        buffer_handle_t bufferHandle;
+        memcpy(&bufferHandle, data + 4, sizeof(buffer_handle_t));
+        if (bufferHandle != codecBuffer.mGraphicBuffer->handle) {
+            // should never happen
+            ALOGE("codecBufferEmptied: buffer's handle is %p, expected %p",
+                    bufferHandle, codecBuffer.mGraphicBuffer->handle);
+            CHECK(!"codecBufferEmptied: mismatched buffer");
+        }
+    }
+
+    // Find matching entry in our cached copy of the BufferQueue slots.
+    // If we find a match, release that slot.  If we don't, the BufferQueue
+    // has dropped that GraphicBuffer, and there's nothing for us to release.
+    //
+    // (We could store "id" in CodecBuffer and avoid the slot search.)
+    int id;
+    for (id = 0; id < BufferQueue::NUM_BUFFER_SLOTS; id++) {
+        if (mBufferSlot[id] == NULL) {
+            continue;
+        }
+
+        if (mBufferSlot[id]->handle == codecBuffer.mGraphicBuffer->handle) {
+            ALOGV("cbi %d matches bq slot %d, handle=%p",
+                    cbi, id, mBufferSlot[id]->handle);
+
+            mBufferQueue->releaseBuffer(id, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR,
+                    Fence::NO_FENCE);
+            break;
+        }
+    }
+    if (id == BufferQueue::NUM_BUFFER_SLOTS) {
+        ALOGV("codecBufferEmptied: no match for emptied buffer in cbi %d",
+                cbi);
+    }
+
+    // Mark the codec buffer as available by clearing the GraphicBuffer ref.
+    codecBuffer.mGraphicBuffer = NULL;
+
+    if (mNumFramesAvailable) {
+        // Fill this codec buffer.
+        CHECK(!mEndOfStream);
+        ALOGV("buffer freed, %d frames avail", mNumFramesAvailable);
+        fillCodecBuffer_l();
+    } else if (mEndOfStream) {
+        // No frames available, but EOS is pending, so use this buffer to
+        // send that.
+        ALOGV("buffer freed, EOS pending");
+        submitEndOfInputStream_l();
+    }
+    return;
+}
+
+status_t GraphicBufferSource::fillCodecBuffer_l() {
+    CHECK(mExecuting && mNumFramesAvailable > 0);
+    int cbi = findAvailableCodecBuffer_l();
+    if (cbi < 0) {
+        // No buffers available, bail.
+        ALOGV("fillCodecBuffer_l: no codec buffers, avail now %d",
+                mNumFramesAvailable);
+    } else {
+        ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%d",
+                mNumFramesAvailable);
+        BufferQueue::BufferItem item;
+        status_t err = mBufferQueue->acquireBuffer(&item);
+        if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
+            // shouldn't happen
+            ALOGW("fillCodecBuffer_l: frame was not available");
+            return err;
+        } else if (err != OK) {
+            // now what? fake end-of-stream?
+            ALOGW("fillCodecBuffer_l: acquireBuffer returned err=%d", err);
+            return err;
+        }
+
+        mNumFramesAvailable--;
+
+        // Wait for it to become available.
+        err = item.mFence->waitForever(1000,
+                "GraphicBufferSource::fillCodecBuffer_l");
+        if (err != OK) {
+            ALOGW("failed to wait for buffer fence: %d", err);
+            // keep going
+        }
+
+        // If this is the first time we're seeing this buffer, add it to our
+        // slot table.
+        if (item.mGraphicBuffer != NULL) {
+            ALOGV("fillCodecBuffer_l: setting mBufferSlot %d", item.mBuf);
+            mBufferSlot[item.mBuf] = item.mGraphicBuffer;
+        }
+
+        err = submitBuffer_l(mBufferSlot[item.mBuf], item.mTimestamp, cbi);
+        if (err != OK) {
+            ALOGV("submitBuffer_l failed, releasing bq buf %d", item.mBuf);
+            mBufferQueue->releaseBuffer(item.mBuf, EGL_NO_DISPLAY,
+                EGL_NO_SYNC_KHR, Fence::NO_FENCE);
+        } else {
+            ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi);
+        }
+    }
+
+    return OK;
+}
+
+void GraphicBufferSource::signalEndOfInputStream() {
+    Mutex::Autolock autoLock(mMutex);
+    ALOGV("signalEndOfInputStream: exec=%d avail=%d",
+            mExecuting, mNumFramesAvailable);
+
+    // Set the end-of-stream flag.  If no frames are pending from the
+    // BufferQueue, and a codec buffer is available, and we're executing,
+    // we initiate the EOS from here.  Otherwise, we'll let
+    // codecBufferEmptied() (or omxExecuting) do it.
+    //
+    // Note: if there are no pending frames and all codec buffers are
+    // available, we *must* submit the EOS from here or we'll just
+    // stall since no future events are expected.
+    mEndOfStream = true;
+
+    if (mExecuting && mNumFramesAvailable == 0) {
+        submitEndOfInputStream_l();
+    }
+}
+
+status_t GraphicBufferSource::submitBuffer_l(sp<GraphicBuffer>& graphicBuffer,
+        int64_t timestamp, int cbi) {
+    ALOGV("submitBuffer_l cbi=%d", cbi);
+    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
+    codecBuffer.mGraphicBuffer = graphicBuffer;
+
+    OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
+    CHECK(header->nAllocLen >= 4 + sizeof(buffer_handle_t));
+    OMX_U8* data = header->pBuffer;
+    const OMX_U32 type = kMetadataBufferTypeGrallocSource;
+    buffer_handle_t handle = codecBuffer.mGraphicBuffer->handle;
+    memcpy(data, &type, 4);
+    memcpy(data + 4, &handle, sizeof(buffer_handle_t));
+
+    status_t err = mNodeInstance->emptyDirectBuffer(header, 0,
+            4 + sizeof(buffer_handle_t), OMX_BUFFERFLAG_ENDOFFRAME,
+            timestamp);
+    if (err != OK) {
+        ALOGW("WARNING: emptyDirectBuffer failed: 0x%x", err);
+        codecBuffer.mGraphicBuffer = NULL;
+        return err;
+    }
+
+    ALOGV("emptyDirectBuffer succeeded, h=%p p=%p bufhandle=%p",
+            header, header->pBuffer, handle);
+    return OK;
+}
+
+void GraphicBufferSource::submitEndOfInputStream_l() {
+    CHECK(mEndOfStream);
+    if (mEndOfStreamSent) {
+        ALOGV("EOS already sent");
+        return;
+    }
+
+    int cbi = findAvailableCodecBuffer_l();
+    if (cbi < 0) {
+        ALOGV("submitEndOfInputStream_l: no codec buffers available");
+        return;
+    }
+
+    // We reject any additional incoming graphic buffers, so there's no need
+    // to stick a placeholder into codecBuffer.mGraphicBuffer to mark it as
+    // in-use.
+    CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi));
+
+    OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader;
+    if (EXTRA_CHECK) {
+        // Guard against implementations that don't check nFilledLen.
+        size_t fillLen = 4 + sizeof(buffer_handle_t);
+        CHECK(header->nAllocLen >= fillLen);
+        OMX_U8* data = header->pBuffer;
+        memset(data, 0xcd, fillLen);
+    }
+
+    uint64_t timestamp = 0; // does this matter?
+
+    status_t err = mNodeInstance->emptyDirectBuffer(header, /*offset*/ 0,
+            /*length*/ 0, OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_EOS,
+            timestamp);
+    if (err != OK) {
+        ALOGW("emptyDirectBuffer EOS failed: 0x%x", err);
+    } else {
+        ALOGV("submitEndOfInputStream_l: buffer submitted, header=%p cbi=%d",
+                header, cbi);
+    }
+}
+
+int GraphicBufferSource::findAvailableCodecBuffer_l() {
+    CHECK(mCodecBuffers.size() > 0);
+
+    for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
+        if (mCodecBuffers[i].mGraphicBuffer == NULL) {
+            return i;
+        }
+    }
+    return -1;
+}
+
+int GraphicBufferSource::findMatchingCodecBuffer_l(
+        const OMX_BUFFERHEADERTYPE* header) {
+    for (int i = (int)mCodecBuffers.size() - 1; i>= 0; --i) {
+        if (mCodecBuffers[i].mHeader == header) {
+            return i;
+        }
+    }
+    return -1;
+}
+
+// BufferQueue::ConsumerListener callback
+void GraphicBufferSource::onFrameAvailable() {
+    Mutex::Autolock autoLock(mMutex);
+
+    ALOGV("onFrameAvailable exec=%d avail=%d", mExecuting, mNumFramesAvailable);
+
+    if (mEndOfStream) {
+        // This should only be possible if a new buffer was queued after
+        // EOS was signaled, i.e. the app is misbehaving.
+        ALOGW("onFrameAvailable: EOS is set, ignoring frame");
+
+        BufferQueue::BufferItem item;
+        status_t err = mBufferQueue->acquireBuffer(&item);
+        if (err == OK) {
+            mBufferQueue->releaseBuffer(item.mBuf, EGL_NO_DISPLAY,
+                EGL_NO_SYNC_KHR, item.mFence);
+        }
+        return;
+    }
+
+    mNumFramesAvailable++;
+
+    if (mExecuting) {
+        fillCodecBuffer_l();
+    }
+}
+
+// BufferQueue::ConsumerListener callback
+void GraphicBufferSource::onBuffersReleased() {
+    Mutex::Autolock lock(mMutex);
+
+    uint32_t slotMask;
+    if (mBufferQueue->getReleasedBuffers(&slotMask) != NO_ERROR) {
+        ALOGW("onBuffersReleased: unable to get released buffer set");
+        slotMask = 0xffffffff;
+    }
+
+    ALOGV("onBuffersReleased: 0x%08x", slotMask);
+
+    for (int i = 0; i < BufferQueue::NUM_BUFFER_SLOTS; i++) {
+        if ((slotMask & 0x01) != 0) {
+            mBufferSlot[i] = NULL;
+        }
+        slotMask >>= 1;
+    }
+}
+
+}  // namespace android