Adding SurfaceEncoder for encoding FilterFrames

Adding SurfaceEncoder which can be used to encode
custom frame data. In a sense, it is reverse
of what SurfaceTexture does.

SurfaceEncoder takes in frames from a native window and
passes them to an encoder, thus acting like a MediaSource.
It uses GRAlloc buffers underneath for passing data.
The client side sets the geometry, format in the beginning,
which cannot be changed while the recording is going on.

Currently, there is no common pixel format that both
GRAlloc and HAL understand.
Also, the encoder cannot encode using the data from the GRAlloc
buffers.

The SurfaceEncoder_test examines mainly the buffer passage
since true encoding cannot be done at this point.
SimpleDummyRecorder 'reads' the frames in the same thread
as the start(), whereas DummyRecorder 'reads' in a separate
thread much like the MPEG4Writer. The test with DummyRecorder
is much closer to the real encoding implementation.

Related to bug id: 4529323

Change-Id: I58ec19a150f8fe4d6195196dc44f55002b46c7c8

4 files changed, 553 insertions, 0 deletions

diff --git a/media/libstagefright/tests/Android.mk b/media/libstagefright/tests/Android.mk
new file mode 100644
index 0000000..89d80a9
--- /dev/null
+++ b/media/libstagefright/tests/Android.mk
@@ -0,0 +1,53 @@
+# Build the unit tests.
+LOCAL_PATH:= $(call my-dir)
+include $(CLEAR_VARS)
+
+ifneq ($(TARGET_SIMULATOR),true)
+
+LOCAL_MODULE := SurfaceEncoder_test
+
+LOCAL_MODULE_TAGS := tests
+
+LOCAL_SRC_FILES := \
+    SurfaceEncoder_test.cpp \
+	DummyRecorder.cpp \
+
+LOCAL_SHARED_LIBRARIES := \
+	libEGL \
+	libGLESv2 \
+	libandroid \
+	libbinder \
+	libcutils \
+	libgui \
+	libstlport \
+	libui \
+	libutils \
+	libstagefright \
+	libstagefright_omx \
+	libstagefright_foundation \
+
+LOCAL_STATIC_LIBRARIES := \
+	libgtest \
+	libgtest_main \
+
+LOCAL_C_INCLUDES := \
+    bionic \
+    bionic/libstdc++/include \
+    external/gtest/include \
+    external/stlport/stlport \
+	frameworks/base/media/libstagefright \
+	frameworks/base/media/libstagefright/include \
+	$(TOP)/frameworks/base/include/media/stagefright/openmax \
+
+include $(BUILD_EXECUTABLE)
+
+endif
+
+# Include subdirectory makefiles
+# ============================================================
+
+# If we're building with ONE_SHOT_MAKEFILE (mm, mmm), then what the framework
+# team really wants is to build the stuff defined by this makefile.
+ifeq (,$(ONE_SHOT_MAKEFILE))
+include $(call first-makefiles-under,$(LOCAL_PATH))
+endif
diff --git a/media/libstagefright/tests/DummyRecorder.cpp b/media/libstagefright/tests/DummyRecorder.cpp
new file mode 100644
index 0000000..8d75d6b
--- /dev/null
+++ b/media/libstagefright/tests/DummyRecorder.cpp
@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "DummyRecorder"
+// #define LOG_NDEBUG 0
+
+#include <media/stagefright/MediaBuffer.h>
+#include <media/stagefright/MediaSource.h>
+#include "DummyRecorder.h"
+
+#include <utils/Log.h>
+
+namespace android {
+
+// static
+void *DummyRecorder::threadWrapper(void *pthis) {
+    LOGV("ThreadWrapper: %p", pthis);
+    DummyRecorder *writer = static_cast<DummyRecorder *>(pthis);
+    writer->readFromSource();
+    return NULL;
+}
+
+
+status_t DummyRecorder::start() {
+    LOGV("Start");
+    mStarted = true;
+
+    mSource->start();
+
+    pthread_attr_t attr;
+    pthread_attr_init(&attr);
+    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
+    int err = pthread_create(&mThread, &attr, threadWrapper, this);
+    pthread_attr_destroy(&attr);
+
+    if (err) {
+        LOGE("Error creating thread!");
+        return -ENODEV;
+    }
+    return OK;
+}
+
+
+status_t DummyRecorder::stop() {
+    LOGV("Stop");
+    mStarted = false;
+
+    mSource->stop();
+    void *dummy;
+    pthread_join(mThread, &dummy);
+    status_t err = (status_t) dummy;
+
+    LOGV("Ending the reading thread");
+    return err;
+}
+
+// pretend to read the source buffers
+void DummyRecorder::readFromSource() {
+    LOGV("ReadFromSource");
+    if (!mStarted) {
+        return;
+    }
+
+    status_t err = OK;
+    MediaBuffer *buffer;
+    LOGV("A fake writer accessing the frames");
+    while (mStarted && (err = mSource->read(&buffer)) == OK){
+        // if not getting a valid buffer from source, then exit
+        if (buffer == NULL) {
+            return;
+        }
+        buffer->release();
+        buffer = NULL;
+    }
+}
+
+
+} // end of namespace android
diff --git a/media/libstagefright/tests/DummyRecorder.h b/media/libstagefright/tests/DummyRecorder.h
new file mode 100644
index 0000000..1cbea1b
--- /dev/null
+++ b/media/libstagefright/tests/DummyRecorder.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef DUMMY_RECORDER_H_
+#define DUMMY_RECORDER_H_
+
+#include <pthread.h>
+#include <utils/String8.h>
+#include <media/stagefright/foundation/ABase.h>
+
+
+namespace android {
+
+class MediaSource;
+class MediaBuffer;
+
+class DummyRecorder {
+    public:
+    // The media source from which this will receive frames
+    sp<MediaSource> mSource;
+    bool mStarted;
+    pthread_t mThread;
+
+    status_t start();
+    status_t stop();
+
+    // actual entry point for the thread
+    void readFromSource();
+
+    // static function to wrap the actual thread entry point
+    static void *threadWrapper(void *pthis);
+
+    DummyRecorder(const sp<MediaSource> &source) : mSource(source)
+                                                    , mStarted(false) {}
+    ~DummyRecorder( ) {}
+
+    private:
+
+    DISALLOW_EVIL_CONSTRUCTORS(DummyRecorder);
+};
+
+} // end of namespace android
+#endif
+
+
diff --git a/media/libstagefright/tests/SurfaceEncoder_test.cpp b/media/libstagefright/tests/SurfaceEncoder_test.cpp
new file mode 100644
index 0000000..65ce7cc
--- /dev/null
+++ b/media/libstagefright/tests/SurfaceEncoder_test.cpp
@@ -0,0 +1,351 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "SurfaceEncoder_test"
+// #define LOG_NDEBUG 0
+
+#include <gtest/gtest.h>
+#include <utils/String8.h>
+#include <utils/Errors.h>
+
+#include <media/stagefright/SurfaceEncoder.h>
+
+#include <gui/SurfaceTextureClient.h>
+#include <ui/GraphicBuffer.h>
+#include <surfaceflinger/ISurfaceComposer.h>
+#include <surfaceflinger/Surface.h>
+#include <surfaceflinger/SurfaceComposerClient.h>
+
+#include <binder/ProcessState.h>
+#include <ui/FramebufferNativeWindow.h>
+
+#include <media/stagefright/MediaDebug.h>
+#include <media/stagefright/MediaDefs.h>
+#include <media/stagefright/MetaData.h>
+#include <media/stagefright/MPEG4Writer.h>
+#include <media/stagefright/OMXClient.h>
+#include <media/stagefright/OMXCodec.h>
+#include <OMX_Component.h>
+
+#include "DummyRecorder.h"
+
+namespace android {
+
+
+class SurfaceEncoderTest : public ::testing::Test {
+public:
+
+    SurfaceEncoderTest( ): mYuvTexWidth(64), mYuvTexHeight(66) { }
+    sp<MPEG4Writer>  setUpWriter(OMXClient &client );
+    void oneBufferPass(int width, int height );
+    static void fillYV12Buffer(uint8_t* buf, int w, int h, int stride) ;
+    static void fillYV12BufferRect(uint8_t* buf, int w, int h,
+                        int stride, const android_native_rect_t& rect) ;
+protected:
+
+    virtual void SetUp() {
+        mSE = new SurfaceEncoder(mYuvTexWidth, mYuvTexHeight);
+        mSE->setSynchronousMode(true);
+        mSTC = new SurfaceTextureClient(mSE);
+        mANW = mSTC;
+
+    }
+
+
+    virtual void TearDown() {
+        mSE.clear();
+        mSTC.clear();
+        mANW.clear();
+    }
+
+    const int mYuvTexWidth;//  = 64;
+    const int mYuvTexHeight;// = 66;
+
+    sp<SurfaceEncoder> mSE;
+    sp<SurfaceTextureClient> mSTC;
+    sp<ANativeWindow> mANW;
+
+};
+
+void SurfaceEncoderTest::oneBufferPass(int width, int height ) {
+    LOGV("One Buffer Pass");
+    ANativeWindowBuffer* anb;
+    ASSERT_EQ(NO_ERROR, mANW->dequeueBuffer(mANW.get(), &anb));
+    ASSERT_TRUE(anb != NULL);
+
+    sp<GraphicBuffer> buf(new GraphicBuffer(anb, false));
+    ASSERT_EQ(NO_ERROR, mANW->lockBuffer(mANW.get(), buf->getNativeBuffer()));
+
+    // Fill the buffer with the a checkerboard pattern
+    uint8_t* img = NULL;
+    buf->lock(GRALLOC_USAGE_SW_WRITE_OFTEN, (void**)(&img));
+    SurfaceEncoderTest::fillYV12Buffer(img, width, height, buf->getStride());
+    buf->unlock();
+
+    ASSERT_EQ(NO_ERROR, mANW->queueBuffer(mANW.get(), buf->getNativeBuffer()));
+}
+
+sp<MPEG4Writer> SurfaceEncoderTest::setUpWriter(OMXClient &client ) {
+    // Writing to a file
+    const char *fileName = "/sdcard/outputSurfEnc.mp4";
+    sp<MetaData> enc_meta = new MetaData;
+    enc_meta->setInt32(kKeyBitRate, 300000);
+    enc_meta->setInt32(kKeyFrameRate, 30);
+
+    enc_meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_VIDEO_MPEG4);
+
+    sp<MetaData> meta = mSE->getFormat();
+
+    int32_t width, height, stride, sliceHeight, colorFormat;
+    CHECK(meta->findInt32(kKeyWidth, &width));
+    CHECK(meta->findInt32(kKeyHeight, &height));
+    CHECK(meta->findInt32(kKeyStride, &stride));
+    CHECK(meta->findInt32(kKeySliceHeight, &sliceHeight));
+    CHECK(meta->findInt32(kKeyColorFormat, &colorFormat));
+
+    enc_meta->setInt32(kKeyWidth, width);
+    enc_meta->setInt32(kKeyHeight, height);
+    enc_meta->setInt32(kKeyIFramesInterval, 1);
+    enc_meta->setInt32(kKeyStride, stride);
+    enc_meta->setInt32(kKeySliceHeight, sliceHeight);
+    // TODO: overwriting the colorformat since the format set by GRAlloc
+    // could be wrong or not be read by OMX
+    enc_meta->setInt32(kKeyColorFormat, OMX_COLOR_FormatYUV420Planar);
+    // colorFormat);
+
+
+    sp<MediaSource> encoder =
+        OMXCodec::Create(
+                client.interface(), enc_meta, true /* createEncoder */, mSE);
+
+    sp<MPEG4Writer> writer = new MPEG4Writer(fileName);
+    writer->addSource(encoder);
+
+    return writer;
+}
+
+// Fill a YV12 buffer with a multi-colored checkerboard pattern
+void SurfaceEncoderTest::fillYV12Buffer(uint8_t* buf, int w, int h, int stride) {
+    const int blockWidth = w > 16 ? w / 16 : 1;
+    const int blockHeight = h > 16 ? h / 16 : 1;
+    const int yuvTexOffsetY = 0;
+    int yuvTexStrideY = stride;
+    int yuvTexOffsetV = yuvTexStrideY * h;
+    int yuvTexStrideV = (yuvTexStrideY/2 + 0xf) & ~0xf;
+    int yuvTexOffsetU = yuvTexOffsetV + yuvTexStrideV * h/2;
+    int yuvTexStrideU = yuvTexStrideV;
+    for (int x = 0; x < w; x++) {
+        for (int y = 0; y < h; y++) {
+            int parityX = (x / blockWidth) & 1;
+            int parityY = (y / blockHeight) & 1;
+            unsigned char intensity = (parityX ^ parityY) ? 63 : 191;
+            buf[yuvTexOffsetY + (y * yuvTexStrideY) + x] = intensity;
+            if (x < w / 2 && y < h / 2) {
+                buf[yuvTexOffsetU + (y * yuvTexStrideU) + x] = intensity;
+                if (x * 2 < w / 2 && y * 2 < h / 2) {
+                    buf[yuvTexOffsetV + (y*2 * yuvTexStrideV) + x*2 + 0] =
+                    buf[yuvTexOffsetV + (y*2 * yuvTexStrideV) + x*2 + 1] =
+                    buf[yuvTexOffsetV + ((y*2+1) * yuvTexStrideV) + x*2 + 0] =
+                    buf[yuvTexOffsetV + ((y*2+1) * yuvTexStrideV) + x*2 + 1] =
+                        intensity;
+                }
+            }
+        }
+    }
+}
+
+// Fill a YV12 buffer with red outside a given rectangle and green inside it.
+void SurfaceEncoderTest::fillYV12BufferRect(uint8_t* buf, int w,
+                  int h, int stride, const android_native_rect_t& rect) {
+    const int yuvTexOffsetY = 0;
+    int yuvTexStrideY = stride;
+    int yuvTexOffsetV = yuvTexStrideY * h;
+    int yuvTexStrideV = (yuvTexStrideY/2 + 0xf) & ~0xf;
+    int yuvTexOffsetU = yuvTexOffsetV + yuvTexStrideV * h/2;
+    int yuvTexStrideU = yuvTexStrideV;
+    for (int x = 0; x < w; x++) {
+        for (int y = 0; y < h; y++) {
+            bool inside = rect.left <= x && x < rect.right &&
+                    rect.top <= y && y < rect.bottom;
+            buf[yuvTexOffsetY + (y * yuvTexStrideY) + x] = inside ? 240 : 64;
+            if (x < w / 2 && y < h / 2) {
+                bool inside = rect.left <= 2*x && 2*x < rect.right &&
+                        rect.top <= 2*y && 2*y < rect.bottom;
+                buf[yuvTexOffsetU + (y * yuvTexStrideU) + x] = 16;
+                buf[yuvTexOffsetV + (y * yuvTexStrideV) + x] =
+                                                inside ? 16 : 255;
+            }
+        }
+    }
+}  ///////// End of class SurfaceEncoderTest
+
+///////////////////////////////////////////////////////////////////
+// Class to imitate the recording     /////////////////////////////
+// ////////////////////////////////////////////////////////////////
+struct SimpleDummyRecorder {
+        sp<MediaSource> mSource;
+
+        SimpleDummyRecorder
+                (const sp<MediaSource> &source): mSource(source) {}
+
+        status_t start() { return mSource->start();}
+        status_t stop()  { return mSource->stop();}
+
+        // fakes reading from a media source
+        status_t readFromSource() {
+            MediaBuffer *buffer;
+            status_t err = mSource->read(&buffer);
+            if (err != OK) {
+                return err;
+            }
+            buffer->release();
+            buffer = NULL;
+            return OK;
+        }
+};
+
+///////////////////////////////////////////////////////////////////
+//           TESTS
+// Just pass one buffer from the native_window to the SurfaceEncoder
+TEST_F(SurfaceEncoderTest, EncodingFromCpuFilledYV12BufferNpotOneBufferPass) {
+    LOGV("Testing OneBufferPass ******************************");
+
+    ASSERT_EQ(NO_ERROR, native_window_set_buffers_geometry(mANW.get(),
+            0, 0, HAL_PIXEL_FORMAT_YV12));
+                                // OMX_COLOR_FormatYUV420Planar)); // ));
+    ASSERT_EQ(NO_ERROR, native_window_set_usage(mANW.get(),
+            GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_SW_WRITE_OFTEN));
+
+    oneBufferPass(mYuvTexWidth, mYuvTexHeight);
+}
+
+// Pass the buffer with the wrong height and weight and should not be accepted
+TEST_F(SurfaceEncoderTest, EncodingFromCpuFilledYV12BufferNpotWrongSizeBufferPass) {
+    LOGV("Testing Wrong size BufferPass ******************************");
+
+    // setting the client side buffer size different than the server size
+    ASSERT_EQ(NO_ERROR, native_window_set_buffers_geometry(mANW.get(),
+             10, 10, HAL_PIXEL_FORMAT_YV12));
+                                // OMX_COLOR_FormatYUV420Planar)); // ));
+    ASSERT_EQ(NO_ERROR, native_window_set_usage(mANW.get(),
+            GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_SW_WRITE_OFTEN));
+
+    ANativeWindowBuffer* anb;
+
+    // make sure we get an error back when dequeuing!
+    ASSERT_NE(NO_ERROR, mANW->dequeueBuffer(mANW.get(), &anb));
+}
+
+
+// pass multiple buffers from the native_window the SurfaceEncoder
+// A dummy writer is used to simulate actual MPEG4Writer
+TEST_F(SurfaceEncoderTest,  EncodingFromCpuFilledYV12BufferNpotMultiBufferPass) {
+    LOGV("Testing MultiBufferPass, Dummy Recorder *********************");
+    ASSERT_EQ(NO_ERROR, native_window_set_buffers_geometry(mANW.get(),
+            0, 0, HAL_PIXEL_FORMAT_YV12));
+    ASSERT_EQ(NO_ERROR, native_window_set_usage(mANW.get(),
+            GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_SW_WRITE_OFTEN));
+
+    SimpleDummyRecorder writer(mSE);
+    writer.start();
+
+    int32_t nFramesCount = 0;
+    while (nFramesCount < 300) {
+        oneBufferPass(mYuvTexWidth, mYuvTexHeight);
+
+        ASSERT_EQ(NO_ERROR, writer.readFromSource());
+
+        nFramesCount++;
+    }
+    writer.stop();
+}
+
+// Delayed pass of multiple buffers from the native_window the SurfaceEncoder
+// A dummy writer is used to simulate actual MPEG4Writer
+TEST_F(SurfaceEncoderTest,  EncodingFromCpuFilledYV12BufferNpotMultiBufferPassLag) {
+    LOGV("Testing MultiBufferPass, Dummy Recorder Lagging **************");
+    ASSERT_EQ(NO_ERROR, native_window_set_buffers_geometry(mANW.get(),
+            0, 0, HAL_PIXEL_FORMAT_YV12));
+    ASSERT_EQ(NO_ERROR, native_window_set_usage(mANW.get(),
+            GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_SW_WRITE_OFTEN));
+
+    SimpleDummyRecorder writer(mSE);
+    writer.start();
+
+    int32_t nFramesCount = 1;
+    const int FRAMES_LAG = mSE->getBufferCount() - 1;
+    while (nFramesCount <= 300) {
+        oneBufferPass(mYuvTexWidth, mYuvTexHeight);
+        // Forcing the writer to lag behind a few frames
+        if (nFramesCount > FRAMES_LAG) {
+            ASSERT_EQ(NO_ERROR, writer.readFromSource());
+        }
+        nFramesCount++;
+    }
+    writer.stop();
+}
+
+// pass multiple buffers from the native_window the SurfaceEncoder
+// A dummy writer (MULTITHREADED) is used to simulate actual MPEG4Writer
+TEST_F(SurfaceEncoderTest, EncodingFromCpuFilledYV12BufferNpotMultiBufferPassThreaded) {
+    LOGV("Testing MultiBufferPass, Dummy Recorder Multi-Threaded **********");
+    ASSERT_EQ(NO_ERROR, native_window_set_buffers_geometry(mANW.get(),
+            0, 0, HAL_PIXEL_FORMAT_YV12));
+    ASSERT_EQ(NO_ERROR, native_window_set_usage(mANW.get(),
+            GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_SW_WRITE_OFTEN));
+
+    DummyRecorder writer(mSE);
+    writer.start();
+
+    int32_t nFramesCount = 0;
+    while (nFramesCount <= 300) {
+        oneBufferPass(mYuvTexWidth, mYuvTexHeight);
+
+        nFramesCount++;
+    }
+    writer.stop();
+}
+
+// Test to examine the actual encoding. Temporarily disabled till the
+// colorformat and encoding from GRAlloc data is resolved
+TEST_F(SurfaceEncoderTest, DISABLED_EncodingFromCpuFilledYV12BufferNpotWrite) {
+    LOGV("Testing the whole pipeline with actual Recorder");
+    ASSERT_EQ(NO_ERROR, native_window_set_buffers_geometry(mANW.get(),
+            0, 0, HAL_PIXEL_FORMAT_YV12)); // OMX_COLOR_FormatYUV420Planar)); // ));
+    ASSERT_EQ(NO_ERROR, native_window_set_usage(mANW.get(),
+            GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_SW_WRITE_OFTEN));
+
+    OMXClient client;
+    CHECK_EQ(OK, client.connect());
+
+    sp<MPEG4Writer> writer = setUpWriter(client);
+    int64_t start = systemTime();
+    CHECK_EQ(OK, writer->start());
+
+    int32_t nFramesCount = 0;
+    while (nFramesCount <= 300) {
+        oneBufferPass(mYuvTexWidth, mYuvTexHeight);
+        nFramesCount++;
+    }
+
+    CHECK_EQ(OK, writer->stop());
+    writer.clear();
+    int64_t end = systemTime();
+    client.disconnect();
+}
+
+
+} // namespace android