Correct MediaCodec + Surface behavior

Assorted tweaks:
- Allow signalEndOfInputStream() before ACodec is in Executing
  state (added message to two more states).
- Return an error if signalEndOfInputStream() is called a second
  time on the same stream.
- Require AndroidOpaque color format in createInputSurface().
- Disallow dequeueInputBuffer() after an input surface has been
  created (boolean flag in MediaCodec tracks it).
- Discard input surface when encoder is re-configure()ed (drop
  OMXNodeInstance's ref when we go back to Loaded).

Bug 7991062

Change-Id: Iff30f3036e14eb5a2f6536910dcf11aba33031ee

1 files changed, 14 insertions, 6 deletions

diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index 79ea04c..0d89c0f 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -69,7 +69,8 @@ MediaCodec::MediaCodec(const sp<ALooper> &looper)
       mDequeueInputTimeoutGeneration(0),
       mDequeueInputReplyID(0),
       mDequeueOutputTimeoutGeneration(0),
-      mDequeueOutputReplyID(0) {
+      mDequeueOutputReplyID(0),
+      mHaveInputSurface(false) {
 }
 
 MediaCodec::~MediaCodec() {
@@ -160,8 +161,6 @@ status_t MediaCodec::createInputSurface(
         sp<IGraphicBufferProducer>* bufferProducer) {
     sp<AMessage> msg = new AMessage(kWhatCreateInputSurface, id());
 
-    // TODO(fadden): require MediaFormat colorFormat == AndroidOpaque
-
     sp<AMessage> response;
     status_t err = PostAndAwaitResponse(msg, &response);
     if (err == NO_ERROR) {
@@ -256,8 +255,6 @@ status_t MediaCodec::queueSecureInputBuffer(
 }
 
 status_t MediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) {
-    // TODO(fadden): fail if an input Surface has been configured
-
     sp<AMessage> msg = new AMessage(kWhatDequeueInputBuffer, id());
     msg->setInt64("timeoutUs", timeoutUs);
 
@@ -604,6 +601,9 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
                     CHECK_EQ(mState, CONFIGURING);
                     setState(CONFIGURED);
 
+                    // reset input surface flag
+                    mHaveInputSurface = false;
+
                     (new AMessage)->postReply(mReplyID);
                     break;
                 }
@@ -618,6 +618,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
                         msg->findObject("input-surface", &obj);
                         CHECK(obj != NULL);
                         response->setObject("input-surface", obj);
+                        mHaveInputSurface = true;
                     } else {
                         response->setInt32("err", err);
                     }
@@ -1029,10 +1030,17 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
 
         case kWhatDequeueInputBuffer:
         {
-            // TODO(fadden): make this fail if we're using an input Surface
             uint32_t replyID;
             CHECK(msg->senderAwaitsResponse(&replyID));
 
+            if (mHaveInputSurface) {
+                ALOGE("dequeueInputBuffer can't be used with input surface");
+                sp<AMessage> response = new AMessage;
+                response->setInt32("err", INVALID_OPERATION);
+                response->postReply(replyID);
+                break;
+            }
+
             if (handleDequeueInputBuffer(replyID, true /* new request */)) {
                 break;
             }