Merge "Optionally repeat the previously submitted frame to the encoder" into klp-dev

4 files changed, 244 insertions, 9 deletions

diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index 36549d1..7b0bce0 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -369,7 +369,8 @@ ACodec::ACodec()
       mChannelMask(0),
       mDequeueCounter(0),
       mStoreMetaDataInOutputBuffers(false),
-      mMetaDataBuffersToSubmit(0) {
+      mMetaDataBuffersToSubmit(0),
+      mRepeatFrameDelayUs(-1ll) {
     mUninitializedState = new UninitializedState(this);
     mLoadedState = new LoadedState(this);
     mLoadedToIdleState = new LoadedToIdleState(this);
@@ -1089,6 +1090,12 @@ status_t ACodec::configureCodec(
         } else {
             mUseMetadataOnEncoderOutput = enable;
         }
+
+        if (!msg->findInt64(
+                    "repeat-previous-frame-after",
+                    &mRepeatFrameDelayUs)) {
+            mRepeatFrameDelayUs = -1ll;
+        }
     }
 
     // Always try to enable dynamic output buffers on native surface
@@ -3611,6 +3618,7 @@ void ACodec::LoadedState::stateEntered() {
 
     mCodec->mDequeueCounter = 0;
     mCodec->mMetaDataBuffersToSubmit = 0;
+    mCodec->mRepeatFrameDelayUs = -1ll;
 
     if (mCodec->mShutdownInProgress) {
         bool keepComponentAllocated = mCodec->mKeepComponentAllocated;
@@ -3742,6 +3750,23 @@ void ACodec::LoadedState::onCreateInputSurface(
 
     err = mCodec->mOMX->createInputSurface(mCodec->mNode, kPortIndexInput,
             &bufferProducer);
+
+    if (err == OK && mCodec->mRepeatFrameDelayUs > 0ll) {
+        err = mCodec->mOMX->setInternalOption(
+                mCodec->mNode,
+                kPortIndexInput,
+                IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY,
+                &mCodec->mRepeatFrameDelayUs,
+                sizeof(mCodec->mRepeatFrameDelayUs));
+
+        if (err != OK) {
+            ALOGE("[%s] Unable to configure option to repeat previous "
+                  "frames (err %d)",
+                  mCodec->mComponentName.c_str(),
+                  err);
+        }
+    }
+
     if (err == OK) {
         notify->setObject("input-surface",
                 new BufferProducerWrapper(bufferProducer));
diff --git a/media/libstagefright/omx/GraphicBufferSource.cpp b/media/libstagefright/omx/GraphicBufferSource.cpp
index 325ffcf..cf43e94 100644
--- a/media/libstagefright/omx/GraphicBufferSource.cpp
+++ b/media/libstagefright/omx/GraphicBufferSource.cpp
@@ -22,6 +22,7 @@
 
 #include <OMX_Core.h>
 #include <media/stagefright/foundation/ADebug.h>
+#include <media/stagefright/foundation/AMessage.h>
 
 #include <media/hardware/MetadataBufferType.h>
 #include <ui/GraphicBuffer.h>
@@ -39,7 +40,13 @@ GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance,
     mSuspended(false),
     mNumFramesAvailable(0),
     mEndOfStream(false),
-    mEndOfStreamSent(false) {
+    mEndOfStreamSent(false),
+    mRepeatAfterUs(-1ll),
+    mRepeatLastFrameGeneration(0),
+    mLatestSubmittedBufferId(-1),
+    mLatestSubmittedBufferFrameNum(0),
+    mLatestSubmittedBufferUseCount(0),
+    mRepeatBufferDeferred(false) {
 
     ALOGV("GraphicBufferSource w=%u h=%u c=%u",
             bufferWidth, bufferHeight, bufferCount);
@@ -123,6 +130,22 @@ void GraphicBufferSource::omxExecuting() {
     if (mEndOfStream && mNumFramesAvailable == 0) {
         submitEndOfInputStream_l();
     }
+
+    if (mRepeatAfterUs > 0ll && mLooper == NULL) {
+        mReflector = new AHandlerReflector<GraphicBufferSource>(this);
+
+        mLooper = new ALooper;
+        mLooper->registerHandler(mReflector);
+        mLooper->start();
+
+        if (mLatestSubmittedBufferId >= 0) {
+            sp<AMessage> msg =
+                new AMessage(kWhatRepeatLastFrame, mReflector->id());
+
+            msg->setInt32("generation", ++mRepeatLastFrameGeneration);
+            msg->post(mRepeatAfterUs);
+        }
+    }
 }
 
 void GraphicBufferSource::omxLoaded(){
@@ -132,6 +155,14 @@ void GraphicBufferSource::omxLoaded(){
         ALOGW("Dropped back down to Loaded without Executing");
     }
 
+    if (mLooper != NULL) {
+        mLooper->unregisterHandler(mReflector->id());
+        mReflector.clear();
+
+        mLooper->stop();
+        mLooper.clear();
+    }
+
     ALOGV("--> loaded; avail=%d eos=%d eosSent=%d",
             mNumFramesAvailable, mEndOfStream, mEndOfStreamSent);
 
@@ -211,8 +242,12 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) {
         ALOGV("cbi %d matches bq slot %d, handle=%p",
                 cbi, id, mBufferSlot[id]->handle);
 
-        mBufferQueue->releaseBuffer(id, codecBuffer.mFrameNumber,
-                EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE);
+        if (id == mLatestSubmittedBufferId) {
+            CHECK_GT(mLatestSubmittedBufferUseCount--, 0);
+        } else {
+            mBufferQueue->releaseBuffer(id, codecBuffer.mFrameNumber,
+                    EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE);
+        }
     } else {
         ALOGV("codecBufferEmptied: no match for emptied buffer in cbi %d",
                 cbi);
@@ -232,7 +267,16 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) {
         // send that.
         ALOGV("buffer freed, EOS pending");
         submitEndOfInputStream_l();
+    } else if (mRepeatBufferDeferred) {
+        bool success = repeatLatestSubmittedBuffer_l();
+        if (success) {
+            ALOGV("deferred repeatLatestSubmittedBuffer_l SUCCESS");
+        } else {
+            ALOGV("deferred repeatLatestSubmittedBuffer_l FAILURE");
+        }
+        mRepeatBufferDeferred = false;
     }
+
     return;
 }
 
@@ -264,6 +308,16 @@ void GraphicBufferSource::suspend(bool suspend) {
     }
 
     mSuspended = false;
+
+    if (mExecuting && mNumFramesAvailable == 0 && mRepeatBufferDeferred) {
+        if (repeatLatestSubmittedBuffer_l()) {
+            ALOGV("suspend/deferred repeatLatestSubmittedBuffer_l SUCCESS");
+
+            mRepeatBufferDeferred = false;
+        } else {
+            ALOGV("suspend/deferred repeatLatestSubmittedBuffer_l FAILURE");
+        }
+    }
 }
 
 bool GraphicBufferSource::fillCodecBuffer_l() {
@@ -318,11 +372,68 @@ bool GraphicBufferSource::fillCodecBuffer_l() {
                 EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE);
     } else {
         ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi);
+        setLatestSubmittedBuffer_l(item);
+    }
+
+    return true;
+}
+
+bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() {
+    CHECK(mExecuting && mNumFramesAvailable == 0);
+
+    if (mLatestSubmittedBufferId < 0 || mSuspended) {
+        return false;
+    }
+
+    int cbi = findAvailableCodecBuffer_l();
+    if (cbi < 0) {
+        // No buffers available, bail.
+        ALOGV("repeatLatestSubmittedBuffer_l: no codec buffers.");
+        return false;
+    }
+
+    BufferQueue::BufferItem item;
+    item.mBuf = mLatestSubmittedBufferId;
+    item.mFrameNumber = mLatestSubmittedBufferFrameNum;
+
+    status_t err = submitBuffer_l(item, cbi);
+
+    if (err != OK) {
+        return false;
     }
 
+    ++mLatestSubmittedBufferUseCount;
+
     return true;
 }
 
+void GraphicBufferSource::setLatestSubmittedBuffer_l(
+        const BufferQueue::BufferItem &item) {
+    ALOGV("setLatestSubmittedBuffer_l");
+
+    if (mLatestSubmittedBufferId >= 0) {
+        if (mLatestSubmittedBufferUseCount == 0) {
+            mBufferQueue->releaseBuffer(
+                    mLatestSubmittedBufferId,
+                    mLatestSubmittedBufferFrameNum,
+                    EGL_NO_DISPLAY,
+                    EGL_NO_SYNC_KHR,
+                    Fence::NO_FENCE);
+        }
+    }
+
+    mLatestSubmittedBufferId = item.mBuf;
+    mLatestSubmittedBufferFrameNum = item.mFrameNumber;
+    mLatestSubmittedBufferUseCount = 1;
+    mRepeatBufferDeferred = false;
+
+    if (mReflector != NULL) {
+        sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id());
+        msg->setInt32("generation", ++mRepeatLastFrameGeneration);
+        msg->post(mRepeatAfterUs);
+    }
+}
+
 status_t GraphicBufferSource::signalEndOfInputStream() {
     Mutex::Autolock autoLock(mMutex);
     ALOGV("signalEndOfInputStream: exec=%d avail=%d eos=%d",
@@ -470,6 +581,9 @@ void GraphicBufferSource::onFrameAvailable() {
 
     mNumFramesAvailable++;
 
+    mRepeatBufferDeferred = false;
+    ++mRepeatLastFrameGeneration;
+
     if (mExecuting) {
         fillCodecBuffer_l();
     }
@@ -495,4 +609,51 @@ void GraphicBufferSource::onBuffersReleased() {
     }
 }
 
+status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs(
+        int64_t repeatAfterUs) {
+    Mutex::Autolock autoLock(mMutex);
+
+    if (mExecuting || repeatAfterUs <= 0ll) {
+        return INVALID_OPERATION;
+    }
+
+    mRepeatAfterUs = repeatAfterUs;
+
+    return OK;
+}
+
+void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) {
+    switch (msg->what()) {
+        case kWhatRepeatLastFrame:
+        {
+            Mutex::Autolock autoLock(mMutex);
+
+            int32_t generation;
+            CHECK(msg->findInt32("generation", &generation));
+
+            if (generation != mRepeatLastFrameGeneration) {
+                // stale
+                break;
+            }
+
+            if (!mExecuting || mNumFramesAvailable > 0) {
+                break;
+            }
+
+            bool success = repeatLatestSubmittedBuffer_l();
+
+            if (success) {
+                ALOGV("repeatLatestSubmittedBuffer_l SUCCESS");
+            } else {
+                ALOGV("repeatLatestSubmittedBuffer_l FAILURE");
+                mRepeatBufferDeferred = true;
+            }
+            break;
+        }
+
+        default:
+            TRESPASS();
+    }
+}
+
 }  // namespace android
diff --git a/media/libstagefright/omx/GraphicBufferSource.h b/media/libstagefright/omx/GraphicBufferSource.h
index ac73770..244a843 100644
--- a/media/libstagefright/omx/GraphicBufferSource.h
+++ b/media/libstagefright/omx/GraphicBufferSource.h
@@ -25,6 +25,8 @@
 #include <OMX_Core.h>
 #include "../include/OMXNodeInstance.h"
 #include <media/stagefright/foundation/ABase.h>
+#include <media/stagefright/foundation/AHandlerReflector.h>
+#include <media/stagefright/foundation/ALooper.h>
 
 namespace android {
 
@@ -89,6 +91,15 @@ public:
     // in the BufferQueue) will be discarded until the suspension is lifted.
     void suspend(bool suspend);
 
+    // Specifies the interval after which we requeue the buffer previously
+    // queued to the encoder. This is useful in the case of surface flinger
+    // providing the input surface if the resulting encoded stream is to
+    // be displayed "live". If we were not to push through the extra frame
+    // the decoder on the remote end would be unable to decode the latest frame.
+    // This API must be called before transitioning the encoder to "executing"
+    // state and once this behaviour is specified it cannot be reset.
+    status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs);
+
 protected:
     // BufferQueue::ConsumerListener interface, called when a new frame of
     // data is available.  If we're executing and a codec buffer is
@@ -147,6 +158,9 @@ private:
     // doing anything if we don't have a codec buffer available.
     void submitEndOfInputStream_l();
 
+    void setLatestSubmittedBuffer_l(const BufferQueue::BufferItem &item);
+    bool repeatLatestSubmittedBuffer_l();
+
     // Lock, covers all member variables.
     mutable Mutex mMutex;
 
@@ -181,6 +195,30 @@ private:
     // Tracks codec buffers.
     Vector<CodecBuffer> mCodecBuffers;
 
+    ////
+    friend class AHandlerReflector<GraphicBufferSource>;
+
+    enum {
+        kWhatRepeatLastFrame,
+    };
+
+    int64_t mRepeatAfterUs;
+
+    sp<ALooper> mLooper;
+    sp<AHandlerReflector<GraphicBufferSource> > mReflector;
+
+    int32_t mRepeatLastFrameGeneration;
+
+    int mLatestSubmittedBufferId;
+    uint64_t mLatestSubmittedBufferFrameNum;
+    int32_t mLatestSubmittedBufferUseCount;
+
+    // The previously submitted buffer should've been repeated but
+    // no codec buffer was available at the time.
+    bool mRepeatBufferDeferred;
+
+    void onMessageReceived(const sp<AMessage> &msg);
+
     DISALLOW_EVIL_CONSTRUCTORS(GraphicBufferSource);
 };
 
diff --git a/media/libstagefright/omx/OMXNodeInstance.cpp b/media/libstagefright/omx/OMXNodeInstance.cpp
index 8d100f1..ef683a0 100644
--- a/media/libstagefright/omx/OMXNodeInstance.cpp
+++ b/media/libstagefright/omx/OMXNodeInstance.cpp
@@ -809,6 +809,7 @@ status_t OMXNodeInstance::setInternalOption(
         size_t size) {
     switch (type) {
         case IOMX::INTERNAL_OPTION_SUSPEND:
+        case IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY:
         {
             const sp<GraphicBufferSource> &bufferSource =
                 getGraphicBufferSource();
@@ -817,12 +818,22 @@ status_t OMXNodeInstance::setInternalOption(
                 return ERROR_UNSUPPORTED;
             }
 
-            if (size != sizeof(bool)) {
-                return INVALID_OPERATION;
-            }
+            if (type == IOMX::INTERNAL_OPTION_SUSPEND) {
+                if (size != sizeof(bool)) {
+                    return INVALID_OPERATION;
+                }
+
+                bool suspend = *(bool *)data;
+                bufferSource->suspend(suspend);
+            } else {
+                if (size != sizeof(int64_t)) {
+                    return INVALID_OPERATION;
+                }
+
+                int64_t delayUs = *(int64_t *)data;
 
-            bool suspend = *(bool *)data;
-            bufferSource->suspend(suspend);
+                return bufferSource->setRepeatPreviousFrameDelayUs(delayUs);
+            }
 
             return OK;
         }