am c69bc40a: Merge "Cap pts gap between adjacent frames to specified value" into klp-dev

* commit 'c69bc40a347c411751cb082b3eb4a82d976a68ec':
  Cap pts gap between adjacent frames to specified value

6 files changed, 169 insertions, 6 deletions

diff --git a/include/media/IOMX.h b/include/media/IOMX.h
index 9c8451c..6643736 100644
--- a/include/media/IOMX.h
+++ b/include/media/IOMX.h
@@ -142,6 +142,7 @@ public:
     enum InternalOptionType {
         INTERNAL_OPTION_SUSPEND,  // data is a bool
         INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY,  // data is an int64_t
+        INTERNAL_OPTION_MAX_TIMESTAMP_GAP, // data is int64_t
     };
     virtual status_t setInternalOption(
             node_id node,
diff --git a/include/media/stagefright/ACodec.h b/include/media/stagefright/ACodec.h
index 510c482..bf3a998 100644
--- a/include/media/stagefright/ACodec.h
+++ b/include/media/stagefright/ACodec.h
@@ -205,6 +205,7 @@ private:
     int32_t mMetaDataBuffersToSubmit;
 
     int64_t mRepeatFrameDelayUs;
+    int64_t mMaxPtsGapUs;
 
     status_t setCyclicIntraMacroblockRefresh(const sp<AMessage> &msg, int32_t mode);
     status_t allocateBuffersOnPort(OMX_U32 portIndex);
diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index fd81a31..a0bde58 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -371,7 +371,8 @@ ACodec::ACodec()
       mDequeueCounter(0),
       mStoreMetaDataInOutputBuffers(false),
       mMetaDataBuffersToSubmit(0),
-      mRepeatFrameDelayUs(-1ll) {
+      mRepeatFrameDelayUs(-1ll),
+      mMaxPtsGapUs(-1l) {
     mUninitializedState = new UninitializedState(this);
     mLoadedState = new LoadedState(this);
     mLoadedToIdleState = new LoadedToIdleState(this);
@@ -1109,6 +1110,10 @@ status_t ACodec::configureCodec(
                     &mRepeatFrameDelayUs)) {
             mRepeatFrameDelayUs = -1ll;
         }
+
+        if (!msg->findInt64("max-pts-gap-to-encoder", &mMaxPtsGapUs)) {
+            mMaxPtsGapUs = -1l;
+        }
     }
 
     // Always try to enable dynamic output buffers on native surface
@@ -3855,6 +3860,21 @@ void ACodec::LoadedState::onCreateInputSurface(
         }
     }
 
+    if (err == OK && mCodec->mMaxPtsGapUs > 0l) {
+        err = mCodec->mOMX->setInternalOption(
+                mCodec->mNode,
+                kPortIndexInput,
+                IOMX::INTERNAL_OPTION_MAX_TIMESTAMP_GAP,
+                &mCodec->mMaxPtsGapUs,
+                sizeof(mCodec->mMaxPtsGapUs));
+
+        if (err != OK) {
+            ALOGE("[%s] Unable to configure max timestamp gap (err %d)",
+                  mCodec->mComponentName.c_str(),
+                  err);
+        }
+    }
+
     if (err == OK) {
         notify->setObject("input-surface",
                 new BufferProducerWrapper(bufferProducer));
diff --git a/media/libstagefright/omx/GraphicBufferSource.cpp b/media/libstagefright/omx/GraphicBufferSource.cpp
index 7492577..20fa7ce 100644
--- a/media/libstagefright/omx/GraphicBufferSource.cpp
+++ b/media/libstagefright/omx/GraphicBufferSource.cpp
@@ -42,7 +42,11 @@ GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance,
     mEndOfStream(false),
     mEndOfStreamSent(false),
     mRepeatAfterUs(-1ll),
+    mMaxTimestampGapUs(-1ll),
+    mPrevOriginalTimeUs(-1ll),
+    mPrevModifiedTimeUs(-1ll),
     mRepeatLastFrameGeneration(0),
+    mRepeatLastFrameTimestamp(-1ll),
     mLatestSubmittedBufferId(-1),
     mLatestSubmittedBufferFrameNum(0),
     mLatestSubmittedBufferUseCount(0),
@@ -299,6 +303,32 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) {
     return;
 }
 
+void GraphicBufferSource::codecBufferFilled(OMX_BUFFERHEADERTYPE* header) {
+    Mutex::Autolock autoLock(mMutex);
+
+    if (mMaxTimestampGapUs > 0ll
+            && !(header->nFlags & OMX_BUFFERFLAG_CODECCONFIG)) {
+        ssize_t index = mOriginalTimeUs.indexOfKey(header->nTimeStamp);
+        if (index >= 0) {
+            ALOGV("OUT timestamp: %lld -> %lld",
+                    header->nTimeStamp, mOriginalTimeUs[index]);
+            header->nTimeStamp = mOriginalTimeUs[index];
+            mOriginalTimeUs.removeItemsAt(index);
+        } else {
+            // giving up the effort as encoder doesn't appear to preserve pts
+            ALOGW("giving up limiting timestamp gap (pts = %lld)",
+                    header->nTimeStamp);
+            mMaxTimestampGapUs = -1ll;
+        }
+        if (mOriginalTimeUs.size() > BufferQueue::NUM_BUFFER_SLOTS) {
+            // something terribly wrong must have happened, giving up...
+            ALOGE("mOriginalTimeUs has too many entries (%d)",
+                    mOriginalTimeUs.size());
+            mMaxTimestampGapUs = -1ll;
+        }
+    }
+}
+
 void GraphicBufferSource::suspend(bool suspend) {
     Mutex::Autolock autoLock(mMutex);
 
@@ -431,6 +461,7 @@ bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() {
     BufferQueue::BufferItem item;
     item.mBuf = mLatestSubmittedBufferId;
     item.mFrameNumber = mLatestSubmittedBufferFrameNum;
+    item.mTimestamp = mRepeatLastFrameTimestamp;
 
     status_t err = submitBuffer_l(item, cbi);
 
@@ -440,6 +471,20 @@ bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() {
 
     ++mLatestSubmittedBufferUseCount;
 
+    /* repeat last frame up to kRepeatLastFrameCount times.
+     * in case of static scene, a single repeat might not get rid of encoder
+     * ghosting completely, refresh a couple more times to get better quality
+     */
+    if (--mRepeatLastFrameCount > 0) {
+        mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
+
+        if (mReflector != NULL) {
+            sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id());
+            msg->setInt32("generation", ++mRepeatLastFrameGeneration);
+            msg->post(mRepeatAfterUs);
+        }
+    }
+
     return true;
 }
 
@@ -460,8 +505,11 @@ void GraphicBufferSource::setLatestSubmittedBuffer_l(
 
     mLatestSubmittedBufferId = item.mBuf;
     mLatestSubmittedBufferFrameNum = item.mFrameNumber;
+    mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
+
     mLatestSubmittedBufferUseCount = 1;
     mRepeatBufferDeferred = false;
+    mRepeatLastFrameCount = kRepeatLastFrameCount;
 
     if (mReflector != NULL) {
         sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id());
@@ -497,6 +545,39 @@ status_t GraphicBufferSource::signalEndOfInputStream() {
     return OK;
 }
 
+int64_t GraphicBufferSource::getTimestamp(const BufferQueue::BufferItem &item) {
+    int64_t timeUs = item.mTimestamp / 1000;
+
+    if (mMaxTimestampGapUs > 0ll) {
+        /* Cap timestamp gap between adjacent frames to specified max
+         *
+         * In the scenario of cast mirroring, encoding could be suspended for
+         * prolonged periods. Limiting the pts gap to workaround the problem
+         * where encoder's rate control logic produces huge frames after a
+         * long period of suspension.
+         */
+
+        int64_t originalTimeUs = timeUs;
+        if (mPrevOriginalTimeUs >= 0ll) {
+            if (originalTimeUs < mPrevOriginalTimeUs) {
+                // Drop the frame if it's going backward in time. Bad timestamp
+                // could disrupt encoder's rate control completely.
+                ALOGV("Dropping frame that's going backward in time");
+                return -1;
+            }
+            int64_t timestampGapUs = originalTimeUs - mPrevOriginalTimeUs;
+            timeUs = (timestampGapUs < mMaxTimestampGapUs ?
+                    timestampGapUs : mMaxTimestampGapUs) + mPrevModifiedTimeUs;
+        }
+        mPrevOriginalTimeUs = originalTimeUs;
+        mPrevModifiedTimeUs = timeUs;
+        mOriginalTimeUs.add(timeUs, originalTimeUs);
+        ALOGV("IN  timestamp: %lld -> %lld", originalTimeUs, timeUs);
+    }
+
+    return timeUs;
+}
+
 status_t GraphicBufferSource::submitBuffer_l(
         const BufferQueue::BufferItem &item, int cbi) {
     ALOGV("submitBuffer_l cbi=%d", cbi);
@@ -513,9 +594,15 @@ status_t GraphicBufferSource::submitBuffer_l(
     memcpy(data, &type, 4);
     memcpy(data + 4, &handle, sizeof(buffer_handle_t));
 
+    int64_t timeUs = getTimestamp(item);
+    if (timeUs < 0ll) {
+        ALOGE("Dropping frame with bad timestamp");
+        return UNKNOWN_ERROR;
+    }
+
     status_t err = mNodeInstance->emptyDirectBuffer(header, 0,
             4 + sizeof(buffer_handle_t), OMX_BUFFERFLAG_ENDOFFRAME,
-            item.mTimestamp / 1000);
+            timeUs);
     if (err != OK) {
         ALOGW("WARNING: emptyDirectBuffer failed: 0x%x", err);
         codecBuffer.mGraphicBuffer = NULL;
@@ -664,6 +751,17 @@ status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs(
     return OK;
 }
 
+status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) {
+    Mutex::Autolock autoLock(mMutex);
+
+    if (mExecuting || maxGapUs <= 0ll) {
+        return INVALID_OPERATION;
+    }
+
+    mMaxTimestampGapUs = maxGapUs;
+
+    return OK;
+}
 void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) {
     switch (msg->what()) {
         case kWhatRepeatLastFrame:
diff --git a/media/libstagefright/omx/GraphicBufferSource.h b/media/libstagefright/omx/GraphicBufferSource.h
index 9e5eee6..3b0e454 100644
--- a/media/libstagefright/omx/GraphicBufferSource.h
+++ b/media/libstagefright/omx/GraphicBufferSource.h
@@ -87,6 +87,10 @@ public:
     // fill it with a new frame of data; otherwise, just mark it as available.
     void codecBufferEmptied(OMX_BUFFERHEADERTYPE* header);
 
+    // Called when omx_message::FILL_BUFFER_DONE is received. (Currently the
+    // buffer source will fix timestamp in the header if needed.)
+    void codecBufferFilled(OMX_BUFFERHEADERTYPE* header);
+
     // This is called after the last input frame has been submitted.  We
     // need to submit an empty buffer with the EOS flag set.  If we don't
     // have a codec buffer ready, we just set the mEndOfStream flag.
@@ -105,6 +109,15 @@ public:
     // state and once this behaviour is specified it cannot be reset.
     status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs);
 
+    // When set, the timestamp fed to the encoder will be modified such that
+    // the gap between two adjacent frames is capped at maxGapUs. Timestamp
+    // will be restored to the original when the encoded frame is returned to
+    // the client.
+    // This is to solve a problem in certain real-time streaming case, where
+    // encoder's rate control logic produces huge frames after a long period
+    // of suspension on input.
+    status_t setMaxTimestampGapUs(int64_t maxGapUs);
+
 protected:
     // BufferQueue::ConsumerListener interface, called when a new frame of
     // data is available.  If we're executing and a codec buffer is
@@ -165,6 +178,7 @@ private:
 
     void setLatestSubmittedBuffer_l(const BufferQueue::BufferItem &item);
     bool repeatLatestSubmittedBuffer_l();
+    int64_t getTimestamp(const BufferQueue::BufferItem &item);
 
     // Lock, covers all member variables.
     mutable Mutex mMutex;
@@ -206,13 +220,22 @@ private:
     enum {
         kWhatRepeatLastFrame,
     };
-
+    enum {
+        kRepeatLastFrameCount = 10,
+    };
     int64_t mRepeatAfterUs;
+    int64_t mMaxTimestampGapUs;
+
+    KeyedVector<int64_t, int64_t> mOriginalTimeUs;
+    int64_t mPrevOriginalTimeUs;
+    int64_t mPrevModifiedTimeUs;
 
     sp<ALooper> mLooper;
     sp<AHandlerReflector<GraphicBufferSource> > mReflector;
 
     int32_t mRepeatLastFrameGeneration;
+    int64_t mRepeatLastFrameTimestamp;
+    int32_t mRepeatLastFrameCount;
 
     int mLatestSubmittedBufferId;
     uint64_t mLatestSubmittedBufferFrameNum;
diff --git a/media/libstagefright/omx/OMXNodeInstance.cpp b/media/libstagefright/omx/OMXNodeInstance.cpp
index 5f104fc..6c5c857 100644
--- a/media/libstagefright/omx/OMXNodeInstance.cpp
+++ b/media/libstagefright/omx/OMXNodeInstance.cpp
@@ -849,6 +849,7 @@ status_t OMXNodeInstance::setInternalOption(
     switch (type) {
         case IOMX::INTERNAL_OPTION_SUSPEND:
         case IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY:
+        case IOMX::INTERNAL_OPTION_MAX_TIMESTAMP_GAP:
         {
             const sp<GraphicBufferSource> &bufferSource =
                 getGraphicBufferSource();
@@ -864,7 +865,8 @@ status_t OMXNodeInstance::setInternalOption(
 
                 bool suspend = *(bool *)data;
                 bufferSource->suspend(suspend);
-            } else {
+            } else if (type ==
+                    IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY){
                 if (size != sizeof(int64_t)) {
                     return INVALID_OPERATION;
                 }
@@ -872,6 +874,14 @@ status_t OMXNodeInstance::setInternalOption(
                 int64_t delayUs = *(int64_t *)data;
 
                 return bufferSource->setRepeatPreviousFrameDelayUs(delayUs);
+            } else {
+                if (size != sizeof(int64_t)) {
+                    return INVALID_OPERATION;
+                }
+
+                int64_t maxGapUs = *(int64_t *)data;
+
+                return bufferSource->setMaxTimestampGapUs(maxGapUs);
             }
 
             return OK;
@@ -883,6 +893,8 @@ status_t OMXNodeInstance::setInternalOption(
 }
 
 void OMXNodeInstance::onMessage(const omx_message &msg) {
+    const sp<GraphicBufferSource>& bufferSource(getGraphicBufferSource());
+
     if (msg.type == omx_message::FILL_BUFFER_DONE) {
         OMX_BUFFERHEADERTYPE *buffer =
             static_cast<OMX_BUFFERHEADERTYPE *>(
@@ -892,10 +904,18 @@ void OMXNodeInstance::onMessage(const omx_message &msg) {
             static_cast<BufferMeta *>(buffer->pAppPrivate);
 
         buffer_meta->CopyFromOMX(buffer);
-    } else if (msg.type == omx_message::EMPTY_BUFFER_DONE) {
-        const sp<GraphicBufferSource>& bufferSource(getGraphicBufferSource());
 
         if (bufferSource != NULL) {
+            // fix up the buffer info (especially timestamp) if needed
+            bufferSource->codecBufferFilled(buffer);
+
+            omx_message newMsg = msg;
+            newMsg.u.extended_buffer_data.timestamp = buffer->nTimeStamp;
+            mObserver->onMessage(newMsg);
+            return;
+        }
+    } else if (msg.type == omx_message::EMPTY_BUFFER_DONE) {
+        if (bufferSource != NULL) {
             // This is one of the buffers used exclusively by
             // GraphicBufferSource.
             // Don't dispatch a message back to ACodec, since it doesn't