summaryrefslogtreecommitdiffstats
path: root/media/libstagefright
diff options
context:
space:
mode:
Diffstat (limited to 'media/libstagefright')
-rw-r--r--media/libstagefright/ACodec.cpp37
-rw-r--r--media/libstagefright/MPEG4Writer.cpp9
-rw-r--r--media/libstagefright/MediaCodec.cpp2
-rw-r--r--media/libstagefright/matroska/MatroskaExtractor.cpp80
-rw-r--r--media/libstagefright/omx/GraphicBufferSource.cpp100
-rw-r--r--media/libstagefright/omx/GraphicBufferSource.h25
-rw-r--r--media/libstagefright/omx/OMXNodeInstance.cpp26
7 files changed, 234 insertions, 45 deletions
diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index eb274a8..c9f89fc 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -365,6 +365,7 @@ ACodec::ACodec()
mIsEncoder(false),
mUseMetadataOnEncoderOutput(false),
mShutdownInProgress(false),
+ mIsConfiguredForAdaptivePlayback(false),
mEncoderDelay(0),
mEncoderPadding(0),
mChannelMaskPresent(false),
@@ -372,7 +373,8 @@ ACodec::ACodec()
mDequeueCounter(0),
mStoreMetaDataInOutputBuffers(false),
mMetaDataBuffersToSubmit(0),
- mRepeatFrameDelayUs(-1ll) {
+ mRepeatFrameDelayUs(-1ll),
+ mMaxPtsGapUs(-1l) {
mUninitializedState = new UninitializedState(this);
mLoadedState = new LoadedState(this);
mLoadedToIdleState = new LoadedToIdleState(this);
@@ -1114,6 +1116,10 @@ status_t ACodec::configureCodec(
&mRepeatFrameDelayUs)) {
mRepeatFrameDelayUs = -1ll;
}
+
+ if (!msg->findInt64("max-pts-gap-to-encoder", &mMaxPtsGapUs)) {
+ mMaxPtsGapUs = -1l;
+ }
}
// Always try to enable dynamic output buffers on native surface
@@ -1121,6 +1127,7 @@ status_t ACodec::configureCodec(
int32_t haveNativeWindow = msg->findObject("native-window", &obj) &&
obj != NULL;
mStoreMetaDataInOutputBuffers = false;
+ mIsConfiguredForAdaptivePlayback = false;
if (!encoder && video && haveNativeWindow) {
err = mOMX->storeMetaDataInBuffers(mNode, kPortIndexOutput, OMX_TRUE);
if (err != OK) {
@@ -1165,12 +1172,14 @@ status_t ACodec::configureCodec(
ALOGW_IF(err != OK,
"[%s] prepareForAdaptivePlayback failed w/ err %d",
mComponentName.c_str(), err);
+ mIsConfiguredForAdaptivePlayback = (err == OK);
}
// allow failure
err = OK;
} else {
ALOGV("[%s] storeMetaDataInBuffers succeeded", mComponentName.c_str());
mStoreMetaDataInOutputBuffers = true;
+ mIsConfiguredForAdaptivePlayback = true;
}
int32_t push;
@@ -3318,11 +3327,11 @@ void ACodec::BaseState::onInputBufferFilled(const sp<AMessage> &msg) {
mCodec->mInputEOSResult = err;
}
break;
-
- default:
- CHECK_EQ((int)mode, (int)FREE_BUFFERS);
- break;
}
+
+ default:
+ CHECK_EQ((int)mode, (int)FREE_BUFFERS);
+ break;
}
}
@@ -3498,7 +3507,7 @@ void ACodec::BaseState::onOutputBufferDrained(const sp<AMessage> &msg) {
int32_t render;
if (mCodec->mNativeWindow != NULL
&& msg->findInt32("render", &render) && render != 0
- && (info->mData == NULL || info->mData->size() != 0)) {
+ && info->mData != NULL && info->mData->size() != 0) {
// The client wants this buffer to be rendered.
status_t err;
@@ -3773,6 +3782,7 @@ void ACodec::LoadedState::stateEntered() {
mCodec->mDequeueCounter = 0;
mCodec->mMetaDataBuffersToSubmit = 0;
mCodec->mRepeatFrameDelayUs = -1ll;
+ mCodec->mIsConfiguredForAdaptivePlayback = false;
if (mCodec->mShutdownInProgress) {
bool keepComponentAllocated = mCodec->mKeepComponentAllocated;
@@ -3921,6 +3931,21 @@ void ACodec::LoadedState::onCreateInputSurface(
}
}
+ if (err == OK && mCodec->mMaxPtsGapUs > 0l) {
+ err = mCodec->mOMX->setInternalOption(
+ mCodec->mNode,
+ kPortIndexInput,
+ IOMX::INTERNAL_OPTION_MAX_TIMESTAMP_GAP,
+ &mCodec->mMaxPtsGapUs,
+ sizeof(mCodec->mMaxPtsGapUs));
+
+ if (err != OK) {
+ ALOGE("[%s] Unable to configure max timestamp gap (err %d)",
+ mCodec->mComponentName.c_str(),
+ err);
+ }
+ }
+
if (err == OK) {
notify->setObject("input-surface",
new BufferProducerWrapper(bufferProducer));
diff --git a/media/libstagefright/MPEG4Writer.cpp b/media/libstagefright/MPEG4Writer.cpp
index a0f17b5..b7a4b75 100644
--- a/media/libstagefright/MPEG4Writer.cpp
+++ b/media/libstagefright/MPEG4Writer.cpp
@@ -972,13 +972,16 @@ void MPEG4Writer::writeFtypBox(MetaData *param) {
if (param && param->findInt32(kKeyFileType, &fileType) &&
fileType != OUTPUT_FORMAT_MPEG_4) {
writeFourcc("3gp4");
+ writeInt32(0);
+ writeFourcc("isom");
+ writeFourcc("3gp4");
} else {
+ writeFourcc("mp42");
+ writeInt32(0);
writeFourcc("isom");
+ writeFourcc("mp42");
}
- writeInt32(0);
- writeFourcc("isom");
- writeFourcc("3gp4");
endBox();
}
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index 8af1aaf..c4c47b3 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -1680,7 +1680,7 @@ status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {
return -EACCES;
}
- if (render && (info->mData == NULL || info->mData->size() != 0)) {
+ if (render && info->mData != NULL && info->mData->size() != 0) {
info->mNotify->setInt32("render", true);
if (mSoftRenderer != NULL) {
diff --git a/media/libstagefright/matroska/MatroskaExtractor.cpp b/media/libstagefright/matroska/MatroskaExtractor.cpp
index d260d0f..dcb1cda 100644
--- a/media/libstagefright/matroska/MatroskaExtractor.cpp
+++ b/media/libstagefright/matroska/MatroskaExtractor.cpp
@@ -716,41 +716,61 @@ bool MatroskaExtractor::isLiveStreaming() const {
return mIsLiveStreaming;
}
+static int bytesForSize(size_t size) {
+ // use at most 28 bits (4 times 7)
+ CHECK(size <= 0xfffffff);
+
+ if (size > 0x1fffff) {
+ return 4;
+ } else if (size > 0x3fff) {
+ return 3;
+ } else if (size > 0x7f) {
+ return 2;
+ }
+ return 1;
+}
+
+static void storeSize(uint8_t *data, size_t &idx, size_t size) {
+ int numBytes = bytesForSize(size);
+ idx += numBytes;
+
+ data += idx;
+ size_t next = 0;
+ while (numBytes--) {
+ *--data = (size & 0x7f) | next;
+ size >>= 7;
+ next = 0x80;
+ }
+}
+
static void addESDSFromCodecPrivate(
const sp<MetaData> &meta,
bool isAudio, const void *priv, size_t privSize) {
- static const uint8_t kStaticESDS[] = {
- 0x03, 22,
- 0x00, 0x00, // ES_ID
- 0x00, // streamDependenceFlag, URL_Flag, OCRstreamFlag
-
- 0x04, 17,
- 0x40, // ObjectTypeIndication
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
- 0x00, 0x00, 0x00, 0x00,
-
- 0x05,
- // CodecSpecificInfo (with size prefix) follows
- };
- // Make sure all sizes can be coded in a single byte.
- CHECK(privSize + 22 - 2 < 128);
- size_t esdsSize = sizeof(kStaticESDS) + privSize + 1;
+ int privSizeBytesRequired = bytesForSize(privSize);
+ int esdsSize2 = 14 + privSizeBytesRequired + privSize;
+ int esdsSize2BytesRequired = bytesForSize(esdsSize2);
+ int esdsSize1 = 4 + esdsSize2BytesRequired + esdsSize2;
+ int esdsSize1BytesRequired = bytesForSize(esdsSize1);
+ size_t esdsSize = 1 + esdsSize1BytesRequired + esdsSize1;
uint8_t *esds = new uint8_t[esdsSize];
- memcpy(esds, kStaticESDS, sizeof(kStaticESDS));
- uint8_t *ptr = esds + sizeof(kStaticESDS);
- *ptr++ = privSize;
- memcpy(ptr, priv, privSize);
-
- // Increment by codecPrivateSize less 2 bytes that are accounted for
- // already in lengths of 22/17
- esds[1] += privSize - 2;
- esds[6] += privSize - 2;
-
- // Set ObjectTypeIndication.
- esds[7] = isAudio ? 0x40 // Audio ISO/IEC 14496-3
- : 0x20; // Visual ISO/IEC 14496-2
+
+ size_t idx = 0;
+ esds[idx++] = 0x03;
+ storeSize(esds, idx, esdsSize1);
+ esds[idx++] = 0x00; // ES_ID
+ esds[idx++] = 0x00; // ES_ID
+ esds[idx++] = 0x00; // streamDependenceFlag, URL_Flag, OCRstreamFlag
+ esds[idx++] = 0x04;
+ storeSize(esds, idx, esdsSize2);
+ esds[idx++] = isAudio ? 0x40 // Audio ISO/IEC 14496-3
+ : 0x20; // Visual ISO/IEC 14496-2
+ for (int i = 0; i < 12; i++) {
+ esds[idx++] = 0x00;
+ }
+ esds[idx++] = 0x05;
+ storeSize(esds, idx, privSize);
+ memcpy(esds + idx, priv, privSize);
meta->setData(kKeyESDS, 0, esds, esdsSize);
diff --git a/media/libstagefright/omx/GraphicBufferSource.cpp b/media/libstagefright/omx/GraphicBufferSource.cpp
index 7492577..20fa7ce 100644
--- a/media/libstagefright/omx/GraphicBufferSource.cpp
+++ b/media/libstagefright/omx/GraphicBufferSource.cpp
@@ -42,7 +42,11 @@ GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance,
mEndOfStream(false),
mEndOfStreamSent(false),
mRepeatAfterUs(-1ll),
+ mMaxTimestampGapUs(-1ll),
+ mPrevOriginalTimeUs(-1ll),
+ mPrevModifiedTimeUs(-1ll),
mRepeatLastFrameGeneration(0),
+ mRepeatLastFrameTimestamp(-1ll),
mLatestSubmittedBufferId(-1),
mLatestSubmittedBufferFrameNum(0),
mLatestSubmittedBufferUseCount(0),
@@ -299,6 +303,32 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) {
return;
}
+void GraphicBufferSource::codecBufferFilled(OMX_BUFFERHEADERTYPE* header) {
+ Mutex::Autolock autoLock(mMutex);
+
+ if (mMaxTimestampGapUs > 0ll
+ && !(header->nFlags & OMX_BUFFERFLAG_CODECCONFIG)) {
+ ssize_t index = mOriginalTimeUs.indexOfKey(header->nTimeStamp);
+ if (index >= 0) {
+ ALOGV("OUT timestamp: %lld -> %lld",
+ header->nTimeStamp, mOriginalTimeUs[index]);
+ header->nTimeStamp = mOriginalTimeUs[index];
+ mOriginalTimeUs.removeItemsAt(index);
+ } else {
+ // giving up the effort as encoder doesn't appear to preserve pts
+ ALOGW("giving up limiting timestamp gap (pts = %lld)",
+ header->nTimeStamp);
+ mMaxTimestampGapUs = -1ll;
+ }
+ if (mOriginalTimeUs.size() > BufferQueue::NUM_BUFFER_SLOTS) {
+ // something terribly wrong must have happened, giving up...
+ ALOGE("mOriginalTimeUs has too many entries (%d)",
+ mOriginalTimeUs.size());
+ mMaxTimestampGapUs = -1ll;
+ }
+ }
+}
+
void GraphicBufferSource::suspend(bool suspend) {
Mutex::Autolock autoLock(mMutex);
@@ -431,6 +461,7 @@ bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() {
BufferQueue::BufferItem item;
item.mBuf = mLatestSubmittedBufferId;
item.mFrameNumber = mLatestSubmittedBufferFrameNum;
+ item.mTimestamp = mRepeatLastFrameTimestamp;
status_t err = submitBuffer_l(item, cbi);
@@ -440,6 +471,20 @@ bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() {
++mLatestSubmittedBufferUseCount;
+ /* repeat last frame up to kRepeatLastFrameCount times.
+ * in case of static scene, a single repeat might not get rid of encoder
+ * ghosting completely, refresh a couple more times to get better quality
+ */
+ if (--mRepeatLastFrameCount > 0) {
+ mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
+
+ if (mReflector != NULL) {
+ sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id());
+ msg->setInt32("generation", ++mRepeatLastFrameGeneration);
+ msg->post(mRepeatAfterUs);
+ }
+ }
+
return true;
}
@@ -460,8 +505,11 @@ void GraphicBufferSource::setLatestSubmittedBuffer_l(
mLatestSubmittedBufferId = item.mBuf;
mLatestSubmittedBufferFrameNum = item.mFrameNumber;
+ mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000;
+
mLatestSubmittedBufferUseCount = 1;
mRepeatBufferDeferred = false;
+ mRepeatLastFrameCount = kRepeatLastFrameCount;
if (mReflector != NULL) {
sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id());
@@ -497,6 +545,39 @@ status_t GraphicBufferSource::signalEndOfInputStream() {
return OK;
}
+int64_t GraphicBufferSource::getTimestamp(const BufferQueue::BufferItem &item) {
+ int64_t timeUs = item.mTimestamp / 1000;
+
+ if (mMaxTimestampGapUs > 0ll) {
+ /* Cap timestamp gap between adjacent frames to specified max
+ *
+ * In the scenario of cast mirroring, encoding could be suspended for
+ * prolonged periods. Limiting the pts gap to workaround the problem
+ * where encoder's rate control logic produces huge frames after a
+ * long period of suspension.
+ */
+
+ int64_t originalTimeUs = timeUs;
+ if (mPrevOriginalTimeUs >= 0ll) {
+ if (originalTimeUs < mPrevOriginalTimeUs) {
+ // Drop the frame if it's going backward in time. Bad timestamp
+ // could disrupt encoder's rate control completely.
+ ALOGV("Dropping frame that's going backward in time");
+ return -1;
+ }
+ int64_t timestampGapUs = originalTimeUs - mPrevOriginalTimeUs;
+ timeUs = (timestampGapUs < mMaxTimestampGapUs ?
+ timestampGapUs : mMaxTimestampGapUs) + mPrevModifiedTimeUs;
+ }
+ mPrevOriginalTimeUs = originalTimeUs;
+ mPrevModifiedTimeUs = timeUs;
+ mOriginalTimeUs.add(timeUs, originalTimeUs);
+ ALOGV("IN timestamp: %lld -> %lld", originalTimeUs, timeUs);
+ }
+
+ return timeUs;
+}
+
status_t GraphicBufferSource::submitBuffer_l(
const BufferQueue::BufferItem &item, int cbi) {
ALOGV("submitBuffer_l cbi=%d", cbi);
@@ -513,9 +594,15 @@ status_t GraphicBufferSource::submitBuffer_l(
memcpy(data, &type, 4);
memcpy(data + 4, &handle, sizeof(buffer_handle_t));
+ int64_t timeUs = getTimestamp(item);
+ if (timeUs < 0ll) {
+ ALOGE("Dropping frame with bad timestamp");
+ return UNKNOWN_ERROR;
+ }
+
status_t err = mNodeInstance->emptyDirectBuffer(header, 0,
4 + sizeof(buffer_handle_t), OMX_BUFFERFLAG_ENDOFFRAME,
- item.mTimestamp / 1000);
+ timeUs);
if (err != OK) {
ALOGW("WARNING: emptyDirectBuffer failed: 0x%x", err);
codecBuffer.mGraphicBuffer = NULL;
@@ -664,6 +751,17 @@ status_t GraphicBufferSource::setRepeatPreviousFrameDelayUs(
return OK;
}
+status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) {
+ Mutex::Autolock autoLock(mMutex);
+
+ if (mExecuting || maxGapUs <= 0ll) {
+ return INVALID_OPERATION;
+ }
+
+ mMaxTimestampGapUs = maxGapUs;
+
+ return OK;
+}
void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) {
switch (msg->what()) {
case kWhatRepeatLastFrame:
diff --git a/media/libstagefright/omx/GraphicBufferSource.h b/media/libstagefright/omx/GraphicBufferSource.h
index 9e5eee6..3b0e454 100644
--- a/media/libstagefright/omx/GraphicBufferSource.h
+++ b/media/libstagefright/omx/GraphicBufferSource.h
@@ -87,6 +87,10 @@ public:
// fill it with a new frame of data; otherwise, just mark it as available.
void codecBufferEmptied(OMX_BUFFERHEADERTYPE* header);
+ // Called when omx_message::FILL_BUFFER_DONE is received. (Currently the
+ // buffer source will fix timestamp in the header if needed.)
+ void codecBufferFilled(OMX_BUFFERHEADERTYPE* header);
+
// This is called after the last input frame has been submitted. We
// need to submit an empty buffer with the EOS flag set. If we don't
// have a codec buffer ready, we just set the mEndOfStream flag.
@@ -105,6 +109,15 @@ public:
// state and once this behaviour is specified it cannot be reset.
status_t setRepeatPreviousFrameDelayUs(int64_t repeatAfterUs);
+ // When set, the timestamp fed to the encoder will be modified such that
+ // the gap between two adjacent frames is capped at maxGapUs. Timestamp
+ // will be restored to the original when the encoded frame is returned to
+ // the client.
+ // This is to solve a problem in certain real-time streaming case, where
+ // encoder's rate control logic produces huge frames after a long period
+ // of suspension on input.
+ status_t setMaxTimestampGapUs(int64_t maxGapUs);
+
protected:
// BufferQueue::ConsumerListener interface, called when a new frame of
// data is available. If we're executing and a codec buffer is
@@ -165,6 +178,7 @@ private:
void setLatestSubmittedBuffer_l(const BufferQueue::BufferItem &item);
bool repeatLatestSubmittedBuffer_l();
+ int64_t getTimestamp(const BufferQueue::BufferItem &item);
// Lock, covers all member variables.
mutable Mutex mMutex;
@@ -206,13 +220,22 @@ private:
enum {
kWhatRepeatLastFrame,
};
-
+ enum {
+ kRepeatLastFrameCount = 10,
+ };
int64_t mRepeatAfterUs;
+ int64_t mMaxTimestampGapUs;
+
+ KeyedVector<int64_t, int64_t> mOriginalTimeUs;
+ int64_t mPrevOriginalTimeUs;
+ int64_t mPrevModifiedTimeUs;
sp<ALooper> mLooper;
sp<AHandlerReflector<GraphicBufferSource> > mReflector;
int32_t mRepeatLastFrameGeneration;
+ int64_t mRepeatLastFrameTimestamp;
+ int32_t mRepeatLastFrameCount;
int mLatestSubmittedBufferId;
uint64_t mLatestSubmittedBufferFrameNum;
diff --git a/media/libstagefright/omx/OMXNodeInstance.cpp b/media/libstagefright/omx/OMXNodeInstance.cpp
index 5f104fc..6c5c857 100644
--- a/media/libstagefright/omx/OMXNodeInstance.cpp
+++ b/media/libstagefright/omx/OMXNodeInstance.cpp
@@ -849,6 +849,7 @@ status_t OMXNodeInstance::setInternalOption(
switch (type) {
case IOMX::INTERNAL_OPTION_SUSPEND:
case IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY:
+ case IOMX::INTERNAL_OPTION_MAX_TIMESTAMP_GAP:
{
const sp<GraphicBufferSource> &bufferSource =
getGraphicBufferSource();
@@ -864,7 +865,8 @@ status_t OMXNodeInstance::setInternalOption(
bool suspend = *(bool *)data;
bufferSource->suspend(suspend);
- } else {
+ } else if (type ==
+ IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY){
if (size != sizeof(int64_t)) {
return INVALID_OPERATION;
}
@@ -872,6 +874,14 @@ status_t OMXNodeInstance::setInternalOption(
int64_t delayUs = *(int64_t *)data;
return bufferSource->setRepeatPreviousFrameDelayUs(delayUs);
+ } else {
+ if (size != sizeof(int64_t)) {
+ return INVALID_OPERATION;
+ }
+
+ int64_t maxGapUs = *(int64_t *)data;
+
+ return bufferSource->setMaxTimestampGapUs(maxGapUs);
}
return OK;
@@ -883,6 +893,8 @@ status_t OMXNodeInstance::setInternalOption(
}
void OMXNodeInstance::onMessage(const omx_message &msg) {
+ const sp<GraphicBufferSource>& bufferSource(getGraphicBufferSource());
+
if (msg.type == omx_message::FILL_BUFFER_DONE) {
OMX_BUFFERHEADERTYPE *buffer =
static_cast<OMX_BUFFERHEADERTYPE *>(
@@ -892,10 +904,18 @@ void OMXNodeInstance::onMessage(const omx_message &msg) {
static_cast<BufferMeta *>(buffer->pAppPrivate);
buffer_meta->CopyFromOMX(buffer);
- } else if (msg.type == omx_message::EMPTY_BUFFER_DONE) {
- const sp<GraphicBufferSource>& bufferSource(getGraphicBufferSource());
if (bufferSource != NULL) {
+ // fix up the buffer info (especially timestamp) if needed
+ bufferSource->codecBufferFilled(buffer);
+
+ omx_message newMsg = msg;
+ newMsg.u.extended_buffer_data.timestamp = buffer->nTimeStamp;
+ mObserver->onMessage(newMsg);
+ return;
+ }
+ } else if (msg.type == omx_message::EMPTY_BUFFER_DONE) {
+ if (bufferSource != NULL) {
// This is one of the buffers used exclusively by
// GraphicBufferSource.
// Don't dispatch a message back to ACodec, since it doesn't
generated by cgit v1.1 at 2024-06-25 19:30:32 +0000