diff options
-rw-r--r-- | include/media/IOMX.h | 27 | ||||
-rw-r--r-- | include/media/stagefright/ACodec.h | 23 | ||||
-rw-r--r-- | include/media/stagefright/foundation/ADebug.h | 25 | ||||
-rw-r--r-- | media/libmedia/IOMX.cpp | 41 | ||||
-rw-r--r-- | media/libstagefright/ACodec.cpp | 207 | ||||
-rw-r--r-- | media/libstagefright/OMXClient.cpp | 18 | ||||
-rw-r--r-- | media/libstagefright/include/OMX.h | 8 | ||||
-rw-r--r-- | media/libstagefright/include/OMXNodeInstance.h | 25 | ||||
-rw-r--r-- | media/libstagefright/omx/GraphicBufferSource.cpp | 74 | ||||
-rw-r--r-- | media/libstagefright/omx/GraphicBufferSource.h | 3 | ||||
-rw-r--r-- | media/libstagefright/omx/OMX.cpp | 12 | ||||
-rw-r--r-- | media/libstagefright/omx/OMXNodeInstance.cpp | 184 | ||||
-rw-r--r-- | media/libstagefright/omx/SoftVideoEncoderOMXComponent.cpp | 24 |
13 files changed, 418 insertions, 253 deletions
diff --git a/include/media/IOMX.h b/include/media/IOMX.h index 46c363a..26cc73e 100644 --- a/include/media/IOMX.h +++ b/include/media/IOMX.h @@ -25,6 +25,8 @@ #include <utils/List.h> #include <utils/String8.h> +#include <media/hardware/MetadataBufferType.h> + #include <OMX_Core.h> #include <OMX_Video.h> @@ -81,8 +83,10 @@ public: virtual status_t getState( node_id node, OMX_STATETYPE* state) = 0; + // This will set *type to previous metadata buffer type on OMX error (not on binder error), and + // new metadata buffer type on success. virtual status_t storeMetaDataInBuffers( - node_id node, OMX_U32 port_index, OMX_BOOL enable) = 0; + node_id node, OMX_U32 port_index, OMX_BOOL enable, MetadataBufferType *type = NULL) = 0; virtual status_t prepareForAdaptivePlayback( node_id node, OMX_U32 portIndex, OMX_BOOL enable, @@ -111,17 +115,23 @@ public: node_id node, OMX_U32 port_index, const sp<GraphicBuffer> &graphicBuffer, buffer_id buffer) = 0; + // This will set *type to resulting metadata buffer type on OMX error (not on binder error) as + // well as on success. virtual status_t createInputSurface( node_id node, OMX_U32 port_index, - sp<IGraphicBufferProducer> *bufferProducer) = 0; + sp<IGraphicBufferProducer> *bufferProducer, + MetadataBufferType *type = NULL) = 0; virtual status_t createPersistentInputSurface( sp<IGraphicBufferProducer> *bufferProducer, sp<IGraphicBufferConsumer> *bufferConsumer) = 0; + // This will set *type to resulting metadata buffer type on OMX error (not on binder error) as + // well as on success. virtual status_t setInputSurface( node_id node, OMX_U32 port_index, - const sp<IGraphicBufferConsumer> &bufferConsumer) = 0; + const sp<IGraphicBufferConsumer> &bufferConsumer, + MetadataBufferType *type) = 0; virtual status_t signalEndOfInputStream(node_id node) = 0; @@ -236,4 +246,15 @@ struct CodecProfileLevel { } // namespace android +inline static const char *asString(android::MetadataBufferType i, const char *def = "??") { + using namespace android; + switch (i) { + case kMetadataBufferTypeCameraSource: return "CameraSource"; + case kMetadataBufferTypeGrallocSource: return "GrallocSource"; + case kMetadataBufferTypeANWBuffer: return "ANWBuffer"; + case kMetadataBufferTypeInvalid: return "Invalid"; + default: return def; + } +} + #endif // ANDROID_IOMX_H_ diff --git a/include/media/stagefright/ACodec.h b/include/media/stagefright/ACodec.h index 4ed97e5..bbecc80 100644 --- a/include/media/stagefright/ACodec.h +++ b/include/media/stagefright/ACodec.h @@ -20,6 +20,7 @@ #include <stdint.h> #include <android/native_window.h> +#include <media/hardware/MetadataBufferType.h> #include <media/IOMX.h> #include <media/stagefright/foundation/AHierarchicalStateMachine.h> #include <media/stagefright/CodecBase.h> @@ -123,7 +124,7 @@ private: kWhatStart = 'star', kWhatRequestIDRFrame = 'ridr', kWhatSetParameters = 'setP', - kWhatSubmitOutputMetaDataBufferIfEOS = 'subm', + kWhatSubmitOutputMetadataBufferIfEOS = 'subm', kWhatOMXDied = 'OMXd', kWhatReleaseCodecInstance = 'relC', }; @@ -207,7 +208,6 @@ private: bool mSentFormat; bool mIsVideo; bool mIsEncoder; - bool mUseMetadataOnEncoderOutput; bool mShutdownInProgress; bool mExplicitShutdown; @@ -222,9 +222,10 @@ private: bool mChannelMaskPresent; int32_t mChannelMask; unsigned mDequeueCounter; - bool mStoreMetaDataInOutputBuffers; + MetadataBufferType mInputMetadataType; + MetadataBufferType mOutputMetadataType; bool mLegacyAdaptiveExperiment; - int32_t mMetaDataBuffersToSubmit; + int32_t mMetadataBuffersToSubmit; size_t mNumUndequeuedBuffers; int64_t mRepeatFrameDelayUs; @@ -249,14 +250,22 @@ private: status_t configureOutputBuffersFromNativeWindow( OMX_U32 *nBufferCount, OMX_U32 *nBufferSize, OMX_U32 *nMinUndequeuedBuffers); - status_t allocateOutputMetaDataBuffers(); - status_t submitOutputMetaDataBuffer(); - void signalSubmitOutputMetaDataBufferIfEOS_workaround(); + status_t allocateOutputMetadataBuffers(); + status_t submitOutputMetadataBuffer(); + void signalSubmitOutputMetadataBufferIfEOS_workaround(); status_t allocateOutputBuffersFromNativeWindow(); status_t cancelBufferToNativeWindow(BufferInfo *info); status_t freeOutputBuffersNotOwnedByComponent(); BufferInfo *dequeueBufferFromNativeWindow(); + inline bool storingMetadataInDecodedBuffers() { + return mOutputMetadataType >= 0 && !mIsEncoder; + } + + inline bool usingMetadataOnEncoderOutput() { + return mOutputMetadataType >= 0 && mIsEncoder; + } + BufferInfo *findBufferByID( uint32_t portIndex, IOMX::buffer_id bufferID, ssize_t *index = NULL); diff --git a/include/media/stagefright/foundation/ADebug.h b/include/media/stagefright/foundation/ADebug.h index a97dd9b..24df85a 100644 --- a/include/media/stagefright/foundation/ADebug.h +++ b/include/media/stagefright/foundation/ADebug.h @@ -24,6 +24,31 @@ #include <media/stagefright/foundation/AString.h> #include <utils/Log.h> +inline static const char *asString(android::status_t i, const char *def = "??") { + using namespace android; + switch (i) { + case NO_ERROR: return "NO_ERROR"; + case UNKNOWN_ERROR: return "UNKNOWN_ERROR"; + case NO_MEMORY: return "NO_MEMORY"; + case INVALID_OPERATION: return "INVALID_OPERATION"; + case BAD_VALUE: return "BAD_VALUE"; + case BAD_TYPE: return "BAD_TYPE"; + case NAME_NOT_FOUND: return "NAME_NOT_FOUND"; + case PERMISSION_DENIED: return "PERMISSION_DENIED"; + case NO_INIT: return "NO_INIT"; + case ALREADY_EXISTS: return "ALREADY_EXISTS"; + case DEAD_OBJECT: return "DEAD_OBJECT"; + case FAILED_TRANSACTION: return "FAILED_TRANSACTION"; + case BAD_INDEX: return "BAD_INDEX"; + case NOT_ENOUGH_DATA: return "NOT_ENOUGH_DATA"; + case WOULD_BLOCK: return "WOULD_BLOCK"; + case TIMED_OUT: return "TIMED_OUT"; + case UNKNOWN_TRANSACTION: return "UNKNOWN_TRANSACTION"; + case FDS_NOT_ALLOWED: return "FDS_NOT_ALLOWED"; + default: return def; + } +} + namespace android { #define LITERAL_TO_STRING_INTERNAL(x) #x diff --git a/media/libmedia/IOMX.cpp b/media/libmedia/IOMX.cpp index d556c33..cac2f7f 100644 --- a/media/libmedia/IOMX.cpp +++ b/media/libmedia/IOMX.cpp @@ -306,7 +306,7 @@ public: virtual status_t createInputSurface( node_id node, OMX_U32 port_index, - sp<IGraphicBufferProducer> *bufferProducer) { + sp<IGraphicBufferProducer> *bufferProducer, MetadataBufferType *type) { Parcel data, reply; status_t err; data.writeInterfaceToken(IOMX::getInterfaceDescriptor()); @@ -318,6 +318,12 @@ public: return err; } + // read type even if createInputSurface failed + int negotiatedType = reply.readInt32(); + if (type != NULL) { + *type = (MetadataBufferType)negotiatedType; + } + err = reply.readInt32(); if (err != OK) { return err; @@ -356,7 +362,7 @@ public: virtual status_t setInputSurface( node_id node, OMX_U32 port_index, - const sp<IGraphicBufferConsumer> &bufferConsumer) { + const sp<IGraphicBufferConsumer> &bufferConsumer, MetadataBufferType *type) { Parcel data, reply; data.writeInterfaceToken(IOMX::getInterfaceDescriptor()); status_t err; @@ -370,6 +376,13 @@ public: ALOGW("binder transaction failed: %d", err); return err; } + + // read type even if setInputSurface failed + int negotiatedType = reply.readInt32(); + if (type != NULL) { + *type = (MetadataBufferType)negotiatedType; + } + return reply.readInt32(); } @@ -388,7 +401,7 @@ public: } virtual status_t storeMetaDataInBuffers( - node_id node, OMX_U32 port_index, OMX_BOOL enable) { + node_id node, OMX_U32 port_index, OMX_BOOL enable, MetadataBufferType *type) { Parcel data, reply; data.writeInterfaceToken(IOMX::getInterfaceDescriptor()); data.writeInt32((int32_t)node); @@ -396,8 +409,13 @@ public: data.writeInt32((uint32_t)enable); remote()->transact(STORE_META_DATA_IN_BUFFERS, data, &reply); - status_t err = reply.readInt32(); - return err; + // read type even storeMetaDataInBuffers failed + int negotiatedType = reply.readInt32(); + if (type != NULL) { + *type = (MetadataBufferType)negotiatedType; + } + + return reply.readInt32(); } virtual status_t prepareForAdaptivePlayback( @@ -818,9 +836,10 @@ status_t BnOMX::onTransact( OMX_U32 port_index = data.readInt32(); sp<IGraphicBufferProducer> bufferProducer; - status_t err = createInputSurface(node, port_index, - &bufferProducer); + MetadataBufferType type; + status_t err = createInputSurface(node, port_index, &bufferProducer, &type); + reply->writeInt32(type); reply->writeInt32(err); if (err == OK) { @@ -859,8 +878,10 @@ status_t BnOMX::onTransact( sp<IGraphicBufferConsumer> bufferConsumer = interface_cast<IGraphicBufferConsumer>(data.readStrongBinder()); - status_t err = setInputSurface(node, port_index, bufferConsumer); + MetadataBufferType type; + status_t err = setInputSurface(node, port_index, bufferConsumer, &type); + reply->writeInt32(type); reply->writeInt32(err); return NO_ERROR; } @@ -885,7 +906,9 @@ status_t BnOMX::onTransact( OMX_U32 port_index = data.readInt32(); OMX_BOOL enable = (OMX_BOOL)data.readInt32(); - status_t err = storeMetaDataInBuffers(node, port_index, enable); + MetadataBufferType type; + status_t err = storeMetaDataInBuffers(node, port_index, enable, &type); + reply->writeInt32(type); reply->writeInt32(err); return NO_ERROR; diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp index cd20979..08045d1 100644 --- a/media/libstagefright/ACodec.cpp +++ b/media/libstagefright/ACodec.cpp @@ -413,7 +413,6 @@ ACodec::ACodec() mSentFormat(false), mIsVideo(false), mIsEncoder(false), - mUseMetadataOnEncoderOutput(false), mShutdownInProgress(false), mExplicitShutdown(false), mEncoderDelay(0), @@ -422,9 +421,10 @@ ACodec::ACodec() mChannelMaskPresent(false), mChannelMask(0), mDequeueCounter(0), - mStoreMetaDataInOutputBuffers(false), + mInputMetadataType(kMetadataBufferTypeInvalid), + mOutputMetadataType(kMetadataBufferTypeInvalid), mLegacyAdaptiveExperiment(false), - mMetaDataBuffersToSubmit(0), + mMetadataBuffersToSubmit(0), mRepeatFrameDelayUs(-1ll), mMaxPtsGapUs(-1ll), mMaxFps(-1), @@ -542,10 +542,10 @@ void ACodec::signalRequestIDRFrame() { // This causes a halt if we already signaled an EOS on the input // port. For now keep submitting an output buffer if there was an // EOS on the input port, but not yet on the output port. -void ACodec::signalSubmitOutputMetaDataBufferIfEOS_workaround() { +void ACodec::signalSubmitOutputMetadataBufferIfEOS_workaround() { if (mPortEOS[kPortIndexInput] && !mPortEOS[kPortIndexOutput] && - mMetaDataBuffersToSubmit > 0) { - (new AMessage(kWhatSubmitOutputMetaDataBufferIfEOS, this))->post(); + mMetadataBuffersToSubmit > 0) { + (new AMessage(kWhatSubmitOutputMetadataBufferIfEOS, this))->post(); } } @@ -620,7 +620,7 @@ status_t ACodec::handleSetSurface(const sp<Surface> &surface) { for (size_t i = 0; i < buffers.size(); ++i) { const BufferInfo &info = buffers[i]; // skip undequeued buffers for meta data mode - if (mStoreMetaDataInOutputBuffers + if (storingMetadataInDecodedBuffers() && !mLegacyAdaptiveExperiment && info.mStatus == BufferInfo::OWNED_BY_NATIVE_WINDOW) { ALOGV("skipping buffer %p", info.mGraphicBuffer->getNativeBuffer()); @@ -638,7 +638,7 @@ status_t ACodec::handleSetSurface(const sp<Surface> &surface) { } // cancel undequeued buffers to new surface - if (!mStoreMetaDataInOutputBuffers || mLegacyAdaptiveExperiment) { + if (!storingMetadataInDecodedBuffers() || mLegacyAdaptiveExperiment) { for (size_t i = 0; i < buffers.size(); ++i) { const BufferInfo &info = buffers[i]; if (info.mStatus == BufferInfo::OWNED_BY_NATIVE_WINDOW) { @@ -674,8 +674,8 @@ status_t ACodec::allocateBuffersOnPort(OMX_U32 portIndex) { status_t err; if (mNativeWindow != NULL && portIndex == kPortIndexOutput) { - if (mStoreMetaDataInOutputBuffers) { - err = allocateOutputMetaDataBuffers(); + if (storingMetadataInDecodedBuffers()) { + err = allocateOutputMetadataBuffers(); } else { err = allocateOutputBuffersFromNativeWindow(); } @@ -688,16 +688,33 @@ status_t ACodec::allocateBuffersOnPort(OMX_U32 portIndex) { mNode, OMX_IndexParamPortDefinition, &def, sizeof(def)); if (err == OK) { - ALOGV("[%s] Allocating %u buffers of size %u on %s port", + MetadataBufferType type = + portIndex == kPortIndexOutput ? mOutputMetadataType : mInputMetadataType; + int32_t bufSize = def.nBufferSize; + if (type == kMetadataBufferTypeGrallocSource) { + bufSize = sizeof(VideoGrallocMetadata); + } else if (type == kMetadataBufferTypeANWBuffer) { + bufSize = sizeof(VideoNativeMetadata); + } + + // If using gralloc or native source input metadata buffers, allocate largest + // metadata size as we prefer to generate native source metadata, but component + // may require gralloc source. + int32_t allottedSize = bufSize; + if (portIndex == kPortIndexInput && type > 0) { + bufSize = max(sizeof(VideoGrallocMetadata), sizeof(VideoNativeMetadata)); + } + + ALOGV("[%s] Allocating %u buffers of size %d/%d (from %u using %s) on %s port", mComponentName.c_str(), - def.nBufferCountActual, def.nBufferSize, + def.nBufferCountActual, bufSize, allottedSize, def.nBufferSize, asString(type), portIndex == kPortIndexInput ? "input" : "output"); - size_t totalSize = def.nBufferCountActual * def.nBufferSize; + size_t totalSize = def.nBufferCountActual * bufSize; mDealer[portIndex] = new MemoryDealer(totalSize, "ACodec"); - for (OMX_U32 i = 0; i < def.nBufferCountActual; ++i) { - sp<IMemory> mem = mDealer[portIndex]->allocate(def.nBufferSize); + for (OMX_U32 i = 0; i < def.nBufferCountActual && err == OK; ++i) { + sp<IMemory> mem = mDealer[portIndex]->allocate(bufSize); if (mem == NULL || mem->pointer() == NULL) { return NO_MEMORY; } @@ -711,27 +728,27 @@ status_t ACodec::allocateBuffersOnPort(OMX_U32 portIndex) { : OMXCodec::kRequiresAllocateBufferOnOutputPorts; if ((portIndex == kPortIndexInput && (mFlags & kFlagIsSecure)) - || mUseMetadataOnEncoderOutput) { + || (portIndex == kPortIndexOutput && usingMetadataOnEncoderOutput())) { mem.clear(); void *ptr; err = mOMX->allocateBuffer( - mNode, portIndex, def.nBufferSize, &info.mBufferID, + mNode, portIndex, bufSize, &info.mBufferID, &ptr); - int32_t bufSize = mUseMetadataOnEncoderOutput ? - (4 + sizeof(buffer_handle_t)) : def.nBufferSize; - info.mData = new ABuffer(ptr, bufSize); } else if (mQuirks & requiresAllocateBufferBit) { err = mOMX->allocateBufferWithBackup( - mNode, portIndex, mem, &info.mBufferID, def.nBufferSize); + mNode, portIndex, mem, &info.mBufferID, allottedSize); } else { - err = mOMX->useBuffer(mNode, portIndex, mem, &info.mBufferID, def.nBufferSize); + err = mOMX->useBuffer(mNode, portIndex, mem, &info.mBufferID, allottedSize); } if (mem != NULL) { - info.mData = new ABuffer(mem->pointer(), def.nBufferSize); + info.mData = new ABuffer(mem->pointer(), bufSize); + if (type == kMetadataBufferTypeANWBuffer) { + ((VideoNativeMetadata *)mem->pointer())->nFenceFd = -1; + } } mBuffers[portIndex].push(info); @@ -896,7 +913,7 @@ status_t ACodec::allocateOutputBuffersFromNativeWindow() { return err; mNumUndequeuedBuffers = minUndequeuedBuffers; - if (!mStoreMetaDataInOutputBuffers) { + if (!storingMetadataInDecodedBuffers()) { static_cast<Surface*>(mNativeWindow.get()) ->getIGraphicBufferProducer()->allowAllocation(true); } @@ -959,7 +976,7 @@ status_t ACodec::allocateOutputBuffersFromNativeWindow() { } } - if (!mStoreMetaDataInOutputBuffers) { + if (!storingMetadataInDecodedBuffers()) { static_cast<Surface*>(mNativeWindow.get()) ->getIGraphicBufferProducer()->allowAllocation(false); } @@ -967,7 +984,7 @@ status_t ACodec::allocateOutputBuffersFromNativeWindow() { return err; } -status_t ACodec::allocateOutputMetaDataBuffers() { +status_t ACodec::allocateOutputMetadataBuffers() { OMX_U32 bufferCount, bufferSize, minUndequeuedBuffers; status_t err = configureOutputBuffersFromNativeWindow( &bufferCount, &bufferSize, &minUndequeuedBuffers); @@ -978,7 +995,9 @@ status_t ACodec::allocateOutputMetaDataBuffers() { ALOGV("[%s] Allocating %u meta buffers on output port", mComponentName.c_str(), bufferCount); - size_t totalSize = bufferCount * 8; + size_t bufSize = mOutputMetadataType == kMetadataBufferTypeANWBuffer ? + sizeof(struct VideoNativeMetadata) : sizeof(struct VideoGrallocMetadata); + size_t totalSize = bufferCount * bufSize; mDealer[kPortIndexOutput] = new MemoryDealer(totalSize, "ACodec"); // Dequeue buffers and send them to OMX @@ -988,11 +1007,13 @@ status_t ACodec::allocateOutputMetaDataBuffers() { info.mGraphicBuffer = NULL; info.mDequeuedAt = mDequeueCounter; - sp<IMemory> mem = mDealer[kPortIndexOutput]->allocate( - sizeof(struct VideoDecoderOutputMetaData)); + sp<IMemory> mem = mDealer[kPortIndexOutput]->allocate(bufSize); if (mem == NULL || mem->pointer() == NULL) { return NO_MEMORY; } + if (mOutputMetadataType == kMetadataBufferTypeANWBuffer) { + ((VideoNativeMetadata *)mem->pointer())->nFenceFd = -1; + } info.mData = new ABuffer(mem->pointer(), mem->size()); // we use useBuffer for metadata regardless of quirks @@ -1044,13 +1065,13 @@ status_t ACodec::allocateOutputMetaDataBuffers() { ->getIGraphicBufferProducer()->allowAllocation(false); } - mMetaDataBuffersToSubmit = bufferCount - minUndequeuedBuffers; + mMetadataBuffersToSubmit = bufferCount - minUndequeuedBuffers; return err; } -status_t ACodec::submitOutputMetaDataBuffer() { - CHECK(mStoreMetaDataInOutputBuffers); - if (mMetaDataBuffersToSubmit == 0) +status_t ACodec::submitOutputMetadataBuffer() { + CHECK(storingMetadataInDecodedBuffers()); + if (mMetadataBuffersToSubmit == 0) return OK; BufferInfo *info = dequeueBufferFromNativeWindow(); @@ -1061,7 +1082,7 @@ status_t ACodec::submitOutputMetaDataBuffer() { ALOGV("[%s] submitting output meta buffer ID %u for graphic buffer %p", mComponentName.c_str(), info->mBufferID, info->mGraphicBuffer.get()); - --mMetaDataBuffersToSubmit; + --mMetadataBuffersToSubmit; status_t err = mOMX->fillBuffer(mNode, info->mBufferID); if (err == OK) { info->mStatus = BufferInfo::OWNED_BY_COMPONENT; @@ -1159,7 +1180,7 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { // same is possible in meta mode, in which case, it will be treated // as a normal buffer, which is not desirable. // TODO: fix this. - if (!stale && (!mStoreMetaDataInOutputBuffers || mLegacyAdaptiveExperiment)) { + if (!stale && (!storingMetadataInDecodedBuffers() || mLegacyAdaptiveExperiment)) { ALOGI("dequeued unrecognized (stale) buffer %p. discarding", buf); stale = true; } @@ -1187,7 +1208,7 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { CHECK(oldest != NULL); // it is impossible to dequeue an unknown buffer in non-meta mode, as the // while loop above does not complete - CHECK(mStoreMetaDataInOutputBuffers); + CHECK(storingMetadataInDecodedBuffers()); // discard buffer in LRU info and replace with new buffer oldest->mGraphicBuffer = new GraphicBuffer(buf, false); @@ -1197,16 +1218,23 @@ ACodec::BufferInfo *ACodec::dequeueBufferFromNativeWindow() { mNode, kPortIndexOutput, oldest->mGraphicBuffer, oldest->mBufferID); - VideoDecoderOutputMetaData *metaData = - reinterpret_cast<VideoDecoderOutputMetaData *>( - oldest->mData->base()); - CHECK_EQ(metaData->eType, kMetadataBufferTypeGrallocSource); - - ALOGV("replaced oldest buffer #%u with age %u (%p/%p stored in %p)", - (unsigned)(oldest - &mBuffers[kPortIndexOutput][0]), - mDequeueCounter - oldest->mDequeuedAt, - metaData->pHandle, - oldest->mGraphicBuffer->handle, oldest->mData->base()); + if (mOutputMetadataType == kMetadataBufferTypeGrallocSource) { + VideoGrallocMetadata *grallocMeta = + reinterpret_cast<VideoGrallocMetadata *>(oldest->mData->base()); + ALOGV("replaced oldest buffer #%u with age %u (%p/%p stored in %p)", + (unsigned)(oldest - &mBuffers[kPortIndexOutput][0]), + mDequeueCounter - oldest->mDequeuedAt, + grallocMeta->hHandle, + oldest->mGraphicBuffer->handle, oldest->mData->base()); + } else if (mOutputMetadataType == kMetadataBufferTypeANWBuffer) { + VideoNativeMetadata *nativeMeta = + reinterpret_cast<VideoNativeMetadata *>(oldest->mData->base()); + ALOGV("replaced oldest buffer #%u with age %u (%p/%p stored in %p)", + (unsigned)(oldest - &mBuffers[kPortIndexOutput][0]), + mDequeueCounter - oldest->mDequeuedAt, + nativeMeta->pBuffer, + oldest->mGraphicBuffer->getNativeBuffer(), oldest->mData->base()); + } return oldest; } @@ -1398,6 +1426,8 @@ status_t ACodec::configureCodec( mIsEncoder = encoder; + mInputMetadataType = kMetadataBufferTypeInvalid; + mOutputMetadataType = kMetadataBufferTypeInvalid; status_t err = setComponentRole(encoder /* isEncoder */, mime); @@ -1416,15 +1446,19 @@ status_t ACodec::configureCodec( if (encoder && msg->findInt32("store-metadata-in-buffers", &storeMeta) && storeMeta != 0) { - err = mOMX->storeMetaDataInBuffers(mNode, kPortIndexInput, OMX_TRUE); - + err = mOMX->storeMetaDataInBuffers(mNode, kPortIndexInput, OMX_TRUE, &mInputMetadataType); if (err != OK) { - ALOGE("[%s] storeMetaDataInBuffers (input) failed w/ err %d", + ALOGE("[%s] storeMetaDataInBuffers (input) failed w/ err %d", mComponentName.c_str(), err); - return err; - } - } + return err; + } + // For this specific case we could be using camera source even if storeMetaDataInBuffers + // returns Gralloc source. Pretend that we are; this will force us to use nBufferSize. + if (mInputMetadataType == kMetadataBufferTypeGrallocSource) { + mInputMetadataType = kMetadataBufferTypeCameraSource; + } + } int32_t prependSPSPPS = 0; if (encoder @@ -1463,14 +1497,10 @@ status_t ACodec::configureCodec( && msg->findInt32("store-metadata-in-buffers-output", &storeMeta) && storeMeta != 0); - err = mOMX->storeMetaDataInBuffers(mNode, kPortIndexOutput, enable); - + err = mOMX->storeMetaDataInBuffers(mNode, kPortIndexOutput, enable, &mOutputMetadataType); if (err != OK) { ALOGE("[%s] storeMetaDataInBuffers (output) failed w/ err %d", mComponentName.c_str(), err); - mUseMetadataOnEncoderOutput = 0; - } else { - mUseMetadataOnEncoderOutput = enable; } if (!msg->findInt64( @@ -1502,7 +1532,6 @@ status_t ACodec::configureCodec( sp<RefBase> obj; bool haveNativeWindow = msg->findObject("native-window", &obj) && obj != NULL && video && !encoder; - mStoreMetaDataInOutputBuffers = false; mLegacyAdaptiveExperiment = false; if (video && !encoder) { inputFormat->setInt32("adaptive-playback", false); @@ -1588,7 +1617,7 @@ status_t ACodec::configureCodec( // Always try to enable dynamic output buffers on native surface err = mOMX->storeMetaDataInBuffers( - mNode, kPortIndexOutput, OMX_TRUE); + mNode, kPortIndexOutput, OMX_TRUE, &mOutputMetadataType); if (err != OK) { ALOGE("[%s] storeMetaDataInBuffers failed w/ err %d", mComponentName.c_str(), err); @@ -1640,7 +1669,7 @@ status_t ACodec::configureCodec( } else { ALOGV("[%s] storeMetaDataInBuffers succeeded", mComponentName.c_str()); - mStoreMetaDataInOutputBuffers = true; + CHECK(storingMetadataInDecodedBuffers()); mLegacyAdaptiveExperiment = ADebug::isExperimentEnabled( "legacy-adaptive", !msg->contains("no-experiments")); @@ -1717,9 +1746,10 @@ status_t ACodec::configureCodec( mNativeWindow.clear(); haveNativeWindow = false; usingSwRenderer = true; - if (mStoreMetaDataInOutputBuffers) { - err = mOMX->storeMetaDataInBuffers(mNode, kPortIndexOutput, OMX_FALSE); - mStoreMetaDataInOutputBuffers = false; + if (storingMetadataInDecodedBuffers()) { + err = mOMX->storeMetaDataInBuffers( + mNode, kPortIndexOutput, OMX_FALSE, &mOutputMetadataType); + mOutputMetadataType = kMetadataBufferTypeInvalid; // just in case // TODO: implement adaptive-playback support for bytebuffer mode. // This is done by SW codecs, but most HW codecs don't support it. inputFormat->setInt32("adaptive-playback", false); @@ -3536,8 +3566,8 @@ void ACodec::waitUntilAllPossibleNativeWindowBuffersAreReturnedToUs() { while (countBuffersOwnedByNativeWindow() > mNumUndequeuedBuffers && dequeueBufferFromNativeWindow() != NULL) { // these buffers will be submitted as regular buffers; account for this - if (mStoreMetaDataInOutputBuffers && mMetaDataBuffersToSubmit > 0) { - --mMetaDataBuffersToSubmit; + if (storingMetadataInDecodedBuffers() && mMetadataBuffersToSubmit > 0) { + --mMetadataBuffersToSubmit; } } } @@ -4628,16 +4658,16 @@ void ACodec::BaseState::onInputBufferFilled(const sp<AMessage> &msg) { mCodec->mBufferStats.add(timeUs, stats); #endif - if (mCodec->mStoreMetaDataInOutputBuffers) { + if (mCodec->storingMetadataInDecodedBuffers()) { // try to submit an output buffer for each input buffer PortMode outputMode = getPortMode(kPortIndexOutput); - ALOGV("MetaDataBuffersToSubmit=%u portMode=%s", - mCodec->mMetaDataBuffersToSubmit, + ALOGV("MetadataBuffersToSubmit=%u portMode=%s", + mCodec->mMetadataBuffersToSubmit, (outputMode == FREE_BUFFERS ? "FREE" : outputMode == KEEP_BUFFERS ? "KEEP" : "RESUBMIT")); if (outputMode == RESUBMIT_BUFFERS) { - mCodec->submitOutputMetaDataBuffer(); + mCodec->submitOutputMetadataBuffer(); } } status_t err2 = mCodec->mOMX->emptyBuffer( @@ -4799,10 +4829,17 @@ bool ACodec::BaseState::onOMXFillBufferDone( if (!mCodec->mSentFormat && rangeLength > 0) { mCodec->sendFormatChange(reply); } - - if (mCodec->mUseMetadataOnEncoderOutput) { - native_handle_t* handle = - *(native_handle_t**)(info->mData->data() + 4); + if (mCodec->usingMetadataOnEncoderOutput()) { + native_handle_t *handle = NULL; + VideoGrallocMetadata &grallocMeta = *(VideoGrallocMetadata *)info->mData->data(); + VideoNativeMetadata &nativeMeta = *(VideoNativeMetadata *)info->mData->data(); + if (info->mData->size() >= sizeof(grallocMeta) + && grallocMeta.eType == kMetadataBufferTypeGrallocSource) { + handle = (native_handle_t *)grallocMeta.hHandle; + } else if (info->mData->size() >= sizeof(nativeMeta) + && nativeMeta.eType == kMetadataBufferTypeANWBuffer) { + handle = (native_handle_t *)nativeMeta.pBuffer->handle; + } info->mData->meta()->setPointer("handle", handle); info->mData->meta()->setInt32("rangeOffset", rangeOffset); info->mData->meta()->setInt32("rangeLength", rangeLength); @@ -4999,7 +5036,8 @@ void ACodec::UninitializedState::stateEntered() { mCodec->mOMX.clear(); mCodec->mQuirks = 0; mCodec->mFlags = 0; - mCodec->mUseMetadataOnEncoderOutput = 0; + mCodec->mInputMetadataType = kMetadataBufferTypeInvalid; + mCodec->mOutputMetadataType = kMetadataBufferTypeInvalid; mCodec->mComponentName.clear(); } @@ -5202,7 +5240,7 @@ void ACodec::LoadedState::stateEntered() { mCodec->mInputEOSResult = OK; mCodec->mDequeueCounter = 0; - mCodec->mMetaDataBuffersToSubmit = 0; + mCodec->mMetadataBuffersToSubmit = 0; mCodec->mRepeatFrameDelayUs = -1ll; mCodec->mInputFormat.clear(); mCodec->mOutputFormat.clear(); @@ -5431,7 +5469,7 @@ void ACodec::LoadedState::onCreateInputSurface( sp<IGraphicBufferProducer> bufferProducer; status_t err = mCodec->mOMX->createInputSurface( - mCodec->mNode, kPortIndexInput, &bufferProducer); + mCodec->mNode, kPortIndexInput, &bufferProducer, &mCodec->mInputMetadataType); if (err == OK) { err = setupInputSurface(); @@ -5463,7 +5501,8 @@ void ACodec::LoadedState::onSetInputSurface( sp<PersistentSurface> surface = static_cast<PersistentSurface *>(obj.get()); status_t err = mCodec->mOMX->setInputSurface( - mCodec->mNode, kPortIndexInput, surface->getBufferConsumer()); + mCodec->mNode, kPortIndexInput, surface->getBufferConsumer(), + &mCodec->mInputMetadataType); if (err == OK) { err = setupInputSurface(); @@ -5680,13 +5719,13 @@ void ACodec::ExecutingState::submitOutputMetaBuffers() { BufferInfo *info = &mCodec->mBuffers[kPortIndexInput].editItemAt(i); if (info->mStatus == BufferInfo::OWNED_BY_COMPONENT) { - if (mCodec->submitOutputMetaDataBuffer() != OK) + if (mCodec->submitOutputMetadataBuffer() != OK) break; } } // *** NOTE: THE FOLLOWING WORKAROUND WILL BE REMOVED *** - mCodec->signalSubmitOutputMetaDataBufferIfEOS_workaround(); + mCodec->signalSubmitOutputMetadataBufferIfEOS_workaround(); } void ACodec::ExecutingState::submitRegularOutputBuffers() { @@ -5731,7 +5770,7 @@ void ACodec::ExecutingState::submitRegularOutputBuffers() { void ACodec::ExecutingState::submitOutputBuffers() { submitRegularOutputBuffers(); - if (mCodec->mStoreMetaDataInOutputBuffers) { + if (mCodec->storingMetadataInDecodedBuffers()) { submitOutputMetaBuffers(); } } @@ -5863,13 +5902,13 @@ bool ACodec::ExecutingState::onMessageReceived(const sp<AMessage> &msg) { } // *** NOTE: THE FOLLOWING WORKAROUND WILL BE REMOVED *** - case kWhatSubmitOutputMetaDataBufferIfEOS: + case kWhatSubmitOutputMetadataBufferIfEOS: { if (mCodec->mPortEOS[kPortIndexInput] && !mCodec->mPortEOS[kPortIndexOutput]) { - status_t err = mCodec->submitOutputMetaDataBuffer(); + status_t err = mCodec->submitOutputMetadataBuffer(); if (err == OK) { - mCodec->signalSubmitOutputMetaDataBufferIfEOS_workaround(); + mCodec->signalSubmitOutputMetadataBufferIfEOS_workaround(); } } return true; @@ -5980,7 +6019,7 @@ bool ACodec::ExecutingState::onOMXEvent( CHECK_EQ(data1, (OMX_U32)kPortIndexOutput); if (data2 == 0 || data2 == OMX_IndexParamPortDefinition) { - mCodec->mMetaDataBuffersToSubmit = 0; + mCodec->mMetadataBuffersToSubmit = 0; CHECK_EQ(mCodec->mOMX->sendCommand( mCodec->mNode, OMX_CommandPortDisable, kPortIndexOutput), diff --git a/media/libstagefright/OMXClient.cpp b/media/libstagefright/OMXClient.cpp index c91a37f..5d04628 100644 --- a/media/libstagefright/OMXClient.cpp +++ b/media/libstagefright/OMXClient.cpp @@ -72,7 +72,7 @@ struct MuxOMX : public IOMX { node_id node, OMX_STATETYPE* state); virtual status_t storeMetaDataInBuffers( - node_id node, OMX_U32 port_index, OMX_BOOL enable); + node_id node, OMX_U32 port_index, OMX_BOOL enable, MetadataBufferType *type); virtual status_t prepareForAdaptivePlayback( node_id node, OMX_U32 port_index, OMX_BOOL enable, @@ -102,7 +102,7 @@ struct MuxOMX : public IOMX { virtual status_t createInputSurface( node_id node, OMX_U32 port_index, - sp<IGraphicBufferProducer> *bufferProducer); + sp<IGraphicBufferProducer> *bufferProducer, MetadataBufferType *type); virtual status_t createPersistentInputSurface( sp<IGraphicBufferProducer> *bufferProducer, @@ -110,7 +110,7 @@ struct MuxOMX : public IOMX { virtual status_t setInputSurface( node_id node, OMX_U32 port_index, - const sp<IGraphicBufferConsumer> &bufferConsumer); + const sp<IGraphicBufferConsumer> &bufferConsumer, MetadataBufferType *type); virtual status_t signalEndOfInputStream(node_id node); @@ -292,8 +292,8 @@ status_t MuxOMX::getState( } status_t MuxOMX::storeMetaDataInBuffers( - node_id node, OMX_U32 port_index, OMX_BOOL enable) { - return getOMX(node)->storeMetaDataInBuffers(node, port_index, enable); + node_id node, OMX_U32 port_index, OMX_BOOL enable, MetadataBufferType *type) { + return getOMX(node)->storeMetaDataInBuffers(node, port_index, enable, type); } status_t MuxOMX::prepareForAdaptivePlayback( @@ -342,9 +342,9 @@ status_t MuxOMX::updateGraphicBufferInMeta( status_t MuxOMX::createInputSurface( node_id node, OMX_U32 port_index, - sp<IGraphicBufferProducer> *bufferProducer) { + sp<IGraphicBufferProducer> *bufferProducer, MetadataBufferType *type) { status_t err = getOMX(node)->createInputSurface( - node, port_index, bufferProducer); + node, port_index, bufferProducer, type); return err; } @@ -358,8 +358,8 @@ status_t MuxOMX::createPersistentInputSurface( status_t MuxOMX::setInputSurface( node_id node, OMX_U32 port_index, - const sp<IGraphicBufferConsumer> &bufferConsumer) { - return getOMX(node)->setInputSurface(node, port_index, bufferConsumer); + const sp<IGraphicBufferConsumer> &bufferConsumer, MetadataBufferType *type) { + return getOMX(node)->setInputSurface(node, port_index, bufferConsumer, type); } status_t MuxOMX::signalEndOfInputStream(node_id node) { diff --git a/media/libstagefright/include/OMX.h b/media/libstagefright/include/OMX.h index fa1e03f..c34954b 100644 --- a/media/libstagefright/include/OMX.h +++ b/media/libstagefright/include/OMX.h @@ -69,7 +69,7 @@ public: node_id node, OMX_U32 port_index, OMX_U32* usage); virtual status_t storeMetaDataInBuffers( - node_id node, OMX_U32 port_index, OMX_BOOL enable); + node_id node, OMX_U32 port_index, OMX_BOOL enable, MetadataBufferType *type); virtual status_t prepareForAdaptivePlayback( node_id node, OMX_U32 portIndex, OMX_BOOL enable, @@ -93,7 +93,8 @@ public: virtual status_t createInputSurface( node_id node, OMX_U32 port_index, - sp<IGraphicBufferProducer> *bufferProducer); + sp<IGraphicBufferProducer> *bufferProducer, + MetadataBufferType *type); virtual status_t createPersistentInputSurface( sp<IGraphicBufferProducer> *bufferProducer, @@ -101,7 +102,8 @@ public: virtual status_t setInputSurface( node_id node, OMX_U32 port_index, - const sp<IGraphicBufferConsumer> &bufferConsumer); + const sp<IGraphicBufferConsumer> &bufferConsumer, + MetadataBufferType *type); virtual status_t signalEndOfInputStream(node_id node); diff --git a/media/libstagefright/include/OMXNodeInstance.h b/media/libstagefright/include/OMXNodeInstance.h index e067598..fe6dccd 100644 --- a/media/libstagefright/include/OMXNodeInstance.h +++ b/media/libstagefright/include/OMXNodeInstance.h @@ -58,7 +58,8 @@ struct OMXNodeInstance { status_t getGraphicBufferUsage(OMX_U32 portIndex, OMX_U32* usage); - status_t storeMetaDataInBuffers(OMX_U32 portIndex, OMX_BOOL enable); + status_t storeMetaDataInBuffers( + OMX_U32 portIndex, OMX_BOOL enable, MetadataBufferType *type); status_t prepareForAdaptivePlayback( OMX_U32 portIndex, OMX_BOOL enable, @@ -81,14 +82,16 @@ struct OMXNodeInstance { OMX::buffer_id buffer); status_t createInputSurface( - OMX_U32 portIndex, sp<IGraphicBufferProducer> *bufferProducer); + OMX_U32 portIndex, sp<IGraphicBufferProducer> *bufferProducer, + MetadataBufferType *type); static status_t createPersistentInputSurface( sp<IGraphicBufferProducer> *bufferProducer, sp<IGraphicBufferConsumer> *bufferConsumer); status_t setInputSurface( - OMX_U32 portIndex, const sp<IGraphicBufferConsumer> &bufferConsumer); + OMX_U32 portIndex, const sp<IGraphicBufferConsumer> &bufferConsumer, + MetadataBufferType *type); status_t signalEndOfInputStream(); @@ -109,9 +112,8 @@ struct OMXNodeInstance { OMX_U32 rangeOffset, OMX_U32 rangeLength, OMX_U32 flags, OMX_TICKS timestamp); - status_t emptyDirectBuffer( - OMX_BUFFERHEADERTYPE *header, - OMX_U32 rangeOffset, OMX_U32 rangeLength, + status_t emptyGraphicBuffer( + OMX_BUFFERHEADERTYPE *header, const sp<GraphicBuffer> &buffer, OMX_U32 flags, OMX_TICKS timestamp); status_t getExtensionIndex( @@ -156,6 +158,7 @@ private: uint32_t mBufferIDCount; KeyedVector<OMX::buffer_id, OMX_BUFFERHEADERTYPE *> mBufferIDToBufferHeader; KeyedVector<OMX_BUFFERHEADERTYPE *, OMX::buffer_id> mBufferHeaderToBufferID; + MetadataBufferType mMetadataType[2]; // For debug support char *mName; @@ -203,15 +206,19 @@ private: OMX_IN OMX_BUFFERHEADERTYPE *pBuffer); status_t storeMetaDataInBuffers_l( - OMX_U32 portIndex, OMX_BOOL enable, - OMX_BOOL useGraphicBuffer, OMX_BOOL *usingGraphicBufferInMeta); + OMX_U32 portIndex, OMX_BOOL enable, MetadataBufferType *type); status_t emptyBuffer_l( OMX_BUFFERHEADERTYPE *header, OMX_U32 flags, OMX_TICKS timestamp, intptr_t debugAddr); + status_t updateGraphicBufferInMeta_l( + OMX_U32 portIndex, const sp<GraphicBuffer> &graphicBuffer, + OMX::buffer_id buffer, OMX_BUFFERHEADERTYPE *header); + status_t createGraphicBufferSource( - OMX_U32 portIndex, sp<IGraphicBufferConsumer> consumer = NULL); + OMX_U32 portIndex, sp<IGraphicBufferConsumer> consumer /* nullable */, + MetadataBufferType *type); sp<GraphicBufferSource> getGraphicBufferSource(); void setGraphicBufferSource(const sp<GraphicBufferSource>& bufferSource); diff --git a/media/libstagefright/omx/GraphicBufferSource.cpp b/media/libstagefright/omx/GraphicBufferSource.cpp index 01cd8f0..f797e63 100644 --- a/media/libstagefright/omx/GraphicBufferSource.cpp +++ b/media/libstagefright/omx/GraphicBufferSource.cpp @@ -29,6 +29,7 @@ #include <media/hardware/MetadataBufferType.h> #include <ui/GraphicBuffer.h> #include <gui/BufferItem.h> +#include <HardwareAPI.h> #include <inttypes.h> #include "FrameDropper.h" @@ -43,7 +44,6 @@ GraphicBufferSource::GraphicBufferSource( uint32_t bufferWidth, uint32_t bufferHeight, uint32_t bufferCount, - bool useGraphicBufferInMeta, const sp<IGraphicBufferConsumer> &consumer) : mInitCheck(UNKNOWN_ERROR), mNodeInstance(nodeInstance), @@ -68,8 +68,7 @@ GraphicBufferSource::GraphicBufferSource( mTimePerCaptureUs(-1ll), mTimePerFrameUs(-1ll), mPrevCaptureUs(-1ll), - mPrevFrameUs(-1ll), - mUseGraphicBufferInMeta(useGraphicBufferInMeta) { + mPrevFrameUs(-1ll) { ALOGV("GraphicBufferSource w=%u h=%u c=%u", bufferWidth, bufferHeight, bufferCount); @@ -262,27 +261,27 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) { return; } - if (EXTRA_CHECK) { + if (EXTRA_CHECK && header->nAllocLen >= sizeof(MetadataBufferType)) { // Pull the graphic buffer handle back out of the buffer, and confirm // that it matches expectations. OMX_U8* data = header->pBuffer; MetadataBufferType type = *(MetadataBufferType *)data; - if (type == kMetadataBufferTypeGrallocSource) { - buffer_handle_t bufferHandle; - memcpy(&bufferHandle, data + 4, sizeof(buffer_handle_t)); - if (bufferHandle != codecBuffer.mGraphicBuffer->handle) { + if (type == kMetadataBufferTypeGrallocSource + && header->nAllocLen >= sizeof(VideoGrallocMetadata)) { + VideoGrallocMetadata &grallocMeta = *(VideoGrallocMetadata *)data; + if (grallocMeta.hHandle != codecBuffer.mGraphicBuffer->handle) { // should never happen ALOGE("codecBufferEmptied: buffer's handle is %p, expected %p", - bufferHandle, codecBuffer.mGraphicBuffer->handle); + grallocMeta.hHandle, codecBuffer.mGraphicBuffer->handle); CHECK(!"codecBufferEmptied: mismatched buffer"); } - } else if (type == kMetadataBufferTypeGraphicBuffer) { - GraphicBuffer *buffer; - memcpy(&buffer, data + 4, sizeof(buffer)); - if (buffer != codecBuffer.mGraphicBuffer.get()) { + } else if (type == kMetadataBufferTypeANWBuffer + && header->nAllocLen >= sizeof(VideoNativeMetadata)) { + VideoNativeMetadata &nativeMeta = *(VideoNativeMetadata *)data; + if (nativeMeta.pBuffer != codecBuffer.mGraphicBuffer->getNativeBuffer()) { // should never happen ALOGE("codecBufferEmptied: buffer is %p, expected %p", - buffer, codecBuffer.mGraphicBuffer.get()); + nativeMeta.pBuffer, codecBuffer.mGraphicBuffer->getNativeBuffer()); CHECK(!"codecBufferEmptied: mismatched buffer"); } } @@ -703,36 +702,17 @@ status_t GraphicBufferSource::submitBuffer_l( codecBuffer.mFrameNumber = item.mFrameNumber; OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader; - CHECK(header->nAllocLen >= 4 + sizeof(buffer_handle_t)); - OMX_U8* data = header->pBuffer; - buffer_handle_t handle; - if (!mUseGraphicBufferInMeta) { - const OMX_U32 type = kMetadataBufferTypeGrallocSource; - handle = codecBuffer.mGraphicBuffer->handle; - memcpy(data, &type, 4); - memcpy(data + 4, &handle, sizeof(buffer_handle_t)); - } else { - // codecBuffer holds a reference to the GraphicBuffer, so - // it is valid while it is with the OMX component - const OMX_U32 type = kMetadataBufferTypeGraphicBuffer; - memcpy(data, &type, 4); - // passing a non-reference-counted graphicBuffer - GraphicBuffer *buffer = codecBuffer.mGraphicBuffer.get(); - handle = buffer->handle; - memcpy(data + 4, &buffer, sizeof(buffer)); - } - - status_t err = mNodeInstance->emptyDirectBuffer(header, 0, - 4 + sizeof(buffer_handle_t), OMX_BUFFERFLAG_ENDOFFRAME, - timeUs); + sp<GraphicBuffer> buffer = codecBuffer.mGraphicBuffer; + status_t err = mNodeInstance->emptyGraphicBuffer( + header, buffer, OMX_BUFFERFLAG_ENDOFFRAME, timeUs); if (err != OK) { - ALOGW("WARNING: emptyDirectBuffer failed: 0x%x", err); + ALOGW("WARNING: emptyNativeWindowBuffer failed: 0x%x", err); codecBuffer.mGraphicBuffer = NULL; return err; } - ALOGV("emptyDirectBuffer succeeded, h=%p p=%p bufhandle=%p", - header, header->pBuffer, handle); + ALOGV("emptyNativeWindowBuffer succeeded, h=%p p=%p buf=%p bufhandle=%p", + header, header->pBuffer, buffer->getNativeBuffer(), buffer->handle); return OK; } @@ -755,19 +735,9 @@ void GraphicBufferSource::submitEndOfInputStream_l() { CodecBuffer& codecBuffer(mCodecBuffers.editItemAt(cbi)); OMX_BUFFERHEADERTYPE* header = codecBuffer.mHeader; - if (EXTRA_CHECK) { - // Guard against implementations that don't check nFilledLen. - size_t fillLen = 4 + sizeof(buffer_handle_t); - CHECK(header->nAllocLen >= fillLen); - OMX_U8* data = header->pBuffer; - memset(data, 0xcd, fillLen); - } - - uint64_t timestamp = 0; // does this matter? - - status_t err = mNodeInstance->emptyDirectBuffer(header, /*offset*/ 0, - /*length*/ 0, OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_EOS, - timestamp); + status_t err = mNodeInstance->emptyGraphicBuffer( + header, NULL /* buffer */, OMX_BUFFERFLAG_ENDOFFRAME | OMX_BUFFERFLAG_EOS, + 0 /* timestamp */); if (err != OK) { ALOGW("emptyDirectBuffer EOS failed: 0x%x", err); } else { diff --git a/media/libstagefright/omx/GraphicBufferSource.h b/media/libstagefright/omx/GraphicBufferSource.h index 1047fb3..21ee96a 100644 --- a/media/libstagefright/omx/GraphicBufferSource.h +++ b/media/libstagefright/omx/GraphicBufferSource.h @@ -55,7 +55,6 @@ public: uint32_t bufferWidth, uint32_t bufferHeight, uint32_t bufferCount, - bool useGraphicBufferInMeta = false, const sp<IGraphicBufferConsumer> &consumer = NULL ); @@ -286,7 +285,7 @@ private: int64_t mPrevCaptureUs; int64_t mPrevFrameUs; - bool mUseGraphicBufferInMeta; + MetadataBufferType mMetadataBufferType; void onMessageReceived(const sp<AMessage> &msg); diff --git a/media/libstagefright/omx/OMX.cpp b/media/libstagefright/omx/OMX.cpp index d46bf9d..4ca827c 100644 --- a/media/libstagefright/omx/OMX.cpp +++ b/media/libstagefright/omx/OMX.cpp @@ -332,8 +332,8 @@ status_t OMX::getGraphicBufferUsage( } status_t OMX::storeMetaDataInBuffers( - node_id node, OMX_U32 port_index, OMX_BOOL enable) { - return findInstance(node)->storeMetaDataInBuffers(port_index, enable); + node_id node, OMX_U32 port_index, OMX_BOOL enable, MetadataBufferType *type) { + return findInstance(node)->storeMetaDataInBuffers(port_index, enable, type); } status_t OMX::prepareForAdaptivePlayback( @@ -373,9 +373,9 @@ status_t OMX::updateGraphicBufferInMeta( status_t OMX::createInputSurface( node_id node, OMX_U32 port_index, - sp<IGraphicBufferProducer> *bufferProducer) { + sp<IGraphicBufferProducer> *bufferProducer, MetadataBufferType *type) { return findInstance(node)->createInputSurface( - port_index, bufferProducer); + port_index, bufferProducer, type); } status_t OMX::createPersistentInputSurface( @@ -387,8 +387,8 @@ status_t OMX::createPersistentInputSurface( status_t OMX::setInputSurface( node_id node, OMX_U32 port_index, - const sp<IGraphicBufferConsumer> &bufferConsumer) { - return findInstance(node)->setInputSurface(port_index, bufferConsumer); + const sp<IGraphicBufferConsumer> &bufferConsumer, MetadataBufferType *type) { + return findInstance(node)->setInputSurface(port_index, bufferConsumer, type); } diff --git a/media/libstagefright/omx/OMXNodeInstance.cpp b/media/libstagefright/omx/OMXNodeInstance.cpp index f5f0f4f..e4b2de4 100644 --- a/media/libstagefright/omx/OMXNodeInstance.cpp +++ b/media/libstagefright/omx/OMXNodeInstance.cpp @@ -32,6 +32,7 @@ #include <gui/BufferQueue.h> #include <HardwareAPI.h> #include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/MediaErrors.h> #include <utils/misc.h> @@ -135,6 +136,18 @@ struct BufferMeta { header->nFilledLen); } + // return either the codec or the backup buffer + sp<ABuffer> getBuffer(const OMX_BUFFERHEADERTYPE *header, bool backup) { + sp<ABuffer> buf; + if (backup && mMem != NULL) { + buf = new ABuffer(mMem->pointer(), mMem->size()); + } else { + buf = new ABuffer(header->pBuffer, header->nAllocLen); + } + buf->setRange(header->nOffset, header->nFilledLen); + return buf; + } + void setGraphicBuffer(const sp<GraphicBuffer> &graphicBuffer) { mGraphicBuffer = graphicBuffer; } @@ -180,6 +193,8 @@ OMXNodeInstance::OMXNodeInstance( mNumPortBuffers[1] = 0; mDebugLevelBumpPendingBuffers[0] = 0; mDebugLevelBumpPendingBuffers[1] = 0; + mMetadataType[0] = kMetadataBufferTypeInvalid; + mMetadataType[1] = kMetadataBufferTypeInvalid; } OMXNodeInstance::~OMXNodeInstance() { @@ -486,63 +501,73 @@ status_t OMXNodeInstance::getGraphicBufferUsage( } status_t OMXNodeInstance::storeMetaDataInBuffers( - OMX_U32 portIndex, - OMX_BOOL enable) { + OMX_U32 portIndex, OMX_BOOL enable, MetadataBufferType *type) { Mutex::Autolock autolock(mLock); CLOG_CONFIG(storeMetaDataInBuffers, "%s:%u en:%d", portString(portIndex), portIndex, enable); - return storeMetaDataInBuffers_l( - portIndex, enable, - OMX_FALSE /* useGraphicBuffer */, NULL /* usingGraphicBufferInMetadata */); + return storeMetaDataInBuffers_l(portIndex, enable, type); } status_t OMXNodeInstance::storeMetaDataInBuffers_l( - OMX_U32 portIndex, - OMX_BOOL enable, - OMX_BOOL useGraphicBuffer, - OMX_BOOL *usingGraphicBufferInMetadata) { + OMX_U32 portIndex, OMX_BOOL enable, MetadataBufferType *type) { + if (portIndex != kPortIndexInput && portIndex != kPortIndexOutput) { + return BAD_VALUE; + } + OMX_INDEXTYPE index; OMX_STRING name = const_cast<OMX_STRING>( "OMX.google.android.index.storeMetaDataInBuffers"); - OMX_STRING graphicBufferName = const_cast<OMX_STRING>( - "OMX.google.android.index.storeGraphicBufferInMetaData"); - if (usingGraphicBufferInMetadata == NULL) { - usingGraphicBufferInMetadata = &useGraphicBuffer; - } + OMX_STRING nativeBufferName = const_cast<OMX_STRING>( + "OMX.google.android.index.storeANWBufferInMetadata"); + MetadataBufferType negotiatedType; - OMX_ERRORTYPE err = - (useGraphicBuffer && portIndex == kPortIndexInput) - ? OMX_GetExtensionIndex(mHandle, graphicBufferName, &index) - : OMX_ErrorBadParameter; - if (err == OMX_ErrorNone) { - *usingGraphicBufferInMetadata = OMX_TRUE; - name = graphicBufferName; - } else { - err = OMX_GetExtensionIndex(mHandle, name, &index); - } + StoreMetaDataInBuffersParams params; + InitOMXParams(¶ms); + params.nPortIndex = portIndex; + params.bStoreMetaData = enable; + OMX_ERRORTYPE err = OMX_GetExtensionIndex(mHandle, nativeBufferName, &index); OMX_ERRORTYPE xerr = err; if (err == OMX_ErrorNone) { - StoreMetaDataInBuffersParams params; - InitOMXParams(¶ms); - params.nPortIndex = portIndex; - params.bStoreMetaData = enable; - err = OMX_SetParameter(mHandle, index, ¶ms); + if (err == OMX_ErrorNone) { + name = nativeBufferName; // set name for debugging + negotiatedType = kMetadataBufferTypeANWBuffer; + } + } + if (err != OMX_ErrorNone) { + err = OMX_GetExtensionIndex(mHandle, name, &index); + xerr = err; + if (err == OMX_ErrorNone) { + negotiatedType = kMetadataBufferTypeGrallocSource; + err = OMX_SetParameter(mHandle, index, ¶ms); + } } // don't log loud error if component does not support metadata mode on the output if (err != OMX_ErrorNone) { - *usingGraphicBufferInMetadata = OMX_FALSE; if (err == OMX_ErrorUnsupportedIndex && portIndex == kPortIndexOutput) { CLOGW("component does not support metadata mode; using fallback"); } else if (xerr != OMX_ErrorNone) { CLOG_ERROR(getExtensionIndex, xerr, "%s", name); } else { - CLOG_ERROR(setParameter, err, "%s(%#x): %s:%u en=%d GB=%d", name, index, - portString(portIndex), portIndex, enable, useGraphicBuffer); + CLOG_ERROR(setParameter, err, "%s(%#x): %s:%u en=%d type=%d", name, index, + portString(portIndex), portIndex, enable, negotiatedType); + } + negotiatedType = mMetadataType[portIndex]; + } else { + if (!enable) { + negotiatedType = kMetadataBufferTypeInvalid; } + mMetadataType[portIndex] = negotiatedType; } + CLOG_CONFIG(storeMetaDataInBuffers, "%s:%u negotiated %s:%d", + portString(portIndex), portIndex, asString(negotiatedType), negotiatedType); + + if (type != NULL) { + *type = negotiatedType; + } + return StatusFromOMXError(err); } @@ -774,37 +799,59 @@ status_t OMXNodeInstance::useGraphicBuffer( return OK; } -status_t OMXNodeInstance::updateGraphicBufferInMeta( +status_t OMXNodeInstance::updateGraphicBufferInMeta_l( OMX_U32 portIndex, const sp<GraphicBuffer>& graphicBuffer, - OMX::buffer_id buffer) { - Mutex::Autolock autoLock(mLock); + OMX::buffer_id buffer, OMX_BUFFERHEADERTYPE *header) { + if (portIndex != kPortIndexInput && portIndex != kPortIndexOutput) { + return BAD_VALUE; + } - OMX_BUFFERHEADERTYPE *header = findBufferHeader(buffer); - VideoDecoderOutputMetaData *metadata = - (VideoDecoderOutputMetaData *)(header->pBuffer); BufferMeta *bufferMeta = (BufferMeta *)(header->pAppPrivate); bufferMeta->setGraphicBuffer(graphicBuffer); - metadata->eType = kMetadataBufferTypeGrallocSource; - metadata->pHandle = graphicBuffer->handle; + if (mMetadataType[portIndex] == kMetadataBufferTypeGrallocSource + && header->nAllocLen >= sizeof(VideoGrallocMetadata)) { + VideoGrallocMetadata &metadata = *(VideoGrallocMetadata *)(header->pBuffer); + metadata.eType = kMetadataBufferTypeGrallocSource; + metadata.hHandle = graphicBuffer == NULL ? NULL : graphicBuffer->handle; + } else if (mMetadataType[portIndex] == kMetadataBufferTypeANWBuffer + && header->nAllocLen >= sizeof(VideoNativeMetadata)) { + VideoNativeMetadata &metadata = *(VideoNativeMetadata *)(header->pBuffer); + metadata.eType = kMetadataBufferTypeANWBuffer; + metadata.pBuffer = graphicBuffer == NULL ? NULL : graphicBuffer->getNativeBuffer(); + metadata.nFenceFd = -1; + } else { + CLOG_BUFFER(updateGraphicBufferInMeta, "%s:%u, %#x bad type (%d) or size (%u)", + portString(portIndex), portIndex, buffer, mMetadataType[portIndex], header->nAllocLen); + return BAD_VALUE; + } + CLOG_BUFFER(updateGraphicBufferInMeta, "%s:%u, %#x := %p", portString(portIndex), portIndex, buffer, graphicBuffer->handle); return OK; } +status_t OMXNodeInstance::updateGraphicBufferInMeta( + OMX_U32 portIndex, const sp<GraphicBuffer>& graphicBuffer, + OMX::buffer_id buffer) { + Mutex::Autolock autoLock(mLock); + OMX_BUFFERHEADERTYPE *header = findBufferHeader(buffer); + return updateGraphicBufferInMeta_l(portIndex, graphicBuffer, buffer, header); +} + status_t OMXNodeInstance::createGraphicBufferSource( - OMX_U32 portIndex, sp<IGraphicBufferConsumer> bufferConsumer) { + OMX_U32 portIndex, sp<IGraphicBufferConsumer> bufferConsumer, MetadataBufferType *type) { status_t err; const sp<GraphicBufferSource>& surfaceCheck = getGraphicBufferSource(); if (surfaceCheck != NULL) { + if (portIndex < NELEM(mMetadataType) && type != NULL) { + *type = mMetadataType[portIndex]; + } return ALREADY_EXISTS; } - // Input buffers will hold meta-data (gralloc references). - OMX_BOOL usingGraphicBuffer = OMX_FALSE; - err = storeMetaDataInBuffers_l( - portIndex, OMX_TRUE, - OMX_TRUE /* useGraphicBuffer */, &usingGraphicBuffer); + // Input buffers will hold meta-data (ANativeWindowBuffer references). + err = storeMetaDataInBuffers_l(portIndex, OMX_TRUE, type); if (err != OK) { return err; } @@ -834,7 +881,6 @@ status_t OMXNodeInstance::createGraphicBufferSource( def.format.video.nFrameWidth, def.format.video.nFrameHeight, def.nBufferCountActual, - usingGraphicBuffer, bufferConsumer); if ((err = bufferSource->initCheck()) != OK) { @@ -846,9 +892,9 @@ status_t OMXNodeInstance::createGraphicBufferSource( } status_t OMXNodeInstance::createInputSurface( - OMX_U32 portIndex, sp<IGraphicBufferProducer> *bufferProducer) { + OMX_U32 portIndex, sp<IGraphicBufferProducer> *bufferProducer, MetadataBufferType *type) { Mutex::Autolock autolock(mLock); - status_t err = createGraphicBufferSource(portIndex); + status_t err = createGraphicBufferSource(portIndex, NULL /* bufferConsumer */, type); if (err != OK) { return err; @@ -886,9 +932,10 @@ status_t OMXNodeInstance::createPersistentInputSurface( } status_t OMXNodeInstance::setInputSurface( - OMX_U32 portIndex, const sp<IGraphicBufferConsumer> &bufferConsumer) { + OMX_U32 portIndex, const sp<IGraphicBufferConsumer> &bufferConsumer, + MetadataBufferType *type) { Mutex::Autolock autolock(mLock); - return createGraphicBufferSource(portIndex, bufferConsumer); + return createGraphicBufferSource(portIndex, bufferConsumer, type); } status_t OMXNodeInstance::signalEndOfInputStream() { @@ -1044,7 +1091,24 @@ status_t OMXNodeInstance::emptyBuffer( BufferMeta *buffer_meta = static_cast<BufferMeta *>(header->pAppPrivate); - buffer_meta->CopyToOMX(header); + sp<ABuffer> backup = buffer_meta->getBuffer(header, true /* backup */); + sp<ABuffer> codec = buffer_meta->getBuffer(header, false /* backup */); + + // convert incoming ANW meta buffers if component is configured for gralloc metadata mode + if (mMetadataType[kPortIndexInput] == kMetadataBufferTypeGrallocSource + && backup->capacity() >= sizeof(VideoNativeMetadata) + && codec->capacity() >= sizeof(VideoGrallocMetadata) + && ((VideoNativeMetadata *)backup->base())->eType + == kMetadataBufferTypeANWBuffer) { + VideoNativeMetadata &backupMeta = *(VideoNativeMetadata *)backup->base(); + VideoGrallocMetadata &codecMeta = *(VideoGrallocMetadata *)codec->base(); + ALOGV("converting ANWB %p to handle %p", backupMeta.pBuffer, backupMeta.pBuffer->handle); + codecMeta.hHandle = backupMeta.pBuffer->handle; + codecMeta.eType = kMetadataBufferTypeGrallocSource; + header->nFilledLen = sizeof(codecMeta); + } else { + buffer_meta->CopyToOMX(header); + } return emptyBuffer_l(header, flags, timestamp, (intptr_t)buffer); } @@ -1106,15 +1170,19 @@ status_t OMXNodeInstance::emptyBuffer_l( } // like emptyBuffer, but the data is already in header->pBuffer -status_t OMXNodeInstance::emptyDirectBuffer( - OMX_BUFFERHEADERTYPE *header, - OMX_U32 rangeOffset, OMX_U32 rangeLength, +status_t OMXNodeInstance::emptyGraphicBuffer( + OMX_BUFFERHEADERTYPE *header, const sp<GraphicBuffer> &graphicBuffer, OMX_U32 flags, OMX_TICKS timestamp) { Mutex::Autolock autoLock(mLock); + OMX::buffer_id buffer = findBufferID(header); + status_t err = updateGraphicBufferInMeta_l(kPortIndexInput, graphicBuffer, buffer, header); + if (err != OK) { + CLOG_ERROR(emptyGraphicBuffer, err, FULL_BUFFER((intptr_t)header->pBuffer, header)); + return err; + } - header->nFilledLen = rangeLength; - header->nOffset = rangeOffset; - + header->nOffset = 0; + header->nFilledLen = graphicBuffer == NULL ? 0 : header->nAllocLen; return emptyBuffer_l(header, flags, timestamp, (intptr_t)header->pBuffer); } diff --git a/media/libstagefright/omx/SoftVideoEncoderOMXComponent.cpp b/media/libstagefright/omx/SoftVideoEncoderOMXComponent.cpp index d4d6217..5f80cbc 100644 --- a/media/libstagefright/omx/SoftVideoEncoderOMXComponent.cpp +++ b/media/libstagefright/omx/SoftVideoEncoderOMXComponent.cpp @@ -155,7 +155,7 @@ void SoftVideoEncoderOMXComponent::updatePortParams() { uint32_t rawBufferSize = inDef->format.video.nStride * inDef->format.video.nSliceHeight * 3 / 2; if (inDef->format.video.eColorFormat == OMX_COLOR_FormatAndroidOpaque) { - inDef->nBufferSize = 4 + max(sizeof(buffer_handle_t), sizeof(GraphicBuffer *)); + inDef->nBufferSize = max(sizeof(VideoNativeMetadata), sizeof(VideoGrallocMetadata)); } else { inDef->nBufferSize = rawBufferSize; } @@ -482,8 +482,8 @@ const uint8_t *SoftVideoEncoderOMXComponent::extractGraphicBuffer( size_t dstVStride = height; MetadataBufferType bufferType = *(MetadataBufferType *)src; - bool usingGraphicBuffer = bufferType == kMetadataBufferTypeGraphicBuffer; - if (!usingGraphicBuffer && bufferType != kMetadataBufferTypeGrallocSource) { + bool usingANWBuffer = bufferType == kMetadataBufferTypeANWBuffer; + if (!usingANWBuffer && bufferType != kMetadataBufferTypeGrallocSource) { ALOGE("Unsupported metadata type (%d)", bufferType); return NULL; } @@ -499,13 +499,14 @@ const uint8_t *SoftVideoEncoderOMXComponent::extractGraphicBuffer( int format; size_t srcStride; size_t srcVStride; - if (usingGraphicBuffer) { - if (srcSize < sizeof(OMX_U32) + sizeof(GraphicBuffer *)) { - ALOGE("Metadata is too small (%zu vs %zu)", srcSize, sizeof(OMX_U32) + sizeof(GraphicBuffer *)); + if (usingANWBuffer) { + if (srcSize < sizeof(VideoNativeMetadata)) { + ALOGE("Metadata is too small (%zu vs %zu)", srcSize, sizeof(VideoNativeMetadata)); return NULL; } - GraphicBuffer *buffer = *(GraphicBuffer **)(src + sizeof(OMX_U32)); + VideoNativeMetadata &nativeMeta = *(VideoNativeMetadata *)src; + ANativeWindowBuffer *buffer = nativeMeta.pBuffer; handle = buffer->handle; format = buffer->format; srcStride = buffer->stride; @@ -519,12 +520,13 @@ const uint8_t *SoftVideoEncoderOMXComponent::extractGraphicBuffer( } else { // TODO: remove this part. Check if anyone uses this. - if (srcSize < sizeof(OMX_U32) + sizeof(buffer_handle_t)) { - ALOGE("Metadata is too small (%zu vs %zu)", srcSize, sizeof(OMX_U32) + sizeof(buffer_handle_t)); + if (srcSize < sizeof(VideoGrallocMetadata)) { + ALOGE("Metadata is too small (%zu vs %zu)", srcSize, sizeof(VideoGrallocMetadata)); return NULL; } - handle = *(buffer_handle_t *)(src + sizeof(OMX_U32)); + VideoGrallocMetadata &grallocMeta = *(VideoGrallocMetadata *)(src); + handle = grallocMeta.hHandle; // assume HAL_PIXEL_FORMAT_RGBA_8888 // there is no way to get the src stride without the graphic buffer format = HAL_PIXEL_FORMAT_RGBA_8888; @@ -606,7 +608,7 @@ const uint8_t *SoftVideoEncoderOMXComponent::extractGraphicBuffer( OMX_ERRORTYPE SoftVideoEncoderOMXComponent::getExtensionIndex( const char *name, OMX_INDEXTYPE *index) { if (!strcmp(name, "OMX.google.android.index.storeMetaDataInBuffers") || - !strcmp(name, "OMX.google.android.index.storeGraphicBufferInMetaData")) { + !strcmp(name, "OMX.google.android.index.storeANWBufferInMetadata")) { *(int32_t*)index = kStoreMetaDataExtensionIndex; return OMX_ErrorNone; } |