From 36dee23baca299f38c134b18f4deb36862bdd89d Mon Sep 17 00:00:00 2001 From: Praveen Chavan Date: Tue, 14 Apr 2015 15:15:04 -0700 Subject: stagefright: Decode video thumbnail using MediaCodec Use MediaCodec (in place of OMXCodec) to decode video thumbnail Change-Id: I05beaa3d67edff51aa17f58444fd34afb3933580 --- .../StagefrightMetadataRetriever.cpp | 305 +++++++++++++-------- 1 file changed, 197 insertions(+), 108 deletions(-) (limited to 'media') diff --git a/media/libstagefright/StagefrightMetadataRetriever.cpp b/media/libstagefright/StagefrightMetadataRetriever.cpp index e9566f2..2054827 100644 --- a/media/libstagefright/StagefrightMetadataRetriever.cpp +++ b/media/libstagefright/StagefrightMetadataRetriever.cpp @@ -20,22 +20,35 @@ #include #include +#include #include "include/StagefrightMetadataRetriever.h" +#include #include + +#include #include +#include #include #include #include +#include +#include +#include +#include #include #include #include -#include +#include + #include namespace android { +static const int64_t kBufferTimeOutUs = 30000ll; // 30 msec +static const size_t kRetryCount = 20; // must be >0 + StagefrightMetadataRetriever::StagefrightMetadataRetriever() : mParsedMetaData(false), mAlbumArt(NULL) { @@ -123,73 +136,52 @@ status_t StagefrightMetadataRetriever::setDataSource( return OK; } -static bool isYUV420PlanarSupported( - OMXClient *client, - const sp &trackMeta) { - - const char *mime; - CHECK(trackMeta->findCString(kKeyMIMEType, &mime)); - - Vector caps; - if (QueryCodecs(client->interface(), mime, - true, /* queryDecoders */ - true, /* hwCodecOnly */ - &caps) == OK) { - - for (size_t j = 0; j < caps.size(); ++j) { - CodecCapabilities cap = caps[j]; - for (size_t i = 0; i < cap.mColorFormats.size(); ++i) { - if (cap.mColorFormats[i] == OMX_COLOR_FormatYUV420Planar) { - return true; - } - } - } - } - return false; -} - -static VideoFrame *extractVideoFrameWithCodecFlags( - OMXClient *client, +static VideoFrame *extractVideoFrame( + const char *componentName, const sp &trackMeta, const sp &source, - uint32_t flags, int64_t frameTimeUs, int seekMode) { sp format = source->getFormat(); - // XXX: - // Once all vendors support OMX_COLOR_FormatYUV420Planar, we can - // remove this check and always set the decoder output color format - if (isYUV420PlanarSupported(client, trackMeta)) { - format->setInt32(kKeyColorFormat, OMX_COLOR_FormatYUV420Planar); - } + sp videoFormat; + convertMetaDataToMessage(trackMeta, &videoFormat); - sp decoder = - OMXCodec::Create( - client->interface(), format, false, source, - NULL, flags | OMXCodec::kClientNeedsFramebuffer); + // TODO: Use Flexible color instead + videoFormat->setInt32("color-format", OMX_COLOR_FormatYUV420Planar); - if (decoder.get() == NULL) { - ALOGV("unable to instantiate video decoder."); + status_t err; + sp looper = new ALooper; + looper->start(); + sp decoder = MediaCodec::CreateByComponentName( + looper, componentName, &err); + if (decoder.get() == NULL || err != OK) { + ALOGW("Failed to instantiate decoder [%s]", componentName); return NULL; } - status_t err = decoder->start(); + err = decoder->configure(videoFormat, NULL /* surface */, NULL /* crypto */, 0 /* flags */); if (err != OK) { - ALOGW("OMXCodec::start returned error %d (0x%08x)\n", err, err); + ALOGW("configure returned error %d (%s)", err, asString(err)); + decoder->release(); return NULL; } - // Read one output buffer, ignore format change notifications - // and spurious empty buffers. + err = decoder->start(); + if (err != OK) { + ALOGW("start returned error %d (%s)", err, asString(err)); + decoder->release(); + return NULL; + } MediaSource::ReadOptions options; if (seekMode < MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC || seekMode > MediaSource::ReadOptions::SEEK_CLOSEST) { ALOGE("Unknown seek mode: %d", seekMode); + decoder->release(); return NULL; } @@ -208,64 +200,155 @@ static VideoFrame *extractVideoFrameWithCodecFlags( options.setSeekTo(frameTimeUs, mode); } - MediaBuffer *buffer = NULL; - do { - if (buffer != NULL) { - buffer->release(); - buffer = NULL; - } - err = decoder->read(&buffer, &options); - options.clearSeekTo(); - } while (err == INFO_FORMAT_CHANGED - || (buffer != NULL && buffer->range_length() == 0)); - + err = source->start(); if (err != OK) { - CHECK(buffer == NULL); + ALOGW("source failed to start: %d (%s)", err, asString(err)); + decoder->release(); + return NULL; + } - ALOGV("decoding frame failed."); - decoder->stop(); + Vector > inputBuffers; + err = decoder->getInputBuffers(&inputBuffers); + if (err != OK) { + ALOGW("failed to get input buffers: %d (%s)", err, asString(err)); + decoder->release(); + return NULL; + } + Vector > outputBuffers; + err = decoder->getOutputBuffers(&outputBuffers); + if (err != OK) { + ALOGW("failed to get output buffers: %d (%s)", err, asString(err)); + decoder->release(); return NULL; } - ALOGV("successfully decoded video frame."); + sp outputFormat = NULL; + bool haveMoreInputs = true; + size_t index, offset, size; + int64_t timeUs; + size_t retriesLeft = kRetryCount; + bool done = false; - int32_t unreadable; - if (buffer->meta_data()->findInt32(kKeyIsUnreadable, &unreadable) - && unreadable != 0) { - ALOGV("video frame is unreadable, decoder does not give us access " - "to the video data."); + do { + size_t inputIndex = -1; + int64_t ptsUs = 0ll; + uint32_t flags = 0; + sp codecBuffer = NULL; + + while (haveMoreInputs) { + err = decoder->dequeueInputBuffer(&inputIndex, kBufferTimeOutUs); + if (err != OK) { + ALOGW("Timed out waiting for input"); + if (retriesLeft) { + err = OK; + } + break; + } + codecBuffer = inputBuffers[inputIndex]; - buffer->release(); - buffer = NULL; + MediaBuffer *mediaBuffer = NULL; - decoder->stop(); + err = source->read(&mediaBuffer, &options); + options.clearSeekTo(); + if (err != OK) { + ALOGW("Input Error or EOS"); + haveMoreInputs = false; + break; + } + + if (mediaBuffer->range_length() > codecBuffer->capacity()) { + ALOGE("buffer size (%zu) too large for codec input size (%zu)", + mediaBuffer->range_length(), codecBuffer->capacity()); + err = BAD_VALUE; + } else { + codecBuffer->setRange(0, mediaBuffer->range_length()); + + CHECK(mediaBuffer->meta_data()->findInt64(kKeyTime, &ptsUs)); + memcpy(codecBuffer->data(), + (const uint8_t*)mediaBuffer->data() + mediaBuffer->range_offset(), + mediaBuffer->range_length()); + } + + mediaBuffer->release(); + break; + } + + if (err == OK && inputIndex < inputBuffers.size()) { + ALOGV("QueueInput: size=%zu ts=%" PRId64 " us flags=%x", + codecBuffer->size(), ptsUs, flags); + err = decoder->queueInputBuffer( + inputIndex, + codecBuffer->offset(), + codecBuffer->size(), + ptsUs, + flags); + + // we don't expect an output from codec config buffer + if (flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) { + continue; + } + } + + while (err == OK) { + // wait for a decoded buffer + err = decoder->dequeueOutputBuffer( + &index, + &offset, + &size, + &timeUs, + &flags, + kBufferTimeOutUs); + + if (err == INFO_FORMAT_CHANGED) { + ALOGV("Received format change"); + err = decoder->getOutputFormat(&outputFormat); + } else if (err == INFO_OUTPUT_BUFFERS_CHANGED) { + ALOGV("Output buffers changed"); + err = decoder->getOutputBuffers(&outputBuffers); + } else { + if (err == -EAGAIN /* INFO_TRY_AGAIN_LATER */ && --retriesLeft > 0) { + ALOGV("Timed-out waiting for output.. retries left = %d", retriesLeft); + err = OK; + } else if (err == OK) { + ALOGV("Received an output buffer"); + done = true; + } else { + ALOGW("Received error %d (%s) instead of output", err, asString(err)); + done = true; + } + break; + } + } + } while (err == OK && !done); + if (err != OK || size <= 0 || outputFormat == NULL) { + ALOGE("Failed to decode thumbnail frame"); + source->stop(); + decoder->stop(); + decoder->release(); return NULL; } - int64_t timeUs; - CHECK(buffer->meta_data()->findInt64(kKeyTime, &timeUs)); + ALOGV("successfully decoded video frame."); + sp videoFrameBuffer = outputBuffers.itemAt(index); + if (thumbNailTime >= 0) { if (timeUs != thumbNailTime) { - const char *mime; - CHECK(trackMeta->findCString(kKeyMIMEType, &mime)); + AString mime; + CHECK(outputFormat->findString("mime", &mime)); - ALOGV("thumbNailTime = %" PRId64 " us, timeUs = %" PRId64 " us, mime = %s", - thumbNailTime, timeUs, mime); + ALOGV("thumbNailTime = %lld us, timeUs = %lld us, mime = %s", + (long long)thumbNailTime, (long long)timeUs, mime.c_str()); } } - sp meta = decoder->getFormat(); - int32_t width, height; - CHECK(meta->findInt32(kKeyWidth, &width)); - CHECK(meta->findInt32(kKeyHeight, &height)); + CHECK(outputFormat->findInt32("width", &width)); + CHECK(outputFormat->findInt32("height", &height)); int32_t crop_left, crop_top, crop_right, crop_bottom; - if (!meta->findRect( - kKeyCropRect, - &crop_left, &crop_top, &crop_right, &crop_bottom)) { + if (!outputFormat->findRect("crop", &crop_left, &crop_top, &crop_right, &crop_bottom)) { crop_left = crop_top = 0; crop_right = width - 1; crop_bottom = height - 1; @@ -285,23 +368,21 @@ static VideoFrame *extractVideoFrameWithCodecFlags( frame->mData = new uint8_t[frame->mSize]; frame->mRotationAngle = rotationAngle; - int32_t displayWidth, displayHeight; - if (meta->findInt32(kKeyDisplayWidth, &displayWidth)) { - frame->mDisplayWidth = displayWidth; - } - if (meta->findInt32(kKeyDisplayHeight, &displayHeight)) { - frame->mDisplayHeight = displayHeight; + int32_t sarWidth, sarHeight; + if (trackMeta->findInt32(kKeySARWidth, &sarWidth) + && trackMeta->findInt32(kKeySARHeight, &sarHeight) + && sarHeight != 0) { + frame->mDisplayWidth = (frame->mDisplayWidth * sarWidth) / sarHeight; } int32_t srcFormat; - CHECK(meta->findInt32(kKeyColorFormat, &srcFormat)); + CHECK(outputFormat->findInt32("color-format", &srcFormat)); - ColorConverter converter( - (OMX_COLOR_FORMATTYPE)srcFormat, OMX_COLOR_Format16bitRGB565); + ColorConverter converter((OMX_COLOR_FORMATTYPE)srcFormat, OMX_COLOR_Format16bitRGB565); if (converter.isValid()) { err = converter.convert( - (const uint8_t *)buffer->data() + buffer->range_offset(), + (const uint8_t *)videoFrameBuffer->data(), width, height, crop_left, crop_top, crop_right, crop_bottom, frame->mData, @@ -309,17 +390,16 @@ static VideoFrame *extractVideoFrameWithCodecFlags( frame->mHeight, 0, 0, frame->mWidth - 1, frame->mHeight - 1); } else { - ALOGE("Unable to instantiate color conversion from format 0x%08x to " - "RGB565", - srcFormat); + ALOGE("Unable to convert from format 0x%08x to RGB565", srcFormat); err = ERROR_UNSUPPORTED; } - buffer->release(); - buffer = NULL; - + videoFrameBuffer.clear(); + source->stop(); + decoder->releaseOutputBuffer(index); decoder->stop(); + decoder->release(); if (err != OK) { ALOGE("Colorconverter failed to convert frame."); @@ -390,20 +470,29 @@ VideoFrame *StagefrightMetadataRetriever::getFrameAtTime( mAlbumArt = MediaAlbumArt::fromData(dataSize, data); } - VideoFrame *frame = - extractVideoFrameWithCodecFlags( - &mClient, trackMeta, source, OMXCodec::kPreferSoftwareCodecs, - timeUs, option); - - if (frame == NULL) { - ALOGV("Software decoder failed to extract thumbnail, " - "trying hardware decoder."); + const char *mime; + CHECK(trackMeta->findCString(kKeyMIMEType, &mime)); - frame = extractVideoFrameWithCodecFlags(&mClient, trackMeta, source, 0, - timeUs, option); + Vector matchingCodecs; + OMXCodec::findMatchingCodecs( + mime, + false, /* encoder */ + NULL, /* matchComponentName */ + OMXCodec::kPreferSoftwareCodecs, + &matchingCodecs); + + for (size_t i = 0; i < matchingCodecs.size(); ++i) { + const char *componentName = matchingCodecs[i].mName.string(); + VideoFrame *frame = + extractVideoFrame(componentName, trackMeta, source, timeUs, option); + + if (frame != NULL) { + return frame; + } + ALOGV("%s failed to extract thumbnail, trying next decoder.", componentName); } - return frame; + return NULL; } MediaAlbumArt *StagefrightMetadataRetriever::extractAlbumArt() { -- cgit v1.1