avextensions: Add support for Codec Extension features

Add support for Codec extension features like
skype, vqzip, b-frame encoding, prefer adaptive playback etc.
Add extensions to support encode batching.

Also enable Werror for libavextensions.

Change-Id: Ia342d4abe4bb1e21b9ff2b8213fb96421485a333

16 files changed, 69 insertions, 17 deletions

diff --git a/include/media/stagefright/ACodec.h b/include/media/stagefright/ACodec.h
index 55833b6..9eaec9b 100644
--- a/include/media/stagefright/ACodec.h
+++ b/include/media/stagefright/ACodec.h
@@ -305,7 +305,7 @@ protected:
             ssize_t *index = NULL);
 
     virtual status_t setComponentRole(bool isEncoder, const char *mime);
-    status_t configureCodec(const char *mime, const sp<AMessage> &msg);
+    virtual status_t configureCodec(const char *mime, const sp<AMessage> &msg);
 
     status_t configureTunneledVideoPlayback(int32_t audioHwSync,
             const sp<ANativeWindow> &nativeWindow);
@@ -321,7 +321,7 @@ protected:
     virtual status_t setupVideoDecoder(
             const char *mime, const sp<AMessage> &msg, bool usingNativeBuffers);
 
-    status_t setupVideoEncoder(
+    virtual status_t setupVideoEncoder(
             const char *mime, const sp<AMessage> &msg);
 
     status_t setVideoFormatOnPort(
@@ -424,11 +424,19 @@ protected:
         DescribeColorFormatParams &describeParams);
 
     status_t requestIDRFrame();
-    status_t setParameters(const sp<AMessage> &params);
+    virtual status_t setParameters(const sp<AMessage> &params);
 
     // Send EOS on input stream.
     void onSignalEndOfInputStream();
 
+    virtual void setBFrames(OMX_VIDEO_PARAM_MPEG4TYPE *mpeg4type) {}
+    virtual void setBFrames(OMX_VIDEO_PARAM_AVCTYPE *h264type,
+        const int32_t iFramesInterval, const int32_t frameRate) {}
+
+    virtual status_t getVQZIPInfo(const sp<AMessage> &msg) {
+        return OK;
+    }
+
     DISALLOW_EVIL_CONSTRUCTORS(ACodec);
 };
 
diff --git a/include/media/stagefright/MediaCodec.h b/include/media/stagefright/MediaCodec.h
index c10963d..b0ae83b 100644
--- a/include/media/stagefright/MediaCodec.h
+++ b/include/media/stagefright/MediaCodec.h
@@ -51,6 +51,7 @@ struct MediaCodec : public AHandler {
         BUFFER_FLAG_SYNCFRAME   = 1,
         BUFFER_FLAG_CODECCONFIG = 2,
         BUFFER_FLAG_EOS         = 4,
+        BUFFER_FLAG_EXTRADATA   = 0x1000,
     };
 
     enum {
diff --git a/media/libavextensions/Android.mk b/media/libavextensions/Android.mk
index 26c9f85..0bc886d 100644
--- a/media/libavextensions/Android.mk
+++ b/media/libavextensions/Android.mk
@@ -15,7 +15,7 @@ LOCAL_C_INCLUDES:= \
         $(TOP)/hardware/qcom/media/mm-core/inc \
         $(TOP)/frameworks/av/media/libstagefright \
 
-LOCAL_CFLAGS += -Wno-multichar
+LOCAL_CFLAGS += -Wno-multichar -Werror
 
 ifeq ($(TARGET_ENABLE_QC_AV_ENHANCEMENTS),true)
        LOCAL_CFLAGS += -DENABLE_AV_ENHANCEMENTS
@@ -42,7 +42,7 @@ LOCAL_C_INCLUDES:= \
         $(TOP)/external/flac/include \
         $(TOP)/hardware/qcom/media/mm-core/inc
 
-LOCAL_CFLAGS += -Wno-multichar
+LOCAL_CFLAGS += -Wno-multichar -Werror
 
 ifeq ($(TARGET_ENABLE_QC_AV_ENHANCEMENTS),true)
        LOCAL_CFLAGS += -DENABLE_AV_ENHANCEMENTS
@@ -75,7 +75,7 @@ LOCAL_C_INCLUDES:= \
         $(TOP)/external/flac/include \
         $(TOP)/hardware/qcom/media/mm-core/inc
 
-LOCAL_CFLAGS += -Wno-multichar
+LOCAL_CFLAGS += -Wno-multichar -Werror
 
 ifeq ($(TARGET_ENABLE_QC_AV_ENHANCEMENTS),true)
        LOCAL_CFLAGS += -DENABLE_AV_ENHANCEMENTS
diff --git a/media/libavextensions/mediaplayerservice/AVNuExtensions.h b/media/libavextensions/mediaplayerservice/AVNuExtensions.h
index 6ab790f..4d1421c 100644
--- a/media/libavextensions/mediaplayerservice/AVNuExtensions.h
+++ b/media/libavextensions/mediaplayerservice/AVNuExtensions.h
@@ -81,6 +81,8 @@ struct AVNuUtils {
     virtual status_t convertToSinkFormatIfNeeded(const sp<ABuffer> &, sp<ABuffer> &,
             audio_format_t sinkFormat, bool isOffload);
 
+    virtual void printFileName(int fd);
+
     // ----- NO TRESSPASSING BEYOND THIS LINE ------
     DECLARE_LOADABLE_SINGLETON(AVNuUtils);
 };
diff --git a/media/libavextensions/mediaplayerservice/AVNuUtils.cpp b/media/libavextensions/mediaplayerservice/AVNuUtils.cpp
index 2e2e511..bc07c90 100644
--- a/media/libavextensions/mediaplayerservice/AVNuUtils.cpp
+++ b/media/libavextensions/mediaplayerservice/AVNuUtils.cpp
@@ -44,7 +44,6 @@
 
 namespace android {
 
-
 sp<MetaData> AVNuUtils::createPCMMetaFromSource(const sp<MetaData> &sMeta) {
     return sMeta;
 }
@@ -93,18 +92,18 @@ void AVNuUtils::setSourcePCMFormat(const sp<MetaData> &) {
 void AVNuUtils::setDecodedPCMFormat(const sp<AMessage> &) {
 
 }
+
 status_t AVNuUtils::convertToSinkFormatIfNeeded(const sp<ABuffer> &, sp<ABuffer> &,
         audio_format_t /*sinkFormat*/, bool /*isOffload*/) {
     return INVALID_OPERATION;
 }
 
+void AVNuUtils::printFileName(int) {}
 
 // ----- NO TRESSPASSING BEYOND THIS LINE ------
-AVNuUtils::AVNuUtils() {
-}
+AVNuUtils::AVNuUtils() {}
 
-AVNuUtils::~AVNuUtils() {
-}
+AVNuUtils::~AVNuUtils() {}
 
 //static
 AVNuUtils *AVNuUtils::sInst =
diff --git a/media/libavextensions/stagefright/AVExtensions.h b/media/libavextensions/stagefright/AVExtensions.h
index 2ebfcde..f2a079f 100644
--- a/media/libavextensions/stagefright/AVExtensions.h
+++ b/media/libavextensions/stagefright/AVExtensions.h
@@ -44,6 +44,8 @@ struct IMediaHTTPConnection;
 struct MediaCodec;
 struct MediaHTTP;
 struct NuCachedSource2;
+class CameraParameters;
+class MediaBuffer;
 
 /*
  * Factory to create objects of base-classes in libstagefright
@@ -97,6 +99,12 @@ struct AVUtils {
 
     virtual bool canOffloadAPE(const sp<MetaData> &meta);
 
+    virtual void extractCustomCameraKeys(
+            const CameraParameters& /*params*/, sp<MetaData> &/*meta*/) {}
+    virtual void printFileName(int /*fd*/) {}
+    virtual void addDecodingTimesFromBatch(MediaBuffer * /*buf*/,
+            List<int64_t> &/*decodeTimeQueue*/) {}
+
     // ----- NO TRESSPASSING BEYOND THIS LINE ------
     DECLARE_LOADABLE_SINGLETON(AVUtils);
 };
diff --git a/media/libavextensions/stagefright/AVUtils.cpp b/media/libavextensions/stagefright/AVUtils.cpp
index 502154f..a683f48 100644
--- a/media/libavextensions/stagefright/AVUtils.cpp
+++ b/media/libavextensions/stagefright/AVUtils.cpp
@@ -101,7 +101,7 @@ sp<MediaCodec> AVUtils::createCustomComponentByName(
     return NULL;
 }
 
-bool AVUtils::canOffloadAPE(const sp<MetaData> &meta) {
+bool AVUtils::canOffloadAPE(const sp<MetaData> &) {
    return true;
 }
 
@@ -110,11 +110,9 @@ bool AVUtils::isEnhancedExtension(const char *) {
 }
 
 // ----- NO TRESSPASSING BEYOND THIS LINE ------
-AVUtils::AVUtils() {
-}
+AVUtils::AVUtils() {}
 
-AVUtils::~AVUtils() {
-}
+AVUtils::~AVUtils() {}
 
 //static
 AVUtils *AVUtils::sInst =
diff --git a/media/libmediaplayerservice/StagefrightRecorder.cpp b/media/libmediaplayerservice/StagefrightRecorder.cpp
index 4c2e8fd..f261d44 100644
--- a/media/libmediaplayerservice/StagefrightRecorder.cpp
+++ b/media/libmediaplayerservice/StagefrightRecorder.cpp
@@ -1563,6 +1563,8 @@ status_t StagefrightRecorder::setupVideoEncoder(
         }
     }
 
+    setupCustomVideoEncoderParams(cameraSource, format);
+
     format->setInt32("bitrate", mVideoBitRate);
     format->setInt32("frame-rate", mFrameRate);
     format->setInt32("i-frame-interval", mIFramesIntervalSec);
diff --git a/media/libmediaplayerservice/StagefrightRecorder.h b/media/libmediaplayerservice/StagefrightRecorder.h
index e005b57..e7b9e55 100644
--- a/media/libmediaplayerservice/StagefrightRecorder.h
+++ b/media/libmediaplayerservice/StagefrightRecorder.h
@@ -39,6 +39,7 @@ class IGraphicBufferConsumer;
 class IGraphicBufferProducer;
 class SurfaceMediaSource;
 struct ALooper;
+struct AMessage;
 
 struct StagefrightRecorder : public MediaRecorderBase {
     StagefrightRecorder(const String16 &opPackageName);
@@ -149,6 +150,8 @@ protected:
     status_t setupCameraSource(sp<CameraSource> *cameraSource);
     status_t setupAudioEncoder(const sp<MediaWriter>& writer);
     status_t setupVideoEncoder(sp<MediaSource> cameraSource, sp<MediaSource> *source);
+    virtual void setupCustomVideoEncoderParams(sp<MediaSource> /*cameraSource*/,
+            sp<AMessage> &/*format*/) {}
 
     // Encoding parameter handling utilities
     status_t setParameter(const String8 &key, const String8 &value);
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
index e9ce52f..a294d36 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
@@ -117,6 +117,7 @@ status_t NuPlayerDriver::setDataSource(int fd, int64_t offset, int64_t length) {
         mCondition.wait(mLock);
     }
 
+    AVNuUtils::get()->printFileName(fd);
     return mAsyncResult;
 }
 
diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index bd3c17c..9af5f4b 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -3235,6 +3235,7 @@ status_t ACodec::setupMPEG4EncoderParameters(const sp<AMessage> &msg) {
         mpeg4type.eLevel = static_cast<OMX_VIDEO_MPEG4LEVELTYPE>(level);
     }
 
+    setBFrames(&mpeg4type);
     err = mOMX->setParameter(
             mNode, OMX_IndexParamVideoMpeg4, &mpeg4type, sizeof(mpeg4type));
 
@@ -3432,6 +3433,8 @@ status_t ACodec::setupAVCEncoderParameters(const sp<AMessage> &msg) {
         err = verifySupportForProfileAndLevel(profile, level);
 
         if (err != OK) {
+            ALOGE("%s does not support profile %x @ level %x",
+                    mComponentName.c_str(), profile, level);
             return err;
         }
 
@@ -3440,11 +3443,14 @@ status_t ACodec::setupAVCEncoderParameters(const sp<AMessage> &msg) {
     }
 
     // XXX
+    // Allow higher profiles to be set since the encoder seems to support
+#if 0
     if (h264type.eProfile != OMX_VIDEO_AVCProfileBaseline) {
         ALOGW("Use baseline profile instead of %d for AVC recording",
             h264type.eProfile);
         h264type.eProfile = OMX_VIDEO_AVCProfileBaseline;
     }
+#endif
 
     if (h264type.eProfile == OMX_VIDEO_AVCProfileBaseline) {
         h264type.nSliceHeaderSpacing = 0;
@@ -3465,6 +3471,7 @@ status_t ACodec::setupAVCEncoderParameters(const sp<AMessage> &msg) {
         h264type.nCabacInitIdc = 0;
     }
 
+    setBFrames(&h264type, iFrameInterval, frameRate);
     if (h264type.nBFrames != 0) {
         h264type.nAllowedPictureTypes |= OMX_VIDEO_PictureTypeB;
     }
@@ -4456,6 +4463,8 @@ void ACodec::sendFormatChange(const sp<AMessage> &reply) {
                 mEncoderPadding * frameSize);
     }
 
+    getVQZIPInfo(notify);
+
     notify->post();
 
     mSentFormat = true;
@@ -5188,6 +5197,7 @@ bool ACodec::BaseState::onOMXFillBufferDone(
                 mCodec->mSkipCutBuffer->submit(info->mData);
             }
             info->mData->meta()->setInt64("timeUs", timeUs);
+            info->mData->meta()->setObject("graphic-buffer", info->mGraphicBuffer);
 
             sp<AMessage> notify = mCodec->mNotify->dup();
             notify->setInt32("what", CodecBase::kWhatDrainThisBuffer);
diff --git a/media/libstagefright/CameraSource.cpp b/media/libstagefright/CameraSource.cpp
index ed3d520..41513fb 100644
--- a/media/libstagefright/CameraSource.cpp
+++ b/media/libstagefright/CameraSource.cpp
@@ -33,6 +33,8 @@
 #include <utils/String8.h>
 #include <cutils/properties.h>
 
+#include <stagefright/AVExtensions.h>
+
 #if LOG_NDEBUG
 #define UNUSED_UNLESS_VERBOSE(x) (void)(x)
 #else
@@ -587,6 +589,8 @@ status_t CameraSource::initWithCameraAccess(
     mMeta->setInt32(kKeyStride,      mVideoSize.width);
     mMeta->setInt32(kKeySliceHeight, mVideoSize.height);
     mMeta->setInt32(kKeyFrameRate,   mVideoFrameRate);
+    AVUtils::get()->extractCustomCameraKeys(params, mMeta);
+
     return OK;
 }
 
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index 4659d92..fca8b2c 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -984,6 +984,9 @@ bool MediaCodec::handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool
         if (omxFlags & OMX_BUFFERFLAG_EOS) {
             flags |= BUFFER_FLAG_EOS;
         }
+        if (omxFlags & OMX_BUFFERFLAG_EXTRADATA) {
+            flags |= BUFFER_FLAG_EXTRADATA;
+        }
 
         response->setInt32("flags", flags);
         response->postReply(replyID);
diff --git a/media/libstagefright/MediaCodecList.cpp b/media/libstagefright/MediaCodecList.cpp
index 5edc04c..b1dd96f 100644
--- a/media/libstagefright/MediaCodecList.cpp
+++ b/media/libstagefright/MediaCodecList.cpp
@@ -939,7 +939,13 @@ status_t MediaCodecList::addLimit(const char **attrs) {
     // complexity: range + default
     bool found;
 
-    if (name == "aspect-ratio" || name == "bitrate" || name == "block-count"
+    // VT specific limits
+    if (name.find("vt-") == 0) {
+        AString value;
+        if (msg->findString("value", &value) && value.size()) {
+            mCurrentInfo->addDetail(name, value);
+        }
+    } else if (name == "aspect-ratio" || name == "bitrate" || name == "block-count"
             || name == "blocks-per-second" || name == "complexity"
             || name == "frame-rate" || name == "quality" || name == "size"
             || name == "measured-blocks-per-second" || name.startsWith("measured-frame-rate-")) {
diff --git a/media/libstagefright/MediaCodecSource.cpp b/media/libstagefright/MediaCodecSource.cpp
index 16a1931..b91660a 100644
--- a/media/libstagefright/MediaCodecSource.cpp
+++ b/media/libstagefright/MediaCodecSource.cpp
@@ -37,6 +37,7 @@
 #include <media/stagefright/PersistentSurface.h>
 #include <media/stagefright/Utils.h>
 #include <OMX_Core.h>
+#include <stagefright/AVExtensions.h>
 
 namespace android {
 
@@ -576,6 +577,9 @@ status_t MediaCodecSource::feedEncoderInputBuffers() {
             // push decoding time for video, or drift time for audio
             if (mIsVideo) {
                 mDecodingTimeQueue.push_back(timeUs);
+                if (mFlags & FLAG_USE_METADATA_INPUT) {
+                    AVUtils::get()->addDecodingTimesFromBatch(mbuf, mDecodingTimeQueue);
+                }
             } else {
 #if DEBUG_DRIFT_TIME
                 if (mFirstSampleTimeUs < 0ll) {
diff --git a/media/libstagefright/StagefrightMetadataRetriever.cpp b/media/libstagefright/StagefrightMetadataRetriever.cpp
index ccf4587..6c795ac 100644
--- a/media/libstagefright/StagefrightMetadataRetriever.cpp
+++ b/media/libstagefright/StagefrightMetadataRetriever.cpp
@@ -45,6 +45,8 @@
 
 #include <CharacterEncodingDetector.h>
 
+#include <stagefright/AVExtensions.h>
+
 namespace android {
 
 static const int64_t kBufferTimeOutUs = 30000ll; // 30 msec
@@ -104,6 +106,7 @@ status_t StagefrightMetadataRetriever::setDataSource(
     fd = dup(fd);
 
     ALOGV("setDataSource(%d, %" PRId64 ", %" PRId64 ")", fd, offset, length);
+    AVUtils::get()->printFileName(fd);
 
     clearMetadata();
     mSource = new FileSource(fd, offset, length);