stagefright: Resurrect PCM offload

 * Forward-port the open-source code from L which was moved to closed-source
   in M. This is being done out of necessity- the architecture chosen by
   Qualcomm is not optimal and doesn't work well with a singular
   codebase which attempts to service a large number of devices.
 * This patch brings in the code to support PCM offload (AudioFlinger
   bypass). This allows for playback of high resolution clips without
   decimation stages, and enables reduced power consumption for audio
   pipelines which take advantage of the Hexagon DSP (effects).

Change-Id: I0ef15fc3df538ab723f3c12ce0ed71d0e607c99e

6 files changed, 195 insertions, 107 deletions

diff --git a/media/libmediaplayerservice/nuplayer/Android.mk b/media/libmediaplayerservice/nuplayer/Android.mk
index cc1bdb4..fbb1276 100644
--- a/media/libmediaplayerservice/nuplayer/Android.mk
+++ b/media/libmediaplayerservice/nuplayer/Android.mk
@@ -27,7 +27,7 @@ LOCAL_C_INCLUDES := \
 	$(TOP)/frameworks/av/media/libavextensions                    \
 	$(TOP)/frameworks/av/include/media                            \
 
-LOCAL_CFLAGS += -Werror -Wall
+LOCAL_CFLAGS += -Werror -Wall #-DLOG_NDEBUG=0
 
 # enable experiments only in userdebug and eng builds
 ifneq (,$(filter userdebug eng,$(TARGET_BUILD_VARIANT)))
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
index de507f1..678db8e 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
@@ -189,6 +189,7 @@ NuPlayer::NuPlayer(pid_t pid)
       mPID(pid),
       mSourceFlags(0),
       mOffloadAudio(false),
+      mOffloadDecodedPCM(false),
       mAudioDecoderGeneration(0),
       mVideoDecoderGeneration(0),
       mRendererGeneration(0),
@@ -1133,10 +1134,12 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
 
             mResetting = true;
 
-            mDeferredActions.push_back(
-                    new FlushDecoderAction(
-                        FLUSH_CMD_SHUTDOWN /* audio */,
-                        FLUSH_CMD_SHUTDOWN /* video */));
+            if (mAudioDecoder != NULL && mFlushingAudio == NONE) {
+                mDeferredActions.push_back(
+                        new FlushDecoderAction(
+                            FLUSH_CMD_SHUTDOWN /* audio */,
+                            FLUSH_CMD_SHUTDOWN /* video */));
+            }
 
             mDeferredActions.push_back(
                     new SimpleAction(&NuPlayer::closeAudioSink));
@@ -1228,9 +1231,16 @@ void NuPlayer::onResume() {
     } else {
         ALOGW("resume called when source is gone or not set");
     }
+    if (mOffloadAudio && !mOffloadDecodedPCM) {
+          // Resuming after a pause timed out event, check if can continue with offload
+          sp<AMessage> videoFormat = mSource->getFormat(false /* audio */);
+          sp<AMessage> format = mSource->getFormat(true /*audio*/);
+          const bool hasVideo = (videoFormat != NULL);
+          tryOpenAudioSinkForOffload(format, hasVideo);
+    } 
     // |mAudioDecoder| may have been released due to the pause timeout, so re-create it if
     // needed.
-    if (audioDecoderStillNeeded() && mAudioDecoder == NULL) {
+    if (audioDecoderStillNeeded() && mAudioDecoder == NULL && !mOffloadDecodedPCM) {
         instantiateDecoder(true /* audio */, &mAudioDecoder);
     }
     if (mRenderer != NULL) {
@@ -1282,6 +1292,7 @@ void NuPlayer::onStart(int64_t startPositionUs) {
     }
 
     mOffloadAudio = false;
+    mOffloadDecodedPCM = false;
     mAudioEOS = false;
     mVideoEOS = false;
     mStarted = true;
@@ -1292,8 +1303,10 @@ void NuPlayer::onStart(int64_t startPositionUs) {
         flags |= Renderer::FLAG_REAL_TIME;
     }
 
+    ALOGV("onStart");
     sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
     AVNuUtils::get()->setSourcePCMFormat(audioMeta);
+    audioMeta->dumpToLog();
     audio_stream_type_t streamType = AUDIO_STREAM_MUSIC;
     if (mAudioSink != NULL) {
         streamType = mAudioSink->getAudioStreamType();
@@ -1690,6 +1703,18 @@ void NuPlayer::flushDecoder(bool audio, bool needShutdown) {
         return;
     }
 
+    FlushStatus *state = audio ? &mFlushingAudio : &mFlushingVideo;
+
+    bool inShutdown = *state != NONE &&
+                      *state != FLUSHING_DECODER &&
+                      *state != FLUSHED;
+
+    // Reject flush if the decoder state is not one of the above
+    if (inShutdown) {
+        ALOGI("flush %s called while in shutdown", audio ? "audio" : "video");
+        return;
+    }
+
     // Make sure we don't continue to scan sources until we finish flushing.
     ++mScanSourcesGeneration;
     if (mScanSourcesPending) {
@@ -2433,4 +2458,32 @@ void NuPlayer::performTearDown(const sp<AMessage> &msg) {
     processDeferredActions();
 }
 
+bool NuPlayer::ifDecodedPCMOffload() {
+    return mOffloadDecodedPCM;
+}
+
+void NuPlayer::setDecodedPcmOffload(bool decodePcmOffload) {
+    mOffloadDecodedPCM = decodePcmOffload;
+}
+
+bool NuPlayer::canOffloadDecodedPCMStream(const sp<MetaData> audioMeta,
+            bool hasVideo, bool isStreaming, audio_stream_type_t streamType) {
+    const char *mime = NULL;
+
+     //For offloading decoded content
+    if (!mOffloadAudio && (audioMeta != NULL)) {
+        audioMeta->findCString(kKeyMIMEType, &mime);
+        sp<MetaData> audioPCMMeta =
+                AVNuUtils::get()->createPCMMetaFromSource(audioMeta);
+
+        ALOGI("canOffloadDecodedPCMStream");
+        audioMeta->dumpToLog();
+        mOffloadDecodedPCM =
+                ((mime && !AVNuUtils::get()->pcmOffloadException(audioMeta)) &&
+                canOffloadStream(audioPCMMeta, hasVideo, isStreaming, streamType));
+        ALOGI("PCM offload decided: %d", mOffloadDecodedPCM);
+    }
+    return mOffloadDecodedPCM;
+}
+
 }  // namespace android
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h
index ee0f3e6..c093b0f 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h
@@ -86,11 +86,12 @@ protected:
     virtual ~NuPlayer();
 
     virtual void onMessageReceived(const sp<AMessage> &msg);
-    virtual bool ifDecodedPCMOffload() {return false;}
-    virtual void setDecodedPcmOffload(bool /*decodePcmOffload*/) {}
-    virtual bool canOffloadDecodedPCMStream(const sp<MetaData> /*meta*/,
-            bool /*hasVideo*/, bool /*isStreaming*/, audio_stream_type_t /*streamType*/) {return false;}
+    virtual bool ifDecodedPCMOffload();
+    virtual void setDecodedPcmOffload(bool decodePcmOffload);
+    virtual bool canOffloadDecodedPCMStream(const sp<MetaData> meta,
+            bool hasVideo, bool isStreaming, audio_stream_type_t streamType);
     static bool IsHTTPLiveURL(const char *url);
+
 public:
     struct NuPlayerStreamListener;
     struct Source;
@@ -151,6 +152,7 @@ protected:
     sp<MediaPlayerBase::AudioSink> mAudioSink;
     sp<DecoderBase> mVideoDecoder;
     bool mOffloadAudio;
+    bool mOffloadDecodedPCM;
     sp<DecoderBase> mAudioDecoder;
     sp<CCDecoder> mCCDecoder;
     sp<Renderer> mRenderer;
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp
index 937936d..0b50d0d 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp
@@ -53,7 +53,8 @@ NuPlayer::DecoderPassThrough::DecoderPassThrough(
       mPendingAudioErr(OK),
       mPendingBuffersToDrain(0),
       mCachedBytes(0),
-      mComponentName("pass through decoder") {
+      mComponentName("pass through decoder"),
+      mPCMFormat(AUDIO_FORMAT_INVALID) {
     ALOGW_IF(renderer == NULL, "expect a non-NULL renderer");
 }
 
@@ -75,6 +76,15 @@ void NuPlayer::DecoderPassThrough::onConfigure(const sp<AMessage> &format) {
     // The audio sink is already opened before the PassThrough decoder is created.
     // Opening again might be relevant if decoder is instantiated after shutdown and
     // format is different.
+    sp<MetaData> audioMeta = mSource->getFormatMeta(true /* audio */);
+    if (AVNuUtils::get()->isRAWFormat(audioMeta)) {
+        mPCMFormat = AVNuUtils::get()->getKeyPCMFormat(audioMeta);
+        if (mPCMFormat != AUDIO_FORMAT_INVALID) {
+            AVNuUtils::get()->setPCMFormat(format, mPCMFormat);
+            AVNuUtils::get()->updateAudioBitWidth(mPCMFormat, format);
+        }
+    }
+
     status_t err = mRenderer->openAudioSink(
             format, true /* offloadOnly */, hasVideo,
             AUDIO_OUTPUT_FLAG_NONE /* flags */, NULL /* isOffloaded */, mSource->isStreaming());
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h
index 629e266..fbc4087 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h
@@ -69,6 +69,7 @@ private:
     size_t  mPendingBuffersToDrain;
     size_t  mCachedBytes;
     AString mComponentName;
+    audio_format_t mPCMFormat;
 
     bool isStaleReply(const sp<AMessage> &msg);
     bool isDoneFetching() const;
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
index 049b79d..9cf6b62 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
@@ -1250,6 +1250,30 @@ void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
 
     if (audio) {
         Mutex::Autolock autoLock(mLock);
+#if 1
+        sp<ABuffer> newBuffer;
+        status_t err = AVNuUtils::get()->convertToSinkFormatIfNeeded(
+                buffer, newBuffer,
+                (offloadingAudio() ? mCurrentOffloadInfo.format : mCurrentPcmInfo.mFormat),
+                offloadingAudio());
+        switch (err) {
+        case NO_INIT:
+            // passthru decoder pushes some buffers before the audio sink
+            // is opened. Since the offload format is known only when the sink
+            // is opened, pcm conversions cannot take place. So, retry.
+            ALOGI("init pending, retrying in 10ms, this shouldn't happen");
+            msg->post(10000LL);
+            return;
+        case OK:
+            break;
+        default:
+            ALOGW("error 0x%x in converting to sink format, drop buffer", err);
+            notifyConsumed->post();
+            return;
+        }
+        CHECK(newBuffer != NULL);
+        entry.mBuffer = newBuffer;
+#endif
         mAudioQueue.push_back(entry);
         postDrainAudioQueue_l();
     } else {
@@ -1727,108 +1751,106 @@ status_t NuPlayer::Renderer::onOpenAudioSink(
     AString mime;
     CHECK(format->findString("mime", &mime));
 
-    if (offloadingAudio()) {
-        audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
-        status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
+    audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
+    status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
 
-        if (err != OK) {
-            ALOGE("Couldn't map mime \"%s\" to a valid "
-                    "audio_format", mime.c_str());
-            onDisableOffloadAudio();
-        } else {
-            audioFormat = AVUtils::get()->updateAudioFormat(audioFormat, format);
-
-            bitWidth = AVUtils::get()->getAudioSampleBits(format);
-            int avgBitRate = -1;
-            format->findInt32("bitrate", &avgBitRate);
-
-            int32_t aacProfile = -1;
-            if (audioFormat == AUDIO_FORMAT_AAC
-                    && format->findInt32("aac-profile", &aacProfile)) {
-                // Redefine AAC format as per aac profile
-                int32_t isADTSSupported;
-                isADTSSupported = AVUtils::get()->mapAACProfileToAudioFormat(format,
-                                          audioFormat,
-                                          aacProfile);
-                if (!isADTSSupported) {
-                    mapAACProfileToAudioFormat(audioFormat,
-                            aacProfile);
-                } else {
-                    ALOGV("Format is AAC ADTS\n");
-                }
+    if (err != OK) {
+        ALOGE("Couldn't map mime \"%s\" to a valid "
+                "audio_format", mime.c_str());
+        onDisableOffloadAudio();
+    } else {
+        audioFormat = AVUtils::get()->updateAudioFormat(audioFormat, format);
+
+        bitWidth = AVUtils::get()->getAudioSampleBits(format);
+        int avgBitRate = -1;
+        format->findInt32("bitrate", &avgBitRate);
+
+        int32_t aacProfile = -1;
+        if (audioFormat == AUDIO_FORMAT_AAC
+                && format->findInt32("aac-profile", &aacProfile)) {
+            // Redefine AAC format as per aac profile
+            int32_t isADTSSupported;
+            isADTSSupported = AVUtils::get()->mapAACProfileToAudioFormat(format,
+                                      audioFormat,
+                                      aacProfile);
+            if (!isADTSSupported) {
+                mapAACProfileToAudioFormat(audioFormat,
+                        aacProfile);
+            } else {
+                ALOGV("Format is AAC ADTS\n");
             }
+        }
 
-            int32_t offloadBufferSize =
-                                    AVUtils::get()->getAudioMaxInputBufferSize(
-                                                   audioFormat,
-                                                   format);
-            audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
-
-            offloadInfo.duration_us = -1;
-            format->findInt64(
-                    "durationUs", &offloadInfo.duration_us);
-            offloadInfo.sample_rate = sampleRate;
-            offloadInfo.channel_mask = channelMask;
-            offloadInfo.format = audioFormat;
-            offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
-            offloadInfo.bit_rate = avgBitRate;
-            offloadInfo.has_video = hasVideo;
-            offloadInfo.is_streaming = isStreaming;
-            offloadInfo.bit_width = bitWidth;
-            offloadInfo.offload_buffer_size = offloadBufferSize;
-
-            if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
-                ALOGV("openAudioSink: no change in offload mode");
-                // no change from previous configuration, everything ok.
-                return OK;
-            }
-            mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
+        int32_t offloadBufferSize =
+                                AVUtils::get()->getAudioMaxInputBufferSize(
+                                               audioFormat,
+                                               format);
+        audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
+
+        offloadInfo.duration_us = -1;
+        format->findInt64(
+                "durationUs", &offloadInfo.duration_us);
+        offloadInfo.sample_rate = sampleRate;
+        offloadInfo.channel_mask = channelMask;
+        offloadInfo.format = audioFormat;
+        offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
+        offloadInfo.bit_rate = avgBitRate;
+        offloadInfo.has_video = hasVideo;
+        offloadInfo.is_streaming = isStreaming;
+        offloadInfo.bit_width = bitWidth;
+        offloadInfo.offload_buffer_size = offloadBufferSize;
+
+        if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
+            ALOGV("openAudioSink: no change in offload mode");
+            // no change from previous configuration, everything ok.
+            return OK;
+        }
+        mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
 
-            ALOGV("openAudioSink: try to open AudioSink in offload mode");
-            uint32_t offloadFlags = flags;
-            offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
-            offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
-            audioSinkChanged = true;
-            mAudioSink->close();
+        ALOGV("openAudioSink: try to open AudioSink in offload mode");
+        uint32_t offloadFlags = flags;
+        offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
+        offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
+        audioSinkChanged = true;
+        mAudioSink->close();
 
-            err = mAudioSink->open(
-                    sampleRate,
-                    numChannels,
-                    (audio_channel_mask_t)channelMask,
-                    audioFormat,
-                    0 /* bufferCount - unused */,
-                    &NuPlayer::Renderer::AudioSinkCallback,
-                    this,
-                    (audio_output_flags_t)offloadFlags,
-                    &offloadInfo);
+        err = mAudioSink->open(
+                sampleRate,
+                numChannels,
+                (audio_channel_mask_t)channelMask,
+                audioFormat,
+                0 /* bufferCount - unused */,
+                &NuPlayer::Renderer::AudioSinkCallback,
+                this,
+                (audio_output_flags_t)offloadFlags,
+                &offloadInfo);
 
-            if (err == OK) {
-                err = mAudioSink->setPlaybackRate(mPlaybackSettings);
-            }
+        if (err == OK) {
+            err = mAudioSink->setPlaybackRate(mPlaybackSettings);
+        }
 
-            if (err == OK) {
-                // If the playback is offloaded to h/w, we pass
-                // the HAL some metadata information.
-                // We don't want to do this for PCM because it
-                // will be going through the AudioFlinger mixer
-                // before reaching the hardware.
-                // TODO
-                mCurrentOffloadInfo = offloadInfo;
-                if (!mPaused) { // for preview mode, don't start if paused
-                    err = mAudioSink->start();
-                }
-                ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
-            }
-            if (err != OK) {
-                // Clean up, fall back to non offload mode.
-                mAudioSink->close();
-                onDisableOffloadAudio();
-                mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
-                ALOGV("openAudioSink: offload failed");
-            } else {
-                mUseAudioCallback = true;  // offload mode transfers data through callback
-                ++mAudioDrainGeneration;  // discard pending kWhatDrainAudioQueue message.
+        if (err == OK) {
+            // If the playback is offloaded to h/w, we pass
+            // the HAL some metadata information.
+            // We don't want to do this for PCM because it
+            // will be going through the AudioFlinger mixer
+            // before reaching the hardware.
+            // TODO
+            mCurrentOffloadInfo = offloadInfo;
+            if (!mPaused) { // for preview mode, don't start if paused
+                err = mAudioSink->start();
             }
+            ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
+        }
+        if (err != OK) {
+            // Clean up, fall back to non offload mode.
+            mAudioSink->close();
+            onDisableOffloadAudio();
+            mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
+            ALOGV("openAudioSink: offload failed");
+        } else {
+            mUseAudioCallback = true;  // offload mode transfers data through callback
+            ++mAudioDrainGeneration;  // discard pending kWhatDrainAudioQueue message.
         }
     }
     if (!offloadOnly && !offloadingAudio()) {
@@ -1839,7 +1861,7 @@ status_t NuPlayer::Renderer::onOpenAudioSink(
         const PcmInfo info = {
                 (audio_channel_mask_t)channelMask,
                 (audio_output_flags_t)pcmFlags,
-                getPCMFormat(format),
+                AVNuUtils::get()->getPCMFormat(format),
                 numChannels,
                 sampleRate
         };
@@ -1874,7 +1896,7 @@ status_t NuPlayer::Renderer::onOpenAudioSink(
                     sampleRate,
                     numChannels,
                     (audio_channel_mask_t)channelMask,
-                    getPCMFormat(format),
+                    AVNuUtils::get()->getPCMFormat(format),
                     0 /* bufferCount - unused */,
                     mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL,
                     mUseAudioCallback ? this : NULL,