From 7137ec7e005a5a6e3c0edb91cfacf16a31f4bf6a Mon Sep 17 00:00:00 2001 From: Chong Zhang Date: Wed, 12 Nov 2014 16:41:05 -0800 Subject: tunnel NuPlayer source and decoder input Bug: 18342383 Change-Id: Ieff1cd3bad2b39d46f127ddd5d5139b919992461 --- .../nuplayer/NuPlayerDecoder.cpp | 1253 ++++++++------------ 1 file changed, 479 insertions(+), 774 deletions(-) (limited to 'media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp') diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp index e695c43..0439a9a 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010 The Android Open Source Project + * Copyright 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,13 +19,12 @@ #include #include +#include "NuPlayerCCDecoder.h" #include "NuPlayerDecoder.h" - #include "NuPlayerRenderer.h" #include "NuPlayerSource.h" #include -#include #include #include #include @@ -34,76 +33,103 @@ #include #include +#include "avc_utils.h" +#include "ATSParser.h" + namespace android { NuPlayer::Decoder::Decoder( const sp ¬ify, const sp &source, const sp &renderer, - const sp &nativeWindow) + const sp &nativeWindow, + const sp &ccDecoder) : mNotify(notify), mNativeWindow(nativeWindow), mSource(source), mRenderer(renderer), + mCCDecoder(ccDecoder), mSkipRenderingUntilMediaTimeUs(-1ll), + mNumFramesTotal(0ll), + mNumFramesDropped(0ll), + mIsAudio(true), + mIsVideoAVC(false), + mIsSecure(false), + mFormatChangePending(false), mBufferGeneration(0), mPaused(true), mComponentName("decoder") { - // Every decoder has its own looper because MediaCodec operations - // are blocking, but NuPlayer needs asynchronous operations. - mDecoderLooper = new ALooper; - mDecoderLooper->setName("NPDecoder"); - mDecoderLooper->start(false, false, ANDROID_PRIORITY_AUDIO); - mCodecLooper = new ALooper; mCodecLooper->setName("NPDecoder-CL"); mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO); } NuPlayer::Decoder::~Decoder() { - mDecoderLooper->unregisterHandler(id()); - mDecoderLooper->stop(); - releaseAndResetMediaBuffers(); } -static -status_t PostAndAwaitResponse( - const sp &msg, sp *response) { - status_t err = msg->postAndAwaitResponse(response); +void NuPlayer::Decoder::getStats( + int64_t *numFramesTotal, + int64_t *numFramesDropped) const { + *numFramesTotal = mNumFramesTotal; + *numFramesDropped = mNumFramesDropped; +} - if (err != OK) { - return err; - } +void NuPlayer::Decoder::onMessageReceived(const sp &msg) { + ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str()); - if (!(*response)->findInt32("err", &err)) { - err = OK; - } + switch (msg->what()) { + case kWhatCodecNotify: + { + if (!isStaleReply(msg)) { + int32_t numInput, numOutput; - return err; -} + if (!msg->findInt32("input-buffers", &numInput)) { + numInput = INT32_MAX; + } -void NuPlayer::Decoder::rememberCodecSpecificData(const sp &format) { - mCSDsForCurrentFormat.clear(); - for (int32_t i = 0; ; ++i) { - AString tag = "csd-"; - tag.append(i); - sp buffer; - if (!format->findBuffer(tag.c_str(), &buffer)) { + if (!msg->findInt32("output-buffers", &numOutput)) { + numOutput = INT32_MAX; + } + + if (!mPaused) { + while (numInput-- > 0 && handleAnInputBuffer()) {} + } + + while (numOutput-- > 0 && handleAnOutputBuffer()) {} + } + + requestCodecNotification(); break; } - mCSDsForCurrentFormat.push(buffer); + + case kWhatRenderBuffer: + { + if (!isStaleReply(msg)) { + onRenderBuffer(msg); + } + break; + } + + default: + DecoderBase::onMessageReceived(msg); + break; } } void NuPlayer::Decoder::onConfigure(const sp &format) { CHECK(mCodec == NULL); + mFormatChangePending = false; + ++mBufferGeneration; AString mime; CHECK(format->findString("mime", &mime)); + mIsAudio = !strncasecmp("audio/", mime.c_str(), 6); + mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str()); + sp surface = NULL; if (mNativeWindow != NULL) { surface = mNativeWindow->getSurfaceTextureClient(); @@ -131,6 +157,7 @@ void NuPlayer::Decoder::onConfigure(const sp &format) { handleError(UNKNOWN_ERROR); return; } + mIsSecure = secure; mCodec->getName(&mComponentName); @@ -183,69 +210,120 @@ void NuPlayer::Decoder::onConfigure(const sp &format) { mPaused = false; } -void NuPlayer::Decoder::releaseAndResetMediaBuffers() { - for (size_t i = 0; i < mMediaBuffers.size(); i++) { - if (mMediaBuffers[i] != NULL) { - mMediaBuffers[i]->release(); - mMediaBuffers.editItemAt(i) = NULL; - } - } - mMediaBuffers.resize(mInputBuffers.size()); - for (size_t i = 0; i < mMediaBuffers.size(); i++) { - mMediaBuffers.editItemAt(i) = NULL; +void NuPlayer::Decoder::onSetRenderer(const sp &renderer) { + bool hadNoRenderer = (mRenderer == NULL); + mRenderer = renderer; + if (hadNoRenderer && mRenderer != NULL) { + requestCodecNotification(); } - mInputBufferIsDequeued.clear(); - mInputBufferIsDequeued.resize(mInputBuffers.size()); - for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) { - mInputBufferIsDequeued.editItemAt(i) = false; +} + +void NuPlayer::Decoder::onGetInputBuffers( + Vector > *dstBuffers) { + dstBuffers->clear(); + for (size_t i = 0; i < mInputBuffers.size(); i++) { + dstBuffers->push(mInputBuffers[i]); } +} - mPendingInputMessages.clear(); - mSkipRenderingUntilMediaTimeUs = -1; +void NuPlayer::Decoder::onResume() { + mPaused = false; } -void NuPlayer::Decoder::requestCodecNotification() { +void NuPlayer::Decoder::onFlush(bool notifyComplete) { + if (mCCDecoder != NULL) { + mCCDecoder->flush(); + } + + if (mRenderer != NULL) { + mRenderer->flush(mIsAudio, notifyComplete); + mRenderer->signalTimeDiscontinuity(); + } + + status_t err = OK; if (mCodec != NULL) { - sp reply = new AMessage(kWhatCodecNotify, id()); - reply->setInt32("generation", mBufferGeneration); - mCodec->requestActivityNotification(reply); + err = mCodec->flush(); + mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator + ++mBufferGeneration; } -} -bool NuPlayer::Decoder::isStaleReply(const sp &msg) { - int32_t generation; - CHECK(msg->findInt32("generation", &generation)); - return generation != mBufferGeneration; -} + if (err != OK) { + ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err); + handleError(err); + // finish with posting kWhatFlushCompleted. + // we attempt to release the buffers even if flush fails. + } + releaseAndResetMediaBuffers(); -void NuPlayer::Decoder::init() { - mDecoderLooper->registerHandler(this); + if (notifyComplete) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatFlushCompleted); + notify->post(); + mPaused = true; + } } -void NuPlayer::Decoder::configure(const sp &format) { - sp msg = new AMessage(kWhatConfigure, id()); - msg->setMessage("format", format); - msg->post(); -} +void NuPlayer::Decoder::onShutdown(bool notifyComplete) { + status_t err = OK; + if (mCodec != NULL) { + err = mCodec->release(); + mCodec = NULL; + ++mBufferGeneration; -void NuPlayer::Decoder::setRenderer(const sp &renderer) { - sp msg = new AMessage(kWhatSetRenderer, id()); - msg->setObject("renderer", renderer); - msg->post(); -} + if (mNativeWindow != NULL) { + // reconnect to surface as MediaCodec disconnected from it + status_t error = + native_window_api_connect( + mNativeWindow->getNativeWindow().get(), + NATIVE_WINDOW_API_MEDIA); + ALOGW_IF(error != NO_ERROR, + "[%s] failed to connect to native window, error=%d", + mComponentName.c_str(), error); + } + mComponentName = "decoder"; + } + + releaseAndResetMediaBuffers(); + + if (err != OK) { + ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err); + handleError(err); + // finish with posting kWhatShutdownCompleted. + } -void NuPlayer::Decoder::signalUpdateFormat(const sp &format) { - sp msg = new AMessage(kWhatUpdateFormat, id()); - msg->setMessage("format", format); - msg->post(); + if (notifyComplete) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatShutdownCompleted); + notify->post(); + mPaused = true; + } } -status_t NuPlayer::Decoder::getInputBuffers(Vector > *buffers) const { - sp msg = new AMessage(kWhatGetInputBuffers, id()); - msg->setPointer("buffers", buffers); +void NuPlayer::Decoder::doRequestBuffers() { + if (mFormatChangePending) { + return; + } + status_t err = OK; + while (!mDequeuedInputBuffers.empty()) { + size_t bufferIx = *mDequeuedInputBuffers.begin(); + sp msg = new AMessage(); + msg->setSize("buffer-ix", bufferIx); + err = fetchInputData(msg); + if (err != OK) { + break; + } + mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin()); + + if (!mPendingInputMessages.empty() + || !onInputBufferFetched(msg)) { + mPendingInputMessages.push_back(msg); + } + } - sp response; - return PostAndAwaitResponse(msg, &response); + if (err == -EWOULDBLOCK + && mSource->feedMoreTSData() == OK) { + scheduleRequestBuffers(); + } } void NuPlayer::Decoder::handleError(int32_t err) @@ -263,6 +341,9 @@ void NuPlayer::Decoder::handleError(int32_t err) } bool NuPlayer::Decoder::handleAnInputBuffer() { + if (mFormatChangePending) { + return false; + } size_t bufferIx = -1; status_t res = mCodec->dequeueInputBuffer(&bufferIx); ALOGV("[%s] dequeued input: %d", @@ -284,22 +365,21 @@ bool NuPlayer::Decoder::handleAnInputBuffer() { } mInputBufferIsDequeued.editItemAt(bufferIx) = true; - sp reply = new AMessage(kWhatInputBufferFilled, id()); - reply->setSize("buffer-ix", bufferIx); - reply->setInt32("generation", mBufferGeneration); - if (!mCSDsToSubmit.isEmpty()) { + sp msg = new AMessage(); + msg->setSize("buffer-ix", bufferIx); + sp buffer = mCSDsToSubmit.itemAt(0); ALOGI("[%s] resubmitting CSD", mComponentName.c_str()); - reply->setBuffer("buffer", buffer); + msg->setBuffer("buffer", buffer); mCSDsToSubmit.removeAt(0); - CHECK(onInputBufferFilled(reply)); + CHECK(onInputBufferFetched(msg)); return true; } while (!mPendingInputMessages.empty()) { sp msg = *mPendingInputMessages.begin(); - if (!onInputBufferFilled(msg)) { + if (!onInputBufferFetched(msg)) { break; } mPendingInputMessages.erase(mPendingInputMessages.begin()); @@ -309,62 +389,310 @@ bool NuPlayer::Decoder::handleAnInputBuffer() { return true; } - sp notify = mNotify->dup(); - notify->setInt32("what", kWhatFillThisBuffer); - notify->setBuffer("buffer", mInputBuffers[bufferIx]); - notify->setMessage("reply", reply); - notify->post(); + mDequeuedInputBuffers.push_back(bufferIx); + + onRequestInputBuffers(); return true; } -bool android::NuPlayer::Decoder::onInputBufferFilled(const sp &msg) { - size_t bufferIx; - CHECK(msg->findSize("buffer-ix", &bufferIx)); - CHECK_LT(bufferIx, mInputBuffers.size()); - sp codecBuffer = mInputBuffers[bufferIx]; +bool NuPlayer::Decoder::handleAnOutputBuffer() { + if (mFormatChangePending) { + return false; + } + size_t bufferIx = -1; + size_t offset; + size_t size; + int64_t timeUs; + uint32_t flags; + status_t res = mCodec->dequeueOutputBuffer( + &bufferIx, &offset, &size, &timeUs, &flags); - sp buffer; - bool hasBuffer = msg->findBuffer("buffer", &buffer); + if (res != OK) { + ALOGV("[%s] dequeued output: %d", mComponentName.c_str(), res); + } else { + ALOGV("[%s] dequeued output: %d (time=%lld flags=%" PRIu32 ")", + mComponentName.c_str(), (int)bufferIx, timeUs, flags); + } - // handle widevine classic source - that fills an arbitrary input buffer - MediaBuffer *mediaBuffer = NULL; - if (hasBuffer) { - mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase()); - if (mediaBuffer != NULL) { - // likely filled another buffer than we requested: adjust buffer index - size_t ix; - for (ix = 0; ix < mInputBuffers.size(); ix++) { - const sp &buf = mInputBuffers[ix]; - if (buf->data() == mediaBuffer->data()) { - // all input buffers are dequeued on start, hence the check - if (!mInputBufferIsDequeued[ix]) { - ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu", - mComponentName.c_str(), ix, bufferIx); - mediaBuffer->release(); - return false; - } + if (res == INFO_OUTPUT_BUFFERS_CHANGED) { + res = mCodec->getOutputBuffers(&mOutputBuffers); + if (res != OK) { + ALOGE("Failed to get output buffers for %s after INFO event (err=%d)", + mComponentName.c_str(), res); + handleError(res); + return false; + } + // NuPlayer ignores this + return true; + } else if (res == INFO_FORMAT_CHANGED) { + sp format = new AMessage(); + res = mCodec->getOutputFormat(&format); + if (res != OK) { + ALOGE("Failed to get output format for %s after INFO event (err=%d)", + mComponentName.c_str(), res); + handleError(res); + return false; + } - // TRICKY: need buffer for the metadata, so instead, set - // codecBuffer to the same (though incorrect) buffer to - // avoid a memcpy into the codecBuffer - codecBuffer = buffer; - codecBuffer->setRange( - mediaBuffer->range_offset(), - mediaBuffer->range_length()); - bufferIx = ix; - break; - } + if (!mIsAudio) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatVideoSizeChanged); + notify->setMessage("format", format); + notify->post(); + } else if (mRenderer != NULL) { + uint32_t flags; + int64_t durationUs; + bool hasVideo = (mSource->getFormat(false /* audio */) != NULL); + if (!hasVideo && + mSource->getDuration(&durationUs) == OK && + durationUs + > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) { + flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER; + } else { + flags = AUDIO_OUTPUT_FLAG_NONE; } - CHECK(ix < mInputBuffers.size()); + + mRenderer->openAudioSink( + format, false /* offloadOnly */, hasVideo, flags); + } + return true; + } else if (res == INFO_DISCONTINUITY) { + // nothing to do + return true; + } else if (res != OK) { + if (res != -EAGAIN) { + ALOGE("Failed to dequeue output buffer for %s (err=%d)", + mComponentName.c_str(), res); + handleError(res); } + return false; } - if (buffer == NULL /* includes !hasBuffer */) { - int32_t streamErr = ERROR_END_OF_STREAM; - CHECK(msg->findInt32("err", &streamErr) || !hasBuffer); - - if (streamErr == OK) { - /* buffers are returned to hold on to */ + CHECK_LT(bufferIx, mOutputBuffers.size()); + sp buffer = mOutputBuffers[bufferIx]; + buffer->setRange(offset, size); + buffer->meta()->clear(); + buffer->meta()->setInt64("timeUs", timeUs); + if (flags & MediaCodec::BUFFER_FLAG_EOS) { + buffer->meta()->setInt32("eos", true); + } + // we do not expect CODECCONFIG or SYNCFRAME for decoder + + sp reply = new AMessage(kWhatRenderBuffer, id()); + reply->setSize("buffer-ix", bufferIx); + reply->setInt32("generation", mBufferGeneration); + + if (mSkipRenderingUntilMediaTimeUs >= 0) { + if (timeUs < mSkipRenderingUntilMediaTimeUs) { + ALOGV("[%s] dropping buffer at time %lld as requested.", + mComponentName.c_str(), (long long)timeUs); + + reply->post(); + return true; + } + + mSkipRenderingUntilMediaTimeUs = -1; + } + + if (mRenderer != NULL) { + // send the buffer to renderer. + mRenderer->queueBuffer(mIsAudio, buffer, reply); + if (flags & MediaCodec::BUFFER_FLAG_EOS) { + mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM); + } + } + + return true; +} + +void NuPlayer::Decoder::releaseAndResetMediaBuffers() { + for (size_t i = 0; i < mMediaBuffers.size(); i++) { + if (mMediaBuffers[i] != NULL) { + mMediaBuffers[i]->release(); + mMediaBuffers.editItemAt(i) = NULL; + } + } + mMediaBuffers.resize(mInputBuffers.size()); + for (size_t i = 0; i < mMediaBuffers.size(); i++) { + mMediaBuffers.editItemAt(i) = NULL; + } + mInputBufferIsDequeued.clear(); + mInputBufferIsDequeued.resize(mInputBuffers.size()); + for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) { + mInputBufferIsDequeued.editItemAt(i) = false; + } + + mPendingInputMessages.clear(); + mDequeuedInputBuffers.clear(); + mSkipRenderingUntilMediaTimeUs = -1; +} + +void NuPlayer::Decoder::requestCodecNotification() { + if (mFormatChangePending) { + return; + } + if (mCodec != NULL) { + sp reply = new AMessage(kWhatCodecNotify, id()); + reply->setInt32("generation", mBufferGeneration); + mCodec->requestActivityNotification(reply); + } +} + +bool NuPlayer::Decoder::isStaleReply(const sp &msg) { + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + return generation != mBufferGeneration; +} + +status_t NuPlayer::Decoder::fetchInputData(sp &reply) { + sp accessUnit; + bool dropAccessUnit; + do { + status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit); + + if (err == -EWOULDBLOCK) { + return err; + } else if (err != OK) { + if (err == INFO_DISCONTINUITY) { + int32_t type; + CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); + + bool formatChange = + (mIsAudio && + (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT)) + || (!mIsAudio && + (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT)); + + bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0; + + ALOGI("%s discontinuity (format=%d, time=%d)", + mIsAudio ? "audio" : "video", formatChange, timeChange); + + bool seamlessFormatChange = false; + sp newFormat = mSource->getFormat(mIsAudio); + if (formatChange) { + seamlessFormatChange = + supportsSeamlessFormatChange(newFormat); + // treat seamless format change separately + formatChange = !seamlessFormatChange; + } + + if (formatChange || timeChange) { + sp msg = mNotify->dup(); + msg->setInt32("what", kWhatInputDiscontinuity); + msg->setInt32("formatChange", formatChange); + msg->post(); + } + + if (formatChange /* not seamless */) { + // must change decoder + // return EOS and wait to be killed + mFormatChangePending = true; + return ERROR_END_OF_STREAM; + } else if (timeChange) { + // need to flush + // TODO: Ideally we shouldn't need a flush upon time + // discontinuity, flushing will cause loss of frames. + // We probably should queue a time change marker to the + // output queue, and handles it in renderer instead. + rememberCodecSpecificData(newFormat); + onFlush(false /* notifyComplete */); + err = OK; + } else if (seamlessFormatChange) { + // reuse existing decoder and don't flush + rememberCodecSpecificData(newFormat); + err = OK; + } else { + // This stream is unaffected by the discontinuity + return -EWOULDBLOCK; + } + } + + reply->setInt32("err", err); + return OK; + } + + if (!mIsAudio) { + ++mNumFramesTotal; + } + + dropAccessUnit = false; + if (!mIsAudio + && !mIsSecure + && mRenderer->getVideoLateByUs() > 100000ll + && mIsVideoAVC + && !IsAVCReferenceFrame(accessUnit)) { + dropAccessUnit = true; + ++mNumFramesDropped; + } + } while (dropAccessUnit); + + // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video"); +#if 0 + int64_t mediaTimeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs)); + ALOGV("feeding %s input buffer at media time %.2f secs", + mIsAudio ? "audio" : "video", + mediaTimeUs / 1E6); +#endif + + if (mCCDecoder != NULL) { + mCCDecoder->decode(accessUnit); + } + + reply->setBuffer("buffer", accessUnit); + + return OK; +} + +bool NuPlayer::Decoder::onInputBufferFetched(const sp &msg) { + size_t bufferIx; + CHECK(msg->findSize("buffer-ix", &bufferIx)); + CHECK_LT(bufferIx, mInputBuffers.size()); + sp codecBuffer = mInputBuffers[bufferIx]; + + sp buffer; + bool hasBuffer = msg->findBuffer("buffer", &buffer); + + // handle widevine classic source - that fills an arbitrary input buffer + MediaBuffer *mediaBuffer = NULL; + if (hasBuffer) { + mediaBuffer = (MediaBuffer *)(buffer->getMediaBufferBase()); + if (mediaBuffer != NULL) { + // likely filled another buffer than we requested: adjust buffer index + size_t ix; + for (ix = 0; ix < mInputBuffers.size(); ix++) { + const sp &buf = mInputBuffers[ix]; + if (buf->data() == mediaBuffer->data()) { + // all input buffers are dequeued on start, hence the check + if (!mInputBufferIsDequeued[ix]) { + ALOGV("[%s] received MediaBuffer for #%zu instead of #%zu", + mComponentName.c_str(), ix, bufferIx); + mediaBuffer->release(); + return false; + } + + // TRICKY: need buffer for the metadata, so instead, set + // codecBuffer to the same (though incorrect) buffer to + // avoid a memcpy into the codecBuffer + codecBuffer = buffer; + codecBuffer->setRange( + mediaBuffer->range_offset(), + mediaBuffer->range_length()); + bufferIx = ix; + break; + } + } + CHECK(ix < mInputBuffers.size()); + } + } + + if (buffer == NULL /* includes !hasBuffer */) { + int32_t streamErr = ERROR_END_OF_STREAM; + CHECK(msg->findInt32("err", &streamErr) || !hasBuffer); + + if (streamErr == OK) { + /* buffers are returned to hold on to */ return true; } @@ -444,127 +772,20 @@ bool android::NuPlayer::Decoder::onInputBufferFilled(const sp &msg) { return true; } -bool NuPlayer::Decoder::handleAnOutputBuffer() { - size_t bufferIx = -1; - size_t offset; - size_t size; - int64_t timeUs; - uint32_t flags; - status_t res = mCodec->dequeueOutputBuffer( - &bufferIx, &offset, &size, &timeUs, &flags); - - if (res != OK) { - ALOGV("[%s] dequeued output: %d", mComponentName.c_str(), res); - } else { - ALOGV("[%s] dequeued output: %d (time=%lld flags=%" PRIu32 ")", - mComponentName.c_str(), (int)bufferIx, timeUs, flags); - } - - if (res == INFO_OUTPUT_BUFFERS_CHANGED) { - res = mCodec->getOutputBuffers(&mOutputBuffers); - if (res != OK) { - ALOGE("Failed to get output buffers for %s after INFO event (err=%d)", - mComponentName.c_str(), res); - handleError(res); - return false; - } - // NuPlayer ignores this - return true; - } else if (res == INFO_FORMAT_CHANGED) { - sp format = new AMessage(); - res = mCodec->getOutputFormat(&format); - if (res != OK) { - ALOGE("Failed to get output format for %s after INFO event (err=%d)", - mComponentName.c_str(), res); - handleError(res); - return false; - } - - if (isVideo()) { - sp notify = mNotify->dup(); - notify->setInt32("what", kWhatVideoSizeChanged); - notify->setMessage("format", format); - notify->post(); - } else if (mRenderer != NULL) { - uint32_t flags; - int64_t durationUs; - bool hasVideo = (mSource->getFormat(false /* audio */) != NULL); - if (!hasVideo && - mSource->getDuration(&durationUs) == OK && - durationUs - > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) { - flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER; - } else { - flags = AUDIO_OUTPUT_FLAG_NONE; - } - - mRenderer->openAudioSink( - format, false /* offloadOnly */, hasVideo, flags); - } - return true; - } else if (res == INFO_DISCONTINUITY) { - // nothing to do - return true; - } else if (res != OK) { - if (res != -EAGAIN) { - ALOGE("Failed to dequeue output buffer for %s (err=%d)", - mComponentName.c_str(), res); - handleError(res); - } - return false; - } - - CHECK_LT(bufferIx, mOutputBuffers.size()); - sp buffer = mOutputBuffers[bufferIx]; - buffer->setRange(offset, size); - buffer->meta()->clear(); - buffer->meta()->setInt64("timeUs", timeUs); - if (flags & MediaCodec::BUFFER_FLAG_EOS) { - buffer->meta()->setInt32("eos", true); - } - // we do not expect CODECCONFIG or SYNCFRAME for decoder - - sp reply = new AMessage(kWhatRenderBuffer, id()); - reply->setSize("buffer-ix", bufferIx); - reply->setInt32("generation", mBufferGeneration); - - if (mSkipRenderingUntilMediaTimeUs >= 0) { - if (timeUs < mSkipRenderingUntilMediaTimeUs) { - ALOGV("[%s] dropping buffer at time %lld as requested.", - mComponentName.c_str(), (long long)timeUs); - - reply->post(); - return true; - } - - mSkipRenderingUntilMediaTimeUs = -1; - } - - if (mRenderer != NULL) { - // send the buffer to renderer. - mRenderer->queueBuffer(!isVideo(), buffer, reply); - if (flags & MediaCodec::BUFFER_FLAG_EOS) { - mRenderer->queueEOS(!isVideo(), ERROR_END_OF_STREAM); - } - } - - return true; -} - void NuPlayer::Decoder::onRenderBuffer(const sp &msg) { status_t err; int32_t render; size_t bufferIx; CHECK(msg->findSize("buffer-ix", &bufferIx)); - if (isVideo()) { + if (!mIsAudio) { int64_t timeUs; sp buffer = mOutputBuffers[bufferIx]; buffer->meta()->findInt64("timeUs", &timeUs); - sp notify = mNotify->dup(); - notify->setInt32("what", kWhatRenderBufferTime); - notify->setInt64("timeUs", timeUs); - notify->post(); + + if (mCCDecoder != NULL && mCCDecoder->isSelected()) { + mCCDecoder->display(timeUs); + } } if (msg->findInt32("render", &render) && render) { @@ -581,208 +802,8 @@ void NuPlayer::Decoder::onRenderBuffer(const sp &msg) { } } -void NuPlayer::Decoder::onFlush() { - if (mRenderer != NULL) { - mRenderer->flush(!isVideo()); - } - - status_t err = OK; - if (mCodec != NULL) { - err = mCodec->flush(); - mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator - ++mBufferGeneration; - } - - if (err != OK) { - ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err); - handleError(err); - // finish with posting kWhatFlushCompleted. - // we attempt to release the buffers even if flush fails. - } - releaseAndResetMediaBuffers(); - - sp notify = mNotify->dup(); - notify->setInt32("what", kWhatFlushCompleted); - notify->post(); - mPaused = true; -} - -void NuPlayer::Decoder::onResume() { - mPaused = false; -} - -void NuPlayer::Decoder::onShutdown() { - status_t err = OK; - if (mCodec != NULL) { - err = mCodec->release(); - mCodec = NULL; - ++mBufferGeneration; - - if (mNativeWindow != NULL) { - // reconnect to surface as MediaCodec disconnected from it - status_t error = - native_window_api_connect( - mNativeWindow->getNativeWindow().get(), - NATIVE_WINDOW_API_MEDIA); - ALOGW_IF(error != NO_ERROR, - "[%s] failed to connect to native window, error=%d", - mComponentName.c_str(), error); - } - mComponentName = "decoder"; - } - - releaseAndResetMediaBuffers(); - - if (err != OK) { - ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err); - handleError(err); - // finish with posting kWhatShutdownCompleted. - } - - sp notify = mNotify->dup(); - notify->setInt32("what", kWhatShutdownCompleted); - notify->post(); - mPaused = true; -} - -void NuPlayer::Decoder::onMessageReceived(const sp &msg) { - ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str()); - - switch (msg->what()) { - case kWhatConfigure: - { - sp format; - CHECK(msg->findMessage("format", &format)); - onConfigure(format); - break; - } - - case kWhatSetRenderer: - { - bool hadNoRenderer = (mRenderer == NULL); - sp obj; - CHECK(msg->findObject("renderer", &obj)); - mRenderer = static_cast(obj.get()); - if (hadNoRenderer && mRenderer != NULL) { - requestCodecNotification(); - } - break; - } - - case kWhatUpdateFormat: - { - sp format; - CHECK(msg->findMessage("format", &format)); - rememberCodecSpecificData(format); - break; - } - - case kWhatGetInputBuffers: - { - uint32_t replyID; - CHECK(msg->senderAwaitsResponse(&replyID)); - - Vector > *dstBuffers; - CHECK(msg->findPointer("buffers", (void **)&dstBuffers)); - - dstBuffers->clear(); - for (size_t i = 0; i < mInputBuffers.size(); i++) { - dstBuffers->push(mInputBuffers[i]); - } - - (new AMessage)->postReply(replyID); - break; - } - - case kWhatCodecNotify: - { - if (!isStaleReply(msg)) { - int32_t numInput, numOutput; - - if (!msg->findInt32("input-buffers", &numInput)) { - numInput = INT32_MAX; - } - - if (!msg->findInt32("output-buffers", &numOutput)) { - numOutput = INT32_MAX; - } - - if (!mPaused) { - while (numInput-- > 0 && handleAnInputBuffer()) {} - } - - while (numOutput-- > 0 && handleAnOutputBuffer()) {} - } - - requestCodecNotification(); - break; - } - - case kWhatInputBufferFilled: - { - if (!isStaleReply(msg)) { - if (!mPendingInputMessages.empty() - || !onInputBufferFilled(msg)) { - mPendingInputMessages.push_back(msg); - } - } - - break; - } - - case kWhatRenderBuffer: - { - if (!isStaleReply(msg)) { - onRenderBuffer(msg); - } - break; - } - - case kWhatFlush: - { - sp format; - if (msg->findMessage("new-format", &format)) { - rememberCodecSpecificData(format); - } - onFlush(); - break; - } - - case kWhatResume: - { - onResume(); - break; - } - - case kWhatShutdown: - { - onShutdown(); - break; - } - - default: - TRESPASS(); - break; - } -} - -void NuPlayer::Decoder::signalFlush(const sp &format) { - sp msg = new AMessage(kWhatFlush, id()); - if (format != NULL) { - msg->setMessage("new-format", format); - } - msg->post(); -} - -void NuPlayer::Decoder::signalResume() { - (new AMessage(kWhatResume, id()))->post(); -} - -void NuPlayer::Decoder::initiateShutdown() { - (new AMessage(kWhatShutdown, id()))->post(); -} - -bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(const sp &targetFormat) const { +bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange( + const sp &targetFormat) const { if (targetFormat == NULL) { return true; } @@ -847,336 +868,20 @@ bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp &targetF return seamless; } -bool NuPlayer::Decoder::isVideo() { - return mNativeWindow != NULL; -} - -struct CCData { - CCData(uint8_t type, uint8_t data1, uint8_t data2) - : mType(type), mData1(data1), mData2(data2) { - } - bool getChannel(size_t *channel) const { - if (mData1 >= 0x10 && mData1 <= 0x1f) { - *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0); - return true; - } - return false; - } - - uint8_t mType; - uint8_t mData1; - uint8_t mData2; -}; - -static bool isNullPad(CCData *cc) { - return cc->mData1 < 0x10 && cc->mData2 < 0x10; -} - -static void dumpBytePair(const sp &ccBuf) { - size_t offset = 0; - AString out; - - while (offset < ccBuf->size()) { - char tmp[128]; - - CCData *cc = (CCData *) (ccBuf->data() + offset); - - if (isNullPad(cc)) { - // 1 null pad or XDS metadata, ignore - offset += sizeof(CCData); - continue; - } - - if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) { - // 2 basic chars - sprintf(tmp, "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2); - } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19) - && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) { - // 1 special char - sprintf(tmp, "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A) - && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){ - // 1 Spanish/French char - sprintf(tmp, "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B) - && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){ - // 1 Portuguese/German/Danish char - sprintf(tmp, "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19) - && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){ - // Mid-Row Codes (Table 69) - sprintf(tmp, "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c) - && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f) - || - ((cc->mData1 == 0x17 || cc->mData1 == 0x1f) - && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){ - // Misc Control Codes (Table 70) - sprintf(tmp, "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else if ((cc->mData1 & 0x70) == 0x10 - && (cc->mData2 & 0x40) == 0x40 - && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) { - // Preamble Address Codes (Table 71) - sprintf(tmp, "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } else { - sprintf(tmp, "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2); - } - - if (out.size() > 0) { - out.append(", "); - } - - out.append(tmp); - - offset += sizeof(CCData); - } - - ALOGI("%s", out.c_str()); -} - -NuPlayer::CCDecoder::CCDecoder(const sp ¬ify) - : mNotify(notify), - mCurrentChannel(0), - mSelectedTrack(-1) { - for (size_t i = 0; i < sizeof(mTrackIndices)/sizeof(mTrackIndices[0]); ++i) { - mTrackIndices[i] = -1; - } -} - -size_t NuPlayer::CCDecoder::getTrackCount() const { - return mFoundChannels.size(); -} - -sp NuPlayer::CCDecoder::getTrackInfo(size_t index) const { - if (!isTrackValid(index)) { - return NULL; - } - - sp format = new AMessage(); - - format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE); - format->setString("language", "und"); - format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608); - //CC1, field 0 channel 0 - bool isDefaultAuto = (mFoundChannels[index] == 0); - format->setInt32("auto", isDefaultAuto); - format->setInt32("default", isDefaultAuto); - format->setInt32("forced", 0); - - return format; -} - -status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) { - if (!isTrackValid(index)) { - return BAD_VALUE; - } - - if (select) { - if (mSelectedTrack == (ssize_t)index) { - ALOGE("track %zu already selected", index); - return BAD_VALUE; - } - ALOGV("selected track %zu", index); - mSelectedTrack = index; - } else { - if (mSelectedTrack != (ssize_t)index) { - ALOGE("track %zu is not selected", index); - return BAD_VALUE; - } - ALOGV("unselected track %zu", index); - mSelectedTrack = -1; - } - - return OK; -} - -bool NuPlayer::CCDecoder::isSelected() const { - return mSelectedTrack >= 0 && mSelectedTrack < (int32_t) getTrackCount(); -} - -bool NuPlayer::CCDecoder::isTrackValid(size_t index) const { - return index < getTrackCount(); -} - -int32_t NuPlayer::CCDecoder::getTrackIndex(size_t channel) const { - if (channel < sizeof(mTrackIndices)/sizeof(mTrackIndices[0])) { - return mTrackIndices[channel]; - } - return -1; -} - -// returns true if a new CC track is found -bool NuPlayer::CCDecoder::extractFromSEI(const sp &accessUnit) { - int64_t timeUs; - CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); - - sp sei; - if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) { - return false; - } - - bool trackAdded = false; - - NALBitReader br(sei->data() + 1, sei->size() - 1); - // sei_message() - while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message() - uint32_t payload_type = 0; - size_t payload_size = 0; - uint8_t last_byte; - - do { - last_byte = br.getBits(8); - payload_type += last_byte; - } while (last_byte == 0xFF); - - do { - last_byte = br.getBits(8); - payload_size += last_byte; - } while (last_byte == 0xFF); - - // sei_payload() - if (payload_type == 4) { - // user_data_registered_itu_t_t35() - - // ATSC A/72: 6.4.2 - uint8_t itu_t_t35_country_code = br.getBits(8); - uint16_t itu_t_t35_provider_code = br.getBits(16); - uint32_t user_identifier = br.getBits(32); - uint8_t user_data_type_code = br.getBits(8); - - payload_size -= 1 + 2 + 4 + 1; - - if (itu_t_t35_country_code == 0xB5 - && itu_t_t35_provider_code == 0x0031 - && user_identifier == 'GA94' - && user_data_type_code == 0x3) { - // MPEG_cc_data() - // ATSC A/53 Part 4: 6.2.3.1 - br.skipBits(1); //process_em_data_flag - bool process_cc_data_flag = br.getBits(1); - br.skipBits(1); //additional_data_flag - size_t cc_count = br.getBits(5); - br.skipBits(8); // em_data; - payload_size -= 2; - - if (process_cc_data_flag) { - AString out; - - sp ccBuf = new ABuffer(cc_count * sizeof(CCData)); - ccBuf->setRange(0, 0); - - for (size_t i = 0; i < cc_count; i++) { - uint8_t marker = br.getBits(5); - CHECK_EQ(marker, 0x1f); - - bool cc_valid = br.getBits(1); - uint8_t cc_type = br.getBits(2); - // remove odd parity bit - uint8_t cc_data_1 = br.getBits(8) & 0x7f; - uint8_t cc_data_2 = br.getBits(8) & 0x7f; - - if (cc_valid - && (cc_type == 0 || cc_type == 1)) { - CCData cc(cc_type, cc_data_1, cc_data_2); - if (!isNullPad(&cc)) { - size_t channel; - if (cc.getChannel(&channel) && getTrackIndex(channel) < 0) { - mTrackIndices[channel] = mFoundChannels.size(); - mFoundChannels.push_back(channel); - trackAdded = true; - } - memcpy(ccBuf->data() + ccBuf->size(), - (void *)&cc, sizeof(cc)); - ccBuf->setRange(0, ccBuf->size() + sizeof(CCData)); - } - } - } - payload_size -= cc_count * 3; - - mCCMap.add(timeUs, ccBuf); - break; - } - } else { - ALOGV("Malformed SEI payload type 4"); - } - } else { - ALOGV("Unsupported SEI payload type %d", payload_type); - } - - // skipping remaining bits of this payload - br.skipBits(payload_size * 8); - } - - return trackAdded; -} - -sp NuPlayer::CCDecoder::filterCCBuf( - const sp &ccBuf, size_t index) { - sp filteredCCBuf = new ABuffer(ccBuf->size()); - filteredCCBuf->setRange(0, 0); - - size_t cc_count = ccBuf->size() / sizeof(CCData); - const CCData* cc_data = (const CCData*)ccBuf->data(); - for (size_t i = 0; i < cc_count; ++i) { - size_t channel; - if (cc_data[i].getChannel(&channel)) { - mCurrentChannel = channel; - } - if (mCurrentChannel == mFoundChannels[index]) { - memcpy(filteredCCBuf->data() + filteredCCBuf->size(), - (void *)&cc_data[i], sizeof(CCData)); - filteredCCBuf->setRange(0, filteredCCBuf->size() + sizeof(CCData)); - } - } - - return filteredCCBuf; -} - -void NuPlayer::CCDecoder::decode(const sp &accessUnit) { - if (extractFromSEI(accessUnit)) { - ALOGI("Found CEA-608 track"); - sp msg = mNotify->dup(); - msg->setInt32("what", kWhatTrackAdded); - msg->post(); - } - // TODO: extract CC from other sources -} - -void NuPlayer::CCDecoder::display(int64_t timeUs) { - if (!isTrackValid(mSelectedTrack)) { - ALOGE("Could not find current track(index=%d)", mSelectedTrack); - return; - } - - ssize_t index = mCCMap.indexOfKey(timeUs); - if (index < 0) { - ALOGV("cc for timestamp %" PRId64 " not found", timeUs); +void NuPlayer::Decoder::rememberCodecSpecificData(const sp &format) { + if (format == NULL) { return; } - - sp ccBuf = filterCCBuf(mCCMap.valueAt(index), mSelectedTrack); - - if (ccBuf->size() > 0) { -#if 0 - dumpBytePair(ccBuf); -#endif - - ccBuf->meta()->setInt32("trackIndex", mSelectedTrack); - ccBuf->meta()->setInt64("timeUs", timeUs); - ccBuf->meta()->setInt64("durationUs", 0ll); - - sp msg = mNotify->dup(); - msg->setInt32("what", kWhatClosedCaptionData); - msg->setBuffer("buffer", ccBuf); - msg->post(); + mCSDsForCurrentFormat.clear(); + for (int32_t i = 0; ; ++i) { + AString tag = "csd-"; + tag.append(i); + sp buffer; + if (!format->findBuffer(tag.c_str(), &buffer)) { + break; + } + mCSDsForCurrentFormat.push(buffer); } - - // remove all entries before timeUs - mCCMap.removeItemsAt(0, index + 1); -} - -void NuPlayer::CCDecoder::flush() { - mCCMap.clear(); } } // namespace android -- cgit v1.1