/* * Copyright (C) 2010 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ //#define LOG_NDEBUG 0 #define LOG_TAG "NuPlayerRenderer" #include #include "NuPlayerRenderer.h" #include #include #include #include #include #include namespace android { // static const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll; NuPlayer::Renderer::Renderer( const sp &sink, const sp ¬ify, uint32_t flags) : mAudioSink(sink), mNotify(notify), mFlags(flags), mNumFramesWritten(0), mDrainAudioQueuePending(false), mDrainVideoQueuePending(false), mAudioQueueGeneration(0), mVideoQueueGeneration(0), mFirstAudioTimeUs(-1), mAnchorTimeMediaUs(-1), mAnchorTimeRealUs(-1), mFlushingAudio(false), mFlushingVideo(false), mHasAudio(false), mHasVideo(false), mSyncQueues(false), mPaused(false), mVideoRenderingStarted(false), mVideoRenderingStartGeneration(0), mAudioRenderingStartGeneration(0), mLastPositionUpdateUs(-1ll), mVideoLateByUs(0ll), mVideoSampleReceived(false) { } NuPlayer::Renderer::~Renderer() { if (offloadingAudio()) { mAudioSink->stop(); mAudioSink->flush(); mAudioSink->close(); } } void NuPlayer::Renderer::queueBuffer( bool audio, const sp &buffer, const sp ¬ifyConsumed) { sp msg = new AMessage(kWhatQueueBuffer, id()); msg->setInt32("audio", static_cast(audio)); msg->setBuffer("buffer", buffer); msg->setMessage("notifyConsumed", notifyConsumed); msg->post(); } void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) { CHECK_NE(finalResult, (status_t)OK); sp msg = new AMessage(kWhatQueueEOS, id()); msg->setInt32("audio", static_cast(audio)); msg->setInt32("finalResult", finalResult); msg->post(); } void NuPlayer::Renderer::flush(bool audio) { { Mutex::Autolock autoLock(mFlushLock); if (audio) { if (mFlushingAudio) { return; } mFlushingAudio = true; } else { if (mFlushingVideo) { return; } mFlushingVideo = true; } } sp msg = new AMessage(kWhatFlush, id()); msg->setInt32("audio", static_cast(audio)); msg->post(); } void NuPlayer::Renderer::signalTimeDiscontinuity() { Mutex::Autolock autoLock(mLock); // CHECK(mAudioQueue.empty()); // CHECK(mVideoQueue.empty()); mAnchorTimeMediaUs = -1; mAnchorTimeRealUs = -1; mSyncQueues = false; } void NuPlayer::Renderer::signalAudioSinkChanged() { (new AMessage(kWhatAudioSinkChanged, id()))->post(); } void NuPlayer::Renderer::signalDisableOffloadAudio() { (new AMessage(kWhatDisableOffloadAudio, id()))->post(); } void NuPlayer::Renderer::pause() { (new AMessage(kWhatPause, id()))->post(); } void NuPlayer::Renderer::resume() { (new AMessage(kWhatResume, id()))->post(); } void NuPlayer::Renderer::onMessageReceived(const sp &msg) { switch (msg->what()) { case kWhatStopAudioSink: { mAudioSink->stop(); break; } case kWhatDrainAudioQueue: { int32_t generation; CHECK(msg->findInt32("generation", &generation)); if (generation != mAudioQueueGeneration) { break; } mDrainAudioQueuePending = false; if (onDrainAudioQueue()) { uint32_t numFramesPlayed; CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK); uint32_t numFramesPendingPlayout = mNumFramesWritten - numFramesPlayed; // This is how long the audio sink will have data to // play back. int64_t delayUs = mAudioSink->msecsPerFrame() * numFramesPendingPlayout * 1000ll; // Let's give it more data after about half that time // has elapsed. // kWhatDrainAudioQueue is used for non-offloading mode, // and mLock is used only for offloading mode. Therefore, // no need to acquire mLock here. postDrainAudioQueue_l(delayUs / 2); } break; } case kWhatDrainVideoQueue: { int32_t generation; CHECK(msg->findInt32("generation", &generation)); if (generation != mVideoQueueGeneration) { break; } mDrainVideoQueuePending = false; onDrainVideoQueue(); postDrainVideoQueue(); break; } case kWhatQueueBuffer: { onQueueBuffer(msg); break; } case kWhatQueueEOS: { onQueueEOS(msg); break; } case kWhatFlush: { onFlush(msg); break; } case kWhatAudioSinkChanged: { onAudioSinkChanged(); break; } case kWhatDisableOffloadAudio: { onDisableOffloadAudio(); break; } case kWhatPause: { onPause(); break; } case kWhatResume: { onResume(); break; } case kWhatAudioOffloadTearDown: { onAudioOffloadTearDown(); break; } default: TRESPASS(); break; } } void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) { if (mDrainAudioQueuePending || mSyncQueues || mPaused || offloadingAudio()) { return; } if (mAudioQueue.empty()) { return; } mDrainAudioQueuePending = true; sp msg = new AMessage(kWhatDrainAudioQueue, id()); msg->setInt32("generation", mAudioQueueGeneration); msg->post(delayUs); } void NuPlayer::Renderer::prepareForMediaRenderingStart() { mAudioRenderingStartGeneration = mAudioQueueGeneration; mVideoRenderingStartGeneration = mVideoQueueGeneration; } void NuPlayer::Renderer::notifyIfMediaRenderingStarted() { if (mVideoRenderingStartGeneration == mVideoQueueGeneration && mAudioRenderingStartGeneration == mAudioQueueGeneration) { mVideoRenderingStartGeneration = -1; mAudioRenderingStartGeneration = -1; sp notify = mNotify->dup(); notify->setInt32("what", kWhatMediaRenderingStart); notify->post(); } } // static size_t NuPlayer::Renderer::AudioSinkCallback( MediaPlayerBase::AudioSink * /* audioSink */, void *buffer, size_t size, void *cookie, MediaPlayerBase::AudioSink::cb_event_t event) { NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie; switch (event) { case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER: { return me->fillAudioBuffer(buffer, size); break; } case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END: { me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM); break; } case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN: { me->notifyAudioOffloadTearDown(); break; } } return 0; } size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) { Mutex::Autolock autoLock(mLock); if (!offloadingAudio() || mPaused) { return 0; } bool hasEOS = false; size_t sizeCopied = 0; bool firstEntry = true; while (sizeCopied < size && !mAudioQueue.empty()) { QueueEntry *entry = &*mAudioQueue.begin(); if (entry->mBuffer == NULL) { // EOS hasEOS = true; mAudioQueue.erase(mAudioQueue.begin()); entry = NULL; break; } if (firstEntry && entry->mOffset == 0) { firstEntry = false; int64_t mediaTimeUs; CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6); if (mFirstAudioTimeUs == -1) { mFirstAudioTimeUs = mediaTimeUs; } uint32_t numFramesPlayed; CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK); // TODO: figure out how to calculate initial latency. // Otherwise, the initial time is not correct till the first sample // is played. mAnchorTimeMediaUs = mFirstAudioTimeUs + (numFramesPlayed * mAudioSink->msecsPerFrame()) * 1000ll; mAnchorTimeRealUs = ALooper::GetNowUs(); } size_t copy = entry->mBuffer->size() - entry->mOffset; size_t sizeRemaining = size - sizeCopied; if (copy > sizeRemaining) { copy = sizeRemaining; } memcpy((char *)buffer + sizeCopied, entry->mBuffer->data() + entry->mOffset, copy); entry->mOffset += copy; if (entry->mOffset == entry->mBuffer->size()) { entry->mNotifyConsumed->post(); mAudioQueue.erase(mAudioQueue.begin()); entry = NULL; } sizeCopied += copy; notifyIfMediaRenderingStarted(); } if (sizeCopied != 0) { notifyPosition(); } if (hasEOS) { (new AMessage(kWhatStopAudioSink, id()))->post(); } return sizeCopied; } bool NuPlayer::Renderer::onDrainAudioQueue() { uint32_t numFramesPlayed; if (mAudioSink->getPosition(&numFramesPlayed) != OK) { return false; } ssize_t numFramesAvailableToWrite = mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed); #if 0 if (numFramesAvailableToWrite == mAudioSink->frameCount()) { ALOGI("audio sink underrun"); } else { ALOGV("audio queue has %d frames left to play", mAudioSink->frameCount() - numFramesAvailableToWrite); } #endif size_t numBytesAvailableToWrite = numFramesAvailableToWrite * mAudioSink->frameSize(); while (numBytesAvailableToWrite > 0 && !mAudioQueue.empty()) { QueueEntry *entry = &*mAudioQueue.begin(); if (entry->mBuffer == NULL) { // EOS int64_t postEOSDelayUs = 0; if (mAudioSink->needsTrailingPadding()) { postEOSDelayUs = getAudioPendingPlayoutUs() + 1000 * mAudioSink->latency(); } notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs); mAudioQueue.erase(mAudioQueue.begin()); entry = NULL; return false; } if (entry->mOffset == 0) { int64_t mediaTimeUs; CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6); mAnchorTimeMediaUs = mediaTimeUs; mAnchorTimeRealUs = ALooper::GetNowUs() + getAudioPendingPlayoutUs() + 1000 * mAudioSink->latency() / 2; } size_t copy = entry->mBuffer->size() - entry->mOffset; if (copy > numBytesAvailableToWrite) { copy = numBytesAvailableToWrite; } ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset, copy); if (written < 0) { // An error in AudioSink write is fatal here. LOG_ALWAYS_FATAL("AudioSink write error(%zd) when writing %zu bytes", written, copy); } entry->mOffset += written; if (entry->mOffset == entry->mBuffer->size()) { entry->mNotifyConsumed->post(); mAudioQueue.erase(mAudioQueue.begin()); entry = NULL; } numBytesAvailableToWrite -= written; size_t copiedFrames = written / mAudioSink->frameSize(); mNumFramesWritten += copiedFrames; notifyIfMediaRenderingStarted(); if (written != (ssize_t)copy) { // A short count was received from AudioSink::write() // // AudioSink write should block until exactly the number of bytes are delivered. // But it may return with a short count (without an error) when: // // 1) Size to be copied is not a multiple of the frame size. We consider this fatal. // 2) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded. // (Case 1) // Must be a multiple of the frame size. If it is not a multiple of a frame size, it // needs to fail, as we should not carry over fractional frames between calls. CHECK_EQ(copy % mAudioSink->frameSize(), 0); // (Case 2) // Return early to the caller. // Beware of calling immediately again as this may busy-loop if you are not careful. ALOGW("AudioSink write short frame count %zd < %zu", written, copy); break; } } notifyPosition(); return !mAudioQueue.empty(); } int64_t NuPlayer::Renderer::getAudioPendingPlayoutUs() { uint32_t numFramesPlayed; CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK); uint32_t numFramesPendingPlayout = mNumFramesWritten - numFramesPlayed; return numFramesPendingPlayout * mAudioSink->msecsPerFrame() * 1000; } void NuPlayer::Renderer::postDrainVideoQueue() { if (mDrainVideoQueuePending || mSyncQueues || (mPaused && mVideoSampleReceived)) { return; } if (mVideoQueue.empty()) { return; } QueueEntry &entry = *mVideoQueue.begin(); sp msg = new AMessage(kWhatDrainVideoQueue, id()); msg->setInt32("generation", mVideoQueueGeneration); int64_t delayUs; if (entry.mBuffer == NULL) { // EOS doesn't carry a timestamp. delayUs = 0; } else if (mFlags & FLAG_REAL_TIME) { int64_t mediaTimeUs; CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); delayUs = mediaTimeUs - ALooper::GetNowUs(); } else { int64_t mediaTimeUs; CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); if (mAnchorTimeMediaUs < 0) { delayUs = 0; if (!mHasAudio) { mAnchorTimeMediaUs = mediaTimeUs; mAnchorTimeRealUs = ALooper::GetNowUs(); } } else { int64_t realTimeUs = (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs; delayUs = realTimeUs - ALooper::GetNowUs(); } } ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs); msg->post(delayUs); mDrainVideoQueuePending = true; } void NuPlayer::Renderer::onDrainVideoQueue() { if (mVideoQueue.empty()) { return; } QueueEntry *entry = &*mVideoQueue.begin(); if (entry->mBuffer == NULL) { // EOS notifyEOS(false /* audio */, entry->mFinalResult); mVideoQueue.erase(mVideoQueue.begin()); entry = NULL; mVideoLateByUs = 0ll; notifyPosition(); return; } int64_t realTimeUs; if (mFlags & FLAG_REAL_TIME) { CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs)); } else { int64_t mediaTimeUs; CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs; } bool tooLate = false; if (!mPaused) { mVideoLateByUs = ALooper::GetNowUs() - realTimeUs; tooLate = (mVideoLateByUs > 40000); if (tooLate) { ALOGV("video late by %lld us (%.2f secs)", mVideoLateByUs, mVideoLateByUs / 1E6); } else { ALOGV("rendering video at media time %.2f secs", (mFlags & FLAG_REAL_TIME ? realTimeUs : (realTimeUs + mAnchorTimeMediaUs - mAnchorTimeRealUs)) / 1E6); } } else { mVideoLateByUs = 0ll; } entry->mNotifyConsumed->setInt32("render", !tooLate); entry->mNotifyConsumed->post(); mVideoQueue.erase(mVideoQueue.begin()); entry = NULL; mVideoSampleReceived = true; if (!mPaused) { if (!mVideoRenderingStarted) { mVideoRenderingStarted = true; notifyVideoRenderingStart(); } notifyIfMediaRenderingStarted(); } notifyPosition(); } void NuPlayer::Renderer::notifyVideoRenderingStart() { sp notify = mNotify->dup(); notify->setInt32("what", kWhatVideoRenderingStart); notify->post(); } void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) { sp notify = mNotify->dup(); notify->setInt32("what", kWhatEOS); notify->setInt32("audio", static_cast(audio)); notify->setInt32("finalResult", finalResult); notify->post(delayUs); } void NuPlayer::Renderer::notifyAudioOffloadTearDown() { (new AMessage(kWhatAudioOffloadTearDown, id()))->post(); } void NuPlayer::Renderer::onQueueBuffer(const sp &msg) { int32_t audio; CHECK(msg->findInt32("audio", &audio)); if (audio) { mHasAudio = true; } else { mHasVideo = true; } if (dropBufferWhileFlushing(audio, msg)) { return; } sp buffer; CHECK(msg->findBuffer("buffer", &buffer)); sp notifyConsumed; CHECK(msg->findMessage("notifyConsumed", ¬ifyConsumed)); QueueEntry entry; entry.mBuffer = buffer; entry.mNotifyConsumed = notifyConsumed; entry.mOffset = 0; entry.mFinalResult = OK; if (audio) { Mutex::Autolock autoLock(mLock); mAudioQueue.push_back(entry); postDrainAudioQueue_l(); } else { mVideoQueue.push_back(entry); postDrainVideoQueue(); } Mutex::Autolock autoLock(mLock); if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) { return; } sp firstAudioBuffer = (*mAudioQueue.begin()).mBuffer; sp firstVideoBuffer = (*mVideoQueue.begin()).mBuffer; if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) { // EOS signalled on either queue. syncQueuesDone_l(); return; } int64_t firstAudioTimeUs; int64_t firstVideoTimeUs; CHECK(firstAudioBuffer->meta() ->findInt64("timeUs", &firstAudioTimeUs)); CHECK(firstVideoBuffer->meta() ->findInt64("timeUs", &firstVideoTimeUs)); int64_t diff = firstVideoTimeUs - firstAudioTimeUs; ALOGV("queueDiff = %.2f secs", diff / 1E6); if (diff > 100000ll) { // Audio data starts More than 0.1 secs before video. // Drop some audio. (*mAudioQueue.begin()).mNotifyConsumed->post(); mAudioQueue.erase(mAudioQueue.begin()); return; } syncQueuesDone_l(); } void NuPlayer::Renderer::syncQueuesDone_l() { if (!mSyncQueues) { return; } mSyncQueues = false; if (!mAudioQueue.empty()) { postDrainAudioQueue_l(); } if (!mVideoQueue.empty()) { postDrainVideoQueue(); } } void NuPlayer::Renderer::onQueueEOS(const sp &msg) { int32_t audio; CHECK(msg->findInt32("audio", &audio)); if (dropBufferWhileFlushing(audio, msg)) { return; } int32_t finalResult; CHECK(msg->findInt32("finalResult", &finalResult)); QueueEntry entry; entry.mOffset = 0; entry.mFinalResult = finalResult; if (audio) { Mutex::Autolock autoLock(mLock); if (mAudioQueue.empty() && mSyncQueues) { syncQueuesDone_l(); } mAudioQueue.push_back(entry); postDrainAudioQueue_l(); } else { if (mVideoQueue.empty() && mSyncQueues) { Mutex::Autolock autoLock(mLock); syncQueuesDone_l(); } mVideoQueue.push_back(entry); postDrainVideoQueue(); } } void NuPlayer::Renderer::onFlush(const sp &msg) { int32_t audio; CHECK(msg->findInt32("audio", &audio)); { Mutex::Autolock autoLock(mFlushLock); if (audio) { mFlushingAudio = false; } else { mFlushingVideo = false; } } // If we're currently syncing the queues, i.e. dropping audio while // aligning the first audio/video buffer times and only one of the // two queues has data, we may starve that queue by not requesting // more buffers from the decoder. If the other source then encounters // a discontinuity that leads to flushing, we'll never find the // corresponding discontinuity on the other queue. // Therefore we'll stop syncing the queues if at least one of them // is flushed. { Mutex::Autolock autoLock(mLock); syncQueuesDone_l(); } ALOGV("flushing %s", audio ? "audio" : "video"); if (audio) { { Mutex::Autolock autoLock(mLock); flushQueue(&mAudioQueue); ++mAudioQueueGeneration; prepareForMediaRenderingStart(); if (offloadingAudio()) { mFirstAudioTimeUs = -1; } } mDrainAudioQueuePending = false; if (offloadingAudio()) { mAudioSink->pause(); mAudioSink->flush(); mAudioSink->start(); } } else { flushQueue(&mVideoQueue); mDrainVideoQueuePending = false; ++mVideoQueueGeneration; prepareForMediaRenderingStart(); } mVideoSampleReceived = false; notifyFlushComplete(audio); } void NuPlayer::Renderer::flushQueue(List *queue) { while (!queue->empty()) { QueueEntry *entry = &*queue->begin(); if (entry->mBuffer != NULL) { entry->mNotifyConsumed->post(); } queue->erase(queue->begin()); entry = NULL; } } void NuPlayer::Renderer::notifyFlushComplete(bool audio) { sp notify = mNotify->dup(); notify->setInt32("what", kWhatFlushComplete); notify->setInt32("audio", static_cast(audio)); notify->post(); } bool NuPlayer::Renderer::dropBufferWhileFlushing( bool audio, const sp &msg) { bool flushing = false; { Mutex::Autolock autoLock(mFlushLock); if (audio) { flushing = mFlushingAudio; } else { flushing = mFlushingVideo; } } if (!flushing) { return false; } sp notifyConsumed; if (msg->findMessage("notifyConsumed", ¬ifyConsumed)) { notifyConsumed->post(); } return true; } void NuPlayer::Renderer::onAudioSinkChanged() { if (offloadingAudio()) { return; } CHECK(!mDrainAudioQueuePending); mNumFramesWritten = 0; uint32_t written; if (mAudioSink->getFramesWritten(&written) == OK) { mNumFramesWritten = written; } } void NuPlayer::Renderer::onDisableOffloadAudio() { Mutex::Autolock autoLock(mLock); mFlags &= ~FLAG_OFFLOAD_AUDIO; ++mAudioQueueGeneration; } void NuPlayer::Renderer::notifyPosition() { if (mAnchorTimeRealUs < 0 || mAnchorTimeMediaUs < 0) { return; } int64_t nowUs = ALooper::GetNowUs(); if (mLastPositionUpdateUs >= 0 && nowUs < mLastPositionUpdateUs + kMinPositionUpdateDelayUs) { return; } mLastPositionUpdateUs = nowUs; int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs; sp notify = mNotify->dup(); notify->setInt32("what", kWhatPosition); notify->setInt64("positionUs", positionUs); notify->setInt64("videoLateByUs", mVideoLateByUs); notify->post(); } void NuPlayer::Renderer::onPause() { if (mPaused) { ALOGW("Renderer::onPause() called while already paused!"); return; } { Mutex::Autolock autoLock(mLock); ++mAudioQueueGeneration; ++mVideoQueueGeneration; prepareForMediaRenderingStart(); mPaused = true; } mDrainAudioQueuePending = false; mDrainVideoQueuePending = false; if (mHasAudio) { mAudioSink->pause(); } ALOGV("now paused audio queue has %d entries, video has %d entries", mAudioQueue.size(), mVideoQueue.size()); } void NuPlayer::Renderer::onResume() { if (!mPaused) { return; } if (mHasAudio) { mAudioSink->start(); } Mutex::Autolock autoLock(mLock); mPaused = false; if (!mAudioQueue.empty()) { postDrainAudioQueue_l(); } if (!mVideoQueue.empty()) { postDrainVideoQueue(); } } void NuPlayer::Renderer::onAudioOffloadTearDown() { uint32_t numFramesPlayed; CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK); int64_t firstAudioTimeUs; { Mutex::Autolock autoLock(mLock); firstAudioTimeUs = mFirstAudioTimeUs; } int64_t currentPositionUs = firstAudioTimeUs + (numFramesPlayed * mAudioSink->msecsPerFrame()) * 1000ll; mAudioSink->stop(); mAudioSink->flush(); sp notify = mNotify->dup(); notify->setInt32("what", kWhatAudioOffloadTearDown); notify->setInt64("positionUs", currentPositionUs); notify->post(); } } // namespace android