/* * Copyright (C) 2010 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ //#define LOG_NDEBUG 0 #define LOG_TAG "NuPlayerRenderer" #include #include "NuPlayerRenderer.h" #include "SoftwareRenderer.h" #include #include #include namespace android { // static const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll; NuPlayer::Renderer::Renderer( const sp &sink, const sp ¬ify, uint32_t flags) : mAudioSink(sink), mSoftRenderer(NULL), mNotify(notify), mFlags(flags), mNumFramesWritten(0), mDrainAudioQueuePending(false), mDrainVideoQueuePending(false), mAudioQueueGeneration(0), mVideoQueueGeneration(0), mAnchorTimeMediaUs(-1), mAnchorTimeRealUs(-1), mFlushingAudio(false), mFlushingVideo(false), mHasAudio(false), mHasVideo(false), mSyncQueues(false), mPaused(false), mVideoRenderingStarted(false), mVideoRenderingStartGeneration(0), mAudioRenderingStartGeneration(0), mLastPositionUpdateUs(-1ll), mVideoLateByUs(0ll) { } NuPlayer::Renderer::~Renderer() { delete mSoftRenderer; } void NuPlayer::Renderer::setSoftRenderer(SoftwareRenderer *softRenderer) { delete mSoftRenderer; mSoftRenderer = softRenderer; } void NuPlayer::Renderer::queueBuffer( bool audio, const sp &buffer, const sp ¬ifyConsumed) { sp msg = new AMessage(kWhatQueueBuffer, id()); msg->setInt32("audio", static_cast(audio)); msg->setBuffer("buffer", buffer); msg->setMessage("notifyConsumed", notifyConsumed); msg->post(); } void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) { CHECK_NE(finalResult, (status_t)OK); sp msg = new AMessage(kWhatQueueEOS, id()); msg->setInt32("audio", static_cast(audio)); msg->setInt32("finalResult", finalResult); msg->post(); } void NuPlayer::Renderer::flush(bool audio) { { Mutex::Autolock autoLock(mFlushLock); if (audio) { CHECK(!mFlushingAudio); mFlushingAudio = true; } else { CHECK(!mFlushingVideo); mFlushingVideo = true; } } sp msg = new AMessage(kWhatFlush, id()); msg->setInt32("audio", static_cast(audio)); msg->post(); } void NuPlayer::Renderer::signalTimeDiscontinuity() { // CHECK(mAudioQueue.empty()); // CHECK(mVideoQueue.empty()); mAnchorTimeMediaUs = -1; mAnchorTimeRealUs = -1; mSyncQueues = false; } void NuPlayer::Renderer::pause() { (new AMessage(kWhatPause, id()))->post(); } void NuPlayer::Renderer::resume() { (new AMessage(kWhatResume, id()))->post(); } void NuPlayer::Renderer::onMessageReceived(const sp &msg) { switch (msg->what()) { case kWhatDrainAudioQueue: { int32_t generation; CHECK(msg->findInt32("generation", &generation)); if (generation != mAudioQueueGeneration) { break; } mDrainAudioQueuePending = false; if (onDrainAudioQueue()) { uint32_t numFramesPlayed; CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK); uint32_t numFramesPendingPlayout = mNumFramesWritten - numFramesPlayed; // This is how long the audio sink will have data to // play back. int64_t delayUs = mAudioSink->msecsPerFrame() * numFramesPendingPlayout * 1000ll; // Let's give it more data after about half that time // has elapsed. postDrainAudioQueue(delayUs / 2); } break; } case kWhatDrainVideoQueue: { int32_t generation; CHECK(msg->findInt32("generation", &generation)); if (generation != mVideoQueueGeneration) { break; } mDrainVideoQueuePending = false; onDrainVideoQueue(); postDrainVideoQueue(); break; } case kWhatQueueBuffer: { onQueueBuffer(msg); break; } case kWhatQueueEOS: { onQueueEOS(msg); break; } case kWhatFlush: { onFlush(msg); break; } case kWhatAudioSinkChanged: { onAudioSinkChanged(); break; } case kWhatPause: { onPause(); break; } case kWhatResume: { onResume(); break; } default: TRESPASS(); break; } } void NuPlayer::Renderer::postDrainAudioQueue(int64_t delayUs) { if (mDrainAudioQueuePending || mSyncQueues || mPaused) { return; } if (mAudioQueue.empty()) { return; } mDrainAudioQueuePending = true; sp msg = new AMessage(kWhatDrainAudioQueue, id()); msg->setInt32("generation", mAudioQueueGeneration); msg->post(delayUs); } void NuPlayer::Renderer::signalAudioSinkChanged() { (new AMessage(kWhatAudioSinkChanged, id()))->post(); } void NuPlayer::Renderer::prepareForMediaRenderingStart() { mAudioRenderingStartGeneration = mAudioQueueGeneration; mVideoRenderingStartGeneration = mVideoQueueGeneration; } void NuPlayer::Renderer::notifyIfMediaRenderingStarted() { if (mVideoRenderingStartGeneration == mVideoQueueGeneration && mAudioRenderingStartGeneration == mAudioQueueGeneration) { mVideoRenderingStartGeneration = -1; mAudioRenderingStartGeneration = -1; sp notify = mNotify->dup(); notify->setInt32("what", kWhatMediaRenderingStart); notify->post(); } } bool NuPlayer::Renderer::onDrainAudioQueue() { uint32_t numFramesPlayed; if (mAudioSink->getPosition(&numFramesPlayed) != OK) { return false; } ssize_t numFramesAvailableToWrite = mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed); #if 0 if (numFramesAvailableToWrite == mAudioSink->frameCount()) { ALOGI("audio sink underrun"); } else { ALOGV("audio queue has %d frames left to play", mAudioSink->frameCount() - numFramesAvailableToWrite); } #endif size_t numBytesAvailableToWrite = numFramesAvailableToWrite * mAudioSink->frameSize(); while (numBytesAvailableToWrite > 0 && !mAudioQueue.empty()) { QueueEntry *entry = &*mAudioQueue.begin(); if (entry->mBuffer == NULL) { // EOS notifyEOS(true /* audio */, entry->mFinalResult); mAudioQueue.erase(mAudioQueue.begin()); entry = NULL; return false; } if (entry->mOffset == 0) { int64_t mediaTimeUs; CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6); mAnchorTimeMediaUs = mediaTimeUs; uint32_t numFramesPlayed; CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK); uint32_t numFramesPendingPlayout = mNumFramesWritten - numFramesPlayed; int64_t realTimeOffsetUs = (mAudioSink->latency() / 2 /* XXX */ + numFramesPendingPlayout * mAudioSink->msecsPerFrame()) * 1000ll; // ALOGI("realTimeOffsetUs = %lld us", realTimeOffsetUs); mAnchorTimeRealUs = ALooper::GetNowUs() + realTimeOffsetUs; } size_t copy = entry->mBuffer->size() - entry->mOffset; if (copy > numBytesAvailableToWrite) { copy = numBytesAvailableToWrite; } CHECK_EQ(mAudioSink->write( entry->mBuffer->data() + entry->mOffset, copy), (ssize_t)copy); entry->mOffset += copy; if (entry->mOffset == entry->mBuffer->size()) { entry->mNotifyConsumed->post(); mAudioQueue.erase(mAudioQueue.begin()); entry = NULL; } numBytesAvailableToWrite -= copy; size_t copiedFrames = copy / mAudioSink->frameSize(); mNumFramesWritten += copiedFrames; notifyIfMediaRenderingStarted(); } notifyPosition(); return !mAudioQueue.empty(); } void NuPlayer::Renderer::postDrainVideoQueue() { if (mDrainVideoQueuePending || mSyncQueues || mPaused) { return; } if (mVideoQueue.empty()) { return; } QueueEntry &entry = *mVideoQueue.begin(); sp msg = new AMessage(kWhatDrainVideoQueue, id()); msg->setInt32("generation", mVideoQueueGeneration); int64_t delayUs; if (entry.mBuffer == NULL) { // EOS doesn't carry a timestamp. delayUs = 0; } else if (mFlags & FLAG_REAL_TIME) { int64_t mediaTimeUs; CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); delayUs = mediaTimeUs - ALooper::GetNowUs(); } else { int64_t mediaTimeUs; CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); if (mAnchorTimeMediaUs < 0) { delayUs = 0; if (!mHasAudio) { mAnchorTimeMediaUs = mediaTimeUs; mAnchorTimeRealUs = ALooper::GetNowUs(); } } else { int64_t realTimeUs = (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs; delayUs = realTimeUs - ALooper::GetNowUs(); } } msg->post(delayUs); mDrainVideoQueuePending = true; } void NuPlayer::Renderer::onDrainVideoQueue() { if (mVideoQueue.empty()) { return; } QueueEntry *entry = &*mVideoQueue.begin(); if (entry->mBuffer == NULL) { // EOS notifyEOS(false /* audio */, entry->mFinalResult); mVideoQueue.erase(mVideoQueue.begin()); entry = NULL; mVideoLateByUs = 0ll; notifyPosition(); return; } int64_t realTimeUs; if (mFlags & FLAG_REAL_TIME) { CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs)); } else { int64_t mediaTimeUs; CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs; } mVideoLateByUs = ALooper::GetNowUs() - realTimeUs; bool tooLate = (mVideoLateByUs > 40000); if (tooLate) { ALOGV("video late by %lld us (%.2f secs)", mVideoLateByUs, mVideoLateByUs / 1E6); } else { ALOGV("rendering video at media time %.2f secs", (mFlags & FLAG_REAL_TIME ? realTimeUs : (realTimeUs + mAnchorTimeMediaUs - mAnchorTimeRealUs)) / 1E6); if (mSoftRenderer != NULL) { mSoftRenderer->render(entry->mBuffer->data(), entry->mBuffer->size(), NULL); } } entry->mNotifyConsumed->setInt32("render", !tooLate); entry->mNotifyConsumed->post(); mVideoQueue.erase(mVideoQueue.begin()); entry = NULL; if (!mVideoRenderingStarted) { mVideoRenderingStarted = true; notifyVideoRenderingStart(); } notifyIfMediaRenderingStarted(); notifyPosition(); } void NuPlayer::Renderer::notifyVideoRenderingStart() { sp notify = mNotify->dup(); notify->setInt32("what", kWhatVideoRenderingStart); notify->post(); } void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult) { sp notify = mNotify->dup(); notify->setInt32("what", kWhatEOS); notify->setInt32("audio", static_cast(audio)); notify->setInt32("finalResult", finalResult); notify->post(); } void NuPlayer::Renderer::onQueueBuffer(const sp &msg) { int32_t audio; CHECK(msg->findInt32("audio", &audio)); if (audio) { mHasAudio = true; } else { mHasVideo = true; } if (dropBufferWhileFlushing(audio, msg)) { return; } sp buffer; CHECK(msg->findBuffer("buffer", &buffer)); sp notifyConsumed; CHECK(msg->findMessage("notifyConsumed", ¬ifyConsumed)); QueueEntry entry; entry.mBuffer = buffer; entry.mNotifyConsumed = notifyConsumed; entry.mOffset = 0; entry.mFinalResult = OK; if (audio) { mAudioQueue.push_back(entry); postDrainAudioQueue(); } else { mVideoQueue.push_back(entry); postDrainVideoQueue(); } if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) { return; } sp firstAudioBuffer = (*mAudioQueue.begin()).mBuffer; sp firstVideoBuffer = (*mVideoQueue.begin()).mBuffer; if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) { // EOS signalled on either queue. syncQueuesDone(); return; } int64_t firstAudioTimeUs; int64_t firstVideoTimeUs; CHECK(firstAudioBuffer->meta() ->findInt64("timeUs", &firstAudioTimeUs)); CHECK(firstVideoBuffer->meta() ->findInt64("timeUs", &firstVideoTimeUs)); int64_t diff = firstVideoTimeUs - firstAudioTimeUs; ALOGV("queueDiff = %.2f secs", diff / 1E6); if (diff > 100000ll) { // Audio data starts More than 0.1 secs before video. // Drop some audio. (*mAudioQueue.begin()).mNotifyConsumed->post(); mAudioQueue.erase(mAudioQueue.begin()); return; } syncQueuesDone(); } void NuPlayer::Renderer::syncQueuesDone() { if (!mSyncQueues) { return; } mSyncQueues = false; if (!mAudioQueue.empty()) { postDrainAudioQueue(); } if (!mVideoQueue.empty()) { postDrainVideoQueue(); } } void NuPlayer::Renderer::onQueueEOS(const sp &msg) { int32_t audio; CHECK(msg->findInt32("audio", &audio)); if (dropBufferWhileFlushing(audio, msg)) { return; } int32_t finalResult; CHECK(msg->findInt32("finalResult", &finalResult)); QueueEntry entry; entry.mOffset = 0; entry.mFinalResult = finalResult; if (audio) { if (mAudioQueue.empty() && mSyncQueues) { syncQueuesDone(); } mAudioQueue.push_back(entry); postDrainAudioQueue(); } else { if (mVideoQueue.empty() && mSyncQueues) { syncQueuesDone(); } mVideoQueue.push_back(entry); postDrainVideoQueue(); } } void NuPlayer::Renderer::onFlush(const sp &msg) { int32_t audio; CHECK(msg->findInt32("audio", &audio)); // If we're currently syncing the queues, i.e. dropping audio while // aligning the first audio/video buffer times and only one of the // two queues has data, we may starve that queue by not requesting // more buffers from the decoder. If the other source then encounters // a discontinuity that leads to flushing, we'll never find the // corresponding discontinuity on the other queue. // Therefore we'll stop syncing the queues if at least one of them // is flushed. syncQueuesDone(); ALOGV("flushing %s", audio ? "audio" : "video"); if (audio) { flushQueue(&mAudioQueue); Mutex::Autolock autoLock(mFlushLock); mFlushingAudio = false; mDrainAudioQueuePending = false; ++mAudioQueueGeneration; prepareForMediaRenderingStart(); } else { flushQueue(&mVideoQueue); Mutex::Autolock autoLock(mFlushLock); mFlushingVideo = false; mDrainVideoQueuePending = false; ++mVideoQueueGeneration; prepareForMediaRenderingStart(); } notifyFlushComplete(audio); } void NuPlayer::Renderer::flushQueue(List *queue) { while (!queue->empty()) { QueueEntry *entry = &*queue->begin(); if (entry->mBuffer != NULL) { entry->mNotifyConsumed->post(); } queue->erase(queue->begin()); entry = NULL; } } void NuPlayer::Renderer::notifyFlushComplete(bool audio) { sp notify = mNotify->dup(); notify->setInt32("what", kWhatFlushComplete); notify->setInt32("audio", static_cast(audio)); notify->post(); } bool NuPlayer::Renderer::dropBufferWhileFlushing( bool audio, const sp &msg) { bool flushing = false; { Mutex::Autolock autoLock(mFlushLock); if (audio) { flushing = mFlushingAudio; } else { flushing = mFlushingVideo; } } if (!flushing) { return false; } sp notifyConsumed; if (msg->findMessage("notifyConsumed", ¬ifyConsumed)) { notifyConsumed->post(); } return true; } void NuPlayer::Renderer::onAudioSinkChanged() { CHECK(!mDrainAudioQueuePending); mNumFramesWritten = 0; uint32_t written; if (mAudioSink->getFramesWritten(&written) == OK) { mNumFramesWritten = written; } } void NuPlayer::Renderer::notifyPosition() { if (mAnchorTimeRealUs < 0 || mAnchorTimeMediaUs < 0) { return; } int64_t nowUs = ALooper::GetNowUs(); if (mLastPositionUpdateUs >= 0 && nowUs < mLastPositionUpdateUs + kMinPositionUpdateDelayUs) { return; } mLastPositionUpdateUs = nowUs; int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs; sp notify = mNotify->dup(); notify->setInt32("what", kWhatPosition); notify->setInt64("positionUs", positionUs); notify->setInt64("videoLateByUs", mVideoLateByUs); notify->post(); } void NuPlayer::Renderer::onPause() { CHECK(!mPaused); mDrainAudioQueuePending = false; ++mAudioQueueGeneration; mDrainVideoQueuePending = false; ++mVideoQueueGeneration; prepareForMediaRenderingStart(); if (mHasAudio) { mAudioSink->pause(); } ALOGV("now paused audio queue has %d entries, video has %d entries", mAudioQueue.size(), mVideoQueue.size()); mPaused = true; } void NuPlayer::Renderer::onResume() { if (!mPaused) { return; } if (mHasAudio) { mAudioSink->start(); } mPaused = false; if (!mAudioQueue.empty()) { postDrainAudioQueue(); } if (!mVideoQueue.empty()) { postDrainVideoQueue(); } } } // namespace android