diff options
28 files changed, 308 insertions, 74 deletions
diff --git a/include/media/IStreamSource.h b/include/media/IStreamSource.h index 39e0a9e..677119b 100644 --- a/include/media/IStreamSource.h +++ b/include/media/IStreamSource.h @@ -37,6 +37,9 @@ struct IStreamSource : public IInterface { enum { // Video PES packets contain exactly one (aligned) access unit. kFlagAlignedVideoData = 1, + + // Timestamps are in ALooper::GetNowUs() units. + kFlagIsRealTimeData = 2, }; virtual uint32_t flags() const { return 0; } }; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp index 2ba6c22..5387e1a 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp @@ -381,9 +381,16 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { mSource->start(); + uint32_t flags = 0; + + if (mSource->isRealTime()) { + flags |= Renderer::FLAG_REAL_TIME; + } + mRenderer = new Renderer( mAudioSink, - new AMessage(kWhatRendererNotify, id())); + new AMessage(kWhatRendererNotify, id()), + flags); looper()->registerHandler(mRenderer); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp index 3c63e80..723af09 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp @@ -378,6 +378,7 @@ status_t NuPlayerDriver::invoke(const Parcel &request, Parcel *reply) { int mode = request.readInt32(); return mPlayer->setVideoScalingMode(mode); } + default: { return INVALID_OPERATION; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp index 1ba76a5..404b56f 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp @@ -31,9 +31,11 @@ const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll; NuPlayer::Renderer::Renderer( const sp<MediaPlayerBase::AudioSink> &sink, - const sp<AMessage> ¬ify) + const sp<AMessage> ¬ify, + uint32_t flags) : mAudioSink(sink), mNotify(notify), + mFlags(flags), mNumFramesWritten(0), mDrainAudioQueuePending(false), mDrainVideoQueuePending(false), @@ -323,6 +325,11 @@ void NuPlayer::Renderer::postDrainVideoQueue() { if (entry.mBuffer == NULL) { // EOS doesn't carry a timestamp. delayUs = 0; + } else if (mFlags & FLAG_REAL_TIME) { + int64_t mediaTimeUs; + CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); + + delayUs = mediaTimeUs - ALooper::GetNowUs(); } else { int64_t mediaTimeUs; CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); @@ -368,12 +375,17 @@ void NuPlayer::Renderer::onDrainVideoQueue() { return; } - int64_t mediaTimeUs; - CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); + int64_t realTimeUs; + if (mFlags & FLAG_REAL_TIME) { + CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs)); + } else { + int64_t mediaTimeUs; + CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); + + realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs; + } - int64_t realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs; mVideoLateByUs = ALooper::GetNowUs() - realTimeUs; - bool tooLate = (mVideoLateByUs > 40000); if (tooLate) { diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h index e4368c7..c9796e2 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h @@ -25,8 +25,12 @@ namespace android { struct ABuffer; struct NuPlayer::Renderer : public AHandler { + enum Flags { + FLAG_REAL_TIME = 1, + }; Renderer(const sp<MediaPlayerBase::AudioSink> &sink, - const sp<AMessage> ¬ify); + const sp<AMessage> ¬ify, + uint32_t flags = 0); void queueBuffer( bool audio, @@ -79,6 +83,7 @@ private: sp<MediaPlayerBase::AudioSink> mAudioSink; sp<AMessage> mNotify; + uint32_t mFlags; List<QueueEntry> mAudioQueue; List<QueueEntry> mVideoQueue; uint32_t mNumFramesWritten; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerSource.h b/media/libmediaplayerservice/nuplayer/NuPlayerSource.h index 8622abe..1cbf575 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerSource.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerSource.h @@ -74,6 +74,10 @@ struct NuPlayer::Source : public AHandler { return INVALID_OPERATION; } + virtual bool isRealTime() const { + return false; + } + protected: virtual ~Source() {} diff --git a/media/libmediaplayerservice/nuplayer/StreamingSource.cpp b/media/libmediaplayerservice/nuplayer/StreamingSource.cpp index df03f86..28f0d50 100644 --- a/media/libmediaplayerservice/nuplayer/StreamingSource.cpp +++ b/media/libmediaplayerservice/nuplayer/StreamingSource.cpp @@ -182,5 +182,9 @@ status_t NuPlayer::StreamingSource::dequeueAccessUnit( return err; } +bool NuPlayer::StreamingSource::isRealTime() const { + return mSource->flags() & IStreamSource::kFlagIsRealTimeData; +} + } // namespace android diff --git a/media/libmediaplayerservice/nuplayer/StreamingSource.h b/media/libmediaplayerservice/nuplayer/StreamingSource.h index 80b061c..412b6c4 100644 --- a/media/libmediaplayerservice/nuplayer/StreamingSource.h +++ b/media/libmediaplayerservice/nuplayer/StreamingSource.h @@ -38,6 +38,8 @@ struct NuPlayer::StreamingSource : public NuPlayer::Source { virtual status_t dequeueAccessUnit(bool audio, sp<ABuffer> *accessUnit); + virtual bool isRealTime() const; + protected: virtual ~StreamingSource(); diff --git a/media/libstagefright/mpeg2ts/ATSParser.cpp b/media/libstagefright/mpeg2ts/ATSParser.cpp index a167b5a..c12572f 100644 --- a/media/libstagefright/mpeg2ts/ATSParser.cpp +++ b/media/libstagefright/mpeg2ts/ATSParser.cpp @@ -452,6 +452,10 @@ int64_t ATSParser::Program::convertPTSToTimestamp(uint64_t PTS) { timeUs += mParser->mAbsoluteTimeAnchorUs; } + if (mParser->mTimeOffsetValid) { + timeUs += mParser->mTimeOffsetUs; + } + return timeUs; } @@ -930,6 +934,8 @@ sp<MediaSource> ATSParser::Stream::getSource(SourceType type) { ATSParser::ATSParser(uint32_t flags) : mFlags(flags), mAbsoluteTimeAnchorUs(-1ll), + mTimeOffsetValid(false), + mTimeOffsetUs(0ll), mNumTSPacketsParsed(0), mNumPCRs(0) { mPSISections.add(0 /* PID */, new PSISection); @@ -960,6 +966,13 @@ void ATSParser::signalDiscontinuity( CHECK(mPrograms.empty()); mAbsoluteTimeAnchorUs = timeUs; return; + } else if (type == DISCONTINUITY_TIME_OFFSET) { + int64_t offset; + CHECK(extra->findInt64("offset", &offset)); + + mTimeOffsetValid = true; + mTimeOffsetUs = offset; + return; } for (size_t i = 0; i < mPrograms.size(); ++i) { diff --git a/media/libstagefright/mpeg2ts/ATSParser.h b/media/libstagefright/mpeg2ts/ATSParser.h index 46edc45..a10edc9 100644 --- a/media/libstagefright/mpeg2ts/ATSParser.h +++ b/media/libstagefright/mpeg2ts/ATSParser.h @@ -39,6 +39,7 @@ struct ATSParser : public RefBase { DISCONTINUITY_AUDIO_FORMAT = 2, DISCONTINUITY_VIDEO_FORMAT = 4, DISCONTINUITY_ABSOLUTE_TIME = 8, + DISCONTINUITY_TIME_OFFSET = 16, DISCONTINUITY_SEEK = DISCONTINUITY_TIME, @@ -106,6 +107,9 @@ private: int64_t mAbsoluteTimeAnchorUs; + bool mTimeOffsetValid; + int64_t mTimeOffsetUs; + size_t mNumTSPacketsParsed; void parseProgramAssociationTable(ABitReader *br); diff --git a/media/libstagefright/omx/GraphicBufferSource.cpp b/media/libstagefright/omx/GraphicBufferSource.cpp index 211e1d1..3854e52 100644 --- a/media/libstagefright/omx/GraphicBufferSource.cpp +++ b/media/libstagefright/omx/GraphicBufferSource.cpp @@ -32,7 +32,7 @@ static const bool EXTRA_CHECK = true; GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance, - uint32_t bufferWidth, uint32_t bufferHeight) : + uint32_t bufferWidth, uint32_t bufferHeight, uint32_t bufferCount) : mInitCheck(UNKNOWN_ERROR), mNodeInstance(nodeInstance), mExecuting(false), @@ -40,20 +40,31 @@ GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance, mEndOfStream(false), mEndOfStreamSent(false) { - ALOGV("GraphicBufferSource w=%u h=%u", bufferWidth, bufferHeight); + ALOGV("GraphicBufferSource w=%u h=%u c=%u", + bufferWidth, bufferHeight, bufferCount); if (bufferWidth == 0 || bufferHeight == 0) { - ALOGE("Invalid dimensions %dx%d", bufferWidth, bufferHeight); + ALOGE("Invalid dimensions %ux%u", bufferWidth, bufferHeight); mInitCheck = BAD_VALUE; return; } + String8 name("GraphicBufferSource"); + mBufferQueue = new BufferQueue(true); + mBufferQueue->setConsumerName(name); mBufferQueue->setDefaultBufferSize(bufferWidth, bufferHeight); mBufferQueue->setSynchronousMode(true); mBufferQueue->setConsumerUsageBits(GRALLOC_USAGE_HW_VIDEO_ENCODER | GRALLOC_USAGE_HW_TEXTURE); + mInitCheck = mBufferQueue->setMaxAcquiredBufferCount(bufferCount); + if (mInitCheck != NO_ERROR) { + ALOGE("Unable to set BQ max acquired buffer count to %u: %d", + bufferCount, mInitCheck); + return; + } + // Note that we can't create an sp<...>(this) in a ctor that will not keep a // reference once the ctor ends, as that would cause the refcount of 'this' // dropping to 0 at the end of the ctor. Since all we need is a wp<...> @@ -64,21 +75,23 @@ GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance, sp<BufferQueue::ConsumerListener> proxy; proxy = new BufferQueue::ProxyConsumerListener(listener); - status_t err = mBufferQueue->consumerConnect(proxy); - if (err != NO_ERROR) { + mInitCheck = mBufferQueue->consumerConnect(proxy); + if (mInitCheck != NO_ERROR) { ALOGE("Error connecting to BufferQueue: %s (%d)", - strerror(-err), err); + strerror(-mInitCheck), mInitCheck); return; } - mInitCheck = OK; + CHECK(mInitCheck == NO_ERROR); } GraphicBufferSource::~GraphicBufferSource() { ALOGV("~GraphicBufferSource"); - status_t err = mBufferQueue->consumerDisconnect(); - if (err != NO_ERROR) { - ALOGW("consumerDisconnect failed: %d", err); + if (mBufferQueue != NULL) { + status_t err = mBufferQueue->consumerDisconnect(); + if (err != NO_ERROR) { + ALOGW("consumerDisconnect failed: %d", err); + } } } @@ -98,8 +111,12 @@ void GraphicBufferSource::omxExecuting() { // one codec buffer simultaneously. (We could instead try to submit // all BQ buffers whenever any codec buffer is freed, but if we get the // initial conditions right that will never be useful.) - while (mNumFramesAvailable && isCodecBufferAvailable_l()) { - fillCodecBuffer_l(); + while (mNumFramesAvailable) { + if (!fillCodecBuffer_l()) { + ALOGV("stop load with frames available (codecAvail=%d)", + isCodecBufferAvailable_l()); + break; + } } ALOGV("done loading initial frames, avail=%d", mNumFramesAvailable); @@ -166,7 +183,7 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) { // see if the GraphicBuffer reference was null, which should only ever // happen for EOS. if (codecBuffer.mGraphicBuffer == NULL) { - CHECK(mEndOfStream); + CHECK(mEndOfStream && mEndOfStreamSent); // No GraphicBuffer to deal with, no additional input or output is // expected, so just return. return; @@ -216,8 +233,9 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) { if (mNumFramesAvailable) { // Fill this codec buffer. - CHECK(!mEndOfStream); - ALOGV("buffer freed, %d frames avail", mNumFramesAvailable); + CHECK(!mEndOfStreamSent); + ALOGV("buffer freed, %d frames avail (eos=%d)", + mNumFramesAvailable, mEndOfStream); fillCodecBuffer_l(); } else if (mEndOfStream) { // No frames available, but EOS is pending, so use this buffer to @@ -228,56 +246,58 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) { return; } -status_t GraphicBufferSource::fillCodecBuffer_l() { +bool GraphicBufferSource::fillCodecBuffer_l() { CHECK(mExecuting && mNumFramesAvailable > 0); + int cbi = findAvailableCodecBuffer_l(); if (cbi < 0) { // No buffers available, bail. ALOGV("fillCodecBuffer_l: no codec buffers, avail now %d", mNumFramesAvailable); - } else { - ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%d", - mNumFramesAvailable); - BufferQueue::BufferItem item; - status_t err = mBufferQueue->acquireBuffer(&item); - if (err == BufferQueue::NO_BUFFER_AVAILABLE) { - // shouldn't happen - ALOGW("fillCodecBuffer_l: frame was not available"); - return err; - } else if (err != OK) { - // now what? fake end-of-stream? - ALOGW("fillCodecBuffer_l: acquireBuffer returned err=%d", err); - return err; - } + return false; + } - mNumFramesAvailable--; + ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%d", + mNumFramesAvailable); + BufferQueue::BufferItem item; + status_t err = mBufferQueue->acquireBuffer(&item); + if (err == BufferQueue::NO_BUFFER_AVAILABLE) { + // shouldn't happen + ALOGW("fillCodecBuffer_l: frame was not available"); + return false; + } else if (err != OK) { + // now what? fake end-of-stream? + ALOGW("fillCodecBuffer_l: acquireBuffer returned err=%d", err); + return false; + } - // Wait for it to become available. - err = item.mFence->waitForever(1000, - "GraphicBufferSource::fillCodecBuffer_l"); - if (err != OK) { - ALOGW("failed to wait for buffer fence: %d", err); - // keep going - } + mNumFramesAvailable--; - // If this is the first time we're seeing this buffer, add it to our - // slot table. - if (item.mGraphicBuffer != NULL) { - ALOGV("fillCodecBuffer_l: setting mBufferSlot %d", item.mBuf); - mBufferSlot[item.mBuf] = item.mGraphicBuffer; - } + // Wait for it to become available. + err = item.mFence->waitForever(1000, + "GraphicBufferSource::fillCodecBuffer_l"); + if (err != OK) { + ALOGW("failed to wait for buffer fence: %d", err); + // keep going + } - err = submitBuffer_l(mBufferSlot[item.mBuf], item.mTimestamp, cbi); - if (err != OK) { - ALOGV("submitBuffer_l failed, releasing bq buf %d", item.mBuf); - mBufferQueue->releaseBuffer(item.mBuf, EGL_NO_DISPLAY, + // If this is the first time we're seeing this buffer, add it to our + // slot table. + if (item.mGraphicBuffer != NULL) { + ALOGV("fillCodecBuffer_l: setting mBufferSlot %d", item.mBuf); + mBufferSlot[item.mBuf] = item.mGraphicBuffer; + } + + err = submitBuffer_l(mBufferSlot[item.mBuf], item.mTimestamp, cbi); + if (err != OK) { + ALOGV("submitBuffer_l failed, releasing bq buf %d", item.mBuf); + mBufferQueue->releaseBuffer(item.mBuf, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE); - } else { - ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi); - } + } else { + ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi); } - return OK; + return true; } status_t GraphicBufferSource::signalEndOfInputStream() { @@ -372,6 +392,7 @@ void GraphicBufferSource::submitEndOfInputStream_l() { } else { ALOGV("submitEndOfInputStream_l: buffer submitted, header=%p cbi=%d", header, cbi); + mEndOfStreamSent = true; } } @@ -400,7 +421,8 @@ int GraphicBufferSource::findMatchingCodecBuffer_l( void GraphicBufferSource::onFrameAvailable() { Mutex::Autolock autoLock(mMutex); - ALOGV("onFrameAvailable exec=%d avail=%d", mExecuting, mNumFramesAvailable); + ALOGV("onFrameAvailable exec=%d avail=%d", + mExecuting, mNumFramesAvailable); if (mEndOfStream) { // This should only be possible if a new buffer was queued after diff --git a/media/libstagefright/omx/GraphicBufferSource.h b/media/libstagefright/omx/GraphicBufferSource.h index 6a34bc5..7f1f22e 100644 --- a/media/libstagefright/omx/GraphicBufferSource.h +++ b/media/libstagefright/omx/GraphicBufferSource.h @@ -47,7 +47,7 @@ namespace android { class GraphicBufferSource : public BufferQueue::ConsumerListener { public: GraphicBufferSource(OMXNodeInstance* nodeInstance, - uint32_t bufferWidth, uint32_t bufferHeight); + uint32_t bufferWidth, uint32_t bufferHeight, uint32_t bufferCount); virtual ~GraphicBufferSource(); // We can't throw an exception if the constructor fails, so we just set @@ -124,7 +124,9 @@ private: // in the onFrameAvailable callback, or if we're in codecBufferEmptied // and mNumFramesAvailable is nonzero). Returns without doing anything if // we don't have a codec buffer available. - status_t fillCodecBuffer_l(); + // + // Returns true if we successfully filled a codec buffer with a BQ buffer. + bool fillCodecBuffer_l(); // Marks the mCodecBuffers entry as in-use, copies the GraphicBuffer // reference into the codec buffer, and submits the data to the codec. diff --git a/media/libstagefright/omx/OMXNodeInstance.cpp b/media/libstagefright/omx/OMXNodeInstance.cpp index f3d8d14..46ff22f 100644 --- a/media/libstagefright/omx/OMXNodeInstance.cpp +++ b/media/libstagefright/omx/OMXNodeInstance.cpp @@ -590,7 +590,8 @@ status_t OMXNodeInstance::createInputSurface( } GraphicBufferSource* bufferSource = new GraphicBufferSource( - this, def.format.video.nFrameWidth, def.format.video.nFrameHeight); + this, def.format.video.nFrameWidth, def.format.video.nFrameHeight, + def.nBufferCountActual); if ((err = bufferSource->initCheck()) != OK) { delete bufferSource; return err; diff --git a/media/libstagefright/wifi-display/ANetworkSession.cpp b/media/libstagefright/wifi-display/ANetworkSession.cpp index cb6011c..23bb04e 100644 --- a/media/libstagefright/wifi-display/ANetworkSession.cpp +++ b/media/libstagefright/wifi-display/ANetworkSession.cpp @@ -27,6 +27,7 @@ #include <net/if.h> #include <netdb.h> #include <netinet/in.h> +#include <sys/ioctl.h> #include <sys/socket.h> #include <media/stagefright/foundation/ABuffer.h> @@ -103,6 +104,8 @@ private: AString mInBuffer; + int64_t mLastStallReportUs; + void notifyError(bool send, status_t err, const char *detail); void notify(NotificationReason reason); @@ -136,7 +139,8 @@ ANetworkSession::Session::Session( mSocket(s), mNotify(notify), mSawReceiveFailure(false), - mSawSendFailure(false) { + mSawSendFailure(false), + mLastStallReportUs(-1ll) { if (mState == CONNECTED) { struct sockaddr_in localAddr; socklen_t localAddrLen = sizeof(localAddr); @@ -507,6 +511,29 @@ status_t ANetworkSession::Session::writeMore() { mSawSendFailure = true; } +#if 1 + int numBytesQueued; + int res = ioctl(mSocket, SIOCOUTQ, &numBytesQueued); + if (res == 0 && numBytesQueued > 50 * 1024) { + if (numBytesQueued > 409600) { + ALOGW("!!! numBytesQueued = %d", numBytesQueued); + } + + int64_t nowUs = ALooper::GetNowUs(); + + if (mLastStallReportUs < 0ll + || nowUs > mLastStallReportUs + 500000ll) { + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("sessionID", mSessionID); + msg->setInt32("reason", kWhatNetworkStall); + msg->setSize("numBytesQueued", numBytesQueued); + msg->post(); + + mLastStallReportUs = nowUs; + } + } +#endif + return err; } diff --git a/media/libstagefright/wifi-display/ANetworkSession.h b/media/libstagefright/wifi-display/ANetworkSession.h index c1acdcc..0d7cbd6 100644 --- a/media/libstagefright/wifi-display/ANetworkSession.h +++ b/media/libstagefright/wifi-display/ANetworkSession.h @@ -83,6 +83,7 @@ struct ANetworkSession : public RefBase { kWhatData, kWhatDatagram, kWhatBinaryData, + kWhatNetworkStall, }; protected: diff --git a/media/libstagefright/wifi-display/Android.mk b/media/libstagefright/wifi-display/Android.mk index 19f560c..f81929c 100644 --- a/media/libstagefright/wifi-display/Android.mk +++ b/media/libstagefright/wifi-display/Android.mk @@ -15,6 +15,7 @@ LOCAL_SRC_FILES:= \ sink/TunnelRenderer.cpp \ sink/WifiDisplaySink.cpp \ SNTPClient.cpp \ + TimeSyncer.cpp \ source/Converter.cpp \ source/MediaPuller.cpp \ source/PlaybackSession.cpp \ diff --git a/media/libstagefright/wifi-display/MediaSender.cpp b/media/libstagefright/wifi-display/MediaSender.cpp index 105c642..e1e957a 100644 --- a/media/libstagefright/wifi-display/MediaSender.cpp +++ b/media/libstagefright/wifi-display/MediaSender.cpp @@ -325,6 +325,15 @@ void MediaSender::onSenderNotify(const sp<AMessage> &msg) { break; } + case kWhatNetworkStall: + { + size_t numBytesQueued; + CHECK(msg->findSize("numBytesQueued", &numBytesQueued)); + + notifyNetworkStall(numBytesQueued); + break; + } + default: TRESPASS(); } @@ -344,6 +353,13 @@ void MediaSender::notifyError(status_t err) { notify->post(); } +void MediaSender::notifyNetworkStall(size_t numBytesQueued) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatNetworkStall); + notify->setSize("numBytesQueued", numBytesQueued); + notify->post(); +} + status_t MediaSender::packetizeAccessUnit( size_t trackIndex, sp<ABuffer> accessUnit, diff --git a/media/libstagefright/wifi-display/MediaSender.h b/media/libstagefright/wifi-display/MediaSender.h index 9a50f9a..447abf7 100644 --- a/media/libstagefright/wifi-display/MediaSender.h +++ b/media/libstagefright/wifi-display/MediaSender.h @@ -42,6 +42,7 @@ struct MediaSender : public AHandler { enum { kWhatInitDone, kWhatError, + kWhatNetworkStall, }; MediaSender( @@ -113,6 +114,7 @@ private: void notifyInitDone(status_t err); void notifyError(status_t err); + void notifyNetworkStall(size_t numBytesQueued); status_t packetizeAccessUnit( size_t trackIndex, diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.cpp b/media/libstagefright/wifi-display/rtp/RTPSender.cpp index 85c5933..8cd712d 100644 --- a/media/libstagefright/wifi-display/rtp/RTPSender.cpp +++ b/media/libstagefright/wifi-display/rtp/RTPSender.cpp @@ -530,6 +530,18 @@ void RTPSender::onNetNotify(bool isRTP, const sp<AMessage> &msg) { } break; } + + case ANetworkSession::kWhatNetworkStall: + { + size_t numBytesQueued; + CHECK(msg->findSize("numBytesQueued", &numBytesQueued)); + + notifyNetworkStall(numBytesQueued); + break; + } + + default: + TRESPASS(); } } @@ -577,6 +589,8 @@ status_t RTPSender::onRTCPData(const sp<ABuffer> &buffer) { case 202: // SDES case 203: + break; + case 204: // APP break; @@ -697,5 +711,12 @@ void RTPSender::notifyError(status_t err) { notify->post(); } +void RTPSender::notifyNetworkStall(size_t numBytesQueued) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatNetworkStall); + notify->setSize("numBytesQueued", numBytesQueued); + notify->post(); +} + } // namespace android diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.h b/media/libstagefright/wifi-display/rtp/RTPSender.h index 2b683a4..83c6223 100644 --- a/media/libstagefright/wifi-display/rtp/RTPSender.h +++ b/media/libstagefright/wifi-display/rtp/RTPSender.h @@ -36,6 +36,7 @@ struct RTPSender : public RTPBase, public AHandler { enum { kWhatInitDone, kWhatError, + kWhatNetworkStall, }; RTPSender( const sp<ANetworkSession> &netSession, @@ -103,6 +104,7 @@ private: void notifyInitDone(status_t err); void notifyError(status_t err); + void notifyNetworkStall(size_t numBytesQueued); DISALLOW_EVIL_CONSTRUCTORS(RTPSender); }; diff --git a/media/libstagefright/wifi-display/sink/TunnelRenderer.cpp b/media/libstagefright/wifi-display/sink/TunnelRenderer.cpp index d9d8a76..6b185db 100644 --- a/media/libstagefright/wifi-display/sink/TunnelRenderer.cpp +++ b/media/libstagefright/wifi-display/sink/TunnelRenderer.cpp @@ -27,6 +27,7 @@ #include <gui/SurfaceComposerClient.h> #include <media/IMediaPlayerService.h> #include <media/IStreamSource.h> +#include <media/mediaplayer.h> #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> @@ -60,6 +61,8 @@ struct TunnelRenderer::StreamSource : public BnStreamSource { void doSomeWork(); + void setTimeOffset(int64_t offset); + protected: virtual ~StreamSource(); @@ -75,6 +78,9 @@ private: size_t mNumDeqeued; + int64_t mTimeOffsetUs; + bool mTimeOffsetChanged; + DISALLOW_EVIL_CONSTRUCTORS(StreamSource); }; @@ -82,7 +88,9 @@ private: TunnelRenderer::StreamSource::StreamSource(TunnelRenderer *owner) : mOwner(owner), - mNumDeqeued(0) { + mNumDeqeued(0), + mTimeOffsetUs(0ll), + mTimeOffsetChanged(false) { } TunnelRenderer::StreamSource::~StreamSource() { @@ -110,7 +118,7 @@ void TunnelRenderer::StreamSource::onBufferAvailable(size_t index) { } uint32_t TunnelRenderer::StreamSource::flags() const { - return kFlagAlignedVideoData; + return kFlagAlignedVideoData | kFlagIsRealTimeData; } void TunnelRenderer::StreamSource::doSomeWork() { @@ -124,21 +132,21 @@ void TunnelRenderer::StreamSource::doSomeWork() { ++mNumDeqeued; - if (mNumDeqeued == 1) { - ALOGI("fixing real time now."); - + if (mTimeOffsetChanged) { sp<AMessage> extra = new AMessage; extra->setInt32( IStreamListener::kKeyDiscontinuityMask, - ATSParser::DISCONTINUITY_ABSOLUTE_TIME); + ATSParser::DISCONTINUITY_TIME_OFFSET); - extra->setInt64("timeUs", ALooper::GetNowUs()); + extra->setInt64("offset", mTimeOffsetUs); mListener->issueCommand( IStreamListener::DISCONTINUITY, false /* synchronous */, extra); + + mTimeOffsetChanged = false; } ALOGV("dequeue TS packet of size %d", srcBuffer->size()); @@ -155,18 +163,32 @@ void TunnelRenderer::StreamSource::doSomeWork() { } } +void TunnelRenderer::StreamSource::setTimeOffset(int64_t offset) { + Mutex::Autolock autoLock(mLock); + + if (offset != mTimeOffsetUs) { + mTimeOffsetUs = offset; + mTimeOffsetChanged = true; + } +} + //////////////////////////////////////////////////////////////////////////////// TunnelRenderer::TunnelRenderer( const sp<IGraphicBufferProducer> &bufferProducer) : mSurfaceTex(bufferProducer), mStartup(true) { + mStreamSource = new StreamSource(this); } TunnelRenderer::~TunnelRenderer() { destroyPlayer(); } +void TunnelRenderer::setTimeOffset(int64_t offset) { + mStreamSource->setTimeOffset(offset); +} + void TunnelRenderer::onMessageReceived(const sp<AMessage> &msg) { switch (msg->what()) { default: @@ -209,8 +231,6 @@ void TunnelRenderer::initPlayer() { sp<IMediaPlayerService> service = interface_cast<IMediaPlayerService>(binder); CHECK(service.get() != NULL); - mStreamSource = new StreamSource(this); - mPlayerClient = new PlayerClient; mPlayer = service->create(mPlayerClient, 0); @@ -226,6 +246,8 @@ void TunnelRenderer::initPlayer() { void TunnelRenderer::destroyPlayer() { mStreamSource.clear(); + mPlayer->setVideoSurfaceTexture(NULL); + mPlayer->stop(); mPlayer.clear(); diff --git a/media/libstagefright/wifi-display/sink/TunnelRenderer.h b/media/libstagefright/wifi-display/sink/TunnelRenderer.h index 8e96665..479e73c 100644 --- a/media/libstagefright/wifi-display/sink/TunnelRenderer.h +++ b/media/libstagefright/wifi-display/sink/TunnelRenderer.h @@ -39,6 +39,12 @@ struct TunnelRenderer : public AHandler { void queueBuffer(const sp<ABuffer> &buffer); sp<ABuffer> dequeueBuffer(); + void setTimeOffset(int64_t offset); + + int64_t getAvgLatenessUs() { + return 0ll; + } + protected: virtual void onMessageReceived(const sp<AMessage> &msg); virtual ~TunnelRenderer(); diff --git a/media/libstagefright/wifi-display/source/Converter.cpp b/media/libstagefright/wifi-display/source/Converter.cpp index 2861aa9..bb8c387 100644 --- a/media/libstagefright/wifi-display/source/Converter.cpp +++ b/media/libstagefright/wifi-display/source/Converter.cpp @@ -55,6 +55,7 @@ Converter::Converter( ,mInSilentMode(false) #endif ,mPrevVideoBitrate(-1) + ,mNumFramesToDrop(0) { AString mime; CHECK(mInputFormat->findString("mime", &mime)); @@ -327,6 +328,13 @@ void Converter::onMessageReceived(const sp<AMessage> &msg) { sp<ABuffer> accessUnit; CHECK(msg->findBuffer("accessUnit", &accessUnit)); + if (mIsVideo && mNumFramesToDrop) { + --mNumFramesToDrop; + ALOGI("dropping frame."); + ReleaseMediaBufferReference(accessUnit); + break; + } + #if 0 void *mbuf; if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf) @@ -422,6 +430,12 @@ void Converter::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatDropAFrame: + { + ++mNumFramesToDrop; + break; + } + default: TRESPASS(); } @@ -690,4 +704,8 @@ void Converter::requestIDRFrame() { (new AMessage(kWhatRequestIDRFrame, id()))->post(); } +void Converter::dropAFrame() { + (new AMessage(kWhatDropAFrame, id()))->post(); +} + } // namespace android diff --git a/media/libstagefright/wifi-display/source/Converter.h b/media/libstagefright/wifi-display/source/Converter.h index 57802bd..a418f69 100644 --- a/media/libstagefright/wifi-display/source/Converter.h +++ b/media/libstagefright/wifi-display/source/Converter.h @@ -51,6 +51,8 @@ struct Converter : public AHandler { void requestIDRFrame(); + void dropAFrame(); + enum { kWhatAccessUnit, kWhatEOS, @@ -63,6 +65,7 @@ struct Converter : public AHandler { kWhatShutdown, kWhatMediaPullerNotify, kWhatEncoderActivity, + kWhatDropAFrame, }; void shutdownAsync(); @@ -102,6 +105,8 @@ private: int32_t mPrevVideoBitrate; + int32_t mNumFramesToDrop; + status_t initEncoder(); void releaseEncoder(); diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.cpp b/media/libstagefright/wifi-display/source/PlaybackSession.cpp index ea195b3..94cb2a4 100644 --- a/media/libstagefright/wifi-display/source/PlaybackSession.cpp +++ b/media/libstagefright/wifi-display/source/PlaybackSession.cpp @@ -515,6 +515,16 @@ void WifiDisplaySource::PlaybackSession::onMessageReceived( } } else if (what == MediaSender::kWhatError) { notifySessionDead(); + } else if (what == MediaSender::kWhatNetworkStall) { + size_t numBytesQueued; + CHECK(msg->findSize("numBytesQueued", &numBytesQueued)); + + if (mVideoTrackIndex >= 0) { + const sp<Track> &videoTrack = + mTracks.valueFor(mVideoTrackIndex); + + videoTrack->converter()->dropAFrame(); + } } else { TRESPASS(); } diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp index b8524f6..c8798c6 100644 --- a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp @@ -23,6 +23,7 @@ #include "Parameters.h" #include "ParsedMessage.h" #include "rtp/RTPSender.h" +#include "TimeSyncer.h" #include <binder/IServiceManager.h> #include <gui/IGraphicBufferProducer.h> @@ -157,6 +158,12 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { } if (err == OK) { + sp<AMessage> notify = new AMessage(kWhatTimeSyncerNotify, id()); + mTimeSyncer = new TimeSyncer(mNetSession, notify); + looper()->registerHandler(mTimeSyncer); + + mTimeSyncer->startServer(8123); + mState = AWAITING_CLIENT_CONNECTION; } @@ -265,6 +272,11 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { break; } + case ANetworkSession::kWhatNetworkStall: + { + break; + } + default: TRESPASS(); } @@ -520,6 +532,11 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatTimeSyncerNotify: + { + break; + } + default: TRESPASS(); } diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.h b/media/libstagefright/wifi-display/source/WifiDisplaySource.h index 724462c..9e72682 100644 --- a/media/libstagefright/wifi-display/source/WifiDisplaySource.h +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.h @@ -30,6 +30,7 @@ namespace android { struct IHDCP; struct IRemoteDisplayClient; struct ParsedMessage; +struct TimeSyncer; // Represents the RTSP server acting as a wifi display source. // Manages incoming connections, sets up Playback sessions as necessary. @@ -81,6 +82,7 @@ private: kWhatHDCPNotify, kWhatFinishStop2, kWhatTeardownTriggerTimedOut, + kWhatTimeSyncerNotify, }; struct ResponseID { @@ -114,6 +116,7 @@ private: VideoFormats mSupportedSourceVideoFormats; sp<ANetworkSession> mNetSession; sp<IRemoteDisplayClient> mClient; + sp<TimeSyncer> mTimeSyncer; struct in_addr mInterfaceAddr; int32_t mSessionID; diff --git a/media/libstagefright/wifi-display/wfd.cpp b/media/libstagefright/wifi-display/wfd.cpp index 3f4216a..0b18484 100644 --- a/media/libstagefright/wifi-display/wfd.cpp +++ b/media/libstagefright/wifi-display/wfd.cpp @@ -321,7 +321,10 @@ int main(int argc, char **argv) { sp<ALooper> looper = new ALooper; sp<WifiDisplaySink> sink = new WifiDisplaySink( - session, surface->getIGraphicBufferProducer()); + 0 /* flags */, + session, + surface->getIGraphicBufferProducer()); + looper->registerHandler(sink); if (connectToPort >= 0) { |