diff options
Diffstat (limited to 'media/libstagefright')
35 files changed, 1415 insertions, 522 deletions
diff --git a/media/libstagefright/CameraSource.cpp b/media/libstagefright/CameraSource.cpp index f8557d0..5a26b06 100644 --- a/media/libstagefright/CameraSource.cpp +++ b/media/libstagefright/CameraSource.cpp @@ -140,7 +140,7 @@ CameraSource *CameraSource::CreateFromCamera( uid_t clientUid, Size videoSize, int32_t frameRate, - const sp<Surface>& surface, + const sp<IGraphicBufferProducer>& surface, bool storeMetaDataInVideoBuffers) { CameraSource *source = new CameraSource(camera, proxy, cameraId, @@ -157,7 +157,7 @@ CameraSource::CameraSource( uid_t clientUid, Size videoSize, int32_t frameRate, - const sp<Surface>& surface, + const sp<IGraphicBufferProducer>& surface, bool storeMetaDataInVideoBuffers) : mCameraFlags(0), mNumInputBuffers(0), @@ -536,7 +536,7 @@ status_t CameraSource::initWithCameraAccess( if (mSurface != NULL) { // This CHECK is good, since we just passed the lock/unlock // check earlier by calling mCamera->setParameters(). - CHECK_EQ((status_t)OK, mCamera->setPreviewDisplay(mSurface)); + CHECK_EQ((status_t)OK, mCamera->setPreviewTexture(mSurface)); } // By default, do not store metadata in video buffers diff --git a/media/libstagefright/CameraSourceTimeLapse.cpp b/media/libstagefright/CameraSourceTimeLapse.cpp index 2ed2223..20214e8 100644 --- a/media/libstagefright/CameraSourceTimeLapse.cpp +++ b/media/libstagefright/CameraSourceTimeLapse.cpp @@ -40,7 +40,7 @@ CameraSourceTimeLapse *CameraSourceTimeLapse::CreateFromCamera( uid_t clientUid, Size videoSize, int32_t videoFrameRate, - const sp<Surface>& surface, + const sp<IGraphicBufferProducer>& surface, int64_t timeBetweenFrameCaptureUs) { CameraSourceTimeLapse *source = new @@ -66,7 +66,7 @@ CameraSourceTimeLapse::CameraSourceTimeLapse( uid_t clientUid, Size videoSize, int32_t videoFrameRate, - const sp<Surface>& surface, + const sp<IGraphicBufferProducer>& surface, int64_t timeBetweenFrameCaptureUs) : CameraSource(camera, proxy, cameraId, clientName, clientUid, videoSize, videoFrameRate, surface, true), diff --git a/media/libstagefright/MPEG4Extractor.cpp b/media/libstagefright/MPEG4Extractor.cpp index b2e60be..56fad60 100644 --- a/media/libstagefright/MPEG4Extractor.cpp +++ b/media/libstagefright/MPEG4Extractor.cpp @@ -2067,17 +2067,30 @@ status_t MPEG4Extractor::updateAudioTrackInfoFromESDS_MPEG4Audio( sampleRate = br.getBits(24); numChannels = br.getBits(4); } else { - static uint32_t kSamplingRate[] = { - 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, - 16000, 12000, 11025, 8000, 7350 - }; - - if (freqIndex == 13 || freqIndex == 14) { - return ERROR_MALFORMED; + numChannels = br.getBits(4); + if (objectType == 5) { + // SBR specific config per 14496-3 table 1.13 + freqIndex = br.getBits(4); + if (freqIndex == 15) { + if (csd_size < 8) { + return ERROR_MALFORMED; + } + sampleRate = br.getBits(24); + } } - sampleRate = kSamplingRate[freqIndex]; - numChannels = br.getBits(4); + if (sampleRate == 0) { + static uint32_t kSamplingRate[] = { + 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, + 16000, 12000, 11025, 8000, 7350 + }; + + if (freqIndex == 13 || freqIndex == 14) { + return ERROR_MALFORMED; + } + + sampleRate = kSamplingRate[freqIndex]; + } } if (numChannels == 0) { diff --git a/media/libstagefright/mpeg2ts/ATSParser.cpp b/media/libstagefright/mpeg2ts/ATSParser.cpp index a167b5a..c12572f 100644 --- a/media/libstagefright/mpeg2ts/ATSParser.cpp +++ b/media/libstagefright/mpeg2ts/ATSParser.cpp @@ -452,6 +452,10 @@ int64_t ATSParser::Program::convertPTSToTimestamp(uint64_t PTS) { timeUs += mParser->mAbsoluteTimeAnchorUs; } + if (mParser->mTimeOffsetValid) { + timeUs += mParser->mTimeOffsetUs; + } + return timeUs; } @@ -930,6 +934,8 @@ sp<MediaSource> ATSParser::Stream::getSource(SourceType type) { ATSParser::ATSParser(uint32_t flags) : mFlags(flags), mAbsoluteTimeAnchorUs(-1ll), + mTimeOffsetValid(false), + mTimeOffsetUs(0ll), mNumTSPacketsParsed(0), mNumPCRs(0) { mPSISections.add(0 /* PID */, new PSISection); @@ -960,6 +966,13 @@ void ATSParser::signalDiscontinuity( CHECK(mPrograms.empty()); mAbsoluteTimeAnchorUs = timeUs; return; + } else if (type == DISCONTINUITY_TIME_OFFSET) { + int64_t offset; + CHECK(extra->findInt64("offset", &offset)); + + mTimeOffsetValid = true; + mTimeOffsetUs = offset; + return; } for (size_t i = 0; i < mPrograms.size(); ++i) { diff --git a/media/libstagefright/mpeg2ts/ATSParser.h b/media/libstagefright/mpeg2ts/ATSParser.h index 46edc45..a10edc9 100644 --- a/media/libstagefright/mpeg2ts/ATSParser.h +++ b/media/libstagefright/mpeg2ts/ATSParser.h @@ -39,6 +39,7 @@ struct ATSParser : public RefBase { DISCONTINUITY_AUDIO_FORMAT = 2, DISCONTINUITY_VIDEO_FORMAT = 4, DISCONTINUITY_ABSOLUTE_TIME = 8, + DISCONTINUITY_TIME_OFFSET = 16, DISCONTINUITY_SEEK = DISCONTINUITY_TIME, @@ -106,6 +107,9 @@ private: int64_t mAbsoluteTimeAnchorUs; + bool mTimeOffsetValid; + int64_t mTimeOffsetUs; + size_t mNumTSPacketsParsed; void parseProgramAssociationTable(ABitReader *br); diff --git a/media/libstagefright/omx/GraphicBufferSource.cpp b/media/libstagefright/omx/GraphicBufferSource.cpp index 211e1d1..3854e52 100644 --- a/media/libstagefright/omx/GraphicBufferSource.cpp +++ b/media/libstagefright/omx/GraphicBufferSource.cpp @@ -32,7 +32,7 @@ static const bool EXTRA_CHECK = true; GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance, - uint32_t bufferWidth, uint32_t bufferHeight) : + uint32_t bufferWidth, uint32_t bufferHeight, uint32_t bufferCount) : mInitCheck(UNKNOWN_ERROR), mNodeInstance(nodeInstance), mExecuting(false), @@ -40,20 +40,31 @@ GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance, mEndOfStream(false), mEndOfStreamSent(false) { - ALOGV("GraphicBufferSource w=%u h=%u", bufferWidth, bufferHeight); + ALOGV("GraphicBufferSource w=%u h=%u c=%u", + bufferWidth, bufferHeight, bufferCount); if (bufferWidth == 0 || bufferHeight == 0) { - ALOGE("Invalid dimensions %dx%d", bufferWidth, bufferHeight); + ALOGE("Invalid dimensions %ux%u", bufferWidth, bufferHeight); mInitCheck = BAD_VALUE; return; } + String8 name("GraphicBufferSource"); + mBufferQueue = new BufferQueue(true); + mBufferQueue->setConsumerName(name); mBufferQueue->setDefaultBufferSize(bufferWidth, bufferHeight); mBufferQueue->setSynchronousMode(true); mBufferQueue->setConsumerUsageBits(GRALLOC_USAGE_HW_VIDEO_ENCODER | GRALLOC_USAGE_HW_TEXTURE); + mInitCheck = mBufferQueue->setMaxAcquiredBufferCount(bufferCount); + if (mInitCheck != NO_ERROR) { + ALOGE("Unable to set BQ max acquired buffer count to %u: %d", + bufferCount, mInitCheck); + return; + } + // Note that we can't create an sp<...>(this) in a ctor that will not keep a // reference once the ctor ends, as that would cause the refcount of 'this' // dropping to 0 at the end of the ctor. Since all we need is a wp<...> @@ -64,21 +75,23 @@ GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance, sp<BufferQueue::ConsumerListener> proxy; proxy = new BufferQueue::ProxyConsumerListener(listener); - status_t err = mBufferQueue->consumerConnect(proxy); - if (err != NO_ERROR) { + mInitCheck = mBufferQueue->consumerConnect(proxy); + if (mInitCheck != NO_ERROR) { ALOGE("Error connecting to BufferQueue: %s (%d)", - strerror(-err), err); + strerror(-mInitCheck), mInitCheck); return; } - mInitCheck = OK; + CHECK(mInitCheck == NO_ERROR); } GraphicBufferSource::~GraphicBufferSource() { ALOGV("~GraphicBufferSource"); - status_t err = mBufferQueue->consumerDisconnect(); - if (err != NO_ERROR) { - ALOGW("consumerDisconnect failed: %d", err); + if (mBufferQueue != NULL) { + status_t err = mBufferQueue->consumerDisconnect(); + if (err != NO_ERROR) { + ALOGW("consumerDisconnect failed: %d", err); + } } } @@ -98,8 +111,12 @@ void GraphicBufferSource::omxExecuting() { // one codec buffer simultaneously. (We could instead try to submit // all BQ buffers whenever any codec buffer is freed, but if we get the // initial conditions right that will never be useful.) - while (mNumFramesAvailable && isCodecBufferAvailable_l()) { - fillCodecBuffer_l(); + while (mNumFramesAvailable) { + if (!fillCodecBuffer_l()) { + ALOGV("stop load with frames available (codecAvail=%d)", + isCodecBufferAvailable_l()); + break; + } } ALOGV("done loading initial frames, avail=%d", mNumFramesAvailable); @@ -166,7 +183,7 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) { // see if the GraphicBuffer reference was null, which should only ever // happen for EOS. if (codecBuffer.mGraphicBuffer == NULL) { - CHECK(mEndOfStream); + CHECK(mEndOfStream && mEndOfStreamSent); // No GraphicBuffer to deal with, no additional input or output is // expected, so just return. return; @@ -216,8 +233,9 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) { if (mNumFramesAvailable) { // Fill this codec buffer. - CHECK(!mEndOfStream); - ALOGV("buffer freed, %d frames avail", mNumFramesAvailable); + CHECK(!mEndOfStreamSent); + ALOGV("buffer freed, %d frames avail (eos=%d)", + mNumFramesAvailable, mEndOfStream); fillCodecBuffer_l(); } else if (mEndOfStream) { // No frames available, but EOS is pending, so use this buffer to @@ -228,56 +246,58 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) { return; } -status_t GraphicBufferSource::fillCodecBuffer_l() { +bool GraphicBufferSource::fillCodecBuffer_l() { CHECK(mExecuting && mNumFramesAvailable > 0); + int cbi = findAvailableCodecBuffer_l(); if (cbi < 0) { // No buffers available, bail. ALOGV("fillCodecBuffer_l: no codec buffers, avail now %d", mNumFramesAvailable); - } else { - ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%d", - mNumFramesAvailable); - BufferQueue::BufferItem item; - status_t err = mBufferQueue->acquireBuffer(&item); - if (err == BufferQueue::NO_BUFFER_AVAILABLE) { - // shouldn't happen - ALOGW("fillCodecBuffer_l: frame was not available"); - return err; - } else if (err != OK) { - // now what? fake end-of-stream? - ALOGW("fillCodecBuffer_l: acquireBuffer returned err=%d", err); - return err; - } + return false; + } - mNumFramesAvailable--; + ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%d", + mNumFramesAvailable); + BufferQueue::BufferItem item; + status_t err = mBufferQueue->acquireBuffer(&item); + if (err == BufferQueue::NO_BUFFER_AVAILABLE) { + // shouldn't happen + ALOGW("fillCodecBuffer_l: frame was not available"); + return false; + } else if (err != OK) { + // now what? fake end-of-stream? + ALOGW("fillCodecBuffer_l: acquireBuffer returned err=%d", err); + return false; + } - // Wait for it to become available. - err = item.mFence->waitForever(1000, - "GraphicBufferSource::fillCodecBuffer_l"); - if (err != OK) { - ALOGW("failed to wait for buffer fence: %d", err); - // keep going - } + mNumFramesAvailable--; - // If this is the first time we're seeing this buffer, add it to our - // slot table. - if (item.mGraphicBuffer != NULL) { - ALOGV("fillCodecBuffer_l: setting mBufferSlot %d", item.mBuf); - mBufferSlot[item.mBuf] = item.mGraphicBuffer; - } + // Wait for it to become available. + err = item.mFence->waitForever(1000, + "GraphicBufferSource::fillCodecBuffer_l"); + if (err != OK) { + ALOGW("failed to wait for buffer fence: %d", err); + // keep going + } - err = submitBuffer_l(mBufferSlot[item.mBuf], item.mTimestamp, cbi); - if (err != OK) { - ALOGV("submitBuffer_l failed, releasing bq buf %d", item.mBuf); - mBufferQueue->releaseBuffer(item.mBuf, EGL_NO_DISPLAY, + // If this is the first time we're seeing this buffer, add it to our + // slot table. + if (item.mGraphicBuffer != NULL) { + ALOGV("fillCodecBuffer_l: setting mBufferSlot %d", item.mBuf); + mBufferSlot[item.mBuf] = item.mGraphicBuffer; + } + + err = submitBuffer_l(mBufferSlot[item.mBuf], item.mTimestamp, cbi); + if (err != OK) { + ALOGV("submitBuffer_l failed, releasing bq buf %d", item.mBuf); + mBufferQueue->releaseBuffer(item.mBuf, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE); - } else { - ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi); - } + } else { + ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi); } - return OK; + return true; } status_t GraphicBufferSource::signalEndOfInputStream() { @@ -372,6 +392,7 @@ void GraphicBufferSource::submitEndOfInputStream_l() { } else { ALOGV("submitEndOfInputStream_l: buffer submitted, header=%p cbi=%d", header, cbi); + mEndOfStreamSent = true; } } @@ -400,7 +421,8 @@ int GraphicBufferSource::findMatchingCodecBuffer_l( void GraphicBufferSource::onFrameAvailable() { Mutex::Autolock autoLock(mMutex); - ALOGV("onFrameAvailable exec=%d avail=%d", mExecuting, mNumFramesAvailable); + ALOGV("onFrameAvailable exec=%d avail=%d", + mExecuting, mNumFramesAvailable); if (mEndOfStream) { // This should only be possible if a new buffer was queued after diff --git a/media/libstagefright/omx/GraphicBufferSource.h b/media/libstagefright/omx/GraphicBufferSource.h index 6a34bc5..7f1f22e 100644 --- a/media/libstagefright/omx/GraphicBufferSource.h +++ b/media/libstagefright/omx/GraphicBufferSource.h @@ -47,7 +47,7 @@ namespace android { class GraphicBufferSource : public BufferQueue::ConsumerListener { public: GraphicBufferSource(OMXNodeInstance* nodeInstance, - uint32_t bufferWidth, uint32_t bufferHeight); + uint32_t bufferWidth, uint32_t bufferHeight, uint32_t bufferCount); virtual ~GraphicBufferSource(); // We can't throw an exception if the constructor fails, so we just set @@ -124,7 +124,9 @@ private: // in the onFrameAvailable callback, or if we're in codecBufferEmptied // and mNumFramesAvailable is nonzero). Returns without doing anything if // we don't have a codec buffer available. - status_t fillCodecBuffer_l(); + // + // Returns true if we successfully filled a codec buffer with a BQ buffer. + bool fillCodecBuffer_l(); // Marks the mCodecBuffers entry as in-use, copies the GraphicBuffer // reference into the codec buffer, and submits the data to the codec. diff --git a/media/libstagefright/omx/OMXNodeInstance.cpp b/media/libstagefright/omx/OMXNodeInstance.cpp index f3d8d14..46ff22f 100644 --- a/media/libstagefright/omx/OMXNodeInstance.cpp +++ b/media/libstagefright/omx/OMXNodeInstance.cpp @@ -590,7 +590,8 @@ status_t OMXNodeInstance::createInputSurface( } GraphicBufferSource* bufferSource = new GraphicBufferSource( - this, def.format.video.nFrameWidth, def.format.video.nFrameHeight); + this, def.format.video.nFrameWidth, def.format.video.nFrameHeight, + def.nBufferCountActual); if ((err = bufferSource->initCheck()) != OK) { delete bufferSource; return err; diff --git a/media/libstagefright/wifi-display/ANetworkSession.cpp b/media/libstagefright/wifi-display/ANetworkSession.cpp index cb6011c..23bb04e 100644 --- a/media/libstagefright/wifi-display/ANetworkSession.cpp +++ b/media/libstagefright/wifi-display/ANetworkSession.cpp @@ -27,6 +27,7 @@ #include <net/if.h> #include <netdb.h> #include <netinet/in.h> +#include <sys/ioctl.h> #include <sys/socket.h> #include <media/stagefright/foundation/ABuffer.h> @@ -103,6 +104,8 @@ private: AString mInBuffer; + int64_t mLastStallReportUs; + void notifyError(bool send, status_t err, const char *detail); void notify(NotificationReason reason); @@ -136,7 +139,8 @@ ANetworkSession::Session::Session( mSocket(s), mNotify(notify), mSawReceiveFailure(false), - mSawSendFailure(false) { + mSawSendFailure(false), + mLastStallReportUs(-1ll) { if (mState == CONNECTED) { struct sockaddr_in localAddr; socklen_t localAddrLen = sizeof(localAddr); @@ -507,6 +511,29 @@ status_t ANetworkSession::Session::writeMore() { mSawSendFailure = true; } +#if 1 + int numBytesQueued; + int res = ioctl(mSocket, SIOCOUTQ, &numBytesQueued); + if (res == 0 && numBytesQueued > 50 * 1024) { + if (numBytesQueued > 409600) { + ALOGW("!!! numBytesQueued = %d", numBytesQueued); + } + + int64_t nowUs = ALooper::GetNowUs(); + + if (mLastStallReportUs < 0ll + || nowUs > mLastStallReportUs + 500000ll) { + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("sessionID", mSessionID); + msg->setInt32("reason", kWhatNetworkStall); + msg->setSize("numBytesQueued", numBytesQueued); + msg->post(); + + mLastStallReportUs = nowUs; + } + } +#endif + return err; } diff --git a/media/libstagefright/wifi-display/ANetworkSession.h b/media/libstagefright/wifi-display/ANetworkSession.h index c1acdcc..0d7cbd6 100644 --- a/media/libstagefright/wifi-display/ANetworkSession.h +++ b/media/libstagefright/wifi-display/ANetworkSession.h @@ -83,6 +83,7 @@ struct ANetworkSession : public RefBase { kWhatData, kWhatDatagram, kWhatBinaryData, + kWhatNetworkStall, }; protected: diff --git a/media/libstagefright/wifi-display/Android.mk b/media/libstagefright/wifi-display/Android.mk index 19f560c..f81929c 100644 --- a/media/libstagefright/wifi-display/Android.mk +++ b/media/libstagefright/wifi-display/Android.mk @@ -15,6 +15,7 @@ LOCAL_SRC_FILES:= \ sink/TunnelRenderer.cpp \ sink/WifiDisplaySink.cpp \ SNTPClient.cpp \ + TimeSyncer.cpp \ source/Converter.cpp \ source/MediaPuller.cpp \ source/PlaybackSession.cpp \ diff --git a/media/libstagefright/wifi-display/MediaReceiver.cpp b/media/libstagefright/wifi-display/MediaReceiver.cpp index 3c92d41..10a2dff 100644 --- a/media/libstagefright/wifi-display/MediaReceiver.cpp +++ b/media/libstagefright/wifi-display/MediaReceiver.cpp @@ -127,7 +127,10 @@ void MediaReceiver::onMessageReceived(const sp<AMessage> &msg) { notifyInitDone(mInitStatus); } - mTSParser = new ATSParser(ATSParser::ALIGNED_VIDEO_DATA); + mTSParser = new ATSParser( + ATSParser::ALIGNED_VIDEO_DATA + | ATSParser::TS_TIMESTAMPS_ARE_ABSOLUTE); + mFormatKnownMask = 0; break; } @@ -306,6 +309,15 @@ void MediaReceiver::postAccessUnit( notify->post(); } +status_t MediaReceiver::notifyLateness(size_t trackIndex, int64_t latenessUs) { + if (trackIndex >= mTrackInfos.size()) { + return -ERANGE; + } + + TrackInfo *info = &mTrackInfos.editItemAt(trackIndex); + return info->mReceiver->notifyLateness(latenessUs); +} + } // namespace android diff --git a/media/libstagefright/wifi-display/MediaReceiver.h b/media/libstagefright/wifi-display/MediaReceiver.h index 7adc3c4..cdfde99 100644 --- a/media/libstagefright/wifi-display/MediaReceiver.h +++ b/media/libstagefright/wifi-display/MediaReceiver.h @@ -60,6 +60,8 @@ struct MediaReceiver : public AHandler { }; status_t initAsync(Mode mode); + status_t notifyLateness(size_t trackIndex, int64_t latenessUs); + protected: virtual void onMessageReceived(const sp<AMessage> &msg); virtual ~MediaReceiver(); diff --git a/media/libstagefright/wifi-display/MediaSender.cpp b/media/libstagefright/wifi-display/MediaSender.cpp index 105c642..e1e957a 100644 --- a/media/libstagefright/wifi-display/MediaSender.cpp +++ b/media/libstagefright/wifi-display/MediaSender.cpp @@ -325,6 +325,15 @@ void MediaSender::onSenderNotify(const sp<AMessage> &msg) { break; } + case kWhatNetworkStall: + { + size_t numBytesQueued; + CHECK(msg->findSize("numBytesQueued", &numBytesQueued)); + + notifyNetworkStall(numBytesQueued); + break; + } + default: TRESPASS(); } @@ -344,6 +353,13 @@ void MediaSender::notifyError(status_t err) { notify->post(); } +void MediaSender::notifyNetworkStall(size_t numBytesQueued) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatNetworkStall); + notify->setSize("numBytesQueued", numBytesQueued); + notify->post(); +} + status_t MediaSender::packetizeAccessUnit( size_t trackIndex, sp<ABuffer> accessUnit, diff --git a/media/libstagefright/wifi-display/MediaSender.h b/media/libstagefright/wifi-display/MediaSender.h index 9a50f9a..447abf7 100644 --- a/media/libstagefright/wifi-display/MediaSender.h +++ b/media/libstagefright/wifi-display/MediaSender.h @@ -42,6 +42,7 @@ struct MediaSender : public AHandler { enum { kWhatInitDone, kWhatError, + kWhatNetworkStall, }; MediaSender( @@ -113,6 +114,7 @@ private: void notifyInitDone(status_t err); void notifyError(status_t err); + void notifyNetworkStall(size_t numBytesQueued); status_t packetizeAccessUnit( size_t trackIndex, diff --git a/media/libstagefright/wifi-display/TimeSyncer.cpp b/media/libstagefright/wifi-display/TimeSyncer.cpp new file mode 100644 index 0000000..64e182e --- /dev/null +++ b/media/libstagefright/wifi-display/TimeSyncer.cpp @@ -0,0 +1,332 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NEBUG 0 +#define LOG_TAG "TimeSyncer" +#include <utils/Log.h> + +#include "TimeSyncer.h" + +#include "ANetworkSession.h" + +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AHandler.h> +#include <media/stagefright/foundation/ALooper.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/Utils.h> + +namespace android { + +TimeSyncer::TimeSyncer( + const sp<ANetworkSession> &netSession, const sp<AMessage> ¬ify) + : mNetSession(netSession), + mNotify(notify), + mIsServer(false), + mConnected(false), + mUDPSession(0), + mSeqNo(0), + mTotalTimeUs(0.0), + mPendingT1(0ll), + mTimeoutGeneration(0) { +} + +TimeSyncer::~TimeSyncer() { +} + +void TimeSyncer::startServer(unsigned localPort) { + sp<AMessage> msg = new AMessage(kWhatStartServer, id()); + msg->setInt32("localPort", localPort); + msg->post(); +} + +void TimeSyncer::startClient(const char *remoteHost, unsigned remotePort) { + sp<AMessage> msg = new AMessage(kWhatStartClient, id()); + msg->setString("remoteHost", remoteHost); + msg->setInt32("remotePort", remotePort); + msg->post(); +} + +void TimeSyncer::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatStartClient: + { + AString remoteHost; + CHECK(msg->findString("remoteHost", &remoteHost)); + + int32_t remotePort; + CHECK(msg->findInt32("remotePort", &remotePort)); + + sp<AMessage> notify = new AMessage(kWhatUDPNotify, id()); + + CHECK_EQ((status_t)OK, + mNetSession->createUDPSession( + 0 /* localPort */, + remoteHost.c_str(), + remotePort, + notify, + &mUDPSession)); + + postSendPacket(); + break; + } + + case kWhatStartServer: + { + mIsServer = true; + + int32_t localPort; + CHECK(msg->findInt32("localPort", &localPort)); + + sp<AMessage> notify = new AMessage(kWhatUDPNotify, id()); + + CHECK_EQ((status_t)OK, + mNetSession->createUDPSession( + localPort, notify, &mUDPSession)); + + break; + } + + case kWhatSendPacket: + { + TimeInfo ti; + memset(&ti, 0, sizeof(ti)); + + ti.mT1 = ALooper::GetNowUs(); + + CHECK_EQ((status_t)OK, + mNetSession->sendRequest( + mUDPSession, &ti, sizeof(ti))); + + mPendingT1 = ti.mT1; + postTimeout(); + break; + } + + case kWhatTimedOut: + { + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + + if (generation != mTimeoutGeneration) { + break; + } + + ALOGI("timed out, sending another request"); + postSendPacket(); + break; + } + + case kWhatUDPNotify: + { + int32_t reason; + CHECK(msg->findInt32("reason", &reason)); + + switch (reason) { + case ANetworkSession::kWhatError: + { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + int32_t err; + CHECK(msg->findInt32("err", &err)); + + AString detail; + CHECK(msg->findString("detail", &detail)); + + ALOGE("An error occurred in session %d (%d, '%s/%s').", + sessionID, + err, + detail.c_str(), + strerror(-err)); + + mNetSession->destroySession(sessionID); + + cancelTimeout(); + + notifyError(err); + break; + } + + case ANetworkSession::kWhatDatagram: + { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + sp<ABuffer> packet; + CHECK(msg->findBuffer("data", &packet)); + + int64_t arrivalTimeUs; + CHECK(packet->meta()->findInt64( + "arrivalTimeUs", &arrivalTimeUs)); + + CHECK_EQ(packet->size(), sizeof(TimeInfo)); + + TimeInfo *ti = (TimeInfo *)packet->data(); + + if (mIsServer) { + if (!mConnected) { + AString fromAddr; + CHECK(msg->findString("fromAddr", &fromAddr)); + + int32_t fromPort; + CHECK(msg->findInt32("fromPort", &fromPort)); + + CHECK_EQ((status_t)OK, + mNetSession->connectUDPSession( + mUDPSession, fromAddr.c_str(), fromPort)); + + mConnected = true; + } + + ti->mT2 = arrivalTimeUs; + ti->mT3 = ALooper::GetNowUs(); + + CHECK_EQ((status_t)OK, + mNetSession->sendRequest( + mUDPSession, ti, sizeof(*ti))); + } else { + if (ti->mT1 != mPendingT1) { + break; + } + + cancelTimeout(); + mPendingT1 = 0; + + ti->mT4 = arrivalTimeUs; + + // One way delay for a packet to travel from client + // to server or back (assumed to be the same either way). + int64_t delay = + (ti->mT2 - ti->mT1 + ti->mT4 - ti->mT3) / 2; + + // Offset between the client clock (T1, T4) and the + // server clock (T2, T3) timestamps. + int64_t offset = + (ti->mT2 - ti->mT1 - ti->mT4 + ti->mT3) / 2; + + mHistory.push_back(*ti); + + ALOGV("delay = %lld us,\toffset %lld us", + delay, + offset); + + if (mHistory.size() < kNumPacketsPerBatch) { + postSendPacket(1000000ll / 30); + } else { + notifyOffset(); + + mHistory.clear(); + postSendPacket(kBatchDelayUs); + } + } + break; + } + + default: + TRESPASS(); + } + + break; + } + + default: + TRESPASS(); + } +} + +void TimeSyncer::postSendPacket(int64_t delayUs) { + (new AMessage(kWhatSendPacket, id()))->post(delayUs); +} + +void TimeSyncer::postTimeout() { + sp<AMessage> msg = new AMessage(kWhatTimedOut, id()); + msg->setInt32("generation", mTimeoutGeneration); + msg->post(kTimeoutDelayUs); +} + +void TimeSyncer::cancelTimeout() { + ++mTimeoutGeneration; +} + +void TimeSyncer::notifyError(status_t err) { + if (mNotify == NULL) { + looper()->stop(); + return; + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatError); + notify->setInt32("err", err); + notify->post(); +} + +// static +int TimeSyncer::CompareRountripTime(const TimeInfo *ti1, const TimeInfo *ti2) { + int64_t rt1 = ti1->mT4 - ti1->mT1; + int64_t rt2 = ti2->mT4 - ti2->mT1; + + if (rt1 < rt2) { + return -1; + } else if (rt1 > rt2) { + return 1; + } + + return 0; +} + +void TimeSyncer::notifyOffset() { + mHistory.sort(CompareRountripTime); + + int64_t sum = 0ll; + size_t count = 0; + + // Only consider the third of the information associated with the best + // (smallest) roundtrip times. + for (size_t i = 0; i < mHistory.size() / 3; ++i) { + const TimeInfo *ti = &mHistory[i]; + +#if 0 + // One way delay for a packet to travel from client + // to server or back (assumed to be the same either way). + int64_t delay = + (ti->mT2 - ti->mT1 + ti->mT4 - ti->mT3) / 2; +#endif + + // Offset between the client clock (T1, T4) and the + // server clock (T2, T3) timestamps. + int64_t offset = + (ti->mT2 - ti->mT1 - ti->mT4 + ti->mT3) / 2; + + ALOGV("(%d) RT: %lld us, offset: %lld us", + i, ti->mT4 - ti->mT1, offset); + + sum += offset; + ++count; + } + + if (mNotify == NULL) { + ALOGI("avg. offset is %lld", sum / count); + return; + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatTimeOffset); + notify->setInt64("offset", sum / count); + notify->post(); +} + +} // namespace android diff --git a/media/libstagefright/wifi-display/TimeSyncer.h b/media/libstagefright/wifi-display/TimeSyncer.h new file mode 100644 index 0000000..0e3aed7 --- /dev/null +++ b/media/libstagefright/wifi-display/TimeSyncer.h @@ -0,0 +1,109 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TIME_SYNCER_H_ + +#define TIME_SYNCER_H_ + +#include <media/stagefright/foundation/AHandler.h> + +namespace android { + +struct ANetworkSession; + +/* + TimeSyncer allows us to synchronize time between a client and a server. + The client sends a UDP packet containing its send-time to the server, + the server sends that packet back to the client amended with information + about when it was received as well as the time the reply was sent back. + Finally the client receives the reply and has now enough information to + compute the clock offset between client and server assuming that packet + exchange is symmetric, i.e. time for a packet client->server and + server->client is roughly equal. + This exchange is repeated a number of times and the average offset computed + over the 30% of packets that had the lowest roundtrip times. + The offset is determined every 10 secs to account for slight differences in + clock frequency. +*/ +struct TimeSyncer : public AHandler { + enum { + kWhatError, + kWhatTimeOffset, + }; + TimeSyncer( + const sp<ANetworkSession> &netSession, + const sp<AMessage> ¬ify); + + void startServer(unsigned localPort); + void startClient(const char *remoteHost, unsigned remotePort); + +protected: + virtual ~TimeSyncer(); + + virtual void onMessageReceived(const sp<AMessage> &msg); + +private: + enum { + kWhatStartServer, + kWhatStartClient, + kWhatUDPNotify, + kWhatSendPacket, + kWhatTimedOut, + }; + + struct TimeInfo { + int64_t mT1; // client timestamp at send + int64_t mT2; // server timestamp at receive + int64_t mT3; // server timestamp at send + int64_t mT4; // client timestamp at receive + }; + + enum { + kNumPacketsPerBatch = 30, + }; + static const int64_t kTimeoutDelayUs = 500000ll; + static const int64_t kBatchDelayUs = 10000000ll; // every 10 secs + + sp<ANetworkSession> mNetSession; + sp<AMessage> mNotify; + + bool mIsServer; + bool mConnected; + int32_t mUDPSession; + uint32_t mSeqNo; + double mTotalTimeUs; + + Vector<TimeInfo> mHistory; + + int64_t mPendingT1; + int32_t mTimeoutGeneration; + + void postSendPacket(int64_t delayUs = 0ll); + + void postTimeout(); + void cancelTimeout(); + + void notifyError(status_t err); + void notifyOffset(); + + static int CompareRountripTime(const TimeInfo *ti1, const TimeInfo *ti2); + + DISALLOW_EVIL_CONSTRUCTORS(TimeSyncer); +}; + +} // namespace android + +#endif // TIME_SYNCER_H_ diff --git a/media/libstagefright/wifi-display/rtp/RTPAssembler.cpp b/media/libstagefright/wifi-display/rtp/RTPAssembler.cpp index d0ab60d..5f189e7 100644 --- a/media/libstagefright/wifi-display/rtp/RTPAssembler.cpp +++ b/media/libstagefright/wifi-display/rtp/RTPAssembler.cpp @@ -53,6 +53,11 @@ void RTPReceiver::TSAssembler::signalDiscontinuity() { } status_t RTPReceiver::TSAssembler::processPacket(const sp<ABuffer> &packet) { + int32_t rtpTime; + CHECK(packet->meta()->findInt32("rtp-time", &rtpTime)); + + packet->meta()->setInt64("timeUs", (rtpTime * 100ll) / 9); + postAccessUnit(packet, mSawDiscontinuity); if (mSawDiscontinuity) { diff --git a/media/libstagefright/wifi-display/rtp/RTPReceiver.cpp b/media/libstagefright/wifi-display/rtp/RTPReceiver.cpp index 29482af..8711b08 100644 --- a/media/libstagefright/wifi-display/rtp/RTPReceiver.cpp +++ b/media/libstagefright/wifi-display/rtp/RTPReceiver.cpp @@ -221,10 +221,12 @@ void RTPReceiver::Source::dequeueMore() { mNumDeclaredLostPrior = mNumDeclaredLost; - ALOGI("lost %lld packets (%.2f %%), declared %d lost\n", - lostInterval, - 100.0f * lostInterval / expectedInterval, - declaredLostInterval); + if (declaredLostInterval > 0) { + ALOGI("lost %lld packets (%.2f %%), declared %d lost\n", + lostInterval, + 100.0f * lostInterval / expectedInterval, + declaredLostInterval); + } } mNextReportTimeUs = nowUs + kReportIntervalUs; @@ -530,6 +532,40 @@ status_t RTPReceiver::connect( return OK; } +status_t RTPReceiver::notifyLateness(int64_t latenessUs) { + sp<ABuffer> buf = new ABuffer(20); + + uint8_t *ptr = buf->data(); + ptr[0] = 0x80 | 0; + ptr[1] = 204; // APP + ptr[2] = 0; + + CHECK((buf->size() % 4) == 0u); + ptr[3] = (buf->size() / 4) - 1; + + ptr[4] = kSourceID >> 24; // SSRC + ptr[5] = (kSourceID >> 16) & 0xff; + ptr[6] = (kSourceID >> 8) & 0xff; + ptr[7] = kSourceID & 0xff; + ptr[8] = 'l'; + ptr[9] = 'a'; + ptr[10] = 't'; + ptr[11] = 'e'; + + ptr[12] = latenessUs >> 56; + ptr[13] = (latenessUs >> 48) & 0xff; + ptr[14] = (latenessUs >> 40) & 0xff; + ptr[15] = (latenessUs >> 32) & 0xff; + ptr[16] = (latenessUs >> 24) & 0xff; + ptr[17] = (latenessUs >> 16) & 0xff; + ptr[18] = (latenessUs >> 8) & 0xff; + ptr[19] = latenessUs & 0xff; + + mNetSession->sendRequest(mRTCPSessionID, buf->data(), buf->size()); + + return OK; +} + void RTPReceiver::onMessageReceived(const sp<AMessage> &msg) { switch (msg->what()) { case kWhatRTPNotify: diff --git a/media/libstagefright/wifi-display/rtp/RTPReceiver.h b/media/libstagefright/wifi-display/rtp/RTPReceiver.h index 2ae864a..ec4671d 100644 --- a/media/libstagefright/wifi-display/rtp/RTPReceiver.h +++ b/media/libstagefright/wifi-display/rtp/RTPReceiver.h @@ -53,6 +53,8 @@ struct RTPReceiver : public RTPBase, public AHandler { int32_t remoteRTPPort, int32_t remoteRTCPPort); + status_t notifyLateness(int64_t latenessUs); + protected: virtual ~RTPReceiver(); virtual void onMessageReceived(const sp<AMessage> &msg); diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.cpp b/media/libstagefright/wifi-display/rtp/RTPSender.cpp index 85c5933..8cd712d 100644 --- a/media/libstagefright/wifi-display/rtp/RTPSender.cpp +++ b/media/libstagefright/wifi-display/rtp/RTPSender.cpp @@ -530,6 +530,18 @@ void RTPSender::onNetNotify(bool isRTP, const sp<AMessage> &msg) { } break; } + + case ANetworkSession::kWhatNetworkStall: + { + size_t numBytesQueued; + CHECK(msg->findSize("numBytesQueued", &numBytesQueued)); + + notifyNetworkStall(numBytesQueued); + break; + } + + default: + TRESPASS(); } } @@ -577,6 +589,8 @@ status_t RTPSender::onRTCPData(const sp<ABuffer> &buffer) { case 202: // SDES case 203: + break; + case 204: // APP break; @@ -697,5 +711,12 @@ void RTPSender::notifyError(status_t err) { notify->post(); } +void RTPSender::notifyNetworkStall(size_t numBytesQueued) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatNetworkStall); + notify->setSize("numBytesQueued", numBytesQueued); + notify->post(); +} + } // namespace android diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.h b/media/libstagefright/wifi-display/rtp/RTPSender.h index 2b683a4..83c6223 100644 --- a/media/libstagefright/wifi-display/rtp/RTPSender.h +++ b/media/libstagefright/wifi-display/rtp/RTPSender.h @@ -36,6 +36,7 @@ struct RTPSender : public RTPBase, public AHandler { enum { kWhatInitDone, kWhatError, + kWhatNetworkStall, }; RTPSender( const sp<ANetworkSession> &netSession, @@ -103,6 +104,7 @@ private: void notifyInitDone(status_t err); void notifyError(status_t err); + void notifyNetworkStall(size_t numBytesQueued); DISALLOW_EVIL_CONSTRUCTORS(RTPSender); }; diff --git a/media/libstagefright/wifi-display/sink/DirectRenderer.cpp b/media/libstagefright/wifi-display/sink/DirectRenderer.cpp index b53252d..12338e9 100644 --- a/media/libstagefright/wifi-display/sink/DirectRenderer.cpp +++ b/media/libstagefright/wifi-display/sink/DirectRenderer.cpp @@ -22,6 +22,7 @@ #include <gui/SurfaceComposerClient.h> #include <gui/Surface.h> +#include <media/AudioTrack.h> #include <media/ICrypto.h> #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> @@ -34,133 +35,208 @@ namespace android { -DirectRenderer::DirectRenderer( - const sp<IGraphicBufferProducer> &bufferProducer) - : mSurfaceTex(bufferProducer), - mVideoDecoderNotificationPending(false), - mRenderPending(false), - mFirstRenderTimeUs(-1ll), - mFirstRenderRealUs(-1ll) { -} +/* + Drives the decoding process using a MediaCodec instance. Input buffers + queued by calls to "queueInputBuffer" are fed to the decoder as soon + as the decoder is ready for them, the client is notified about output + buffers as the decoder spits them out. +*/ +struct DirectRenderer::DecoderContext : public AHandler { + enum { + kWhatOutputBufferReady, + }; + DecoderContext(const sp<AMessage> ¬ify); -DirectRenderer::~DirectRenderer() { - if (mVideoDecoder != NULL) { - mVideoDecoder->release(); - mVideoDecoder.clear(); + status_t init( + const sp<AMessage> &format, + const sp<IGraphicBufferProducer> &surfaceTex); - mVideoDecoderLooper->stop(); - mVideoDecoderLooper.clear(); - } -} + void queueInputBuffer(const sp<ABuffer> &accessUnit); -void DirectRenderer::onMessageReceived(const sp<AMessage> &msg) { - switch (msg->what()) { - case kWhatVideoDecoderNotify: - { - onVideoDecoderNotify(); - break; - } + status_t renderOutputBufferAndRelease(size_t index); + status_t releaseOutputBuffer(size_t index); - case kWhatRender: - { - onRender(); - break; - } +protected: + virtual ~DecoderContext(); - default: - TRESPASS(); - } + virtual void onMessageReceived(const sp<AMessage> &msg); + +private: + enum { + kWhatDecoderNotify, + }; + + sp<AMessage> mNotify; + sp<ALooper> mDecoderLooper; + sp<MediaCodec> mDecoder; + Vector<sp<ABuffer> > mDecoderInputBuffers; + Vector<sp<ABuffer> > mDecoderOutputBuffers; + List<size_t> mDecoderInputBuffersAvailable; + bool mDecoderNotificationPending; + + List<sp<ABuffer> > mAccessUnits; + + void onDecoderNotify(); + void scheduleDecoderNotification(); + void queueDecoderInputBuffers(); + + void queueOutputBuffer( + size_t index, int64_t timeUs, const sp<ABuffer> &buffer); + + DISALLOW_EVIL_CONSTRUCTORS(DecoderContext); +}; + +//////////////////////////////////////////////////////////////////////////////// + +/* + A "push" audio renderer. The primary function of this renderer is to use + an AudioTrack in push mode and making sure not to block the event loop + be ensuring that calls to AudioTrack::write never block. This is done by + estimating an upper bound of data that can be written to the AudioTrack + buffer without delay. +*/ +struct DirectRenderer::AudioRenderer : public AHandler { + AudioRenderer(const sp<DecoderContext> &decoderContext); + + void queueInputBuffer( + size_t index, int64_t timeUs, const sp<ABuffer> &buffer); + +protected: + virtual ~AudioRenderer(); + virtual void onMessageReceived(const sp<AMessage> &msg); + +private: + enum { + kWhatPushAudio, + }; + + struct BufferInfo { + size_t mIndex; + int64_t mTimeUs; + sp<ABuffer> mBuffer; + }; + + sp<DecoderContext> mDecoderContext; + sp<AudioTrack> mAudioTrack; + + List<BufferInfo> mInputBuffers; + bool mPushPending; + + size_t mNumFramesWritten; + + void schedulePushIfNecessary(); + void onPushAudio(); + + ssize_t writeNonBlocking(const uint8_t *data, size_t size); + + DISALLOW_EVIL_CONSTRUCTORS(AudioRenderer); +}; + +//////////////////////////////////////////////////////////////////////////////// + +DirectRenderer::DecoderContext::DecoderContext(const sp<AMessage> ¬ify) + : mNotify(notify), + mDecoderNotificationPending(false) { } -void DirectRenderer::setFormat( - size_t trackIndex, const sp<AMessage> &format) { - if (trackIndex == 1) { - // Ignore audio for now. - return; +DirectRenderer::DecoderContext::~DecoderContext() { + if (mDecoder != NULL) { + mDecoder->release(); + mDecoder.clear(); + + mDecoderLooper->stop(); + mDecoderLooper.clear(); } +} - CHECK(mVideoDecoder == NULL); +status_t DirectRenderer::DecoderContext::init( + const sp<AMessage> &format, + const sp<IGraphicBufferProducer> &surfaceTex) { + CHECK(mDecoder == NULL); AString mime; CHECK(format->findString("mime", &mime)); - mVideoDecoderLooper = new ALooper; - mVideoDecoderLooper->setName("video codec looper"); + mDecoderLooper = new ALooper; + mDecoderLooper->setName("video codec looper"); - mVideoDecoderLooper->start( + mDecoderLooper->start( false /* runOnCallingThread */, false /* canCallJava */, PRIORITY_DEFAULT); - mVideoDecoder = MediaCodec::CreateByType( - mVideoDecoderLooper, mime.c_str(), false /* encoder */); + mDecoder = MediaCodec::CreateByType( + mDecoderLooper, mime.c_str(), false /* encoder */); - CHECK(mVideoDecoder != NULL); + CHECK(mDecoder != NULL); - status_t err = mVideoDecoder->configure( + status_t err = mDecoder->configure( format, - mSurfaceTex == NULL - ? NULL : new Surface(mSurfaceTex), + surfaceTex == NULL + ? NULL : new Surface(surfaceTex), NULL /* crypto */, 0 /* flags */); CHECK_EQ(err, (status_t)OK); - err = mVideoDecoder->start(); + err = mDecoder->start(); CHECK_EQ(err, (status_t)OK); - err = mVideoDecoder->getInputBuffers( - &mVideoDecoderInputBuffers); + err = mDecoder->getInputBuffers( + &mDecoderInputBuffers); CHECK_EQ(err, (status_t)OK); - scheduleVideoDecoderNotification(); + err = mDecoder->getOutputBuffers( + &mDecoderOutputBuffers); + CHECK_EQ(err, (status_t)OK); + + scheduleDecoderNotification(); + + return OK; } -void DirectRenderer::queueAccessUnit( - size_t trackIndex, const sp<ABuffer> &accessUnit) { - if (trackIndex == 1) { - // Ignore audio for now. - return; - } +void DirectRenderer::DecoderContext::queueInputBuffer( + const sp<ABuffer> &accessUnit) { + CHECK(mDecoder != NULL); - if (mVideoDecoder == NULL) { - sp<AMessage> format = new AMessage; - format->setString("mime", "video/avc"); - format->setInt32("width", 640); - format->setInt32("height", 360); + mAccessUnits.push_back(accessUnit); + queueDecoderInputBuffers(); +} - setFormat(0, format); - } +status_t DirectRenderer::DecoderContext::renderOutputBufferAndRelease( + size_t index) { + return mDecoder->renderOutputBufferAndRelease(index); +} - mVideoAccessUnits.push_back(accessUnit); - queueVideoDecoderInputBuffers(); +status_t DirectRenderer::DecoderContext::releaseOutputBuffer(size_t index) { + return mDecoder->releaseOutputBuffer(index); } -void DirectRenderer::queueVideoDecoderInputBuffers() { - if (mVideoDecoder == NULL) { +void DirectRenderer::DecoderContext::queueDecoderInputBuffers() { + if (mDecoder == NULL) { return; } bool submittedMore = false; - while (!mVideoAccessUnits.empty() - && !mVideoDecoderInputBuffersAvailable.empty()) { - size_t index = *mVideoDecoderInputBuffersAvailable.begin(); + while (!mAccessUnits.empty() + && !mDecoderInputBuffersAvailable.empty()) { + size_t index = *mDecoderInputBuffersAvailable.begin(); - mVideoDecoderInputBuffersAvailable.erase( - mVideoDecoderInputBuffersAvailable.begin()); + mDecoderInputBuffersAvailable.erase( + mDecoderInputBuffersAvailable.begin()); - sp<ABuffer> srcBuffer = *mVideoAccessUnits.begin(); - mVideoAccessUnits.erase(mVideoAccessUnits.begin()); + sp<ABuffer> srcBuffer = *mAccessUnits.begin(); + mAccessUnits.erase(mAccessUnits.begin()); const sp<ABuffer> &dstBuffer = - mVideoDecoderInputBuffers.itemAt(index); + mDecoderInputBuffers.itemAt(index); memcpy(dstBuffer->data(), srcBuffer->data(), srcBuffer->size()); int64_t timeUs; CHECK(srcBuffer->meta()->findInt64("timeUs", &timeUs)); - status_t err = mVideoDecoder->queueInputBuffer( + status_t err = mDecoder->queueInputBuffer( index, 0 /* offset */, srcBuffer->size(), @@ -172,19 +248,33 @@ void DirectRenderer::queueVideoDecoderInputBuffers() { } if (submittedMore) { - scheduleVideoDecoderNotification(); + scheduleDecoderNotification(); } } -void DirectRenderer::onVideoDecoderNotify() { - mVideoDecoderNotificationPending = false; +void DirectRenderer::DecoderContext::onMessageReceived( + const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatDecoderNotify: + { + onDecoderNotify(); + break; + } + + default: + TRESPASS(); + } +} + +void DirectRenderer::DecoderContext::onDecoderNotify() { + mDecoderNotificationPending = false; for (;;) { size_t index; - status_t err = mVideoDecoder->dequeueInputBuffer(&index); + status_t err = mDecoder->dequeueInputBuffer(&index); if (err == OK) { - mVideoDecoderInputBuffersAvailable.push_back(index); + mDecoderInputBuffersAvailable.push_back(index); } else if (err == -EAGAIN) { break; } else { @@ -192,7 +282,7 @@ void DirectRenderer::onVideoDecoderNotify() { } } - queueVideoDecoderInputBuffers(); + queueDecoderInputBuffers(); for (;;) { size_t index; @@ -200,7 +290,7 @@ void DirectRenderer::onVideoDecoderNotify() { size_t size; int64_t timeUs; uint32_t flags; - status_t err = mVideoDecoder->dequeueOutputBuffer( + status_t err = mDecoder->dequeueOutputBuffer( &index, &offset, &size, @@ -208,9 +298,12 @@ void DirectRenderer::onVideoDecoderNotify() { &flags); if (err == OK) { - queueOutputBuffer(index, timeUs); + queueOutputBuffer( + index, timeUs, mDecoderOutputBuffers.itemAt(index)); } else if (err == INFO_OUTPUT_BUFFERS_CHANGED) { - // We don't care. + err = mDecoder->getOutputBuffers( + &mDecoderOutputBuffers); + CHECK_EQ(err, (status_t)OK); } else if (err == INFO_FORMAT_CHANGED) { // We don't care. } else if (err == -EAGAIN) { @@ -220,75 +313,336 @@ void DirectRenderer::onVideoDecoderNotify() { } } - scheduleVideoDecoderNotification(); + scheduleDecoderNotification(); } -void DirectRenderer::queueOutputBuffer(size_t index, int64_t timeUs) { -#if 0 - OutputInfo info; +void DirectRenderer::DecoderContext::scheduleDecoderNotification() { + if (mDecoderNotificationPending) { + return; + } + + sp<AMessage> notify = + new AMessage(kWhatDecoderNotify, id()); + + mDecoder->requestActivityNotification(notify); + mDecoderNotificationPending = true; +} + +void DirectRenderer::DecoderContext::queueOutputBuffer( + size_t index, int64_t timeUs, const sp<ABuffer> &buffer) { + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("what", kWhatOutputBufferReady); + msg->setSize("index", index); + msg->setInt64("timeUs", timeUs); + msg->setBuffer("buffer", buffer); + msg->post(); +} + +//////////////////////////////////////////////////////////////////////////////// + +DirectRenderer::AudioRenderer::AudioRenderer( + const sp<DecoderContext> &decoderContext) + : mDecoderContext(decoderContext), + mPushPending(false), + mNumFramesWritten(0) { + mAudioTrack = new AudioTrack( + AUDIO_STREAM_DEFAULT, + 48000.0f, + AUDIO_FORMAT_PCM, + AUDIO_CHANNEL_OUT_STEREO, + (int)0 /* frameCount */); + + CHECK_EQ((status_t)OK, mAudioTrack->initCheck()); + + mAudioTrack->start(); +} + +DirectRenderer::AudioRenderer::~AudioRenderer() { +} + +void DirectRenderer::AudioRenderer::queueInputBuffer( + size_t index, int64_t timeUs, const sp<ABuffer> &buffer) { + BufferInfo info; info.mIndex = index; info.mTimeUs = timeUs; - mOutputBuffers.push_back(info); + info.mBuffer = buffer; - scheduleRenderIfNecessary(); -#else - status_t err = mVideoDecoder->renderOutputBufferAndRelease(index); - CHECK_EQ(err, (status_t)OK); -#endif + mInputBuffers.push_back(info); + schedulePushIfNecessary(); +} + +void DirectRenderer::AudioRenderer::onMessageReceived( + const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatPushAudio: + { + onPushAudio(); + break; + } + + default: + break; + } } -void DirectRenderer::scheduleRenderIfNecessary() { - if (mRenderPending || mOutputBuffers.empty()) { +void DirectRenderer::AudioRenderer::schedulePushIfNecessary() { + if (mPushPending || mInputBuffers.empty()) { return; } - mRenderPending = true; + mPushPending = true; - int64_t timeUs = (*mOutputBuffers.begin()).mTimeUs; - int64_t nowUs = ALooper::GetNowUs(); + uint32_t numFramesPlayed; + CHECK_EQ(mAudioTrack->getPosition(&numFramesPlayed), + (status_t)OK); + + uint32_t numFramesPendingPlayout = mNumFramesWritten - numFramesPlayed; + + // This is how long the audio sink will have data to + // play back. + const float msecsPerFrame = 1000.0f / mAudioTrack->getSampleRate(); + + int64_t delayUs = + msecsPerFrame * numFramesPendingPlayout * 1000ll; + + // Let's give it more data after about half that time + // has elapsed. + (new AMessage(kWhatPushAudio, id()))->post(delayUs / 2); +} + +void DirectRenderer::AudioRenderer::onPushAudio() { + mPushPending = false; + + while (!mInputBuffers.empty()) { + const BufferInfo &info = *mInputBuffers.begin(); - if (mFirstRenderTimeUs < 0ll) { - mFirstRenderTimeUs = timeUs; - mFirstRenderRealUs = nowUs; + ssize_t n = writeNonBlocking( + info.mBuffer->data(), info.mBuffer->size()); + + if (n < (ssize_t)info.mBuffer->size()) { + CHECK_GE(n, 0); + + info.mBuffer->setRange( + info.mBuffer->offset() + n, info.mBuffer->size() - n); + break; + } + + mDecoderContext->releaseOutputBuffer(info.mIndex); + + mInputBuffers.erase(mInputBuffers.begin()); + } + + schedulePushIfNecessary(); +} + +ssize_t DirectRenderer::AudioRenderer::writeNonBlocking( + const uint8_t *data, size_t size) { + uint32_t numFramesPlayed; + status_t err = mAudioTrack->getPosition(&numFramesPlayed); + if (err != OK) { + return err; + } + + ssize_t numFramesAvailableToWrite = + mAudioTrack->frameCount() - (mNumFramesWritten - numFramesPlayed); + + size_t numBytesAvailableToWrite = + numFramesAvailableToWrite * mAudioTrack->frameSize(); + + if (size > numBytesAvailableToWrite) { + size = numBytesAvailableToWrite; } - int64_t whenUs = timeUs - mFirstRenderTimeUs + mFirstRenderRealUs; - int64_t delayUs = whenUs - nowUs; + CHECK_EQ(mAudioTrack->write(data, size), (ssize_t)size); - (new AMessage(kWhatRender, id()))->post(delayUs); + size_t numFramesWritten = size / mAudioTrack->frameSize(); + mNumFramesWritten += numFramesWritten; + + return size; } -void DirectRenderer::onRender() { - mRenderPending = false; +//////////////////////////////////////////////////////////////////////////////// - int64_t nowUs = ALooper::GetNowUs(); +DirectRenderer::DirectRenderer( + const sp<IGraphicBufferProducer> &bufferProducer) + : mSurfaceTex(bufferProducer), + mVideoRenderPending(false), + mLatencySum(0ll), + mLatencyCount(0), + mNumFramesLate(0), + mNumFrames(0) { +} - while (!mOutputBuffers.empty()) { - const OutputInfo &info = *mOutputBuffers.begin(); +DirectRenderer::~DirectRenderer() { +} - if (info.mTimeUs > nowUs) { +int64_t DirectRenderer::getAvgLatenessUs() { + if (mLatencyCount == 0) { + return 0ll; + } + + int64_t avgLatencyUs = mLatencySum / mLatencyCount; + + mLatencySum = 0ll; + mLatencyCount = 0; + + if (mNumFrames > 0) { + ALOGI("%d / %d frames late", mNumFramesLate, mNumFrames); + mNumFramesLate = 0; + mNumFrames = 0; + } + + return avgLatencyUs; +} + +void DirectRenderer::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatDecoderNotify: + { + onDecoderNotify(msg); break; } - status_t err = mVideoDecoder->renderOutputBufferAndRelease(info.mIndex); - CHECK_EQ(err, (status_t)OK); + case kWhatRenderVideo: + { + onRenderVideo(); + break; + } + + default: + TRESPASS(); + } +} + +void DirectRenderer::setFormat(size_t trackIndex, const sp<AMessage> &format) { + CHECK_LT(trackIndex, 2u); + + CHECK(mDecoderContext[trackIndex] == NULL); + + sp<AMessage> notify = new AMessage(kWhatDecoderNotify, id()); + notify->setSize("trackIndex", trackIndex); + + mDecoderContext[trackIndex] = new DecoderContext(notify); + looper()->registerHandler(mDecoderContext[trackIndex]); + + CHECK_EQ((status_t)OK, + mDecoderContext[trackIndex]->init( + format, trackIndex == 0 ? mSurfaceTex : NULL)); + + if (trackIndex == 1) { + // Audio + mAudioRenderer = new AudioRenderer(mDecoderContext[1]); + looper()->registerHandler(mAudioRenderer); + } +} + +void DirectRenderer::queueAccessUnit( + size_t trackIndex, const sp<ABuffer> &accessUnit) { + CHECK_LT(trackIndex, 2u); + + if (mDecoderContext[trackIndex] == NULL) { + CHECK_EQ(trackIndex, 0u); + + sp<AMessage> format = new AMessage; + format->setString("mime", "video/avc"); + format->setInt32("width", 640); + format->setInt32("height", 360); - mOutputBuffers.erase(mOutputBuffers.begin()); + setFormat(trackIndex, format); } - scheduleRenderIfNecessary(); + mDecoderContext[trackIndex]->queueInputBuffer(accessUnit); } -void DirectRenderer::scheduleVideoDecoderNotification() { - if (mVideoDecoderNotificationPending) { +void DirectRenderer::onDecoderNotify(const sp<AMessage> &msg) { + size_t trackIndex; + CHECK(msg->findSize("trackIndex", &trackIndex)); + + int32_t what; + CHECK(msg->findInt32("what", &what)); + + switch (what) { + case DecoderContext::kWhatOutputBufferReady: + { + size_t index; + CHECK(msg->findSize("index", &index)); + + int64_t timeUs; + CHECK(msg->findInt64("timeUs", &timeUs)); + + sp<ABuffer> buffer; + CHECK(msg->findBuffer("buffer", &buffer)); + + queueOutputBuffer(trackIndex, index, timeUs, buffer); + break; + } + + default: + TRESPASS(); + } +} + +void DirectRenderer::queueOutputBuffer( + size_t trackIndex, + size_t index, int64_t timeUs, const sp<ABuffer> &buffer) { + if (trackIndex == 1) { + // Audio + mAudioRenderer->queueInputBuffer(index, timeUs, buffer); return; } - sp<AMessage> notify = - new AMessage(kWhatVideoDecoderNotify, id()); + OutputInfo info; + info.mIndex = index; + info.mTimeUs = timeUs; + info.mBuffer = buffer; + mVideoOutputBuffers.push_back(info); + + scheduleVideoRenderIfNecessary(); +} + +void DirectRenderer::scheduleVideoRenderIfNecessary() { + if (mVideoRenderPending || mVideoOutputBuffers.empty()) { + return; + } + + mVideoRenderPending = true; + + int64_t timeUs = (*mVideoOutputBuffers.begin()).mTimeUs; + int64_t nowUs = ALooper::GetNowUs(); + + int64_t delayUs = timeUs - nowUs; + + (new AMessage(kWhatRenderVideo, id()))->post(delayUs); +} + +void DirectRenderer::onRenderVideo() { + mVideoRenderPending = false; + + int64_t nowUs = ALooper::GetNowUs(); + + while (!mVideoOutputBuffers.empty()) { + const OutputInfo &info = *mVideoOutputBuffers.begin(); + + if (info.mTimeUs > nowUs) { + break; + } + + if (info.mTimeUs + 15000ll < nowUs) { + ++mNumFramesLate; + } + ++mNumFrames; + + mLatencySum += nowUs - info.mTimeUs; + ++mLatencyCount; + + status_t err = + mDecoderContext[0]->renderOutputBufferAndRelease(info.mIndex); + CHECK_EQ(err, (status_t)OK); + + mVideoOutputBuffers.erase(mVideoOutputBuffers.begin()); + } - mVideoDecoder->requestActivityNotification(notify); - mVideoDecoderNotificationPending = true; + scheduleVideoRenderIfNecessary(); } } // namespace android diff --git a/media/libstagefright/wifi-display/sink/DirectRenderer.h b/media/libstagefright/wifi-display/sink/DirectRenderer.h index 7219080..92c176a 100644 --- a/media/libstagefright/wifi-display/sink/DirectRenderer.h +++ b/media/libstagefright/wifi-display/sink/DirectRenderer.h @@ -23,57 +23,61 @@ namespace android { struct ABuffer; +struct AudioTrack; struct IGraphicBufferProducer; struct MediaCodec; -// An experimental renderer that only supports video and decodes video data -// as soon as it arrives using a MediaCodec instance, rendering it without -// delay. Primarily meant to finetune packet loss discovery and minimize -// latency. +// Renders audio and video data queued by calls to "queueAccessUnit". struct DirectRenderer : public AHandler { DirectRenderer(const sp<IGraphicBufferProducer> &bufferProducer); void setFormat(size_t trackIndex, const sp<AMessage> &format); void queueAccessUnit(size_t trackIndex, const sp<ABuffer> &accessUnit); + int64_t getAvgLatenessUs(); + protected: virtual void onMessageReceived(const sp<AMessage> &msg); virtual ~DirectRenderer(); private: + struct DecoderContext; + struct AudioRenderer; + enum { - kWhatVideoDecoderNotify, - kWhatRender, + kWhatDecoderNotify, + kWhatRenderVideo, }; struct OutputInfo { size_t mIndex; int64_t mTimeUs; + sp<ABuffer> mBuffer; }; sp<IGraphicBufferProducer> mSurfaceTex; - sp<ALooper> mVideoDecoderLooper; - sp<MediaCodec> mVideoDecoder; - Vector<sp<ABuffer> > mVideoDecoderInputBuffers; - List<size_t> mVideoDecoderInputBuffersAvailable; - bool mVideoDecoderNotificationPending; + sp<DecoderContext> mDecoderContext[2]; + List<OutputInfo> mVideoOutputBuffers; + + bool mVideoRenderPending; + + sp<AudioRenderer> mAudioRenderer; - List<sp<ABuffer> > mVideoAccessUnits; + int64_t mLatencySum; + size_t mLatencyCount; - List<OutputInfo> mOutputBuffers; - bool mRenderPending; - int64_t mFirstRenderTimeUs; - int64_t mFirstRenderRealUs; + int32_t mNumFramesLate; + int32_t mNumFrames; - void onVideoDecoderNotify(); - void onRender(); + void onDecoderNotify(const sp<AMessage> &msg); - void queueVideoDecoderInputBuffers(); - void scheduleVideoDecoderNotification(); - void scheduleRenderIfNecessary(); + void queueOutputBuffer( + size_t trackIndex, + size_t index, int64_t timeUs, const sp<ABuffer> &buffer); - void queueOutputBuffer(size_t index, int64_t timeUs); + void scheduleVideoRenderIfNecessary(); + void onRenderVideo(); DISALLOW_EVIL_CONSTRUCTORS(DirectRenderer); }; diff --git a/media/libstagefright/wifi-display/sink/TunnelRenderer.cpp b/media/libstagefright/wifi-display/sink/TunnelRenderer.cpp index d9d8a76..6b185db 100644 --- a/media/libstagefright/wifi-display/sink/TunnelRenderer.cpp +++ b/media/libstagefright/wifi-display/sink/TunnelRenderer.cpp @@ -27,6 +27,7 @@ #include <gui/SurfaceComposerClient.h> #include <media/IMediaPlayerService.h> #include <media/IStreamSource.h> +#include <media/mediaplayer.h> #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> @@ -60,6 +61,8 @@ struct TunnelRenderer::StreamSource : public BnStreamSource { void doSomeWork(); + void setTimeOffset(int64_t offset); + protected: virtual ~StreamSource(); @@ -75,6 +78,9 @@ private: size_t mNumDeqeued; + int64_t mTimeOffsetUs; + bool mTimeOffsetChanged; + DISALLOW_EVIL_CONSTRUCTORS(StreamSource); }; @@ -82,7 +88,9 @@ private: TunnelRenderer::StreamSource::StreamSource(TunnelRenderer *owner) : mOwner(owner), - mNumDeqeued(0) { + mNumDeqeued(0), + mTimeOffsetUs(0ll), + mTimeOffsetChanged(false) { } TunnelRenderer::StreamSource::~StreamSource() { @@ -110,7 +118,7 @@ void TunnelRenderer::StreamSource::onBufferAvailable(size_t index) { } uint32_t TunnelRenderer::StreamSource::flags() const { - return kFlagAlignedVideoData; + return kFlagAlignedVideoData | kFlagIsRealTimeData; } void TunnelRenderer::StreamSource::doSomeWork() { @@ -124,21 +132,21 @@ void TunnelRenderer::StreamSource::doSomeWork() { ++mNumDeqeued; - if (mNumDeqeued == 1) { - ALOGI("fixing real time now."); - + if (mTimeOffsetChanged) { sp<AMessage> extra = new AMessage; extra->setInt32( IStreamListener::kKeyDiscontinuityMask, - ATSParser::DISCONTINUITY_ABSOLUTE_TIME); + ATSParser::DISCONTINUITY_TIME_OFFSET); - extra->setInt64("timeUs", ALooper::GetNowUs()); + extra->setInt64("offset", mTimeOffsetUs); mListener->issueCommand( IStreamListener::DISCONTINUITY, false /* synchronous */, extra); + + mTimeOffsetChanged = false; } ALOGV("dequeue TS packet of size %d", srcBuffer->size()); @@ -155,18 +163,32 @@ void TunnelRenderer::StreamSource::doSomeWork() { } } +void TunnelRenderer::StreamSource::setTimeOffset(int64_t offset) { + Mutex::Autolock autoLock(mLock); + + if (offset != mTimeOffsetUs) { + mTimeOffsetUs = offset; + mTimeOffsetChanged = true; + } +} + //////////////////////////////////////////////////////////////////////////////// TunnelRenderer::TunnelRenderer( const sp<IGraphicBufferProducer> &bufferProducer) : mSurfaceTex(bufferProducer), mStartup(true) { + mStreamSource = new StreamSource(this); } TunnelRenderer::~TunnelRenderer() { destroyPlayer(); } +void TunnelRenderer::setTimeOffset(int64_t offset) { + mStreamSource->setTimeOffset(offset); +} + void TunnelRenderer::onMessageReceived(const sp<AMessage> &msg) { switch (msg->what()) { default: @@ -209,8 +231,6 @@ void TunnelRenderer::initPlayer() { sp<IMediaPlayerService> service = interface_cast<IMediaPlayerService>(binder); CHECK(service.get() != NULL); - mStreamSource = new StreamSource(this); - mPlayerClient = new PlayerClient; mPlayer = service->create(mPlayerClient, 0); @@ -226,6 +246,8 @@ void TunnelRenderer::initPlayer() { void TunnelRenderer::destroyPlayer() { mStreamSource.clear(); + mPlayer->setVideoSurfaceTexture(NULL); + mPlayer->stop(); mPlayer.clear(); diff --git a/media/libstagefright/wifi-display/sink/TunnelRenderer.h b/media/libstagefright/wifi-display/sink/TunnelRenderer.h index 8e96665..479e73c 100644 --- a/media/libstagefright/wifi-display/sink/TunnelRenderer.h +++ b/media/libstagefright/wifi-display/sink/TunnelRenderer.h @@ -39,6 +39,12 @@ struct TunnelRenderer : public AHandler { void queueBuffer(const sp<ABuffer> &buffer); sp<ABuffer> dequeueBuffer(); + void setTimeOffset(int64_t offset); + + int64_t getAvgLatenessUs() { + return 0ll; + } + protected: virtual void onMessageReceived(const sp<AMessage> &msg); virtual ~TunnelRenderer(); diff --git a/media/libstagefright/wifi-display/sink/WifiDisplaySink.cpp b/media/libstagefright/wifi-display/sink/WifiDisplaySink.cpp index 158c2da..62021c0 100644 --- a/media/libstagefright/wifi-display/sink/WifiDisplaySink.cpp +++ b/media/libstagefright/wifi-display/sink/WifiDisplaySink.cpp @@ -23,22 +23,24 @@ #include "DirectRenderer.h" #include "MediaReceiver.h" #include "ParsedMessage.h" +#include "TimeSyncer.h" #include "TunnelRenderer.h" +#include <cutils/properties.h> #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> #include <media/stagefright/MediaErrors.h> -#include <cutils/properties.h> - namespace android { WifiDisplaySink::WifiDisplaySink( + uint32_t flags, const sp<ANetworkSession> &netSession, const sp<IGraphicBufferProducer> &bufferProducer, const sp<AMessage> ¬ify) : mState(UNDEFINED), + mFlags(flags), mNetSession(netSession), mSurfaceTex(bufferProducer), mNotify(notify), @@ -46,7 +48,11 @@ WifiDisplaySink::WifiDisplaySink( mUsingTCPInterleaving(false), mSessionID(0), mNextCSeq(1), - mIDRFrameRequestPending(false) { + mIDRFrameRequestPending(false), + mTimeOffsetUs(0ll), + mTimeOffsetValid(false), + mTargetLatencyUs(-1ll), + mSetupDeferred(false) { // We support any and all resolutions, but prefer 720p30 mSinkSupportedVideoFormats.setNativeResolution( VideoFormats::RESOLUTION_CEA, 5); // 1280 x 720 p30 @@ -199,6 +205,16 @@ void WifiDisplaySink::onMessageReceived(const sp<AMessage> &msg) { { ALOGI("We're now connected."); mState = CONNECTED; + + if (mFlags & FLAG_SPECIAL_MODE) { + sp<AMessage> notify = new AMessage( + kWhatTimeSyncerNotify, id()); + + mTimeSyncer = new TimeSyncer(mNetSession, notify); + looper()->registerHandler(mTimeSyncer); + + mTimeSyncer->startClient(mRTSPHost.c_str(), 8123); + } break; } @@ -226,6 +242,41 @@ void WifiDisplaySink::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatTimeSyncerNotify: + { + int32_t what; + CHECK(msg->findInt32("what", &what)); + + if (what == TimeSyncer::kWhatTimeOffset) { + CHECK(msg->findInt64("offset", &mTimeOffsetUs)); + mTimeOffsetValid = true; + + if (mSetupDeferred) { + CHECK_EQ((status_t)OK, + sendSetup( + mSessionID, + "rtsp://x.x.x.x:x/wfd1.0/streamid=0")); + + mSetupDeferred = false; + } + } + break; + } + + case kWhatReportLateness: + { + int64_t latenessUs = mRenderer->getAvgLatenessUs(); + + ALOGI("avg. lateness = %lld ms", + (latenessUs + mTargetLatencyUs) / 1000ll); + + mMediaReceiver->notifyLateness( + 0 /* trackIndex */, latenessUs); + + msg->post(kReportLatenessEveryUs); + break; + } + default: TRESPASS(); } @@ -266,15 +317,44 @@ void WifiDisplaySink::onMediaReceiverNotify(const sp<AMessage> &msg) { looper()->registerHandler(mRenderer); } + CHECK(mTimeOffsetValid); + + int64_t latencyUs = 200000ll; // 200ms by default + + char val[PROPERTY_VALUE_MAX]; + if (property_get("media.wfd-sink.latency", val, NULL)) { + char *end; + int64_t x = strtoll(val, &end, 10); + + if (end > val && *end == '\0' && x >= 0ll) { + latencyUs = x; + } + } + + if (latencyUs != mTargetLatencyUs) { + mTargetLatencyUs = latencyUs; + + ALOGI("Assuming %lld ms of latency.", latencyUs / 1000ll); + } + sp<ABuffer> accessUnit; CHECK(msg->findBuffer("accessUnit", &accessUnit)); -#if USE_TUNNEL_RENDERER - mRenderer->queueBuffer(accessUnit); -#else + int64_t timeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + + // We are the timesync _client_, + // client time = server time - time offset. + timeUs += mTargetLatencyUs - mTimeOffsetUs; + + accessUnit->meta()->setInt64("timeUs", timeUs); + size_t trackIndex; CHECK(msg->findSize("trackIndex", &trackIndex)); +#if USE_TUNNEL_RENDERER + mRenderer->queueBuffer(accessUnit); +#else sp<AMessage> format; if (msg->findMessage("format", &format)) { mRenderer->setFormat(trackIndex, format); @@ -445,6 +525,8 @@ status_t WifiDisplaySink::onReceivePlayResponse( mState = PLAYING; + (new AMessage(kWhatReportLateness, id()))->post(kReportLatenessEveryUs); + return OK; } @@ -555,6 +637,8 @@ void WifiDisplaySink::onGetParameterRequest( mUsingTCPTransport = true; mUsingTCPInterleaving = true; } + } else if (mFlags & FLAG_SPECIAL_MODE) { + mUsingTCPTransport = true; } body = "wfd_video_formats: "; @@ -735,12 +819,16 @@ void WifiDisplaySink::onSetParameterRequest( const char *content = data->getContent(); if (strstr(content, "wfd_trigger_method: SETUP\r\n") != NULL) { - status_t err = - sendSetup( - sessionID, - "rtsp://x.x.x.x:x/wfd1.0/streamid=0"); + if ((mFlags & FLAG_SPECIAL_MODE) && !mTimeOffsetValid) { + mSetupDeferred = true; + } else { + status_t err = + sendSetup( + sessionID, + "rtsp://x.x.x.x:x/wfd1.0/streamid=0"); - CHECK_EQ(err, (status_t)OK); + CHECK_EQ(err, (status_t)OK); + } } AString response = "RTSP/1.0 200 OK\r\n"; diff --git a/media/libstagefright/wifi-display/sink/WifiDisplaySink.h b/media/libstagefright/wifi-display/sink/WifiDisplaySink.h index 01af58b..2b8c6f7 100644 --- a/media/libstagefright/wifi-display/sink/WifiDisplaySink.h +++ b/media/libstagefright/wifi-display/sink/WifiDisplaySink.h @@ -31,6 +31,7 @@ struct AMessage; struct DirectRenderer; struct MediaReceiver; struct ParsedMessage; +struct TimeSyncer; struct TunnelRenderer; #define USE_TUNNEL_RENDERER 0 @@ -43,11 +44,16 @@ struct WifiDisplaySink : public AHandler { kWhatDisconnected, }; + enum Flags { + FLAG_SPECIAL_MODE = 1, + }; + // If no notification message is specified (notify == NULL) // the sink will stop its looper() once the session ends, // otherwise it will post an appropriate notification but leave // the looper() running. WifiDisplaySink( + uint32_t flags, const sp<ANetworkSession> &netSession, const sp<IGraphicBufferProducer> &bufferProducer = NULL, const sp<AMessage> ¬ify = NULL); @@ -73,6 +79,8 @@ private: kWhatRTSPNotify, kWhatStop, kWhatMediaReceiverNotify, + kWhatTimeSyncerNotify, + kWhatReportLateness, }; struct ResponseID { @@ -89,11 +97,15 @@ private: typedef status_t (WifiDisplaySink::*HandleRTSPResponseFunc)( int32_t sessionID, const sp<ParsedMessage> &msg); + static const int64_t kReportLatenessEveryUs = 1000000ll; + State mState; + uint32_t mFlags; VideoFormats mSinkSupportedVideoFormats; sp<ANetworkSession> mNetSession; sp<IGraphicBufferProducer> mSurfaceTex; sp<AMessage> mNotify; + sp<TimeSyncer> mTimeSyncer; bool mUsingTCPTransport; bool mUsingTCPInterleaving; AString mRTSPHost; @@ -117,6 +129,13 @@ private: bool mIDRFrameRequestPending; + int64_t mTimeOffsetUs; + bool mTimeOffsetValid; + + int64_t mTargetLatencyUs; + + bool mSetupDeferred; + status_t sendM2(int32_t sessionID); status_t sendSetup(int32_t sessionID, const char *uri); status_t sendPlay(int32_t sessionID, const char *uri); diff --git a/media/libstagefright/wifi-display/source/Converter.cpp b/media/libstagefright/wifi-display/source/Converter.cpp index 2861aa9..bb8c387 100644 --- a/media/libstagefright/wifi-display/source/Converter.cpp +++ b/media/libstagefright/wifi-display/source/Converter.cpp @@ -55,6 +55,7 @@ Converter::Converter( ,mInSilentMode(false) #endif ,mPrevVideoBitrate(-1) + ,mNumFramesToDrop(0) { AString mime; CHECK(mInputFormat->findString("mime", &mime)); @@ -327,6 +328,13 @@ void Converter::onMessageReceived(const sp<AMessage> &msg) { sp<ABuffer> accessUnit; CHECK(msg->findBuffer("accessUnit", &accessUnit)); + if (mIsVideo && mNumFramesToDrop) { + --mNumFramesToDrop; + ALOGI("dropping frame."); + ReleaseMediaBufferReference(accessUnit); + break; + } + #if 0 void *mbuf; if (accessUnit->meta()->findPointer("mediaBuffer", &mbuf) @@ -422,6 +430,12 @@ void Converter::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatDropAFrame: + { + ++mNumFramesToDrop; + break; + } + default: TRESPASS(); } @@ -690,4 +704,8 @@ void Converter::requestIDRFrame() { (new AMessage(kWhatRequestIDRFrame, id()))->post(); } +void Converter::dropAFrame() { + (new AMessage(kWhatDropAFrame, id()))->post(); +} + } // namespace android diff --git a/media/libstagefright/wifi-display/source/Converter.h b/media/libstagefright/wifi-display/source/Converter.h index 57802bd..a418f69 100644 --- a/media/libstagefright/wifi-display/source/Converter.h +++ b/media/libstagefright/wifi-display/source/Converter.h @@ -51,6 +51,8 @@ struct Converter : public AHandler { void requestIDRFrame(); + void dropAFrame(); + enum { kWhatAccessUnit, kWhatEOS, @@ -63,6 +65,7 @@ struct Converter : public AHandler { kWhatShutdown, kWhatMediaPullerNotify, kWhatEncoderActivity, + kWhatDropAFrame, }; void shutdownAsync(); @@ -102,6 +105,8 @@ private: int32_t mPrevVideoBitrate; + int32_t mNumFramesToDrop; + status_t initEncoder(); void releaseEncoder(); diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.cpp b/media/libstagefright/wifi-display/source/PlaybackSession.cpp index ea195b3..94cb2a4 100644 --- a/media/libstagefright/wifi-display/source/PlaybackSession.cpp +++ b/media/libstagefright/wifi-display/source/PlaybackSession.cpp @@ -515,6 +515,16 @@ void WifiDisplaySource::PlaybackSession::onMessageReceived( } } else if (what == MediaSender::kWhatError) { notifySessionDead(); + } else if (what == MediaSender::kWhatNetworkStall) { + size_t numBytesQueued; + CHECK(msg->findSize("numBytesQueued", &numBytesQueued)); + + if (mVideoTrackIndex >= 0) { + const sp<Track> &videoTrack = + mTracks.valueFor(mVideoTrackIndex); + + videoTrack->converter()->dropAFrame(); + } } else { TRESPASS(); } diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp index b8524f6..c8798c6 100644 --- a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp @@ -23,6 +23,7 @@ #include "Parameters.h" #include "ParsedMessage.h" #include "rtp/RTPSender.h" +#include "TimeSyncer.h" #include <binder/IServiceManager.h> #include <gui/IGraphicBufferProducer.h> @@ -157,6 +158,12 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { } if (err == OK) { + sp<AMessage> notify = new AMessage(kWhatTimeSyncerNotify, id()); + mTimeSyncer = new TimeSyncer(mNetSession, notify); + looper()->registerHandler(mTimeSyncer); + + mTimeSyncer->startServer(8123); + mState = AWAITING_CLIENT_CONNECTION; } @@ -265,6 +272,11 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { break; } + case ANetworkSession::kWhatNetworkStall: + { + break; + } + default: TRESPASS(); } @@ -520,6 +532,11 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatTimeSyncerNotify: + { + break; + } + default: TRESPASS(); } diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.h b/media/libstagefright/wifi-display/source/WifiDisplaySource.h index 724462c..9e72682 100644 --- a/media/libstagefright/wifi-display/source/WifiDisplaySource.h +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.h @@ -30,6 +30,7 @@ namespace android { struct IHDCP; struct IRemoteDisplayClient; struct ParsedMessage; +struct TimeSyncer; // Represents the RTSP server acting as a wifi display source. // Manages incoming connections, sets up Playback sessions as necessary. @@ -81,6 +82,7 @@ private: kWhatHDCPNotify, kWhatFinishStop2, kWhatTeardownTriggerTimedOut, + kWhatTimeSyncerNotify, }; struct ResponseID { @@ -114,6 +116,7 @@ private: VideoFormats mSupportedSourceVideoFormats; sp<ANetworkSession> mNetSession; sp<IRemoteDisplayClient> mClient; + sp<TimeSyncer> mTimeSyncer; struct in_addr mInterfaceAddr; int32_t mSessionID; diff --git a/media/libstagefright/wifi-display/udptest.cpp b/media/libstagefright/wifi-display/udptest.cpp index 86437e0..111846d 100644 --- a/media/libstagefright/wifi-display/udptest.cpp +++ b/media/libstagefright/wifi-display/udptest.cpp @@ -19,292 +19,13 @@ #include <utils/Log.h> #include "ANetworkSession.h" +#include "TimeSyncer.h" #include <binder/ProcessState.h> -#include <media/stagefright/foundation/ABuffer.h> -#include <media/stagefright/foundation/ADebug.h> -#include <media/stagefright/foundation/AHandler.h> -#include <media/stagefright/foundation/ALooper.h> #include <media/stagefright/foundation/AMessage.h> -#include <media/stagefright/Utils.h> namespace android { -struct TestHandler : public AHandler { - TestHandler(const sp<ANetworkSession> &netSession); - - void startServer(unsigned localPort); - void startClient(const char *remoteHost, unsigned remotePort); - -protected: - virtual ~TestHandler(); - - virtual void onMessageReceived(const sp<AMessage> &msg); - -private: - enum { - kWhatStartServer, - kWhatStartClient, - kWhatUDPNotify, - kWhatSendPacket, - kWhatTimedOut, - }; - - struct TimeInfo { - int64_t mT1; // client timestamp at send - int64_t mT2; // server timestamp at receive - int64_t mT3; // server timestamp at send - int64_t mT4; // client timestamp at receive - }; - - static const int64_t kTimeoutDelayUs = 1000000ll; - - sp<ANetworkSession> mNetSession; - - bool mIsServer; - bool mConnected; - int32_t mUDPSession; - uint32_t mSeqNo; - double mTotalTimeUs; - int32_t mCount; - int64_t mSumOffsets; - - int64_t mPendingT1; - int32_t mTimeoutGeneration; - - void postSendPacket(int64_t delayUs = 0ll); - - void postTimeout(); - void cancelTimeout(); - - DISALLOW_EVIL_CONSTRUCTORS(TestHandler); -}; - -TestHandler::TestHandler(const sp<ANetworkSession> &netSession) - : mNetSession(netSession), - mIsServer(false), - mConnected(false), - mUDPSession(0), - mSeqNo(0), - mTotalTimeUs(0.0), - mCount(0), - mSumOffsets(0ll), - mPendingT1(0ll), - mTimeoutGeneration(0) { -} - -TestHandler::~TestHandler() { -} - -void TestHandler::startServer(unsigned localPort) { - sp<AMessage> msg = new AMessage(kWhatStartServer, id()); - msg->setInt32("localPort", localPort); - msg->post(); -} - -void TestHandler::startClient(const char *remoteHost, unsigned remotePort) { - sp<AMessage> msg = new AMessage(kWhatStartClient, id()); - msg->setString("remoteHost", remoteHost); - msg->setInt32("remotePort", remotePort); - msg->post(); -} - -void TestHandler::onMessageReceived(const sp<AMessage> &msg) { - switch (msg->what()) { - case kWhatStartClient: - { - AString remoteHost; - CHECK(msg->findString("remoteHost", &remoteHost)); - - int32_t remotePort; - CHECK(msg->findInt32("remotePort", &remotePort)); - - sp<AMessage> notify = new AMessage(kWhatUDPNotify, id()); - - CHECK_EQ((status_t)OK, - mNetSession->createUDPSession( - 0 /* localPort */, - remoteHost.c_str(), - remotePort, - notify, - &mUDPSession)); - - postSendPacket(); - break; - } - - case kWhatStartServer: - { - mIsServer = true; - - int32_t localPort; - CHECK(msg->findInt32("localPort", &localPort)); - - sp<AMessage> notify = new AMessage(kWhatUDPNotify, id()); - - CHECK_EQ((status_t)OK, - mNetSession->createUDPSession( - localPort, notify, &mUDPSession)); - - break; - } - - case kWhatSendPacket: - { - TimeInfo ti; - memset(&ti, 0, sizeof(ti)); - - ti.mT1 = ALooper::GetNowUs(); - - CHECK_EQ((status_t)OK, - mNetSession->sendRequest( - mUDPSession, &ti, sizeof(ti))); - - mPendingT1 = ti.mT1; - postTimeout(); - break; - } - - case kWhatTimedOut: - { - int32_t generation; - CHECK(msg->findInt32("generation", &generation)); - - if (generation != mTimeoutGeneration) { - break; - } - - ALOGI("timed out, sending another request"); - postSendPacket(); - break; - } - - case kWhatUDPNotify: - { - int32_t reason; - CHECK(msg->findInt32("reason", &reason)); - - switch (reason) { - case ANetworkSession::kWhatError: - { - int32_t sessionID; - CHECK(msg->findInt32("sessionID", &sessionID)); - - int32_t err; - CHECK(msg->findInt32("err", &err)); - - AString detail; - CHECK(msg->findString("detail", &detail)); - - ALOGE("An error occurred in session %d (%d, '%s/%s').", - sessionID, - err, - detail.c_str(), - strerror(-err)); - - mNetSession->destroySession(sessionID); - - cancelTimeout(); - looper()->stop(); - break; - } - - case ANetworkSession::kWhatDatagram: - { - int32_t sessionID; - CHECK(msg->findInt32("sessionID", &sessionID)); - - sp<ABuffer> packet; - CHECK(msg->findBuffer("data", &packet)); - - int64_t arrivalTimeUs; - CHECK(packet->meta()->findInt64( - "arrivalTimeUs", &arrivalTimeUs)); - - CHECK_EQ(packet->size(), sizeof(TimeInfo)); - - TimeInfo *ti = (TimeInfo *)packet->data(); - - if (mIsServer) { - if (!mConnected) { - AString fromAddr; - CHECK(msg->findString("fromAddr", &fromAddr)); - - int32_t fromPort; - CHECK(msg->findInt32("fromPort", &fromPort)); - - CHECK_EQ((status_t)OK, - mNetSession->connectUDPSession( - mUDPSession, fromAddr.c_str(), fromPort)); - - mConnected = true; - } - - ti->mT2 = arrivalTimeUs; - ti->mT3 = ALooper::GetNowUs(); - - CHECK_EQ((status_t)OK, - mNetSession->sendRequest( - mUDPSession, ti, sizeof(*ti))); - } else { - if (ti->mT1 != mPendingT1) { - break; - } - - cancelTimeout(); - mPendingT1 = 0; - - ti->mT4 = arrivalTimeUs; - - // One way delay for a packet to travel from client - // to server or back (assumed to be the same either way). - int64_t delay = - (ti->mT2 - ti->mT1 + ti->mT4 - ti->mT3) / 2; - - // Offset between the client clock (T1, T4) and the - // server clock (T2, T3) timestamps. - int64_t offset = - (ti->mT2 - ti->mT1 - ti->mT4 + ti->mT3) / 2; - - mSumOffsets += offset; - ++mCount; - - printf("delay = %lld us,\toffset %lld us\n", - delay, - offset); - fflush(stdout); - - postSendPacket(1000000ll / 30); - } - break; - } - - default: - TRESPASS(); - } - - break; - } - - default: - TRESPASS(); - } -} - -void TestHandler::postSendPacket(int64_t delayUs) { - (new AMessage(kWhatSendPacket, id()))->post(delayUs); -} - -void TestHandler::postTimeout() { - sp<AMessage> msg = new AMessage(kWhatTimedOut, id()); - msg->setInt32("generation", mTimeoutGeneration); - msg->post(kTimeoutDelayUs); -} - -void TestHandler::cancelTimeout() { - ++mTimeoutGeneration; -} - } // namespace android static void usage(const char *me) { @@ -379,7 +100,7 @@ int main(int argc, char **argv) { sp<ALooper> looper = new ALooper; - sp<TestHandler> handler = new TestHandler(netSession); + sp<TimeSyncer> handler = new TimeSyncer(netSession, NULL /* notify */); looper->registerHandler(handler); if (localPort >= 0) { diff --git a/media/libstagefright/wifi-display/wfd.cpp b/media/libstagefright/wifi-display/wfd.cpp index 3f4216a..0b18484 100644 --- a/media/libstagefright/wifi-display/wfd.cpp +++ b/media/libstagefright/wifi-display/wfd.cpp @@ -321,7 +321,10 @@ int main(int argc, char **argv) { sp<ALooper> looper = new ALooper; sp<WifiDisplaySink> sink = new WifiDisplaySink( - session, surface->getIGraphicBufferProducer()); + 0 /* flags */, + session, + surface->getIGraphicBufferProducer()); + looper->registerHandler(sink); if (connectToPort >= 0) { |