diff options
Diffstat (limited to 'media')
41 files changed, 3204 insertions, 146 deletions
diff --git a/media/libmedia/Android.mk b/media/libmedia/Android.mk index bcce063..76308e8 100644 --- a/media/libmedia/Android.mk +++ b/media/libmedia/Android.mk @@ -27,6 +27,8 @@ LOCAL_SRC_FILES:= \ IMediaRecorderClient.cpp \ IMediaPlayer.cpp \ IMediaRecorder.cpp \ + IRemoteDisplay.cpp \ + IRemoteDisplayClient.cpp \ IStreamSource.cpp \ Metadata.cpp \ mediarecorder.cpp \ diff --git a/media/libmedia/IMediaPlayerService.cpp b/media/libmedia/IMediaPlayerService.cpp index d3e2e19..c2ec439 100644 --- a/media/libmedia/IMediaPlayerService.cpp +++ b/media/libmedia/IMediaPlayerService.cpp @@ -24,9 +24,12 @@ #include <media/IMediaPlayerService.h> #include <media/IMediaRecorder.h> #include <media/IOMX.h> +#include <media/IRemoteDisplay.h> +#include <media/IRemoteDisplayClient.h> #include <media/IStreamSource.h> #include <utils/Errors.h> // for status_t +#include <utils/String8.h> namespace android { @@ -40,7 +43,8 @@ enum { MAKE_CRYPTO, ENABLE_REMOTE_DISPLAY, ADD_BATTERY_DATA, - PULL_BATTERY_DATA + PULL_BATTERY_DATA, + LISTEN_FOR_REMOTE_DISPLAY, }; class BpMediaPlayerService: public BpInterface<IMediaPlayerService> @@ -148,6 +152,17 @@ public: data.writeInterfaceToken(IMediaPlayerService::getInterfaceDescriptor()); return remote()->transact(PULL_BATTERY_DATA, data, reply); } + + virtual sp<IRemoteDisplay> listenForRemoteDisplay(const sp<IRemoteDisplayClient>& client, + const String8& iface) + { + Parcel data, reply; + data.writeInterfaceToken(IMediaPlayerService::getInterfaceDescriptor()); + data.writeStrongBinder(client->asBinder()); + data.writeString8(iface); + remote()->transact(LISTEN_FOR_REMOTE_DISPLAY, data, &reply); + return interface_cast<IRemoteDisplay>(reply.readStrongBinder()); + } }; IMPLEMENT_META_INTERFACE(MediaPlayerService, "android.media.IMediaPlayerService"); @@ -242,6 +257,15 @@ status_t BnMediaPlayerService::onTransact( pullBatteryData(reply); return NO_ERROR; } break; + case LISTEN_FOR_REMOTE_DISPLAY: { + CHECK_INTERFACE(IMediaPlayerService, data, reply); + sp<IRemoteDisplayClient> client( + interface_cast<IRemoteDisplayClient>(data.readStrongBinder())); + String8 iface(data.readString8()); + sp<IRemoteDisplay> display(listenForRemoteDisplay(client, iface)); + reply->writeStrongBinder(display->asBinder()); + return NO_ERROR; + } break; default: return BBinder::onTransact(code, data, reply, flags); } diff --git a/media/libmedia/IRemoteDisplay.cpp b/media/libmedia/IRemoteDisplay.cpp new file mode 100644 index 0000000..5d6ab34 --- /dev/null +++ b/media/libmedia/IRemoteDisplay.cpp @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdint.h> +#include <sys/types.h> + +#include <media/IRemoteDisplay.h> + +namespace android { + +enum { + DISCONNECT = IBinder::FIRST_CALL_TRANSACTION, +}; + +class BpRemoteDisplay: public BpInterface<IRemoteDisplay> +{ +public: + BpRemoteDisplay(const sp<IBinder>& impl) + : BpInterface<IRemoteDisplay>(impl) + { + } + + status_t disconnect() + { + Parcel data, reply; + data.writeInterfaceToken(IRemoteDisplay::getInterfaceDescriptor()); + remote()->transact(DISCONNECT, data, &reply); + return reply.readInt32(); + } +}; + +IMPLEMENT_META_INTERFACE(RemoteDisplay, "android.media.IRemoteDisplay"); + +// ---------------------------------------------------------------------- + +status_t BnRemoteDisplay::onTransact( + uint32_t code, const Parcel& data, Parcel* reply, uint32_t flags) +{ + switch (code) { + case DISCONNECT: { + CHECK_INTERFACE(IRemoteDisplay, data, reply); + reply->writeInt32(disconnect()); + return NO_ERROR; + } + default: + return BBinder::onTransact(code, data, reply, flags); + } +} + +}; // namespace android diff --git a/media/libmedia/IRemoteDisplayClient.cpp b/media/libmedia/IRemoteDisplayClient.cpp new file mode 100644 index 0000000..4a1b570 --- /dev/null +++ b/media/libmedia/IRemoteDisplayClient.cpp @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdint.h> +#include <sys/types.h> + +#include <media/IRemoteDisplayClient.h> +#include <gui/ISurfaceTexture.h> +#include <utils/String8.h> + +namespace android { + +enum { + ON_DISPLAY_CONNECTED = IBinder::FIRST_CALL_TRANSACTION, + ON_DISPLAY_DISCONNECTED, + ON_DISPLAY_ERROR, +}; + +class BpRemoteDisplayClient: public BpInterface<IRemoteDisplayClient> +{ +public: + BpRemoteDisplayClient(const sp<IBinder>& impl) + : BpInterface<IRemoteDisplayClient>(impl) + { + } + + void onDisplayConnected(const sp<ISurfaceTexture>& surfaceTexture, + uint32_t width, uint32_t height, uint32_t flags) + { + Parcel data, reply; + data.writeInterfaceToken(IRemoteDisplayClient::getInterfaceDescriptor()); + data.writeStrongBinder(surfaceTexture->asBinder()); + data.writeInt32(width); + data.writeInt32(height); + data.writeInt32(flags); + remote()->transact(ON_DISPLAY_CONNECTED, data, &reply, IBinder::FLAG_ONEWAY); + } + + void onDisplayDisconnected() + { + Parcel data, reply; + data.writeInterfaceToken(IRemoteDisplayClient::getInterfaceDescriptor()); + remote()->transact(ON_DISPLAY_DISCONNECTED, data, &reply, IBinder::FLAG_ONEWAY); + } + + void onDisplayError(int32_t error) + { + Parcel data, reply; + data.writeInterfaceToken(IRemoteDisplayClient::getInterfaceDescriptor()); + data.writeInt32(error); + remote()->transact(ON_DISPLAY_ERROR, data, &reply, IBinder::FLAG_ONEWAY); + } +}; + +IMPLEMENT_META_INTERFACE(RemoteDisplayClient, "android.media.IRemoteDisplayClient"); + +// ---------------------------------------------------------------------- + +status_t BnRemoteDisplayClient::onTransact( + uint32_t code, const Parcel& data, Parcel* reply, uint32_t flags) +{ + switch (code) { + case ON_DISPLAY_CONNECTED: { + CHECK_INTERFACE(IRemoteDisplayClient, data, reply); + sp<ISurfaceTexture> surfaceTexture( + interface_cast<ISurfaceTexture>(data.readStrongBinder())); + uint32_t width = data.readInt32(); + uint32_t height = data.readInt32(); + uint32_t flags = data.readInt32(); + onDisplayConnected(surfaceTexture, width, height, flags); + return NO_ERROR; + } + case ON_DISPLAY_DISCONNECTED: { + CHECK_INTERFACE(IRemoteDisplayClient, data, reply); + onDisplayDisconnected(); + return NO_ERROR; + } + case ON_DISPLAY_ERROR: { + CHECK_INTERFACE(IRemoteDisplayClient, data, reply); + int32_t error = data.readInt32(); + onDisplayError(error); + return NO_ERROR; + } + default: + return BBinder::onTransact(code, data, reply, flags); + } +} + +}; // namespace android diff --git a/media/libmediaplayerservice/MediaPlayerService.cpp b/media/libmediaplayerservice/MediaPlayerService.cpp index 166bae9..9005500 100644 --- a/media/libmediaplayerservice/MediaPlayerService.cpp +++ b/media/libmediaplayerservice/MediaPlayerService.cpp @@ -44,6 +44,8 @@ #include <utils/SystemClock.h> #include <utils/Vector.h> +#include <media/IRemoteDisplay.h> +#include <media/IRemoteDisplayClient.h> #include <media/MediaPlayerInterface.h> #include <media/mediarecorder.h> #include <media/MediaMetadataRetrieverInterface.h> @@ -279,6 +281,11 @@ sp<ICrypto> MediaPlayerService::makeCrypto() { return new Crypto; } +sp<IRemoteDisplay> MediaPlayerService::listenForRemoteDisplay( + const sp<IRemoteDisplayClient>& client, const String8& iface) { + return new RemoteDisplay(client, iface.string());; +} + status_t MediaPlayerService::enableRemoteDisplay(const char *iface) { Mutex::Autolock autoLock(mLock); @@ -287,20 +294,12 @@ status_t MediaPlayerService::enableRemoteDisplay(const char *iface) { return INVALID_OPERATION; } - mRemoteDisplay = new RemoteDisplay; - - status_t err = mRemoteDisplay->start(iface); - - if (err != OK) { - mRemoteDisplay.clear(); - return err; - } - + mRemoteDisplay = new RemoteDisplay(NULL /* client */, iface); return OK; } if (mRemoteDisplay != NULL) { - mRemoteDisplay->stop(); + mRemoteDisplay->disconnect(); mRemoteDisplay.clear(); } diff --git a/media/libmediaplayerservice/MediaPlayerService.h b/media/libmediaplayerservice/MediaPlayerService.h index 2577c58..ca8a96f 100644 --- a/media/libmediaplayerservice/MediaPlayerService.h +++ b/media/libmediaplayerservice/MediaPlayerService.h @@ -41,6 +41,8 @@ class AudioTrack; class IMediaRecorder; class IMediaMetadataRetriever; class IOMX; +class IRemoteDisplay; +class IRemoteDisplayClient; class MediaRecorderClient; struct RemoteDisplay; @@ -248,6 +250,9 @@ public: virtual sp<IMemory> decode(int fd, int64_t offset, int64_t length, uint32_t *pSampleRate, int* pNumChannels, audio_format_t* pFormat); virtual sp<IOMX> getOMX(); virtual sp<ICrypto> makeCrypto(); + + virtual sp<IRemoteDisplay> listenForRemoteDisplay(const sp<IRemoteDisplayClient>& client, + const String8& iface); virtual status_t enableRemoteDisplay(const char *iface); virtual status_t dump(int fd, const Vector<String16>& args); diff --git a/media/libmediaplayerservice/RemoteDisplay.cpp b/media/libmediaplayerservice/RemoteDisplay.cpp index 49f7278..1cc605e 100644 --- a/media/libmediaplayerservice/RemoteDisplay.cpp +++ b/media/libmediaplayerservice/RemoteDisplay.cpp @@ -19,29 +19,27 @@ #include "ANetworkSession.h" #include "source/WifiDisplaySource.h" +#include <media/IRemoteDisplayClient.h> + namespace android { -RemoteDisplay::RemoteDisplay() - : mInitCheck(NO_INIT), - mLooper(new ALooper), +RemoteDisplay::RemoteDisplay( + const sp<IRemoteDisplayClient> &client, const char *iface) + : mLooper(new ALooper), mNetSession(new ANetworkSession), - mSource(new WifiDisplaySource(mNetSession)) { + mSource(new WifiDisplaySource(mNetSession, client)) { mLooper->registerHandler(mSource); -} - -RemoteDisplay::~RemoteDisplay() { -} -status_t RemoteDisplay::start(const char *iface) { mNetSession->start(); mLooper->start(); mSource->start(iface); +} - return OK; +RemoteDisplay::~RemoteDisplay() { } -status_t RemoteDisplay::stop() { +status_t RemoteDisplay::disconnect() { mSource->stop(); mLooper->stop(); @@ -51,4 +49,3 @@ status_t RemoteDisplay::stop() { } } // namespace android - diff --git a/media/libmediaplayerservice/RemoteDisplay.h b/media/libmediaplayerservice/RemoteDisplay.h index 3607d06..63c5286 100644 --- a/media/libmediaplayerservice/RemoteDisplay.h +++ b/media/libmediaplayerservice/RemoteDisplay.h @@ -18,6 +18,7 @@ #define REMOTE_DISPLAY_H_ +#include <media/IRemoteDisplay.h> #include <media/stagefright/foundation/ABase.h> #include <utils/Errors.h> #include <utils/RefBase.h> @@ -26,20 +27,18 @@ namespace android { struct ALooper; struct ANetworkSession; +struct IRemoteDisplayClient; struct WifiDisplaySource; -struct RemoteDisplay : public RefBase { - RemoteDisplay(); +struct RemoteDisplay : public BnRemoteDisplay { + RemoteDisplay(const sp<IRemoteDisplayClient> &client, const char *iface); - status_t start(const char *iface); - status_t stop(); + virtual status_t disconnect(); protected: virtual ~RemoteDisplay(); private: - status_t mInitCheck; - sp<ALooper> mNetLooper; sp<ALooper> mLooper; sp<ANetworkSession> mNetSession; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp index a02732b..dc1e351 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp @@ -68,7 +68,8 @@ NuPlayer::NuPlayer() mSkipRenderingVideoUntilMediaTimeUs(-1ll), mVideoLateByUs(0ll), mNumFramesTotal(0ll), - mNumFramesDropped(0ll) { + mNumFramesDropped(0ll), + mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW) { } NuPlayer::~NuPlayer() { @@ -217,6 +218,9 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { CHECK(msg->findObject("native-window", &obj)); mNativeWindow = static_cast<NativeWindowWrapper *>(obj.get()); + + // XXX - ignore error from setVideoScalingMode for now + setVideoScalingMode(mVideoScalingMode); break; } @@ -293,8 +297,8 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { break; } - if (mAudioDecoder == NULL && mAudioSink != NULL || - mVideoDecoder == NULL && mNativeWindow != NULL) { + if ((mAudioDecoder == NULL && mAudioSink != NULL) + || (mVideoDecoder == NULL && mNativeWindow != NULL)) { msg->post(100000ll); mScanSourcesPending = true; } @@ -957,4 +961,18 @@ sp<AMessage> NuPlayer::Source::getFormat(bool audio) { return NULL; } +status_t NuPlayer::setVideoScalingMode(int32_t mode) { + mVideoScalingMode = mode; + if (mNativeWindow != NULL) { + status_t ret = native_window_set_scaling_mode( + mNativeWindow->getNativeWindow().get(), mVideoScalingMode); + if (ret != OK) { + ALOGE("Failed to set scaling mode (%d): %s", + -ret, strerror(-ret)); + return ret; + } + } + return OK; +} + } // namespace android diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h index 996806e..36d3a9c 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h @@ -55,6 +55,8 @@ struct NuPlayer : public AHandler { // Will notify the driver through "notifySeekComplete" once finished. void seekToAsync(int64_t seekTimeUs); + status_t setVideoScalingMode(int32_t mode); + protected: virtual ~NuPlayer(); @@ -130,6 +132,8 @@ private: int64_t mVideoLateByUs; int64_t mNumFramesTotal, mNumFramesDropped; + int32_t mVideoScalingMode; + status_t instantiateDecoder(bool audio, sp<Decoder> *decoder); status_t feedDecoderInputData(bool audio, const sp<AMessage> &msg); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp index 441cbf3..d03601f 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp @@ -259,7 +259,29 @@ player_type NuPlayerDriver::playerType() { } status_t NuPlayerDriver::invoke(const Parcel &request, Parcel *reply) { - return INVALID_OPERATION; + if (reply == NULL) { + ALOGE("reply is a NULL pointer"); + return BAD_VALUE; + } + + int32_t methodId; + status_t ret = request.readInt32(&methodId); + if (ret != OK) { + ALOGE("Failed to retrieve the requested method to invoke"); + return ret; + } + + switch (methodId) { + case INVOKE_ID_SET_VIDEO_SCALING_MODE: + { + int mode = request.readInt32(); + return mPlayer->setVideoScalingMode(mode); + } + default: + { + return INVALID_OPERATION; + } + } } void NuPlayerDriver::setAudioSink(const sp<AudioSink> &audioSink) { diff --git a/media/libmediaplayerservice/nuplayer/StreamingSource.cpp b/media/libmediaplayerservice/nuplayer/StreamingSource.cpp index b696aa4..a1fd2ed 100644 --- a/media/libmediaplayerservice/nuplayer/StreamingSource.cpp +++ b/media/libmediaplayerservice/nuplayer/StreamingSource.cpp @@ -42,7 +42,15 @@ NuPlayer::StreamingSource::~StreamingSource() { void NuPlayer::StreamingSource::start() { mStreamListener = new NuPlayerStreamListener(mSource, 0); - mTSParser = new ATSParser(ATSParser::TS_TIMESTAMPS_ARE_ABSOLUTE); + + uint32_t sourceFlags = mSource->flags(); + + uint32_t parserFlags = ATSParser::TS_TIMESTAMPS_ARE_ABSOLUTE; + if (sourceFlags & IStreamSource::kFlagAlignedVideoData) { + parserFlags |= ATSParser::ALIGNED_VIDEO_DATA; + } + + mTSParser = new ATSParser(parserFlags); mStreamListener->start(); } @@ -138,7 +146,17 @@ status_t NuPlayer::StreamingSource::dequeueAccessUnit( return finalResult == OK ? -EWOULDBLOCK : finalResult; } - return source->dequeueAccessUnit(accessUnit); + status_t err = source->dequeueAccessUnit(accessUnit); + +#if !defined(LOG_NDEBUG) || LOG_NDEBUG == 0 + if (err == OK) { + int64_t timeUs; + CHECK((*accessUnit)->meta()->findInt64("timeUs", &timeUs)); + ALOGV("dequeueAccessUnit timeUs=%lld us", timeUs); + } +#endif + + return err; } } // namespace android diff --git a/media/libmediaplayerservice/nuplayer/mp4/MP4Source.cpp b/media/libmediaplayerservice/nuplayer/mp4/MP4Source.cpp index c80d13f..ffb3a65 100644 --- a/media/libmediaplayerservice/nuplayer/mp4/MP4Source.cpp +++ b/media/libmediaplayerservice/nuplayer/mp4/MP4Source.cpp @@ -93,6 +93,10 @@ struct StreamSource : public FragmentedMP4Parser::Source { return total; } + bool isSeekable() { + return false; + } + private: sp<NuPlayer::NuPlayerStreamListener> mListener; off64_t mPosition; diff --git a/media/libstagefright/Android.mk b/media/libstagefright/Android.mk index 1522e75..f40982e 100644 --- a/media/libstagefright/Android.mk +++ b/media/libstagefright/Android.mk @@ -19,6 +19,7 @@ LOCAL_SRC_FILES:= \ ESDS.cpp \ FileSource.cpp \ FLACExtractor.cpp \ + FragmentedMP4Extractor.cpp \ HTTPBase.cpp \ JPEGSource.cpp \ MP3Extractor.cpp \ diff --git a/media/libstagefright/DRMExtractor.cpp b/media/libstagefright/DRMExtractor.cpp index 524c3aa..63cb430 100644 --- a/media/libstagefright/DRMExtractor.cpp +++ b/media/libstagefright/DRMExtractor.cpp @@ -15,11 +15,6 @@ */ #include "include/DRMExtractor.h" -#include "include/AMRExtractor.h" -#include "include/MP3Extractor.h" -#include "include/MPEG4Extractor.h" -#include "include/WAVExtractor.h" -#include "include/OggExtractor.h" #include <arpa/inet.h> #include <utils/String8.h> diff --git a/media/libstagefright/DataSource.cpp b/media/libstagefright/DataSource.cpp index 1de808e..9d0eea2 100644 --- a/media/libstagefright/DataSource.cpp +++ b/media/libstagefright/DataSource.cpp @@ -20,17 +20,18 @@ #include "include/chromium_http_stub.h" #endif +#include "include/AACExtractor.h" +#include "include/DRMExtractor.h" +#include "include/FLACExtractor.h" +#include "include/FragmentedMP4Extractor.h" +#include "include/HTTPBase.h" #include "include/MP3Extractor.h" -#include "include/MPEG4Extractor.h" -#include "include/WAVExtractor.h" -#include "include/OggExtractor.h" #include "include/MPEG2PSExtractor.h" #include "include/MPEG2TSExtractor.h" +#include "include/MPEG4Extractor.h" #include "include/NuCachedSource2.h" -#include "include/HTTPBase.h" -#include "include/DRMExtractor.h" -#include "include/FLACExtractor.h" -#include "include/AACExtractor.h" +#include "include/OggExtractor.h" +#include "include/WAVExtractor.h" #include "include/WVMExtractor.h" #include "matroska/MatroskaExtractor.h" @@ -110,6 +111,7 @@ void DataSource::RegisterSniffer(SnifferFunc func) { // static void DataSource::RegisterDefaultSniffers() { RegisterSniffer(SniffMPEG4); + RegisterSniffer(SniffFragmentedMP4); RegisterSniffer(SniffMatroska); RegisterSniffer(SniffOgg); RegisterSniffer(SniffWAV); diff --git a/media/libstagefright/FragmentedMP4Extractor.cpp b/media/libstagefright/FragmentedMP4Extractor.cpp new file mode 100644 index 0000000..82712ef --- /dev/null +++ b/media/libstagefright/FragmentedMP4Extractor.cpp @@ -0,0 +1,460 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "FragmentedMP4Extractor" +#include <utils/Log.h> + +#include "include/FragmentedMP4Extractor.h" +#include "include/SampleTable.h" +#include "include/ESDS.h" + +#include <arpa/inet.h> + +#include <ctype.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include <cutils/properties.h> // for property_get + +#include <media/stagefright/foundation/ABitReader.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/DataSource.h> +#include <media/stagefright/MediaBuffer.h> +#include <media/stagefright/MediaBufferGroup.h> +#include <media/stagefright/MediaDefs.h> +#include <media/stagefright/MediaSource.h> +#include <media/stagefright/MetaData.h> +#include <media/stagefright/Utils.h> +#include <utils/String8.h> + +namespace android { + +class FragmentedMPEG4Source : public MediaSource { +public: + // Caller retains ownership of the Parser + FragmentedMPEG4Source(bool audio, + const sp<MetaData> &format, + const sp<FragmentedMP4Parser> &parser, + const sp<FragmentedMP4Extractor> &extractor); + + virtual status_t start(MetaData *params = NULL); + virtual status_t stop(); + + virtual sp<MetaData> getFormat(); + + virtual status_t read( + MediaBuffer **buffer, const ReadOptions *options = NULL); + +protected: + virtual ~FragmentedMPEG4Source(); + +private: + Mutex mLock; + + sp<MetaData> mFormat; + sp<FragmentedMP4Parser> mParser; + sp<FragmentedMP4Extractor> mExtractor; + bool mIsAudioTrack; + uint32_t mCurrentSampleIndex; + + bool mIsAVC; + size_t mNALLengthSize; + + bool mStarted; + + MediaBufferGroup *mGroup; + + bool mWantsNALFragments; + + uint8_t *mSrcBuffer; + + FragmentedMPEG4Source(const FragmentedMPEG4Source &); + FragmentedMPEG4Source &operator=(const FragmentedMPEG4Source &); +}; + + +FragmentedMP4Extractor::FragmentedMP4Extractor(const sp<DataSource> &source) + : mLooper(new ALooper), + mParser(new FragmentedMP4Parser()), + mDataSource(source), + mInitCheck(NO_INIT), + mFileMetaData(new MetaData) { + ALOGV("FragmentedMP4Extractor"); + mLooper->registerHandler(mParser); + mLooper->start(false /* runOnCallingThread */); + mParser->start(mDataSource); + + bool hasVideo = mParser->getFormat(false /* audio */, true /* synchronous */) != NULL; + bool hasAudio = mParser->getFormat(true /* audio */, true /* synchronous */) != NULL; + + ALOGV("number of tracks: %d", countTracks()); + + if (hasVideo) { + mFileMetaData->setCString( + kKeyMIMEType, MEDIA_MIMETYPE_CONTAINER_MPEG4); + } else if (hasAudio) { + mFileMetaData->setCString(kKeyMIMEType, "audio/mp4"); + } else { + ALOGE("no audio and no video, no idea what file type this is"); + } + // tracks are numbered such that video track is first, audio track is second + if (hasAudio && hasVideo) { + mTrackCount = 2; + mAudioTrackIndex = 1; + } else if (hasAudio) { + mTrackCount = 1; + mAudioTrackIndex = 0; + } else if (hasVideo) { + mTrackCount = 1; + mAudioTrackIndex = -1; + } else { + mTrackCount = 0; + mAudioTrackIndex = -1; + } +} + +FragmentedMP4Extractor::~FragmentedMP4Extractor() { + ALOGV("~FragmentedMP4Extractor"); + mLooper->stop(); +} + +uint32_t FragmentedMP4Extractor::flags() const { + return CAN_PAUSE | + (mParser->isSeekable() ? (CAN_SEEK_BACKWARD | CAN_SEEK_FORWARD | CAN_SEEK) : 0); +} + +sp<MetaData> FragmentedMP4Extractor::getMetaData() { + return mFileMetaData; +} + +size_t FragmentedMP4Extractor::countTracks() { + return mTrackCount; +} + + +sp<MetaData> FragmentedMP4Extractor::getTrackMetaData( + size_t index, uint32_t flags) { + if (index >= countTracks()) { + return NULL; + } + + sp<AMessage> msg = mParser->getFormat(index == mAudioTrackIndex, true /* synchronous */); + + if (msg == NULL) { + ALOGV("got null format for track %d", index); + return NULL; + } + + sp<MetaData> meta = new MetaData(); + convertMessageToMetaData(msg, meta); + return meta; +} + +static void MakeFourCCString(uint32_t x, char *s) { + s[0] = x >> 24; + s[1] = (x >> 16) & 0xff; + s[2] = (x >> 8) & 0xff; + s[3] = x & 0xff; + s[4] = '\0'; +} + +sp<MediaSource> FragmentedMP4Extractor::getTrack(size_t index) { + if (index >= countTracks()) { + return NULL; + } + return new FragmentedMPEG4Source(index == mAudioTrackIndex, getTrackMetaData(index, 0), mParser, this); +} + + +//////////////////////////////////////////////////////////////////////////////// + +FragmentedMPEG4Source::FragmentedMPEG4Source( + bool audio, + const sp<MetaData> &format, + const sp<FragmentedMP4Parser> &parser, + const sp<FragmentedMP4Extractor> &extractor) + : mFormat(format), + mParser(parser), + mExtractor(extractor), + mIsAudioTrack(audio), + mStarted(false), + mGroup(NULL), + mWantsNALFragments(false), + mSrcBuffer(NULL) { +} + +FragmentedMPEG4Source::~FragmentedMPEG4Source() { + if (mStarted) { + stop(); + } +} + +status_t FragmentedMPEG4Source::start(MetaData *params) { + Mutex::Autolock autoLock(mLock); + + CHECK(!mStarted); + + int32_t val; + if (params && params->findInt32(kKeyWantsNALFragments, &val) + && val != 0) { + mWantsNALFragments = true; + } else { + mWantsNALFragments = false; + } + ALOGV("caller wants NAL fragments: %s", mWantsNALFragments ? "yes" : "no"); + + mGroup = new MediaBufferGroup; + + int32_t max_size = 65536; + // XXX CHECK(mFormat->findInt32(kKeyMaxInputSize, &max_size)); + + mGroup->add_buffer(new MediaBuffer(max_size)); + + mSrcBuffer = new uint8_t[max_size]; + + mStarted = true; + + return OK; +} + +status_t FragmentedMPEG4Source::stop() { + Mutex::Autolock autoLock(mLock); + + CHECK(mStarted); + + delete[] mSrcBuffer; + mSrcBuffer = NULL; + + delete mGroup; + mGroup = NULL; + + mStarted = false; + mCurrentSampleIndex = 0; + + return OK; +} + +sp<MetaData> FragmentedMPEG4Source::getFormat() { + Mutex::Autolock autoLock(mLock); + + return mFormat; +} + + +status_t FragmentedMPEG4Source::read( + MediaBuffer **out, const ReadOptions *options) { + int64_t seekTimeUs; + ReadOptions::SeekMode mode; + if (options && options->getSeekTo(&seekTimeUs, &mode)) { + mParser->seekTo(mIsAudioTrack, seekTimeUs); + } + MediaBuffer *buffer = NULL; + mGroup->acquire_buffer(&buffer); + sp<ABuffer> parseBuffer; + + status_t ret = mParser->dequeueAccessUnit(mIsAudioTrack, &parseBuffer, true /* synchronous */); + if (ret != OK) { + buffer->release(); + ALOGV("returning %d", ret); + return ret; + } + sp<AMessage> meta = parseBuffer->meta(); + int64_t timeUs; + CHECK(meta->findInt64("timeUs", &timeUs)); + buffer->meta_data()->setInt64(kKeyTime, timeUs); + buffer->set_range(0, parseBuffer->size()); + memcpy(buffer->data(), parseBuffer->data(), parseBuffer->size()); + *out = buffer; + return OK; +} + + +static bool isCompatibleBrand(uint32_t fourcc) { + static const uint32_t kCompatibleBrands[] = { + FOURCC('i', 's', 'o', 'm'), + FOURCC('i', 's', 'o', '2'), + FOURCC('a', 'v', 'c', '1'), + FOURCC('3', 'g', 'p', '4'), + FOURCC('m', 'p', '4', '1'), + FOURCC('m', 'p', '4', '2'), + + // Won't promise that the following file types can be played. + // Just give these file types a chance. + FOURCC('q', 't', ' ', ' '), // Apple's QuickTime + FOURCC('M', 'S', 'N', 'V'), // Sony's PSP + + FOURCC('3', 'g', '2', 'a'), // 3GPP2 + FOURCC('3', 'g', '2', 'b'), + }; + + for (size_t i = 0; + i < sizeof(kCompatibleBrands) / sizeof(kCompatibleBrands[0]); + ++i) { + if (kCompatibleBrands[i] == fourcc) { + return true; + } + } + + return false; +} + +// Attempt to actually parse the 'ftyp' atom and determine if a suitable +// compatible brand is present. +// Also try to identify where this file's metadata ends +// (end of the 'moov' atom) and report it to the caller as part of +// the metadata. +static bool Sniff( + const sp<DataSource> &source, String8 *mimeType, float *confidence, + sp<AMessage> *meta) { + // We scan up to 128k bytes to identify this file as an MP4. + static const off64_t kMaxScanOffset = 128ll * 1024ll; + + off64_t offset = 0ll; + bool foundGoodFileType = false; + bool isFragmented = false; + off64_t moovAtomEndOffset = -1ll; + bool done = false; + + while (!done && offset < kMaxScanOffset) { + uint32_t hdr[2]; + if (source->readAt(offset, hdr, 8) < 8) { + return false; + } + + uint64_t chunkSize = ntohl(hdr[0]); + uint32_t chunkType = ntohl(hdr[1]); + off64_t chunkDataOffset = offset + 8; + + if (chunkSize == 1) { + if (source->readAt(offset + 8, &chunkSize, 8) < 8) { + return false; + } + + chunkSize = ntoh64(chunkSize); + chunkDataOffset += 8; + + if (chunkSize < 16) { + // The smallest valid chunk is 16 bytes long in this case. + return false; + } + } else if (chunkSize < 8) { + // The smallest valid chunk is 8 bytes long. + return false; + } + + off64_t chunkDataSize = offset + chunkSize - chunkDataOffset; + + char chunkstring[5]; + MakeFourCCString(chunkType, chunkstring); + ALOGV("saw chunk type %s, size %lld @ %lld", chunkstring, chunkSize, offset); + switch (chunkType) { + case FOURCC('f', 't', 'y', 'p'): + { + if (chunkDataSize < 8) { + return false; + } + + uint32_t numCompatibleBrands = (chunkDataSize - 8) / 4; + for (size_t i = 0; i < numCompatibleBrands + 2; ++i) { + if (i == 1) { + // Skip this index, it refers to the minorVersion, + // not a brand. + continue; + } + + uint32_t brand; + if (source->readAt( + chunkDataOffset + 4 * i, &brand, 4) < 4) { + return false; + } + + brand = ntohl(brand); + char brandstring[5]; + MakeFourCCString(brand, brandstring); + ALOGV("Brand: %s", brandstring); + + if (isCompatibleBrand(brand)) { + foundGoodFileType = true; + break; + } + } + + if (!foundGoodFileType) { + return false; + } + + break; + } + + case FOURCC('m', 'o', 'o', 'v'): + { + moovAtomEndOffset = offset + chunkSize; + break; + } + + case FOURCC('m', 'o', 'o', 'f'): + { + // this is kind of broken, since we might not actually find a + // moof box in the first 128k. + isFragmented = true; + done = true; + break; + } + + default: + break; + } + + offset += chunkSize; + } + + if (!foundGoodFileType || !isFragmented) { + return false; + } + + *mimeType = MEDIA_MIMETYPE_CONTAINER_MPEG4; + *confidence = 0.5f; // slightly more than MPEG4Extractor + + if (moovAtomEndOffset >= 0) { + *meta = new AMessage; + (*meta)->setInt64("meta-data-size", moovAtomEndOffset); + (*meta)->setInt32("fragmented", 1); // tell MediaExtractor what to instantiate + + ALOGV("found metadata size: %lld", moovAtomEndOffset); + } + + return true; +} + +// used by DataSource::RegisterDefaultSniffers +bool SniffFragmentedMP4( + const sp<DataSource> &source, String8 *mimeType, float *confidence, + sp<AMessage> *meta) { + ALOGV("SniffFragmentedMP4"); + char prop[PROPERTY_VALUE_MAX]; + if (property_get("media.stagefright.use-fragmp4", prop, NULL) + && (!strcmp(prop, "1") || !strcasecmp(prop, "true"))) { + return Sniff(source, mimeType, confidence, meta); + } + + return false; +} + +} // namespace android diff --git a/media/libstagefright/MPEG4Extractor.cpp b/media/libstagefright/MPEG4Extractor.cpp index a572541..7d49ef0 100644 --- a/media/libstagefright/MPEG4Extractor.cpp +++ b/media/libstagefright/MPEG4Extractor.cpp @@ -14,6 +14,7 @@ * limitations under the License. */ +//#define LOG_NDEBUG 0 #define LOG_TAG "MPEG4Extractor" #include <utils/Log.h> @@ -408,7 +409,7 @@ char* MPEG4Extractor::getDrmTrackInfo(size_t trackID, int *len) { } // Reads an encoded integer 7 bits at a time until it encounters the high bit clear. -int32_t readSize(off64_t offset, +static int32_t readSize(off64_t offset, const sp<DataSource> DataSource, uint8_t *numOfBytes) { uint32_t size = 0; uint8_t data; diff --git a/media/libstagefright/MediaExtractor.cpp b/media/libstagefright/MediaExtractor.cpp index 9ab6611..b18c916 100644 --- a/media/libstagefright/MediaExtractor.cpp +++ b/media/libstagefright/MediaExtractor.cpp @@ -21,6 +21,7 @@ #include "include/AMRExtractor.h" #include "include/MP3Extractor.h" #include "include/MPEG4Extractor.h" +#include "include/FragmentedMP4Extractor.h" #include "include/WAVExtractor.h" #include "include/OggExtractor.h" #include "include/MPEG2PSExtractor.h" @@ -93,7 +94,12 @@ sp<MediaExtractor> MediaExtractor::Create( MediaExtractor *ret = NULL; if (!strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_MPEG4) || !strcasecmp(mime, "audio/mp4")) { - ret = new MPEG4Extractor(source); + int fragmented = 0; + if (meta != NULL && meta->findInt32("fragmented", &fragmented) && fragmented) { + ret = new FragmentedMP4Extractor(source); + } else { + ret = new MPEG4Extractor(source); + } } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_MPEG)) { ret = new MP3Extractor(source, meta); } else if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AMR_NB) diff --git a/media/libstagefright/MetaData.cpp b/media/libstagefright/MetaData.cpp index 755594a..a01ec97 100644 --- a/media/libstagefright/MetaData.cpp +++ b/media/libstagefright/MetaData.cpp @@ -22,6 +22,8 @@ #include <string.h> #include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AString.h> +#include <media/stagefright/foundation/hexdump.h> #include <media/stagefright/MetaData.h> namespace android { @@ -318,6 +320,12 @@ String8 MetaData::typed_data::asString() const { default: out = String8::format("(unknown type %d, size %d)", mType, mSize); + if (mSize <= 48) { // if it's less than three lines of hex data, dump it + AString foo; + hexdump(data, mSize, 0, &foo); + out.append("\n"); + out.append(foo.c_str()); + } break; } return out; diff --git a/media/libstagefright/SurfaceMediaSource.cpp b/media/libstagefright/SurfaceMediaSource.cpp index c478b28..867f76d 100644 --- a/media/libstagefright/SurfaceMediaSource.cpp +++ b/media/libstagefright/SurfaceMediaSource.cpp @@ -167,6 +167,10 @@ status_t SurfaceMediaSource::start(MetaData *params) return OK; } +status_t SurfaceMediaSource::setMaxAcquiredBufferCount(size_t count) { + return mBufferQueue->setMaxAcquiredBufferCount(count); +} + status_t SurfaceMediaSource::stop() { diff --git a/media/libstagefright/Utils.cpp b/media/libstagefright/Utils.cpp index 2a16f66..74e9222 100644 --- a/media/libstagefright/Utils.cpp +++ b/media/libstagefright/Utils.cpp @@ -241,5 +241,196 @@ status_t convertMetaDataToMessage( return OK; } +static size_t reassembleAVCC(const sp<ABuffer> &csd0, const sp<ABuffer> csd1, char *avcc) { + + avcc[0] = 1; // version + avcc[1] = 0x64; // profile + avcc[2] = 0; // unused (?) + avcc[3] = 0xd; // level + avcc[4] = 0xff; // reserved+size + + size_t i = 0; + int numparams = 0; + int lastparamoffset = 0; + int avccidx = 6; + do { + if (i >= csd0->size() - 4 || + memcmp(csd0->data() + i, "\x00\x00\x00\x01", 4) == 0) { + if (i >= csd0->size() - 4) { + // there can't be another param here, so use all the rest + i = csd0->size(); + } + ALOGV("block at %d, last was %d", i, lastparamoffset); + if (lastparamoffset > 0) { + int size = i - lastparamoffset; + avcc[avccidx++] = size >> 8; + avcc[avccidx++] = size & 0xff; + memcpy(avcc+avccidx, csd0->data() + lastparamoffset, size); + avccidx += size; + numparams++; + } + i += 4; + lastparamoffset = i; + } else { + i++; + } + } while(i < csd0->size()); + ALOGV("csd0 contains %d params", numparams); + + avcc[5] = 0xe0 | numparams; + //and now csd-1 + i = 0; + numparams = 0; + lastparamoffset = 0; + int numpicparamsoffset = avccidx; + avccidx++; + do { + if (i >= csd1->size() - 4 || + memcmp(csd1->data() + i, "\x00\x00\x00\x01", 4) == 0) { + if (i >= csd1->size() - 4) { + // there can't be another param here, so use all the rest + i = csd1->size(); + } + ALOGV("block at %d, last was %d", i, lastparamoffset); + if (lastparamoffset > 0) { + int size = i - lastparamoffset; + avcc[avccidx++] = size >> 8; + avcc[avccidx++] = size & 0xff; + memcpy(avcc+avccidx, csd1->data() + lastparamoffset, size); + avccidx += size; + numparams++; + } + i += 4; + lastparamoffset = i; + } else { + i++; + } + } while(i < csd1->size()); + avcc[numpicparamsoffset] = numparams; + return avccidx; +} + +static void reassembleESDS(const sp<ABuffer> &csd0, char *esds) { + int csd0size = csd0->size(); + esds[0] = 3; // kTag_ESDescriptor; + int esdescriptorsize = 26 + csd0size; + CHECK(esdescriptorsize < 268435456); // 7 bits per byte, so max is 2^28-1 + esds[1] = 0x80 | (esdescriptorsize >> 21); + esds[2] = 0x80 | ((esdescriptorsize >> 14) & 0x7f); + esds[3] = 0x80 | ((esdescriptorsize >> 7) & 0x7f); + esds[4] = (esdescriptorsize & 0x7f); + esds[5] = esds[6] = 0; // es id + esds[7] = 0; // flags + esds[8] = 4; // kTag_DecoderConfigDescriptor + int configdescriptorsize = 18 + csd0size; + esds[9] = 0x80 | (configdescriptorsize >> 21); + esds[10] = 0x80 | ((configdescriptorsize >> 14) & 0x7f); + esds[11] = 0x80 | ((configdescriptorsize >> 7) & 0x7f); + esds[12] = (configdescriptorsize & 0x7f); + esds[13] = 0x40; // objectTypeIndication + esds[14] = 0x15; // not sure what 14-25 mean, they are ignored by ESDS.cpp, + esds[15] = 0x00; // but the actual values here were taken from a real file. + esds[16] = 0x18; + esds[17] = 0x00; + esds[18] = 0x00; + esds[19] = 0x00; + esds[20] = 0xfa; + esds[21] = 0x00; + esds[22] = 0x00; + esds[23] = 0x00; + esds[24] = 0xfa; + esds[25] = 0x00; + esds[26] = 5; // kTag_DecoderSpecificInfo; + esds[27] = 0x80 | (csd0size >> 21); + esds[28] = 0x80 | ((csd0size >> 14) & 0x7f); + esds[29] = 0x80 | ((csd0size >> 7) & 0x7f); + esds[30] = (csd0size & 0x7f); + memcpy((void*)&esds[31], csd0->data(), csd0size); + // data following this is ignored, so don't bother appending it + +} + +void convertMessageToMetaData(const sp<AMessage> &msg, sp<MetaData> &meta) { + AString mime; + if (msg->findString("mime", &mime)) { + meta->setCString(kKeyMIMEType, mime.c_str()); + } else { + ALOGW("did not find mime type"); + } + + int64_t durationUs; + if (msg->findInt64("durationUs", &durationUs)) { + meta->setInt64(kKeyDuration, durationUs); + } + + if (mime.startsWith("video/")) { + int32_t width; + int32_t height; + if (msg->findInt32("width", &width) && msg->findInt32("height", &height)) { + meta->setInt32(kKeyWidth, width); + meta->setInt32(kKeyHeight, height); + } else { + ALOGW("did not find width and/or height"); + } + } else if (mime.startsWith("audio/")) { + int32_t numChannels; + if (msg->findInt32("channel-count", &numChannels)) { + meta->setInt32(kKeyChannelCount, numChannels); + } + int32_t sampleRate; + if (msg->findInt32("sample-rate", &sampleRate)) { + meta->setInt32(kKeySampleRate, sampleRate); + } + int32_t channelMask; + if (msg->findInt32("channel-mask", &channelMask)) { + meta->setInt32(kKeyChannelMask, channelMask); + } + int32_t delay = 0; + if (msg->findInt32("encoder-delay", &delay)) { + meta->setInt32(kKeyEncoderDelay, delay); + } + int32_t padding = 0; + if (msg->findInt32("encoder-padding", &padding)) { + meta->setInt32(kKeyEncoderPadding, padding); + } + + int32_t isADTS; + if (msg->findInt32("is-adts", &isADTS)) { + meta->setInt32(kKeyIsADTS, isADTS); + } + } + + int32_t maxInputSize; + if (msg->findInt32("max-input-size", &maxInputSize)) { + meta->setInt32(kKeyMaxInputSize, maxInputSize); + } + + // reassemble the csd data into its original form + sp<ABuffer> csd0; + if (msg->findBuffer("csd-0", &csd0)) { + if (mime.startsWith("video/")) { // do we need to be stricter than this? + sp<ABuffer> csd1; + if (msg->findBuffer("csd-1", &csd1)) { + char avcc[1024]; // that oughta be enough, right? + size_t outsize = reassembleAVCC(csd0, csd1, avcc); + meta->setData(kKeyAVCC, kKeyAVCC, avcc, outsize); + } + } else if (mime.startsWith("audio/")) { + int csd0size = csd0->size(); + char esds[csd0size + 31]; + reassembleESDS(csd0, esds); + meta->setData(kKeyESDS, kKeyESDS, esds, sizeof(esds)); + } + } + + // XXX TODO add whatever other keys there are + +#if 0 + ALOGI("converted %s to:", msg->debugString(0).c_str()); + meta->dumpToLog(); +#endif +} + + } // namespace android diff --git a/media/libstagefright/include/FragmentedMP4Extractor.h b/media/libstagefright/include/FragmentedMP4Extractor.h new file mode 100644 index 0000000..763cd3a --- /dev/null +++ b/media/libstagefright/include/FragmentedMP4Extractor.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FRAGMENTED_MP4_EXTRACTOR_H_ + +#define FRAGMENTED_MP4_EXTRACTOR_H_ + +#include "include/FragmentedMP4Parser.h" + +#include <media/stagefright/MediaExtractor.h> +#include <utils/Vector.h> +#include <utils/String8.h> + +namespace android { + +struct AMessage; +class DataSource; +class SampleTable; +class String8; + +class FragmentedMP4Extractor : public MediaExtractor { +public: + // Extractor assumes ownership of "source". + FragmentedMP4Extractor(const sp<DataSource> &source); + + virtual size_t countTracks(); + virtual sp<MediaSource> getTrack(size_t index); + virtual sp<MetaData> getTrackMetaData(size_t index, uint32_t flags); + virtual sp<MetaData> getMetaData(); + virtual uint32_t flags() const; + +protected: + virtual ~FragmentedMP4Extractor(); + +private: + sp<ALooper> mLooper; + sp<FragmentedMP4Parser> mParser; + sp<DataSource> mDataSource; + status_t mInitCheck; + size_t mAudioTrackIndex; + size_t mTrackCount; + + sp<MetaData> mFileMetaData; + + Vector<uint32_t> mPath; + + FragmentedMP4Extractor(const FragmentedMP4Extractor &); + FragmentedMP4Extractor &operator=(const FragmentedMP4Extractor &); +}; + +bool SniffFragmentedMP4( + const sp<DataSource> &source, String8 *mimeType, float *confidence, + sp<AMessage> *); + +} // namespace android + +#endif // MPEG4_EXTRACTOR_H_ diff --git a/media/libstagefright/include/FragmentedMP4Parser.h b/media/libstagefright/include/FragmentedMP4Parser.h index bd8fe32..0edafb9 100644 --- a/media/libstagefright/include/FragmentedMP4Parser.h +++ b/media/libstagefright/include/FragmentedMP4Parser.h @@ -19,6 +19,7 @@ #define PARSER_H_ #include <media/stagefright/foundation/AHandler.h> +#include <media/stagefright/DataSource.h> #include <utils/Vector.h> namespace android { @@ -30,6 +31,7 @@ struct FragmentedMP4Parser : public AHandler { Source() {} virtual ssize_t readAt(off64_t offset, void *data, size_t size) = 0; + virtual bool isSeekable() = 0; protected: virtual ~Source() {} @@ -42,9 +44,12 @@ struct FragmentedMP4Parser : public AHandler { void start(const char *filename); void start(const sp<Source> &source); + void start(sp<DataSource> &source); - sp<AMessage> getFormat(bool audio); - status_t dequeueAccessUnit(bool audio, sp<ABuffer> *accessUnit); + sp<AMessage> getFormat(bool audio, bool synchronous = false); + status_t dequeueAccessUnit(bool audio, sp<ABuffer> *accessUnit, bool synchronous = false); + status_t seekTo(bool audio, int64_t timeUs); + bool isSeekable() const; virtual void onMessageReceived(const sp<AMessage> &msg); @@ -58,6 +63,7 @@ private: kWhatReadMore, kWhatGetFormat, kWhatDequeueAccessUnit, + kWhatSeekTo, }; struct TrackFragment; @@ -97,6 +103,11 @@ private: off64_t mOffset; }; + struct SidxEntry { + size_t mSize; + uint32_t mDurationUs; + }; + struct TrackInfo { enum Flags { kTrackEnabled = 0x01, @@ -107,6 +118,7 @@ private: uint32_t mTrackID; uint32_t mFlags; uint32_t mDuration; // This is the duration in terms of movie timescale! + uint64_t mSidxDuration; // usec, from sidx box, which can use a different timescale uint32_t mMediaTimeScale; @@ -121,6 +133,7 @@ private: uint32_t mDecodingTime; + Vector<SidxEntry> mSidx; sp<StaticTrackFragment> mStaticFragment; List<sp<TrackFragment> > mFragments; }; @@ -151,6 +164,8 @@ private: sp<Source> mSource; off_t mBufferPos; bool mSuspended; + bool mDoneWithMoov; + off_t mFirstMoofOffset; // used as the starting point for offsets calculated from the sidx box sp<ABuffer> mBuffer; Vector<Container> mStack; KeyedVector<uint32_t, TrackInfo> mTracks; // TrackInfo by trackID @@ -164,6 +179,7 @@ private: status_t onProceed(); status_t onDequeueAccessUnit(size_t trackIndex, sp<ABuffer> *accessUnit); + status_t onSeekTo(bool wantAudio, int64_t position); void enter(off64_t offset, uint32_t type, uint64_t size); @@ -222,6 +238,9 @@ private: status_t parseMediaData( uint32_t type, size_t offset, uint64_t size); + status_t parseSegmentIndex( + uint32_t type, size_t offset, uint64_t size); + TrackInfo *editTrack(uint32_t trackID, bool createIfNecessary = false); ssize_t findTrack(bool wantAudio) const; diff --git a/media/libstagefright/mp4/FragmentedMP4Parser.cpp b/media/libstagefright/mp4/FragmentedMP4Parser.cpp index e130a80..7fe4e63 100644 --- a/media/libstagefright/mp4/FragmentedMP4Parser.cpp +++ b/media/libstagefright/mp4/FragmentedMP4Parser.cpp @@ -18,8 +18,8 @@ #define LOG_TAG "FragmentedMP4Parser" #include <utils/Log.h> -#include "include/FragmentedMP4Parser.h" #include "include/ESDS.h" +#include "include/FragmentedMP4Parser.h" #include "TrackFragment.h" @@ -31,6 +31,7 @@ #include <media/stagefright/MediaErrors.h> #include <media/stagefright/Utils.h> + namespace android { static const char *Fourcc2String(uint32_t fourcc) { @@ -121,6 +122,8 @@ const FragmentedMP4Parser::DispatchEntry FragmentedMP4Parser::kDispatchTable[] = }, { FOURCC('m', 'f', 'r', 'a'), 0, NULL }, + + { FOURCC('s', 'i', 'd', 'x'), 0, &FragmentedMP4Parser::parseSegmentIndex }, }; struct FileSource : public FragmentedMP4Parser::Source { @@ -134,15 +137,92 @@ struct FileSource : public FragmentedMP4Parser::Source { return fread(data, 1, size, mFile); } + virtual bool isSeekable() { + return true; + } + private: FILE *mFile; DISALLOW_EVIL_CONSTRUCTORS(FileSource); }; +struct ReadTracker : public RefBase { + ReadTracker(off64_t size) { + allocSize = 1 + size / 8192; // 1 bit per kilobyte + bitmap = (char*) calloc(1, allocSize); + } + virtual ~ReadTracker() { + dumpToLog(); + free(bitmap); + } + void mark(off64_t offset, size_t size) { + int firstbit = offset / 1024; + int lastbit = (offset + size - 1) / 1024; + for (int i = firstbit; i <= lastbit; i++) { + bitmap[i/8] |= (0x80 >> (i & 7)); + } + } + + private: + void dumpToLog() { + // 96 chars per line, each char represents one kilobyte, 1 kb per bit + int numlines = allocSize / 12; + char buf[97]; + char *cur = bitmap; + for (int i = 0; i < numlines; i++ && cur) { + for (int j = 0; j < 12; j++) { + for (int k = 0; k < 8; k++) { + buf[(j * 8) + k] = (*cur & (0x80 >> k)) ? 'X' : '.'; + } + cur++; + } + buf[96] = '\0'; + ALOGI("%5dk: %s", i * 96, buf); + } + } + + size_t allocSize; + char *bitmap; +}; + +struct DataSourceSource : public FragmentedMP4Parser::Source { + DataSourceSource(sp<DataSource> &source) + : mDataSource(source) { + CHECK(mDataSource != NULL); +#if 0 + off64_t size; + if (source->getSize(&size) == OK) { + mReadTracker = new ReadTracker(size); + } else { + ALOGE("couldn't get data source size"); + } +#endif + } + + virtual ssize_t readAt(off64_t offset, void *data, size_t size) { + if (mReadTracker != NULL) { + mReadTracker->mark(offset, size); + } + return mDataSource->readAt(offset, data, size); + } + + virtual bool isSeekable() { + return true; + } + + private: + sp<DataSource> mDataSource; + sp<ReadTracker> mReadTracker; + + DISALLOW_EVIL_CONSTRUCTORS(DataSourceSource); +}; + FragmentedMP4Parser::FragmentedMP4Parser() : mBufferPos(0), mSuspended(false), + mDoneWithMoov(false), + mFirstMoofOffset(0), mFinalResult(OK) { } @@ -153,54 +233,142 @@ void FragmentedMP4Parser::start(const char *filename) { sp<AMessage> msg = new AMessage(kWhatStart, id()); msg->setObject("source", new FileSource(filename)); msg->post(); + ALOGV("Parser::start(%s)", filename); } void FragmentedMP4Parser::start(const sp<Source> &source) { sp<AMessage> msg = new AMessage(kWhatStart, id()); msg->setObject("source", source); msg->post(); + ALOGV("Parser::start(Source)"); +} + +void FragmentedMP4Parser::start(sp<DataSource> &source) { + sp<AMessage> msg = new AMessage(kWhatStart, id()); + msg->setObject("source", new DataSourceSource(source)); + msg->post(); + ALOGV("Parser::start(DataSource)"); } -sp<AMessage> FragmentedMP4Parser::getFormat(bool audio) { - sp<AMessage> msg = new AMessage(kWhatGetFormat, id()); - msg->setInt32("audio", audio); +sp<AMessage> FragmentedMP4Parser::getFormat(bool audio, bool synchronous) { - sp<AMessage> response; - status_t err = msg->postAndAwaitResponse(&response); + while (true) { + bool moovDone = mDoneWithMoov; + sp<AMessage> msg = new AMessage(kWhatGetFormat, id()); + msg->setInt32("audio", audio); - if (err != OK) { - return NULL; - } + sp<AMessage> response; + status_t err = msg->postAndAwaitResponse(&response); - if (response->findInt32("err", &err) && err != OK) { - return NULL; - } + if (err != OK) { + ALOGV("getFormat post failed: %d", err); + return NULL; + } + + if (response->findInt32("err", &err) && err != OK) { + if (synchronous && err == -EWOULDBLOCK && !moovDone) { + resumeIfNecessary(); + ALOGV("@getFormat parser not ready yet, retrying"); + usleep(10000); + continue; + } + ALOGV("getFormat failed: %d", err); + return NULL; + } - sp<AMessage> format; - CHECK(response->findMessage("format", &format)); + sp<AMessage> format; + CHECK(response->findMessage("format", &format)); - ALOGV("returning format %s", format->debugString().c_str()); - return format; + ALOGV("returning format %s", format->debugString().c_str()); + return format; + } } -status_t FragmentedMP4Parser::dequeueAccessUnit(bool audio, sp<ABuffer> *accessUnit) { - sp<AMessage> msg = new AMessage(kWhatDequeueAccessUnit, id()); - msg->setInt32("audio", audio); +status_t FragmentedMP4Parser::seekTo(bool wantAudio, int64_t timeUs) { + sp<AMessage> msg = new AMessage(kWhatSeekTo, id()); + msg->setInt32("audio", wantAudio); + msg->setInt64("position", timeUs); sp<AMessage> response; status_t err = msg->postAndAwaitResponse(&response); + return err; +} - if (err != OK) { - return err; +bool FragmentedMP4Parser::isSeekable() const { + while (mFirstMoofOffset == 0 && mFinalResult == OK) { + usleep(10000); + } + bool seekable = mSource->isSeekable(); + for (size_t i = 0; seekable && i < mTracks.size(); i++) { + const TrackInfo *info = &mTracks.valueAt(i); + seekable &= !info->mSidx.empty(); } + return seekable; +} - if (response->findInt32("err", &err) && err != OK) { - return err; +status_t FragmentedMP4Parser::onSeekTo(bool wantAudio, int64_t position) { + status_t err = -EINVAL; + ssize_t trackIndex = findTrack(wantAudio); + if (trackIndex < 0) { + err = trackIndex; + } else { + TrackInfo *info = &mTracks.editValueAt(trackIndex); + + int numSidxEntries = info->mSidx.size(); + int64_t totalTime = 0; + off_t totalOffset = mFirstMoofOffset; + for (int i = 0; i < numSidxEntries; i++) { + const SidxEntry *se = &info->mSidx[i]; + totalTime += se->mDurationUs; + if (totalTime > position) { + mBuffer->setRange(0,0); + mBufferPos = totalOffset; + if (mFinalResult == ERROR_END_OF_STREAM) { + mFinalResult = OK; + mSuspended = true; // force resume + resumeIfNecessary(); + } + info->mFragments.clear(); + info->mDecodingTime = position * info->mMediaTimeScale / 1000000ll; + return OK; + } + totalOffset += se->mSize; + } } + ALOGV("seekTo out of range"); + return err; +} - CHECK(response->findBuffer("accessUnit", accessUnit)); +status_t FragmentedMP4Parser::dequeueAccessUnit(bool audio, sp<ABuffer> *accessUnit, + bool synchronous) { - return OK; + while (true) { + sp<AMessage> msg = new AMessage(kWhatDequeueAccessUnit, id()); + msg->setInt32("audio", audio); + + sp<AMessage> response; + status_t err = msg->postAndAwaitResponse(&response); + + if (err != OK) { + ALOGV("dequeue fail 1: %d", err); + return err; + } + + if (response->findInt32("err", &err) && err != OK) { + if (synchronous && err == -EWOULDBLOCK) { + resumeIfNecessary(); + ALOGV("Parser not ready yet, retrying"); + usleep(10000); + continue; + } + ALOGV("dequeue fail 2: %d, %d", err, synchronous); + return err; + } + + CHECK(response->findBuffer("accessUnit", accessUnit)); + + return OK; + } } ssize_t FragmentedMP4Parser::findTrack(bool wantAudio) const { @@ -272,7 +440,7 @@ void FragmentedMP4Parser::onMessageReceived(const sp<AMessage> &msg) { size_t maxBytesToRead = mBuffer->capacity() - mBuffer->size(); if (maxBytesToRead < needed) { - ALOGI("resizing buffer."); + ALOGV("resizing buffer."); sp<ABuffer> newBuffer = new ABuffer((mBuffer->size() + needed + 1023) & ~1023); @@ -290,7 +458,7 @@ void FragmentedMP4Parser::onMessageReceived(const sp<AMessage> &msg) { mBuffer->data() + mBuffer->size(), needed); if (n < (ssize_t)needed) { - ALOGI("%s", "Reached EOF"); + ALOGV("Reached EOF when reading %d @ %d + %d", needed, mBufferPos, mBuffer->size()); if (n < 0) { mFinalResult = n; } else if (n == 0) { @@ -321,8 +489,16 @@ void FragmentedMP4Parser::onMessageReceived(const sp<AMessage> &msg) { } else { TrackInfo *info = &mTracks.editValueAt(trackIndex); + sp<AMessage> format = info->mSampleDescs.itemAt(0).mFormat; + if (info->mSidxDuration) { + format->setInt64("durationUs", info->mSidxDuration); + } else { + // this is probably going to be zero. Oh well... + format->setInt64("durationUs", + 1000000ll * info->mDuration / info->mMediaTimeScale); + } response->setMessage( - "format", info->mSampleDescs.itemAt(0).mFormat); + "format", format); err = OK; } @@ -366,6 +542,30 @@ void FragmentedMP4Parser::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatSeekTo: + { + ALOGV("kWhatSeekTo"); + int32_t wantAudio; + CHECK(msg->findInt32("audio", &wantAudio)); + int64_t position; + CHECK(msg->findInt64("position", &position)); + + status_t err = -EWOULDBLOCK; + sp<AMessage> response = new AMessage; + + ssize_t trackIndex = findTrack(wantAudio); + + if (trackIndex < 0) { + err = trackIndex; + } else { + err = onSeekTo(wantAudio, position); + } + response->setInt32("err", err); + uint32_t replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + response->postReply(replyID); + break; + } default: TRESPASS(); } @@ -429,6 +629,12 @@ status_t FragmentedMP4Parser::onProceed() { if ((i < kNumDispatchers && kDispatchTable[i].mHandler == 0) || isSampleEntryBox || ptype == FOURCC('i', 'l', 's', 't')) { // This is a container box. + if (type == FOURCC('m', 'o', 'o', 'f')) { + if (mFirstMoofOffset == 0) { + ALOGV("first moof @ %08x", mBufferPos + offset); + mFirstMoofOffset = mBufferPos + offset - 8; // point at the size + } + } if (type == FOURCC('m', 'e', 't', 'a')) { if ((err = need(offset + 4)) < OK) { return err; @@ -589,7 +795,7 @@ void FragmentedMP4Parser::resumeIfNecessary() { return; } - ALOGI("resuming."); + ALOGV("resuming."); mSuspended = false; (new AMessage(kWhatProceed, id()))->post(); @@ -647,7 +853,7 @@ status_t FragmentedMP4Parser::onDequeueAccessUnit( int cmp = CompareSampleLocation(sampleInfo, mdatInfo); - if (cmp < 0) { + if (cmp < 0 && !mSource->isSeekable()) { return -EPIPE; } else if (cmp == 0) { if (i > 0) { @@ -669,6 +875,8 @@ status_t FragmentedMP4Parser::onDequeueAccessUnit( size_t numDroppable = 0; bool done = false; + // XXX FIXME: if one of the tracks is not advanced (e.g. if you play an audio+video + // file with sf2), then mMediaData will not be pruned and keeps growing for (size_t i = 0; !done && i < mMediaData.size(); ++i) { const MediaDataInfo &mdatInfo = mMediaData.itemAt(i); @@ -896,6 +1104,8 @@ void FragmentedMP4Parser::skip(off_t distance) { static_cast<DynamicTrackFragment *>( fragment.get())->signalCompletion(); + } else if (container->mType == FOURCC('m', 'o', 'o', 'v')) { + mDoneWithMoov = true; } container = NULL; @@ -953,6 +1163,10 @@ status_t FragmentedMP4Parser::parseTrackHeader( TrackInfo *info = editTrack(trackID, true /* createIfNecessary */); info->mFlags = flags; info->mDuration = duration; + if (info->mDuration == 0xffffffff) { + // ffmpeg sets this to -1, which is incorrect. + info->mDuration = 0; + } info->mStaticFragment = new StaticTrackFragment; @@ -1363,13 +1577,100 @@ status_t FragmentedMP4Parser::parseMediaData( info->mOffset = mBufferPos + offset; if (mMediaData.size() > 10) { - ALOGI("suspending for now."); + ALOGV("suspending for now."); mSuspended = true; } return OK; } +status_t FragmentedMP4Parser::parseSegmentIndex( + uint32_t type, size_t offset, uint64_t size) { + ALOGV("sidx box type %d, offset %d, size %d", type, int(offset), int(size)); +// AString sidxstr; +// hexdump(mBuffer->data() + offset, size, 0 /* indent */, &sidxstr); +// ALOGV("raw sidx:"); +// ALOGV("%s", sidxstr.c_str()); + if (offset + 12 > size) { + return -EINVAL; + } + + uint32_t flags = readU32(offset); + + uint32_t version = flags >> 24; + flags &= 0xffffff; + + ALOGV("sidx version %d", version); + + uint32_t referenceId = readU32(offset + 4); + uint32_t timeScale = readU32(offset + 8); + ALOGV("sidx refid/timescale: %d/%d", referenceId, timeScale); + + uint64_t earliestPresentationTime; + uint64_t firstOffset; + + offset += 12; + + if (version == 0) { + if (offset + 8 > size) { + return -EINVAL; + } + earliestPresentationTime = readU32(offset); + firstOffset = readU32(offset + 4); + offset += 8; + } else { + if (offset + 16 > size) { + return -EINVAL; + } + earliestPresentationTime = readU64(offset); + firstOffset = readU64(offset + 8); + offset += 16; + } + ALOGV("sidx pres/off: %Ld/%Ld", earliestPresentationTime, firstOffset); + + if (offset + 4 > size) { + return -EINVAL; + } + if (readU16(offset) != 0) { // reserved + return -EINVAL; + } + int32_t referenceCount = readU16(offset + 2); + offset += 4; + ALOGV("refcount: %d", referenceCount); + + if (offset + referenceCount * 12 > size) { + return -EINVAL; + } + + TrackInfo *info = editTrack(mCurrentTrackID); + uint64_t total_duration = 0; + for (int i = 0; i < referenceCount; i++) { + uint32_t d1 = readU32(offset); + uint32_t d2 = readU32(offset + 4); + uint32_t d3 = readU32(offset + 8); + + if (d1 & 0x80000000) { + ALOGW("sub-sidx boxes not supported yet"); + } + bool sap = d3 & 0x80000000; + bool saptype = d3 >> 28; + if (!sap || saptype > 2) { + ALOGW("not a stream access point, or unsupported type"); + } + total_duration += d2; + offset += 12; + ALOGV(" item %d, %08x %08x %08x", i, d1, d2, d3); + SidxEntry se; + se.mSize = d1 & 0x7fffffff; + se.mDurationUs = 1000000LL * d2 / timeScale; + info->mSidx.add(se); + } + + info->mSidxDuration = total_duration * 1000000 / timeScale; + ALOGV("duration: %lld", info->mSidxDuration); + return OK; +} + status_t FragmentedMP4Parser::parseTrackExtends( uint32_t type, size_t offset, uint64_t size) { if (offset + 24 > size) { @@ -1407,6 +1708,7 @@ FragmentedMP4Parser::TrackInfo *FragmentedMP4Parser::editTrack( info.mTrackID = trackID; info.mFlags = 0; info.mDuration = 0xffffffff; + info.mSidxDuration = 0; info.mMediaTimeScale = 0; info.mMediaHandlerType = 0; info.mDefaultSampleDescriptionIndex = 0; diff --git a/media/libstagefright/mpeg2ts/ATSParser.cpp b/media/libstagefright/mpeg2ts/ATSParser.cpp index 5f3e300..d988356 100644 --- a/media/libstagefright/mpeg2ts/ATSParser.cpp +++ b/media/libstagefright/mpeg2ts/ATSParser.cpp @@ -51,7 +51,8 @@ struct ATSParser::Program : public RefBase { unsigned pid, ABitReader *br, status_t *err); bool parsePID( - unsigned pid, unsigned payload_unit_start_indicator, + unsigned pid, unsigned continuity_counter, + unsigned payload_unit_start_indicator, ABitReader *br, status_t *err); void signalDiscontinuity( @@ -77,6 +78,10 @@ struct ATSParser::Program : public RefBase { return mProgramMapPID; } + uint32_t parserFlags() const { + return mParser->mFlags; + } + private: ATSParser *mParser; unsigned mProgramNumber; @@ -91,13 +96,17 @@ private: }; struct ATSParser::Stream : public RefBase { - Stream(Program *program, unsigned elementaryPID, unsigned streamType); + Stream(Program *program, + unsigned elementaryPID, + unsigned streamType, + unsigned PCR_PID); unsigned type() const { return mStreamType; } unsigned pid() const { return mElementaryPID; } void setPID(unsigned pid) { mElementaryPID = pid; } status_t parse( + unsigned continuity_counter, unsigned payload_unit_start_indicator, ABitReader *br); @@ -115,6 +124,8 @@ private: Program *mProgram; unsigned mElementaryPID; unsigned mStreamType; + unsigned mPCR_PID; + int32_t mExpectedContinuityCounter; sp<ABuffer> mBuffer; sp<AnotherPacketSource> mSource; @@ -184,7 +195,8 @@ bool ATSParser::Program::parsePSISection( } bool ATSParser::Program::parsePID( - unsigned pid, unsigned payload_unit_start_indicator, + unsigned pid, unsigned continuity_counter, + unsigned payload_unit_start_indicator, ABitReader *br, status_t *err) { *err = OK; @@ -194,7 +206,7 @@ bool ATSParser::Program::parsePID( } *err = mStreams.editValueAt(index)->parse( - payload_unit_start_indicator, br); + continuity_counter, payload_unit_start_indicator, br); return true; } @@ -241,7 +253,10 @@ status_t ATSParser::Program::parseProgramMap(ABitReader *br) { MY_LOGV(" section_number = %u", br->getBits(8)); MY_LOGV(" last_section_number = %u", br->getBits(8)); MY_LOGV(" reserved = %u", br->getBits(3)); - MY_LOGV(" PCR_PID = 0x%04x", br->getBits(13)); + + unsigned PCR_PID = br->getBits(13); + ALOGV(" PCR_PID = 0x%04x", PCR_PID); + MY_LOGV(" reserved = %u", br->getBits(4)); unsigned program_info_length = br->getBits(12); @@ -382,7 +397,9 @@ status_t ATSParser::Program::parseProgramMap(ABitReader *br) { ssize_t index = mStreams.indexOfKey(info.mPID); if (index < 0) { - sp<Stream> stream = new Stream(this, info.mPID, info.mType); + sp<Stream> stream = new Stream( + this, info.mPID, info.mType, PCR_PID); + mStreams.add(info.mPID, stream); } } @@ -419,21 +436,35 @@ int64_t ATSParser::Program::convertPTSToTimestamp(uint64_t PTS) { } } - return (PTS * 100) / 9; + int64_t timeUs = (PTS * 100) / 9; + + if (mParser->mAbsoluteTimeAnchorUs >= 0ll) { + timeUs += mParser->mAbsoluteTimeAnchorUs; + } + + return timeUs; } //////////////////////////////////////////////////////////////////////////////// ATSParser::Stream::Stream( - Program *program, unsigned elementaryPID, unsigned streamType) + Program *program, + unsigned elementaryPID, + unsigned streamType, + unsigned PCR_PID) : mProgram(program), mElementaryPID(elementaryPID), mStreamType(streamType), + mPCR_PID(PCR_PID), + mExpectedContinuityCounter(-1), mPayloadStarted(false), mQueue(NULL) { switch (mStreamType) { case STREAMTYPE_H264: - mQueue = new ElementaryStreamQueue(ElementaryStreamQueue::H264); + mQueue = new ElementaryStreamQueue( + ElementaryStreamQueue::H264, + (mProgram->parserFlags() & ALIGNED_VIDEO_DATA) + ? ElementaryStreamQueue::kFlag_AlignedData : 0); break; case STREAMTYPE_MPEG2_AUDIO_ADTS: mQueue = new ElementaryStreamQueue(ElementaryStreamQueue::AAC); @@ -473,11 +504,25 @@ ATSParser::Stream::~Stream() { } status_t ATSParser::Stream::parse( + unsigned continuity_counter, unsigned payload_unit_start_indicator, ABitReader *br) { if (mQueue == NULL) { return OK; } + if (mExpectedContinuityCounter >= 0 + && (unsigned)mExpectedContinuityCounter != continuity_counter) { + ALOGI("discontinuity on stream pid 0x%04x", mElementaryPID); + + mPayloadStarted = false; + mBuffer->setRange(0, 0); + mExpectedContinuityCounter = -1; + + return OK; + } + + mExpectedContinuityCounter = (continuity_counter + 1) & 0x0f; + if (payload_unit_start_indicator) { if (mPayloadStarted) { // Otherwise we run the danger of receiving the trailing bytes @@ -664,8 +709,7 @@ status_t ATSParser::Stream::parsePES(ABitReader *br) { PTS |= br->getBits(15); CHECK_EQ(br->getBits(1), 1u); - ALOGV("PTS = %llu", PTS); - // ALOGI("PTS = %.2f secs", PTS / 90000.0f); + ALOGV("PTS = 0x%016llx (%.2f)", PTS, PTS / 90000.0); optional_bytes_remaining -= 5; @@ -847,7 +891,10 @@ sp<MediaSource> ATSParser::Stream::getSource(SourceType type) { //////////////////////////////////////////////////////////////////////////////// ATSParser::ATSParser(uint32_t flags) - : mFlags(flags) { + : mFlags(flags), + mAbsoluteTimeAnchorUs(-1ll), + mNumTSPacketsParsed(0), + mNumPCRs(0) { mPSISections.add(0 /* PID */, new PSISection); } @@ -863,6 +910,15 @@ status_t ATSParser::feedTSPacket(const void *data, size_t size) { void ATSParser::signalDiscontinuity( DiscontinuityType type, const sp<AMessage> &extra) { + if (type == DISCONTINUITY_ABSOLUTE_TIME) { + int64_t timeUs; + CHECK(extra->findInt64("timeUs", &timeUs)); + + CHECK(mPrograms.empty()); + mAbsoluteTimeAnchorUs = timeUs; + return; + } + for (size_t i = 0; i < mPrograms.size(); ++i) { mPrograms.editItemAt(i)->signalDiscontinuity(type, extra); } @@ -942,6 +998,7 @@ void ATSParser::parseProgramAssociationTable(ABitReader *br) { status_t ATSParser::parsePID( ABitReader *br, unsigned PID, + unsigned continuity_counter, unsigned payload_unit_start_indicator) { ssize_t sectionIndex = mPSISections.indexOfKey(PID); @@ -1002,7 +1059,8 @@ status_t ATSParser::parsePID( for (size_t i = 0; i < mPrograms.size(); ++i) { status_t err; if (mPrograms.editItemAt(i)->parsePID( - PID, payload_unit_start_indicator, br, &err)) { + PID, continuity_counter, payload_unit_start_indicator, + br, &err)) { if (err != OK) { return err; } @@ -1019,10 +1077,55 @@ status_t ATSParser::parsePID( return OK; } -void ATSParser::parseAdaptationField(ABitReader *br) { +void ATSParser::parseAdaptationField(ABitReader *br, unsigned PID) { unsigned adaptation_field_length = br->getBits(8); + if (adaptation_field_length > 0) { - br->skipBits(adaptation_field_length * 8); // XXX + unsigned discontinuity_indicator = br->getBits(1); + + if (discontinuity_indicator) { + ALOGV("PID 0x%04x: discontinuity_indicator = 1 (!!!)", PID); + } + + br->skipBits(2); + unsigned PCR_flag = br->getBits(1); + + size_t numBitsRead = 4; + + if (PCR_flag) { + br->skipBits(4); + uint64_t PCR_base = br->getBits(32); + PCR_base = (PCR_base << 1) | br->getBits(1); + + br->skipBits(6); + unsigned PCR_ext = br->getBits(9); + + // The number of bytes from the start of the current + // MPEG2 transport stream packet up and including + // the final byte of this PCR_ext field. + size_t byteOffsetFromStartOfTSPacket = + (188 - br->numBitsLeft() / 8); + + uint64_t PCR = PCR_base * 300 + PCR_ext; + + ALOGV("PID 0x%04x: PCR = 0x%016llx (%.2f)", + PID, PCR, PCR / 27E6); + + // The number of bytes received by this parser up to and + // including the final byte of this PCR_ext field. + size_t byteOffsetFromStart = + mNumTSPacketsParsed * 188 + byteOffsetFromStartOfTSPacket; + + for (size_t i = 0; i < mPrograms.size(); ++i) { + updatePCR(PID, PCR, byteOffsetFromStart); + } + + numBitsRead += 52; + } + + CHECK_GE(adaptation_field_length * 8, numBitsRead); + + br->skipBits(adaptation_field_length * 8 - numBitsRead); } } @@ -1048,19 +1151,24 @@ status_t ATSParser::parseTS(ABitReader *br) { ALOGV("adaptation_field_control = %u", adaptation_field_control); unsigned continuity_counter = br->getBits(4); - ALOGV("continuity_counter = %u", continuity_counter); + ALOGV("PID = 0x%04x, continuity_counter = %u", PID, continuity_counter); // ALOGI("PID = 0x%04x, continuity_counter = %u", PID, continuity_counter); if (adaptation_field_control == 2 || adaptation_field_control == 3) { - parseAdaptationField(br); + parseAdaptationField(br, PID); } + status_t err = OK; + if (adaptation_field_control == 1 || adaptation_field_control == 3) { - return parsePID(br, PID, payload_unit_start_indicator); + err = parsePID( + br, PID, continuity_counter, payload_unit_start_indicator); } - return OK; + ++mNumTSPacketsParsed; + + return err; } sp<MediaSource> ATSParser::getSource(SourceType type) { @@ -1091,6 +1199,31 @@ bool ATSParser::PTSTimeDeltaEstablished() { return mPrograms.editItemAt(0)->PTSTimeDeltaEstablished(); } +void ATSParser::updatePCR( + unsigned PID, uint64_t PCR, size_t byteOffsetFromStart) { + ALOGV("PCR 0x%016llx @ %d", PCR, byteOffsetFromStart); + + if (mNumPCRs == 2) { + mPCR[0] = mPCR[1]; + mPCRBytes[0] = mPCRBytes[1]; + mSystemTimeUs[0] = mSystemTimeUs[1]; + mNumPCRs = 1; + } + + mPCR[mNumPCRs] = PCR; + mPCRBytes[mNumPCRs] = byteOffsetFromStart; + mSystemTimeUs[mNumPCRs] = ALooper::GetNowUs(); + + ++mNumPCRs; + + if (mNumPCRs == 2) { + double transportRate = + (mPCRBytes[1] - mPCRBytes[0]) * 27E6 / (mPCR[1] - mPCR[0]); + + ALOGV("transportRate = %.2f bytes/sec", transportRate); + } +} + //////////////////////////////////////////////////////////////////////////////// ATSParser::PSISection::PSISection() { diff --git a/media/libstagefright/mpeg2ts/ATSParser.h b/media/libstagefright/mpeg2ts/ATSParser.h index 9ef2939..5ccbab7 100644 --- a/media/libstagefright/mpeg2ts/ATSParser.h +++ b/media/libstagefright/mpeg2ts/ATSParser.h @@ -38,6 +38,7 @@ struct ATSParser : public RefBase { DISCONTINUITY_TIME = 1, DISCONTINUITY_AUDIO_FORMAT = 2, DISCONTINUITY_VIDEO_FORMAT = 4, + DISCONTINUITY_ABSOLUTE_TIME = 8, DISCONTINUITY_SEEK = DISCONTINUITY_TIME, @@ -54,7 +55,9 @@ struct ATSParser : public RefBase { // If this flag is _not_ specified, the first PTS encountered in a // program of this stream will be assumed to correspond to media time 0 // instead. - TS_TIMESTAMPS_ARE_ABSOLUTE = 1 + TS_TIMESTAMPS_ARE_ABSOLUTE = 1, + // Video PES packets contain exactly one (aligned) access unit. + ALIGNED_VIDEO_DATA = 2, }; ATSParser(uint32_t flags = 0); @@ -100,17 +103,29 @@ private: // Keyed by PID KeyedVector<unsigned, sp<PSISection> > mPSISections; + int64_t mAbsoluteTimeAnchorUs; + + size_t mNumTSPacketsParsed; + void parseProgramAssociationTable(ABitReader *br); void parseProgramMap(ABitReader *br); void parsePES(ABitReader *br); status_t parsePID( ABitReader *br, unsigned PID, + unsigned continuity_counter, unsigned payload_unit_start_indicator); - void parseAdaptationField(ABitReader *br); + void parseAdaptationField(ABitReader *br, unsigned PID); status_t parseTS(ABitReader *br); + void updatePCR(unsigned PID, uint64_t PCR, size_t byteOffsetFromStart); + + uint64_t mPCR[2]; + size_t mPCRBytes[2]; + int64_t mSystemTimeUs[2]; + size_t mNumPCRs; + DISALLOW_EVIL_CONSTRUCTORS(ATSParser); }; diff --git a/media/libstagefright/wifi-display/Android.mk b/media/libstagefright/wifi-display/Android.mk index b035a51..0e59b9e 100644 --- a/media/libstagefright/wifi-display/Android.mk +++ b/media/libstagefright/wifi-display/Android.mk @@ -5,6 +5,10 @@ include $(CLEAR_VARS) LOCAL_SRC_FILES:= \ ANetworkSession.cpp \ ParsedMessage.cpp \ + sink/LinearRegression.cpp \ + sink/RTPSink.cpp \ + sink/TunnelRenderer.cpp \ + sink/WifiDisplaySink.cpp \ source/Converter.cpp \ source/PlaybackSession.cpp \ source/RepeaterSource.cpp \ diff --git a/media/libstagefright/wifi-display/sink/LinearRegression.cpp b/media/libstagefright/wifi-display/sink/LinearRegression.cpp new file mode 100644 index 0000000..8cfce37 --- /dev/null +++ b/media/libstagefright/wifi-display/sink/LinearRegression.cpp @@ -0,0 +1,110 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "LinearRegression" +#include <utils/Log.h> + +#include "LinearRegression.h" + +#include <math.h> +#include <string.h> + +namespace android { + +LinearRegression::LinearRegression(size_t historySize) + : mHistorySize(historySize), + mCount(0), + mHistory(new Point[mHistorySize]), + mSumX(0.0), + mSumY(0.0) { +} + +LinearRegression::~LinearRegression() { + delete[] mHistory; + mHistory = NULL; +} + +void LinearRegression::addPoint(float x, float y) { + if (mCount == mHistorySize) { + const Point &oldest = mHistory[0]; + + mSumX -= oldest.mX; + mSumY -= oldest.mY; + + memmove(&mHistory[0], &mHistory[1], (mHistorySize - 1) * sizeof(Point)); + --mCount; + } + + Point *newest = &mHistory[mCount++]; + newest->mX = x; + newest->mY = y; + + mSumX += x; + mSumY += y; +} + +bool LinearRegression::approxLine(float *n1, float *n2, float *b) const { + static const float kEpsilon = 1.0E-4; + + if (mCount < 2) { + return false; + } + + float sumX2 = 0.0f; + float sumY2 = 0.0f; + float sumXY = 0.0f; + + float meanX = mSumX / (float)mCount; + float meanY = mSumY / (float)mCount; + + for (size_t i = 0; i < mCount; ++i) { + const Point &p = mHistory[i]; + + float x = p.mX - meanX; + float y = p.mY - meanY; + + sumX2 += x * x; + sumY2 += y * y; + sumXY += x * y; + } + + float T = sumX2 + sumY2; + float D = sumX2 * sumY2 - sumXY * sumXY; + float root = sqrt(T * T * 0.25 - D); + + float L1 = T * 0.5 - root; + + if (fabs(sumXY) > kEpsilon) { + *n1 = 1.0; + *n2 = (2.0 * L1 - sumX2) / sumXY; + + float mag = sqrt((*n1) * (*n1) + (*n2) * (*n2)); + + *n1 /= mag; + *n2 /= mag; + } else { + *n1 = 0.0; + *n2 = 1.0; + } + + *b = (*n1) * meanX + (*n2) * meanY; + + return true; +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/sink/LinearRegression.h b/media/libstagefright/wifi-display/sink/LinearRegression.h new file mode 100644 index 0000000..ca6f5a1 --- /dev/null +++ b/media/libstagefright/wifi-display/sink/LinearRegression.h @@ -0,0 +1,52 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LINEAR_REGRESSION_H_ + +#define LINEAR_REGRESSION_H_ + +#include <sys/types.h> +#include <media/stagefright/foundation/ABase.h> + +namespace android { + +// Helper class to fit a line to a set of points minimizing the sum of +// squared (orthogonal) distances from line to individual points. +struct LinearRegression { + LinearRegression(size_t historySize); + ~LinearRegression(); + + void addPoint(float x, float y); + + bool approxLine(float *n1, float *n2, float *b) const; + +private: + struct Point { + float mX, mY; + }; + + size_t mHistorySize; + size_t mCount; + Point *mHistory; + + float mSumX, mSumY; + + DISALLOW_EVIL_CONSTRUCTORS(LinearRegression); +}; + +} // namespace android + +#endif // LINEAR_REGRESSION_H_ diff --git a/media/libstagefright/wifi-display/sink/RTPSink.cpp b/media/libstagefright/wifi-display/sink/RTPSink.cpp new file mode 100644 index 0000000..0918034 --- /dev/null +++ b/media/libstagefright/wifi-display/sink/RTPSink.cpp @@ -0,0 +1,806 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "RTPSink" +#include <utils/Log.h> + +#include "RTPSink.h" + +#include "ANetworkSession.h" +#include "TunnelRenderer.h" + +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/foundation/hexdump.h> +#include <media/stagefright/MediaErrors.h> +#include <media/stagefright/Utils.h> + +namespace android { + +struct RTPSink::Source : public RefBase { + Source(uint16_t seq, const sp<ABuffer> &buffer, + const sp<AMessage> queueBufferMsg); + + bool updateSeq(uint16_t seq, const sp<ABuffer> &buffer); + + void addReportBlock(uint32_t ssrc, const sp<ABuffer> &buf); + +protected: + virtual ~Source(); + +private: + static const uint32_t kMinSequential = 2; + static const uint32_t kMaxDropout = 3000; + static const uint32_t kMaxMisorder = 100; + static const uint32_t kRTPSeqMod = 1u << 16; + + sp<AMessage> mQueueBufferMsg; + + uint16_t mMaxSeq; + uint32_t mCycles; + uint32_t mBaseSeq; + uint32_t mBadSeq; + uint32_t mProbation; + uint32_t mReceived; + uint32_t mExpectedPrior; + uint32_t mReceivedPrior; + + void initSeq(uint16_t seq); + void queuePacket(const sp<ABuffer> &buffer); + + DISALLOW_EVIL_CONSTRUCTORS(Source); +}; + +//////////////////////////////////////////////////////////////////////////////// + +RTPSink::Source::Source( + uint16_t seq, const sp<ABuffer> &buffer, + const sp<AMessage> queueBufferMsg) + : mQueueBufferMsg(queueBufferMsg), + mProbation(kMinSequential) { + initSeq(seq); + mMaxSeq = seq - 1; + + buffer->setInt32Data(mCycles | seq); + queuePacket(buffer); +} + +RTPSink::Source::~Source() { +} + +void RTPSink::Source::initSeq(uint16_t seq) { + mMaxSeq = seq; + mCycles = 0; + mBaseSeq = seq; + mBadSeq = kRTPSeqMod + 1; + mReceived = 0; + mExpectedPrior = 0; + mReceivedPrior = 0; +} + +bool RTPSink::Source::updateSeq(uint16_t seq, const sp<ABuffer> &buffer) { + uint16_t udelta = seq - mMaxSeq; + + if (mProbation) { + // Startup phase + + if (seq == mMaxSeq + 1) { + buffer->setInt32Data(mCycles | seq); + queuePacket(buffer); + + --mProbation; + mMaxSeq = seq; + if (mProbation == 0) { + initSeq(seq); + ++mReceived; + + return true; + } + } else { + // Packet out of sequence, restart startup phase + + mProbation = kMinSequential - 1; + mMaxSeq = seq; + +#if 0 + mPackets.clear(); + mTotalBytesQueued = 0; + ALOGI("XXX cleared packets"); +#endif + + buffer->setInt32Data(mCycles | seq); + queuePacket(buffer); + } + + return false; + } + + if (udelta < kMaxDropout) { + // In order, with permissible gap. + + if (seq < mMaxSeq) { + // Sequence number wrapped - count another 64K cycle + mCycles += kRTPSeqMod; + } + + mMaxSeq = seq; + } else if (udelta <= kRTPSeqMod - kMaxMisorder) { + // The sequence number made a very large jump + + if (seq == mBadSeq) { + // Two sequential packets -- assume that the other side + // restarted without telling us so just re-sync + // (i.e. pretend this was the first packet) + + initSeq(seq); + } else { + mBadSeq = (seq + 1) & (kRTPSeqMod - 1); + + return false; + } + } else { + // Duplicate or reordered packet. + } + + ++mReceived; + + buffer->setInt32Data(mCycles | seq); + queuePacket(buffer); + + return true; +} + +void RTPSink::Source::queuePacket(const sp<ABuffer> &buffer) { + sp<AMessage> msg = mQueueBufferMsg->dup(); + msg->setBuffer("buffer", buffer); + msg->post(); +} + +void RTPSink::Source::addReportBlock( + uint32_t ssrc, const sp<ABuffer> &buf) { + uint32_t extMaxSeq = mMaxSeq | mCycles; + uint32_t expected = extMaxSeq - mBaseSeq + 1; + + int64_t lost = (int64_t)expected - (int64_t)mReceived; + if (lost > 0x7fffff) { + lost = 0x7fffff; + } else if (lost < -0x800000) { + lost = -0x800000; + } + + uint32_t expectedInterval = expected - mExpectedPrior; + mExpectedPrior = expected; + + uint32_t receivedInterval = mReceived - mReceivedPrior; + mReceivedPrior = mReceived; + + int64_t lostInterval = expectedInterval - receivedInterval; + + uint8_t fractionLost; + if (expectedInterval == 0 || lostInterval <=0) { + fractionLost = 0; + } else { + fractionLost = (lostInterval << 8) / expectedInterval; + } + + uint8_t *ptr = buf->data() + buf->size(); + + ptr[0] = ssrc >> 24; + ptr[1] = (ssrc >> 16) & 0xff; + ptr[2] = (ssrc >> 8) & 0xff; + ptr[3] = ssrc & 0xff; + + ptr[4] = fractionLost; + + ptr[5] = (lost >> 16) & 0xff; + ptr[6] = (lost >> 8) & 0xff; + ptr[7] = lost & 0xff; + + ptr[8] = extMaxSeq >> 24; + ptr[9] = (extMaxSeq >> 16) & 0xff; + ptr[10] = (extMaxSeq >> 8) & 0xff; + ptr[11] = extMaxSeq & 0xff; + + // XXX TODO: + + ptr[12] = 0x00; // interarrival jitter + ptr[13] = 0x00; + ptr[14] = 0x00; + ptr[15] = 0x00; + + ptr[16] = 0x00; // last SR + ptr[17] = 0x00; + ptr[18] = 0x00; + ptr[19] = 0x00; + + ptr[20] = 0x00; // delay since last SR + ptr[21] = 0x00; + ptr[22] = 0x00; + ptr[23] = 0x00; +} + +//////////////////////////////////////////////////////////////////////////////// + +RTPSink::RTPSink( + const sp<ANetworkSession> &netSession, + const sp<ISurfaceTexture> &surfaceTex) + : mNetSession(netSession), + mSurfaceTex(surfaceTex), + mRTPPort(0), + mRTPSessionID(0), + mRTCPSessionID(0), + mFirstArrivalTimeUs(-1ll), + mNumPacketsReceived(0ll), + mRegression(1000), + mMaxDelayMs(-1ll) { +} + +RTPSink::~RTPSink() { + if (mRTCPSessionID != 0) { + mNetSession->destroySession(mRTCPSessionID); + } + + if (mRTPSessionID != 0) { + mNetSession->destroySession(mRTPSessionID); + } +} + +status_t RTPSink::init(bool useTCPInterleaving) { + if (useTCPInterleaving) { + return OK; + } + + int clientRtp; + + sp<AMessage> rtpNotify = new AMessage(kWhatRTPNotify, id()); + sp<AMessage> rtcpNotify = new AMessage(kWhatRTCPNotify, id()); + for (clientRtp = 15550;; clientRtp += 2) { + int32_t rtpSession; + status_t err = mNetSession->createUDPSession( + clientRtp, rtpNotify, &rtpSession); + + if (err != OK) { + ALOGI("failed to create RTP socket on port %d", clientRtp); + continue; + } + + int32_t rtcpSession; + err = mNetSession->createUDPSession( + clientRtp + 1, rtcpNotify, &rtcpSession); + + if (err == OK) { + mRTPPort = clientRtp; + mRTPSessionID = rtpSession; + mRTCPSessionID = rtcpSession; + break; + } + + ALOGI("failed to create RTCP socket on port %d", clientRtp + 1); + mNetSession->destroySession(rtpSession); + } + + if (mRTPPort == 0) { + return UNKNOWN_ERROR; + } + + return OK; +} + +int32_t RTPSink::getRTPPort() const { + return mRTPPort; +} + +void RTPSink::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatRTPNotify: + case kWhatRTCPNotify: + { + int32_t reason; + CHECK(msg->findInt32("reason", &reason)); + + switch (reason) { + case ANetworkSession::kWhatError: + { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + int32_t err; + CHECK(msg->findInt32("err", &err)); + + AString detail; + CHECK(msg->findString("detail", &detail)); + + ALOGE("An error occurred in session %d (%d, '%s/%s').", + sessionID, + err, + detail.c_str(), + strerror(-err)); + + mNetSession->destroySession(sessionID); + + if (sessionID == mRTPSessionID) { + mRTPSessionID = 0; + } else if (sessionID == mRTCPSessionID) { + mRTCPSessionID = 0; + } + break; + } + + case ANetworkSession::kWhatDatagram: + { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + sp<ABuffer> data; + CHECK(msg->findBuffer("data", &data)); + + status_t err; + if (msg->what() == kWhatRTPNotify) { + err = parseRTP(data); + } else { + err = parseRTCP(data); + } + break; + } + + default: + TRESPASS(); + } + break; + } + + case kWhatSendRR: + { + onSendRR(); + break; + } + + case kWhatPacketLost: + { + onPacketLost(msg); + break; + } + + case kWhatInject: + { + int32_t isRTP; + CHECK(msg->findInt32("isRTP", &isRTP)); + + sp<ABuffer> buffer; + CHECK(msg->findBuffer("buffer", &buffer)); + + status_t err; + if (isRTP) { + err = parseRTP(buffer); + } else { + err = parseRTCP(buffer); + } + break; + } + + default: + TRESPASS(); + } +} + +status_t RTPSink::injectPacket(bool isRTP, const sp<ABuffer> &buffer) { + sp<AMessage> msg = new AMessage(kWhatInject, id()); + msg->setInt32("isRTP", isRTP); + msg->setBuffer("buffer", buffer); + msg->post(); + + return OK; +} + +status_t RTPSink::parseRTP(const sp<ABuffer> &buffer) { + size_t size = buffer->size(); + if (size < 12) { + // Too short to be a valid RTP header. + return ERROR_MALFORMED; + } + + const uint8_t *data = buffer->data(); + + if ((data[0] >> 6) != 2) { + // Unsupported version. + return ERROR_UNSUPPORTED; + } + + if (data[0] & 0x20) { + // Padding present. + + size_t paddingLength = data[size - 1]; + + if (paddingLength + 12 > size) { + // If we removed this much padding we'd end up with something + // that's too short to be a valid RTP header. + return ERROR_MALFORMED; + } + + size -= paddingLength; + } + + int numCSRCs = data[0] & 0x0f; + + size_t payloadOffset = 12 + 4 * numCSRCs; + + if (size < payloadOffset) { + // Not enough data to fit the basic header and all the CSRC entries. + return ERROR_MALFORMED; + } + + if (data[0] & 0x10) { + // Header eXtension present. + + if (size < payloadOffset + 4) { + // Not enough data to fit the basic header, all CSRC entries + // and the first 4 bytes of the extension header. + + return ERROR_MALFORMED; + } + + const uint8_t *extensionData = &data[payloadOffset]; + + size_t extensionLength = + 4 * (extensionData[2] << 8 | extensionData[3]); + + if (size < payloadOffset + 4 + extensionLength) { + return ERROR_MALFORMED; + } + + payloadOffset += 4 + extensionLength; + } + + uint32_t srcId = U32_AT(&data[8]); + uint32_t rtpTime = U32_AT(&data[4]); + uint16_t seqNo = U16_AT(&data[2]); + + int64_t arrivalTimeUs; + CHECK(buffer->meta()->findInt64("arrivalTimeUs", &arrivalTimeUs)); + + if (mFirstArrivalTimeUs < 0ll) { + mFirstArrivalTimeUs = arrivalTimeUs; + } + arrivalTimeUs -= mFirstArrivalTimeUs; + + int64_t arrivalTimeMedia = (arrivalTimeUs * 9ll) / 100ll; + + ALOGV("seqNo: %d, SSRC 0x%08x, diff %lld", + seqNo, srcId, rtpTime - arrivalTimeMedia); + + mRegression.addPoint((float)rtpTime, (float)arrivalTimeMedia); + + ++mNumPacketsReceived; + + float n1, n2, b; + if (mRegression.approxLine(&n1, &n2, &b)) { + ALOGV("Line %lld: %.2f %.2f %.2f, slope %.2f", + mNumPacketsReceived, n1, n2, b, -n1 / n2); + + float expectedArrivalTimeMedia = (b - n1 * (float)rtpTime) / n2; + float latenessMs = (arrivalTimeMedia - expectedArrivalTimeMedia) / 90.0; + + if (mMaxDelayMs < 0ll || latenessMs > mMaxDelayMs) { + mMaxDelayMs = latenessMs; + ALOGI("packet was %.2f ms late", latenessMs); + } + } + + sp<AMessage> meta = buffer->meta(); + meta->setInt32("ssrc", srcId); + meta->setInt32("rtp-time", rtpTime); + meta->setInt32("PT", data[1] & 0x7f); + meta->setInt32("M", data[1] >> 7); + + buffer->setRange(payloadOffset, size - payloadOffset); + + ssize_t index = mSources.indexOfKey(srcId); + if (index < 0) { + if (mRenderer == NULL) { + sp<AMessage> notifyLost = new AMessage(kWhatPacketLost, id()); + notifyLost->setInt32("ssrc", srcId); + + mRenderer = new TunnelRenderer(notifyLost, mSurfaceTex); + looper()->registerHandler(mRenderer); + } + + sp<AMessage> queueBufferMsg = + new AMessage(TunnelRenderer::kWhatQueueBuffer, mRenderer->id()); + + sp<Source> source = new Source(seqNo, buffer, queueBufferMsg); + mSources.add(srcId, source); + } else { + mSources.valueAt(index)->updateSeq(seqNo, buffer); + } + + return OK; +} + +status_t RTPSink::parseRTCP(const sp<ABuffer> &buffer) { + const uint8_t *data = buffer->data(); + size_t size = buffer->size(); + + while (size > 0) { + if (size < 8) { + // Too short to be a valid RTCP header + return ERROR_MALFORMED; + } + + if ((data[0] >> 6) != 2) { + // Unsupported version. + return ERROR_UNSUPPORTED; + } + + if (data[0] & 0x20) { + // Padding present. + + size_t paddingLength = data[size - 1]; + + if (paddingLength + 12 > size) { + // If we removed this much padding we'd end up with something + // that's too short to be a valid RTP header. + return ERROR_MALFORMED; + } + + size -= paddingLength; + } + + size_t headerLength = 4 * (data[2] << 8 | data[3]) + 4; + + if (size < headerLength) { + // Only received a partial packet? + return ERROR_MALFORMED; + } + + switch (data[1]) { + case 200: + { + parseSR(data, headerLength); + break; + } + + case 201: // RR + case 202: // SDES + case 204: // APP + break; + + case 205: // TSFB (transport layer specific feedback) + case 206: // PSFB (payload specific feedback) + // hexdump(data, headerLength); + break; + + case 203: + { + parseBYE(data, headerLength); + break; + } + + default: + { + ALOGW("Unknown RTCP packet type %u of size %d", + (unsigned)data[1], headerLength); + break; + } + } + + data += headerLength; + size -= headerLength; + } + + return OK; +} + +status_t RTPSink::parseBYE(const uint8_t *data, size_t size) { + size_t SC = data[0] & 0x3f; + + if (SC == 0 || size < (4 + SC * 4)) { + // Packet too short for the minimal BYE header. + return ERROR_MALFORMED; + } + + uint32_t id = U32_AT(&data[4]); + + return OK; +} + +status_t RTPSink::parseSR(const uint8_t *data, size_t size) { + size_t RC = data[0] & 0x1f; + + if (size < (7 + RC * 6) * 4) { + // Packet too short for the minimal SR header. + return ERROR_MALFORMED; + } + + uint32_t id = U32_AT(&data[4]); + uint64_t ntpTime = U64_AT(&data[8]); + uint32_t rtpTime = U32_AT(&data[16]); + + ALOGV("SR: ssrc 0x%08x, ntpTime 0x%016llx, rtpTime 0x%08x", + id, ntpTime, rtpTime); + + return OK; +} + +status_t RTPSink::connect( + const char *host, int32_t remoteRtpPort, int32_t remoteRtcpPort) { + ALOGI("connecting RTP/RTCP sockets to %s:{%d,%d}", + host, remoteRtpPort, remoteRtcpPort); + + status_t err = + mNetSession->connectUDPSession(mRTPSessionID, host, remoteRtpPort); + + if (err != OK) { + return err; + } + + err = mNetSession->connectUDPSession(mRTCPSessionID, host, remoteRtcpPort); + + if (err != OK) { + return err; + } + +#if 0 + sp<ABuffer> buf = new ABuffer(1500); + memset(buf->data(), 0, buf->size()); + + mNetSession->sendRequest( + mRTPSessionID, buf->data(), buf->size()); + + mNetSession->sendRequest( + mRTCPSessionID, buf->data(), buf->size()); +#endif + + scheduleSendRR(); + + return OK; +} + +void RTPSink::scheduleSendRR() { + (new AMessage(kWhatSendRR, id()))->post(2000000ll); +} + +void RTPSink::addSDES(const sp<ABuffer> &buffer) { + uint8_t *data = buffer->data() + buffer->size(); + data[0] = 0x80 | 1; + data[1] = 202; // SDES + data[4] = 0xde; // SSRC + data[5] = 0xad; + data[6] = 0xbe; + data[7] = 0xef; + + size_t offset = 8; + + data[offset++] = 1; // CNAME + + AString cname = "stagefright@somewhere"; + data[offset++] = cname.size(); + + memcpy(&data[offset], cname.c_str(), cname.size()); + offset += cname.size(); + + data[offset++] = 6; // TOOL + + AString tool = "stagefright/1.0"; + data[offset++] = tool.size(); + + memcpy(&data[offset], tool.c_str(), tool.size()); + offset += tool.size(); + + data[offset++] = 0; + + if ((offset % 4) > 0) { + size_t count = 4 - (offset % 4); + switch (count) { + case 3: + data[offset++] = 0; + case 2: + data[offset++] = 0; + case 1: + data[offset++] = 0; + } + } + + size_t numWords = (offset / 4) - 1; + data[2] = numWords >> 8; + data[3] = numWords & 0xff; + + buffer->setRange(buffer->offset(), buffer->size() + offset); +} + +void RTPSink::onSendRR() { + sp<ABuffer> buf = new ABuffer(1500); + buf->setRange(0, 0); + + uint8_t *ptr = buf->data(); + ptr[0] = 0x80 | 0; + ptr[1] = 201; // RR + ptr[2] = 0; + ptr[3] = 1; + ptr[4] = 0xde; // SSRC + ptr[5] = 0xad; + ptr[6] = 0xbe; + ptr[7] = 0xef; + + buf->setRange(0, 8); + + size_t numReportBlocks = 0; + for (size_t i = 0; i < mSources.size(); ++i) { + uint32_t ssrc = mSources.keyAt(i); + sp<Source> source = mSources.valueAt(i); + + if (numReportBlocks > 31 || buf->size() + 24 > buf->capacity()) { + // Cannot fit another report block. + break; + } + + source->addReportBlock(ssrc, buf); + ++numReportBlocks; + } + + ptr[0] |= numReportBlocks; // 5 bit + + size_t sizeInWordsMinus1 = 1 + 6 * numReportBlocks; + ptr[2] = sizeInWordsMinus1 >> 8; + ptr[3] = sizeInWordsMinus1 & 0xff; + + buf->setRange(0, (sizeInWordsMinus1 + 1) * 4); + + addSDES(buf); + + mNetSession->sendRequest(mRTCPSessionID, buf->data(), buf->size()); + + scheduleSendRR(); +} + +void RTPSink::onPacketLost(const sp<AMessage> &msg) { + uint32_t srcId; + CHECK(msg->findInt32("ssrc", (int32_t *)&srcId)); + + int32_t seqNo; + CHECK(msg->findInt32("seqNo", &seqNo)); + + int32_t blp = 0; + + sp<ABuffer> buf = new ABuffer(1500); + buf->setRange(0, 0); + + uint8_t *ptr = buf->data(); + ptr[0] = 0x80 | 1; // generic NACK + ptr[1] = 205; // RTPFB + ptr[2] = 0; + ptr[3] = 3; + ptr[4] = 0xde; // sender SSRC + ptr[5] = 0xad; + ptr[6] = 0xbe; + ptr[7] = 0xef; + ptr[8] = (srcId >> 24) & 0xff; + ptr[9] = (srcId >> 16) & 0xff; + ptr[10] = (srcId >> 8) & 0xff; + ptr[11] = (srcId & 0xff); + ptr[12] = (seqNo >> 8) & 0xff; + ptr[13] = (seqNo & 0xff); + ptr[14] = (blp >> 8) & 0xff; + ptr[15] = (blp & 0xff); + + buf->setRange(0, 16); + + mNetSession->sendRequest(mRTCPSessionID, buf->data(), buf->size()); +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/sink/RTPSink.h b/media/libstagefright/wifi-display/sink/RTPSink.h new file mode 100644 index 0000000..a1d127d --- /dev/null +++ b/media/libstagefright/wifi-display/sink/RTPSink.h @@ -0,0 +1,98 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RTP_SINK_H_ + +#define RTP_SINK_H_ + +#include <media/stagefright/foundation/AHandler.h> + +#include "LinearRegression.h" + +#include <gui/Surface.h> + +namespace android { + +struct ABuffer; +struct ANetworkSession; +struct TunnelRenderer; + +// Creates a pair of sockets for RTP/RTCP traffic, instantiates a renderer +// for incoming transport stream data and occasionally sends statistics over +// the RTCP channel. +struct RTPSink : public AHandler { + RTPSink(const sp<ANetworkSession> &netSession, + const sp<ISurfaceTexture> &surfaceTex); + + // If TCP interleaving is used, no UDP sockets are created, instead + // incoming RTP/RTCP packets (arriving on the RTSP control connection) + // are manually injected by WifiDisplaySink. + status_t init(bool useTCPInterleaving); + + status_t connect( + const char *host, int32_t remoteRtpPort, int32_t remoteRtcpPort); + + int32_t getRTPPort() const; + + status_t injectPacket(bool isRTP, const sp<ABuffer> &buffer); + +protected: + virtual void onMessageReceived(const sp<AMessage> &msg); + virtual ~RTPSink(); + +private: + enum { + kWhatRTPNotify, + kWhatRTCPNotify, + kWhatSendRR, + kWhatPacketLost, + kWhatInject, + }; + + struct Source; + struct StreamSource; + + sp<ANetworkSession> mNetSession; + sp<ISurfaceTexture> mSurfaceTex; + KeyedVector<uint32_t, sp<Source> > mSources; + + int32_t mRTPPort; + int32_t mRTPSessionID; + int32_t mRTCPSessionID; + + int64_t mFirstArrivalTimeUs; + int64_t mNumPacketsReceived; + LinearRegression mRegression; + int64_t mMaxDelayMs; + + sp<TunnelRenderer> mRenderer; + + status_t parseRTP(const sp<ABuffer> &buffer); + status_t parseRTCP(const sp<ABuffer> &buffer); + status_t parseBYE(const uint8_t *data, size_t size); + status_t parseSR(const uint8_t *data, size_t size); + + void addSDES(const sp<ABuffer> &buffer); + void onSendRR(); + void onPacketLost(const sp<AMessage> &msg); + void scheduleSendRR(); + + DISALLOW_EVIL_CONSTRUCTORS(RTPSink); +}; + +} // namespace android + +#endif // RTP_SINK_H_ diff --git a/media/libstagefright/wifi-display/sink/TunnelRenderer.cpp b/media/libstagefright/wifi-display/sink/TunnelRenderer.cpp new file mode 100644 index 0000000..bc35aef --- /dev/null +++ b/media/libstagefright/wifi-display/sink/TunnelRenderer.cpp @@ -0,0 +1,396 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "TunnelRenderer" +#include <utils/Log.h> + +#include "TunnelRenderer.h" + +#include "ATSParser.h" + +#include <binder/IMemory.h> +#include <binder/IServiceManager.h> +#include <gui/SurfaceComposerClient.h> +#include <media/IMediaPlayerService.h> +#include <media/IStreamSource.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <ui/DisplayInfo.h> + +namespace android { + +struct TunnelRenderer::PlayerClient : public BnMediaPlayerClient { + PlayerClient() {} + + virtual void notify(int msg, int ext1, int ext2, const Parcel *obj) { + ALOGI("notify %d, %d, %d", msg, ext1, ext2); + } + +protected: + virtual ~PlayerClient() {} + +private: + DISALLOW_EVIL_CONSTRUCTORS(PlayerClient); +}; + +struct TunnelRenderer::StreamSource : public BnStreamSource { + StreamSource(TunnelRenderer *owner); + + virtual void setListener(const sp<IStreamListener> &listener); + virtual void setBuffers(const Vector<sp<IMemory> > &buffers); + + virtual void onBufferAvailable(size_t index); + + virtual uint32_t flags() const; + + void doSomeWork(); + +protected: + virtual ~StreamSource(); + +private: + mutable Mutex mLock; + + TunnelRenderer *mOwner; + + sp<IStreamListener> mListener; + + Vector<sp<IMemory> > mBuffers; + List<size_t> mIndicesAvailable; + + size_t mNumDeqeued; + + DISALLOW_EVIL_CONSTRUCTORS(StreamSource); +}; + +//////////////////////////////////////////////////////////////////////////////// + +TunnelRenderer::StreamSource::StreamSource(TunnelRenderer *owner) + : mOwner(owner), + mNumDeqeued(0) { +} + +TunnelRenderer::StreamSource::~StreamSource() { +} + +void TunnelRenderer::StreamSource::setListener( + const sp<IStreamListener> &listener) { + mListener = listener; +} + +void TunnelRenderer::StreamSource::setBuffers( + const Vector<sp<IMemory> > &buffers) { + mBuffers = buffers; +} + +void TunnelRenderer::StreamSource::onBufferAvailable(size_t index) { + CHECK_LT(index, mBuffers.size()); + + { + Mutex::Autolock autoLock(mLock); + mIndicesAvailable.push_back(index); + } + + doSomeWork(); +} + +uint32_t TunnelRenderer::StreamSource::flags() const { + return kFlagAlignedVideoData; +} + +void TunnelRenderer::StreamSource::doSomeWork() { + Mutex::Autolock autoLock(mLock); + + while (!mIndicesAvailable.empty()) { + sp<ABuffer> srcBuffer = mOwner->dequeueBuffer(); + if (srcBuffer == NULL) { + break; + } + + ++mNumDeqeued; + + if (mNumDeqeued == 1) { + ALOGI("fixing real time now."); + + sp<AMessage> extra = new AMessage; + + extra->setInt32( + IStreamListener::kKeyDiscontinuityMask, + ATSParser::DISCONTINUITY_ABSOLUTE_TIME); + + extra->setInt64("timeUs", ALooper::GetNowUs()); + + mListener->issueCommand( + IStreamListener::DISCONTINUITY, + false /* synchronous */, + extra); + } + + ALOGV("dequeue TS packet of size %d", srcBuffer->size()); + + size_t index = *mIndicesAvailable.begin(); + mIndicesAvailable.erase(mIndicesAvailable.begin()); + + sp<IMemory> mem = mBuffers.itemAt(index); + CHECK_LE(srcBuffer->size(), mem->size()); + CHECK_EQ((srcBuffer->size() % 188), 0u); + + memcpy(mem->pointer(), srcBuffer->data(), srcBuffer->size()); + mListener->queueBuffer(index, srcBuffer->size()); + } +} + +//////////////////////////////////////////////////////////////////////////////// + +TunnelRenderer::TunnelRenderer( + const sp<AMessage> ¬ifyLost, + const sp<ISurfaceTexture> &surfaceTex) + : mNotifyLost(notifyLost), + mSurfaceTex(surfaceTex), + mTotalBytesQueued(0ll), + mLastDequeuedExtSeqNo(-1), + mFirstFailedAttemptUs(-1ll), + mRequestedRetransmission(false) { +} + +TunnelRenderer::~TunnelRenderer() { + destroyPlayer(); +} + +void TunnelRenderer::queueBuffer(const sp<ABuffer> &buffer) { + Mutex::Autolock autoLock(mLock); + + mTotalBytesQueued += buffer->size(); + + if (mPackets.empty()) { + mPackets.push_back(buffer); + return; + } + + int32_t newExtendedSeqNo = buffer->int32Data(); + + List<sp<ABuffer> >::iterator firstIt = mPackets.begin(); + List<sp<ABuffer> >::iterator it = --mPackets.end(); + for (;;) { + int32_t extendedSeqNo = (*it)->int32Data(); + + if (extendedSeqNo == newExtendedSeqNo) { + // Duplicate packet. + return; + } + + if (extendedSeqNo < newExtendedSeqNo) { + // Insert new packet after the one at "it". + mPackets.insert(++it, buffer); + return; + } + + if (it == firstIt) { + // Insert new packet before the first existing one. + mPackets.insert(it, buffer); + return; + } + + --it; + } +} + +sp<ABuffer> TunnelRenderer::dequeueBuffer() { + Mutex::Autolock autoLock(mLock); + + sp<ABuffer> buffer; + int32_t extSeqNo; + while (!mPackets.empty()) { + buffer = *mPackets.begin(); + extSeqNo = buffer->int32Data(); + + if (mLastDequeuedExtSeqNo < 0 || extSeqNo > mLastDequeuedExtSeqNo) { + break; + } + + // This is a retransmission of a packet we've already returned. + + mTotalBytesQueued -= buffer->size(); + buffer.clear(); + extSeqNo = -1; + + mPackets.erase(mPackets.begin()); + } + + if (mPackets.empty()) { + if (mFirstFailedAttemptUs < 0ll) { + mFirstFailedAttemptUs = ALooper::GetNowUs(); + mRequestedRetransmission = false; + } else { + ALOGV("no packets available for %.2f secs", + (ALooper::GetNowUs() - mFirstFailedAttemptUs) / 1E6); + } + + return NULL; + } + + if (mLastDequeuedExtSeqNo < 0 || extSeqNo == mLastDequeuedExtSeqNo + 1) { + if (mRequestedRetransmission) { + ALOGI("Recovered after requesting retransmission of %d", + extSeqNo); + } + + mLastDequeuedExtSeqNo = extSeqNo; + mFirstFailedAttemptUs = -1ll; + mRequestedRetransmission = false; + + mPackets.erase(mPackets.begin()); + + mTotalBytesQueued -= buffer->size(); + + return buffer; + } + + if (mFirstFailedAttemptUs < 0ll) { + mFirstFailedAttemptUs = ALooper::GetNowUs(); + + ALOGI("failed to get the correct packet the first time."); + return NULL; + } + + if (mFirstFailedAttemptUs + 50000ll > ALooper::GetNowUs()) { + // We're willing to wait a little while to get the right packet. + + if (!mRequestedRetransmission) { + ALOGI("requesting retransmission of seqNo %d", + (mLastDequeuedExtSeqNo + 1) & 0xffff); + + sp<AMessage> notify = mNotifyLost->dup(); + notify->setInt32("seqNo", (mLastDequeuedExtSeqNo + 1) & 0xffff); + notify->post(); + + mRequestedRetransmission = true; + } else { + ALOGI("still waiting for the correct packet to arrive."); + } + + return NULL; + } + + ALOGI("dropping packet. extSeqNo %d didn't arrive in time", + mLastDequeuedExtSeqNo + 1); + + // Permanent failure, we never received the packet. + mLastDequeuedExtSeqNo = extSeqNo; + mFirstFailedAttemptUs = -1ll; + mRequestedRetransmission = false; + + mTotalBytesQueued -= buffer->size(); + + mPackets.erase(mPackets.begin()); + + return buffer; +} + +void TunnelRenderer::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatQueueBuffer: + { + sp<ABuffer> buffer; + CHECK(msg->findBuffer("buffer", &buffer)); + + queueBuffer(buffer); + + if (mStreamSource == NULL) { + if (mTotalBytesQueued > 0ll) { + initPlayer(); + } else { + ALOGI("Have %lld bytes queued...", mTotalBytesQueued); + } + } else { + mStreamSource->doSomeWork(); + } + break; + } + + default: + TRESPASS(); + } +} + +void TunnelRenderer::initPlayer() { + if (mSurfaceTex == NULL) { + mComposerClient = new SurfaceComposerClient; + CHECK_EQ(mComposerClient->initCheck(), (status_t)OK); + + DisplayInfo info; + SurfaceComposerClient::getDisplayInfo(0, &info); + ssize_t displayWidth = info.w; + ssize_t displayHeight = info.h; + + mSurfaceControl = + mComposerClient->createSurface( + String8("A Surface"), + displayWidth, + displayHeight, + PIXEL_FORMAT_RGB_565, + 0); + + CHECK(mSurfaceControl != NULL); + CHECK(mSurfaceControl->isValid()); + + SurfaceComposerClient::openGlobalTransaction(); + CHECK_EQ(mSurfaceControl->setLayer(INT_MAX), (status_t)OK); + CHECK_EQ(mSurfaceControl->show(), (status_t)OK); + SurfaceComposerClient::closeGlobalTransaction(); + + mSurface = mSurfaceControl->getSurface(); + CHECK(mSurface != NULL); + } + + sp<IServiceManager> sm = defaultServiceManager(); + sp<IBinder> binder = sm->getService(String16("media.player")); + sp<IMediaPlayerService> service = interface_cast<IMediaPlayerService>(binder); + CHECK(service.get() != NULL); + + mStreamSource = new StreamSource(this); + + mPlayerClient = new PlayerClient; + + mPlayer = service->create(getpid(), mPlayerClient, 0); + CHECK(mPlayer != NULL); + CHECK_EQ(mPlayer->setDataSource(mStreamSource), (status_t)OK); + + mPlayer->setVideoSurfaceTexture( + mSurfaceTex != NULL ? mSurfaceTex : mSurface->getSurfaceTexture()); + + mPlayer->start(); +} + +void TunnelRenderer::destroyPlayer() { + mStreamSource.clear(); + + mPlayer->stop(); + mPlayer.clear(); + + if (mSurfaceTex == NULL) { + mSurface.clear(); + mSurfaceControl.clear(); + + mComposerClient->dispose(); + mComposerClient.clear(); + } +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/sink/TunnelRenderer.h b/media/libstagefright/wifi-display/sink/TunnelRenderer.h new file mode 100644 index 0000000..c9597e0 --- /dev/null +++ b/media/libstagefright/wifi-display/sink/TunnelRenderer.h @@ -0,0 +1,84 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TUNNEL_RENDERER_H_ + +#define TUNNEL_RENDERER_H_ + +#include <gui/Surface.h> +#include <media/stagefright/foundation/AHandler.h> + +namespace android { + +struct ABuffer; +struct SurfaceComposerClient; +struct SurfaceControl; +struct Surface; +struct IMediaPlayer; +struct IStreamListener; + +// This class reassembles incoming RTP packets into the correct order +// and sends the resulting transport stream to a mediaplayer instance +// for playback. +struct TunnelRenderer : public AHandler { + TunnelRenderer( + const sp<AMessage> ¬ifyLost, + const sp<ISurfaceTexture> &surfaceTex); + + sp<ABuffer> dequeueBuffer(); + + enum { + kWhatQueueBuffer, + }; + +protected: + virtual void onMessageReceived(const sp<AMessage> &msg); + virtual ~TunnelRenderer(); + +private: + struct PlayerClient; + struct StreamSource; + + mutable Mutex mLock; + + sp<AMessage> mNotifyLost; + sp<ISurfaceTexture> mSurfaceTex; + + List<sp<ABuffer> > mPackets; + int64_t mTotalBytesQueued; + + sp<SurfaceComposerClient> mComposerClient; + sp<SurfaceControl> mSurfaceControl; + sp<Surface> mSurface; + sp<PlayerClient> mPlayerClient; + sp<IMediaPlayer> mPlayer; + sp<StreamSource> mStreamSource; + + int32_t mLastDequeuedExtSeqNo; + int64_t mFirstFailedAttemptUs; + bool mRequestedRetransmission; + + void initPlayer(); + void destroyPlayer(); + + void queueBuffer(const sp<ABuffer> &buffer); + + DISALLOW_EVIL_CONSTRUCTORS(TunnelRenderer); +}; + +} // namespace android + +#endif // TUNNEL_RENDERER_H_ diff --git a/media/libstagefright/wifi-display/source/Converter.cpp b/media/libstagefright/wifi-display/source/Converter.cpp index ee05e45..b8b8688 100644 --- a/media/libstagefright/wifi-display/source/Converter.cpp +++ b/media/libstagefright/wifi-display/source/Converter.cpp @@ -54,6 +54,10 @@ status_t Converter::initCheck() const { return mInitCheck; } +size_t Converter::getInputBufferCount() const { + return mEncoderInputBuffers.size(); +} + sp<AMessage> Converter::getOutputFormat() const { return mOutputFormat; } diff --git a/media/libstagefright/wifi-display/source/Converter.h b/media/libstagefright/wifi-display/source/Converter.h index 6700a32..67471c7 100644 --- a/media/libstagefright/wifi-display/source/Converter.h +++ b/media/libstagefright/wifi-display/source/Converter.h @@ -36,6 +36,8 @@ struct Converter : public AHandler { status_t initCheck() const; + size_t getInputBufferCount() const; + sp<AMessage> getOutputFormat() const; void feedAccessUnit(const sp<ABuffer> &accessUnit); diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.cpp b/media/libstagefright/wifi-display/source/PlaybackSession.cpp index f9223d6..6c01c7b 100644 --- a/media/libstagefright/wifi-display/source/PlaybackSession.cpp +++ b/media/libstagefright/wifi-display/source/PlaybackSession.cpp @@ -113,9 +113,11 @@ void WifiDisplaySource::PlaybackSession::Track::setPacketizerTrackIndex(size_t i WifiDisplaySource::PlaybackSession::PlaybackSession( const sp<ANetworkSession> &netSession, - const sp<AMessage> ¬ify) + const sp<AMessage> ¬ify, + bool legacyMode) : mNetSession(netSession), mNotify(notify), + mLegacyMode(legacyMode), mLastLifesignUs(), mTSQueue(new ABuffer(12 + kMaxNumTSPacketsPerRTPPacket * 188)), mPrevTimeUs(-1ll), @@ -240,11 +242,6 @@ WifiDisplaySource::PlaybackSession::~PlaybackSession() { mPacketizer.clear(); - sp<IServiceManager> sm = defaultServiceManager(); - sp<IBinder> binder = sm->getService(String16("SurfaceFlinger")); - sp<ISurfaceComposer> service = interface_cast<ISurfaceComposer>(binder); - CHECK(service != NULL); - if (mSerializer != NULL) { mSerializer->stop(); @@ -257,7 +254,14 @@ WifiDisplaySource::PlaybackSession::~PlaybackSession() { mSerializerLooper.clear(); } - service->connectDisplay(NULL); + if (mLegacyMode) { + sp<IServiceManager> sm = defaultServiceManager(); + sp<IBinder> binder = sm->getService(String16("SurfaceFlinger")); + sp<ISurfaceComposer> service = interface_cast<ISurfaceComposer>(binder); + CHECK(service != NULL); + + service->connectDisplay(NULL); + } if (mRTCPSessionID != 0) { mNetSession->destroySession(mRTCPSessionID); @@ -598,28 +602,7 @@ status_t WifiDisplaySource::PlaybackSession::setupPacketizer() { SurfaceComposerClient::getDisplayInfo(0, &info); // sp<SurfaceMediaSource> source = new SurfaceMediaSource(info.w, info.h); - sp<SurfaceMediaSource> source = new SurfaceMediaSource(720, 1280); - - sp<IServiceManager> sm = defaultServiceManager(); - sp<IBinder> binder = sm->getService(String16("SurfaceFlinger")); - sp<ISurfaceComposer> service = interface_cast<ISurfaceComposer>(binder); - CHECK(service != NULL); - - service->connectDisplay(source->getBufferQueue()); - -#if 0 - { - ALOGI("reading buffer"); - - CHECK_EQ((status_t)OK, source->start()); - MediaBuffer *mbuf; - CHECK_EQ((status_t)OK, source->read(&mbuf)); - mbuf->release(); - mbuf = NULL; - - ALOGI("got buffer"); - } -#endif + sp<SurfaceMediaSource> source = new SurfaceMediaSource(width(), height()); #if 0 ssize_t index = mSerializer->addSource(source); @@ -644,10 +627,29 @@ status_t WifiDisplaySource::PlaybackSession::setupPacketizer() { sp<Converter> converter = new Converter(notify, mCodecLooper, format); + CHECK_EQ(converter->initCheck(), (status_t)OK); + + size_t numInputBuffers = converter->getInputBufferCount(); + ALOGI("numInputBuffers to the encoder is %d", numInputBuffers); looper()->registerHandler(converter); mTracks.add(index, new Track(converter)); + + sp<IServiceManager> sm = defaultServiceManager(); + sp<IBinder> binder = sm->getService(String16("SurfaceFlinger")); + sp<ISurfaceComposer> service = interface_cast<ISurfaceComposer>(binder); + CHECK(service != NULL); + + // Add one reference to account for the serializer. + err = source->setMaxAcquiredBufferCount(numInputBuffers + 1); + CHECK_EQ(err, (status_t)OK); + + mBufferQueue = source->getBufferQueue(); + + if (mLegacyMode) { + service->connectDisplay(mBufferQueue); + } #endif #if 0 @@ -679,6 +681,18 @@ status_t WifiDisplaySource::PlaybackSession::setupPacketizer() { return OK; } +sp<ISurfaceTexture> WifiDisplaySource::PlaybackSession::getSurfaceTexture() { + return mBufferQueue; +} + +int32_t WifiDisplaySource::PlaybackSession::width() const { + return 720; +} + +int32_t WifiDisplaySource::PlaybackSession::height() const { + return 1280; +} + void WifiDisplaySource::PlaybackSession::scheduleSendSR() { if (mSendSRPending) { return; diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.h b/media/libstagefright/wifi-display/source/PlaybackSession.h index a6c9f27..5c228f6 100644 --- a/media/libstagefright/wifi-display/source/PlaybackSession.h +++ b/media/libstagefright/wifi-display/source/PlaybackSession.h @@ -23,6 +23,8 @@ namespace android { struct ABuffer; +struct BufferQueue; +struct ISurfaceTexture; struct Serializer; struct TSPacketizer; @@ -32,7 +34,9 @@ struct TSPacketizer; // display. struct WifiDisplaySource::PlaybackSession : public AHandler { PlaybackSession( - const sp<ANetworkSession> &netSession, const sp<AMessage> ¬ify); + const sp<ANetworkSession> &netSession, + const sp<AMessage> ¬ify, + bool legacyMode); status_t init( const char *clientIP, int32_t clientRtp, int32_t clientRtcp, @@ -46,6 +50,10 @@ struct WifiDisplaySource::PlaybackSession : public AHandler { status_t play(); status_t pause(); + sp<ISurfaceTexture> getSurfaceTexture(); + int32_t width() const; + int32_t height() const; + enum { kWhatSessionDead, kWhatBinaryData, @@ -73,6 +81,7 @@ private: sp<ANetworkSession> mNetSession; sp<AMessage> mNotify; + bool mLegacyMode; int64_t mLastLifesignUs; @@ -80,6 +89,7 @@ private: sp<Serializer> mSerializer; sp<TSPacketizer> mPacketizer; sp<ALooper> mCodecLooper; + sp<BufferQueue> mBufferQueue; KeyedVector<size_t, sp<Track> > mTracks; diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp index a998dcd..0786f2b 100644 --- a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp @@ -22,6 +22,9 @@ #include "PlaybackSession.h" #include "ParsedMessage.h" +#include <gui/ISurfaceTexture.h> + +#include <media/IRemoteDisplayClient.h> #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> @@ -32,8 +35,11 @@ namespace android { -WifiDisplaySource::WifiDisplaySource(const sp<ANetworkSession> &netSession) +WifiDisplaySource::WifiDisplaySource( + const sp<ANetworkSession> &netSession, + const sp<IRemoteDisplayClient> &client) : mNetSession(netSession), + mClient(client), mSessionID(0), mReaperPending(false), mNextCSeq(1) { @@ -201,6 +207,10 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { mPlaybackSessions.removeItemsAt(i); } + if (mClient != NULL) { + mClient->onDisplayDisconnected(); + } + status_t err = OK; sp<AMessage> response = new AMessage; @@ -768,7 +778,8 @@ void WifiDisplaySource::onSetupRequest( notify->setInt32("sessionID", sessionID); sp<PlaybackSession> playbackSession = - new PlaybackSession(mNetSession, notify); + new PlaybackSession( + mNetSession, notify, mClient == NULL /* legacyMode */); looper()->registerHandler(playbackSession); @@ -869,6 +880,14 @@ void WifiDisplaySource::onPlayRequest( err = mNetSession->sendRequest(sessionID, response.c_str()); CHECK_EQ(err, (status_t)OK); + + if (mClient != NULL) { + mClient->onDisplayConnected( + playbackSession->getSurfaceTexture(), + playbackSession->width(), + playbackSession->height(), + 0 /* flags */); + } } void WifiDisplaySource::onPauseRequest( diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.h b/media/libstagefright/wifi-display/source/WifiDisplaySource.h index f56347d..99eb4f5 100644 --- a/media/libstagefright/wifi-display/source/WifiDisplaySource.h +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.h @@ -24,6 +24,7 @@ namespace android { +struct IRemoteDisplayClient; struct ParsedMessage; // Represents the RTSP server acting as a wifi display source. @@ -31,7 +32,9 @@ struct ParsedMessage; struct WifiDisplaySource : public AHandler { static const unsigned kWifiDisplayDefaultPort = 7236; - WifiDisplaySource(const sp<ANetworkSession> &netSession); + WifiDisplaySource( + const sp<ANetworkSession> &netSession, + const sp<IRemoteDisplayClient> &client); status_t start(const char *iface); status_t stop(); @@ -74,6 +77,7 @@ private: kPlaybackSessionTimeoutSecs * 1000000ll; sp<ANetworkSession> mNetSession; + sp<IRemoteDisplayClient> mClient; int32_t mSessionID; struct ClientInfo { diff --git a/media/libstagefright/wifi-display/wfd.cpp b/media/libstagefright/wifi-display/wfd.cpp index 5e7d9fd..d886f14 100644 --- a/media/libstagefright/wifi-display/wfd.cpp +++ b/media/libstagefright/wifi-display/wfd.cpp @@ -18,11 +18,8 @@ #define LOG_TAG "wfd" #include <utils/Log.h> -#define SUPPORT_SINK 0 - -#if SUPPORT_SINK #include "sink/WifiDisplaySink.h" -#endif +#include "source/WifiDisplaySource.h" #include <binder/ProcessState.h> #include <binder/IServiceManager.h> @@ -49,10 +46,8 @@ static void enableDisableRemoteDisplay(const char *iface) { static void usage(const char *me) { fprintf(stderr, "usage:\n" -#if SUPPORT_SINK " %s -c host[:port]\tconnect to wifi source\n" " -u uri \tconnect to an rtsp uri\n" -#endif " -e ip[:port] \tenable remote display\n" " -d \tdisable remote display\n", me); @@ -72,7 +67,6 @@ int main(int argc, char **argv) { int res; while ((res = getopt(argc, argv, "hc:l:u:e:d")) >= 0) { switch (res) { -#if SUPPORT_SINK case 'c': { const char *colonPos = strrchr(optarg, ':'); @@ -100,7 +94,6 @@ int main(int argc, char **argv) { uri = optarg; break; } -#endif case 'e': { @@ -124,7 +117,6 @@ int main(int argc, char **argv) { } } -#if SUPPORT_SINK if (connectToPort < 0 && uri.empty()) { fprintf(stderr, "You need to select either source host or uri.\n"); @@ -154,7 +146,6 @@ int main(int argc, char **argv) { } looper->start(true /* runOnCallingThread */); -#endif return 0; } |