diff options
Diffstat (limited to 'media')
181 files changed, 8516 insertions, 2842 deletions
diff --git a/media/libeffects/factory/EffectsFactory.c b/media/libeffects/factory/EffectsFactory.c index 6d30d64..c310fe2 100644 --- a/media/libeffects/factory/EffectsFactory.c +++ b/media/libeffects/factory/EffectsFactory.c @@ -28,6 +28,7 @@ static list_elem_t *gEffectList; // list of effect_entry_t: all currently created effects static list_elem_t *gLibraryList; // list of lib_entry_t: all currently loaded libraries +static list_elem_t *gSkippedEffects; // list of effects skipped because of duplicate uuid // list of effect_descriptor and list of sub effects : all currently loaded // It does not contain effects without sub effects. static list_sub_elem_t *gSubEffectList; @@ -63,10 +64,10 @@ static int findEffect(const effect_uuid_t *type, lib_entry_t **lib, effect_descriptor_t **desc); // To search a subeffect in the gSubEffectList -int findSubEffect(const effect_uuid_t *uuid, +static int findSubEffect(const effect_uuid_t *uuid, lib_entry_t **lib, effect_descriptor_t **desc); -static void dumpEffectDescriptor(effect_descriptor_t *desc, char *str, size_t len); +static void dumpEffectDescriptor(effect_descriptor_t *desc, char *str, size_t len, int indent); static int stringToUuid(const char *str, effect_uuid_t *uuid); static int uuidToString(const effect_uuid_t *uuid, char *str, size_t maxLen); @@ -237,8 +238,8 @@ int EffectQueryEffect(uint32_t index, effect_descriptor_t *pDescriptor) } #if (LOG_NDEBUG == 0) - char str[256]; - dumpEffectDescriptor(pDescriptor, str, 256); + char str[512]; + dumpEffectDescriptor(pDescriptor, str, sizeof(str), 0 /* indent */); ALOGV("EffectQueryEffect() desc:%s", str); #endif pthread_mutex_unlock(&gLibLock); @@ -503,15 +504,31 @@ int loadLibrary(cnode *root, const char *name) audio_effect_library_t *desc; list_elem_t *e; lib_entry_t *l; + char path[PATH_MAX]; + char *str; + size_t len; node = config_find(root, PATH_TAG); if (node == NULL) { return -EINVAL; } + // audio_effects.conf always specifies 32 bit lib path: convert to 64 bit path if needed + strlcpy(path, node->value, PATH_MAX); +#ifdef __LP64__ + str = strstr(path, "/lib/"); + if (str == NULL) + return -EINVAL; + len = str - path; + path[len] = '\0'; + strlcat(path, "/lib64/", PATH_MAX); + strlcat(path, node->value + len + strlen("/lib/"), PATH_MAX); +#endif + if (strlen(path) >= PATH_MAX - 1) + return -EINVAL; - hdl = dlopen(node->value, RTLD_NOW); + hdl = dlopen(path, RTLD_NOW); if (hdl == NULL) { - ALOGW("loadLibrary() failed to open %s", node->value); + ALOGW("loadLibrary() failed to open %s", path); goto error; } @@ -535,7 +552,7 @@ int loadLibrary(cnode *root, const char *name) // add entry for library in gLibraryList l = malloc(sizeof(lib_entry_t)); l->name = strndup(name, PATH_MAX); - l->path = strndup(node->value, PATH_MAX); + l->path = strndup(path, PATH_MAX); l->handle = hdl; l->desc = desc; l->effects = NULL; @@ -547,7 +564,7 @@ int loadLibrary(cnode *root, const char *name) e->next = gLibraryList; gLibraryList = e; pthread_mutex_unlock(&gLibLock); - ALOGV("getLibrary() linked library %p for path %s", l, node->value); + ALOGV("getLibrary() linked library %p for path %s", l, path); return 0; @@ -595,8 +612,8 @@ int addSubEffect(cnode *root) return -EINVAL; } #if (LOG_NDEBUG==0) - char s[256]; - dumpEffectDescriptor(d, s, 256); + char s[512]; + dumpEffectDescriptor(d, s, sizeof(s), 0 /* indent */); ALOGV("addSubEffect() read descriptor %p:%s",d, s); #endif if (EFFECT_API_VERSION_MAJOR(d->apiVersion) != @@ -660,6 +677,13 @@ int loadEffect(cnode *root) ALOGW("loadEffect() invalid uuid %s", node->value); return -EINVAL; } + lib_entry_t *tmp; + bool skip = false; + if (findEffect(NULL, &uuid, &tmp, NULL) == 0) { + ALOGW("skipping duplicate uuid %s %s", node->value, + node->next ? "and its sub-effects" : ""); + skip = true; + } d = malloc(sizeof(effect_descriptor_t)); if (l->desc->get_descriptor(&uuid, d) != 0) { @@ -670,8 +694,8 @@ int loadEffect(cnode *root) return -EINVAL; } #if (LOG_NDEBUG==0) - char s[256]; - dumpEffectDescriptor(d, s, 256); + char s[512]; + dumpEffectDescriptor(d, s, sizeof(s), 0 /* indent */); ALOGV("loadEffect() read descriptor %p:%s",d, s); #endif if (EFFECT_API_VERSION_MAJOR(d->apiVersion) != @@ -682,8 +706,14 @@ int loadEffect(cnode *root) } e = malloc(sizeof(list_elem_t)); e->object = d; - e->next = l->effects; - l->effects = e; + if (skip) { + e->next = gSkippedEffects; + gSkippedEffects = e; + return -EINVAL; + } else { + e->next = l->effects; + l->effects = e; + } // After the UUID node in the config_tree, if node->next is valid, // that would be sub effect node. @@ -876,22 +906,30 @@ int findEffect(const effect_uuid_t *type, return ret; } -void dumpEffectDescriptor(effect_descriptor_t *desc, char *str, size_t len) { +void dumpEffectDescriptor(effect_descriptor_t *desc, char *str, size_t len, int indent) { char s[256]; + char ss[256]; + char idt[indent + 1]; - snprintf(str, len, "\nEffect Descriptor %p:\n", desc); - strncat(str, "- TYPE: ", len); - uuidToString(&desc->uuid, s, 256); - snprintf(str, len, "- UUID: %s\n", s); - uuidToString(&desc->type, s, 256); - snprintf(str, len, "- TYPE: %s\n", s); - sprintf(s, "- apiVersion: %08X\n- flags: %08X\n", - desc->apiVersion, desc->flags); - strncat(str, s, len); - sprintf(s, "- name: %s\n", desc->name); - strncat(str, s, len); - sprintf(s, "- implementor: %s\n", desc->implementor); - strncat(str, s, len); + memset(idt, ' ', indent); + idt[indent] = 0; + + str[0] = 0; + + snprintf(s, sizeof(s), "%s%s / %s\n", idt, desc->name, desc->implementor); + strlcat(str, s, len); + + uuidToString(&desc->uuid, s, sizeof(s)); + snprintf(ss, sizeof(ss), "%s UUID: %s\n", idt, s); + strlcat(str, ss, len); + + uuidToString(&desc->type, s, sizeof(s)); + snprintf(ss, sizeof(ss), "%s TYPE: %s\n", idt, s); + strlcat(str, ss, len); + + sprintf(s, "%s apiVersion: %08X\n%s flags: %08X\n", idt, + desc->apiVersion, idt, desc->flags); + strlcat(str, s, len); } int stringToUuid(const char *str, effect_uuid_t *uuid) @@ -934,3 +972,40 @@ int uuidToString(const effect_uuid_t *uuid, char *str, size_t maxLen) return 0; } +int EffectDumpEffects(int fd) { + char s[512]; + list_elem_t *e = gLibraryList; + lib_entry_t *l = NULL; + effect_descriptor_t *d = NULL; + int found = 0; + int ret = 0; + + while (e) { + l = (lib_entry_t *)e->object; + list_elem_t *efx = l->effects; + dprintf(fd, "Library %s\n", l->name); + if (!efx) { + dprintf(fd, " (no effects)\n"); + } + while (efx) { + d = (effect_descriptor_t *)efx->object; + dumpEffectDescriptor(d, s, sizeof(s), 2); + dprintf(fd, "%s", s); + efx = efx->next; + } + e = e->next; + } + + e = gSkippedEffects; + if (e) { + dprintf(fd, "Skipped effects\n"); + while(e) { + d = (effect_descriptor_t *)e->object; + dumpEffectDescriptor(d, s, sizeof(s), 2 /* indent */); + dprintf(fd, "%s", s); + e = e->next; + } + } + return ret; +} + diff --git a/media/libmedia/Android.mk b/media/libmedia/Android.mk index 6c585fb..3b260d6 100644 --- a/media/libmedia/Android.mk +++ b/media/libmedia/Android.mk @@ -36,6 +36,8 @@ LOCAL_SRC_FILES:= \ IMediaRecorder.cpp \ IRemoteDisplay.cpp \ IRemoteDisplayClient.cpp \ + IResourceManagerClient.cpp \ + IResourceManagerService.cpp \ IStreamSource.cpp \ MediaCodecInfo.cpp \ Metadata.cpp \ @@ -53,6 +55,8 @@ LOCAL_SRC_FILES:= \ CharacterEncodingDetector.cpp \ IMediaDeathNotifier.cpp \ MediaProfiles.cpp \ + MediaResource.cpp \ + MediaResourcePolicy.cpp \ IEffect.cpp \ IEffectClient.cpp \ AudioEffect.cpp \ @@ -61,15 +65,11 @@ LOCAL_SRC_FILES:= \ StringArray.cpp \ AudioPolicy.cpp -LOCAL_SRC_FILES += ../libnbaio/roundup.c - LOCAL_SHARED_LIBRARIES := \ libui liblog libcutils libutils libbinder libsonivox libicuuc libicui18n libexpat \ libcamera_client libstagefright_foundation \ libgui libdl libaudioutils libnbaio -LOCAL_STATIC_LIBRARIES += libinstantssq - LOCAL_WHOLE_STATIC_LIBRARIES := libmedia_helper LOCAL_MODULE:= libmedia @@ -85,12 +85,3 @@ LOCAL_C_INCLUDES := \ include $(BUILD_SHARED_LIBRARY) -include $(CLEAR_VARS) - -LOCAL_SRC_FILES += SingleStateQueue.cpp -LOCAL_CFLAGS += -DSINGLE_STATE_QUEUE_INSTANTIATIONS='"SingleStateQueueInstantiations.cpp"' - -LOCAL_MODULE := libinstantssq -LOCAL_MODULE_TAGS := optional - -include $(BUILD_STATIC_LIBRARY) diff --git a/media/libmedia/AudioEffect.cpp b/media/libmedia/AudioEffect.cpp index af103c1..7d8222f 100644 --- a/media/libmedia/AudioEffect.cpp +++ b/media/libmedia/AudioEffect.cpp @@ -486,4 +486,4 @@ status_t AudioEffect::guidToString(const effect_uuid_t *guid, char *str, size_t } -}; // namespace android +} // namespace android diff --git a/media/libmedia/AudioParameter.cpp b/media/libmedia/AudioParameter.cpp index 33dbf0b..8c8cf45 100644 --- a/media/libmedia/AudioParameter.cpp +++ b/media/libmedia/AudioParameter.cpp @@ -180,4 +180,4 @@ status_t AudioParameter::getAt(size_t index, String8& key, String8& value) } } -}; // namespace android +} // namespace android diff --git a/media/libmedia/AudioPolicy.cpp b/media/libmedia/AudioPolicy.cpp index d2d0971..c7dafcb 100644 --- a/media/libmedia/AudioPolicy.cpp +++ b/media/libmedia/AudioPolicy.cpp @@ -112,4 +112,4 @@ status_t AudioMix::writeToParcel(Parcel *parcel) const return NO_ERROR; } -}; // namespace android +} // namespace android diff --git a/media/libmedia/AudioRecord.cpp b/media/libmedia/AudioRecord.cpp index 07ca14f..100a914 100644 --- a/media/libmedia/AudioRecord.cpp +++ b/media/libmedia/AudioRecord.cpp @@ -112,7 +112,9 @@ AudioRecord::~AudioRecord() mCblkMemory.clear(); mBufferMemory.clear(); IPCThreadState::self()->flushCommands(); - AudioSystem::releaseAudioSessionId(mSessionId, -1); + ALOGV("~AudioRecord, releasing session id %d", + mSessionId); + AudioSystem::releaseAudioSessionId(mSessionId, -1 /*pid*/); } } @@ -159,8 +161,6 @@ status_t AudioRecord::set( } mTransfer = transferType; - AutoMutex lock(mLock); - // invariant that mAudioRecord != 0 is true only after set() returns successfully if (mAudioRecord != 0) { ALOGE("Track already in use"); @@ -233,6 +233,7 @@ status_t AudioRecord::set( if (cbf != NULL) { mAudioRecordThread = new AudioRecordThread(*this, threadCanCallJava); mAudioRecordThread->run("AudioRecord", ANDROID_PRIORITY_AUDIO); + // thread begins in paused state, and will not reference us until start() } // create the IAudioRecord @@ -286,7 +287,6 @@ status_t AudioRecord::start(AudioSystem::sync_event_t event, int triggerSession) status_t status = NO_ERROR; if (!(flags & CBLK_INVALID)) { - ALOGV("mAudioRecord->start()"); status = mAudioRecord->start(event, triggerSession); if (status == DEAD_OBJECT) { flags |= CBLK_INVALID; @@ -352,6 +352,10 @@ status_t AudioRecord::setMarkerPosition(uint32_t marker) mMarkerPosition = marker; mMarkerReached = false; + sp<AudioRecordThread> t = mAudioRecordThread; + if (t != 0) { + t->wake(); + } return NO_ERROR; } @@ -378,6 +382,10 @@ status_t AudioRecord::setPositionUpdatePeriod(uint32_t updatePeriod) mNewPosition = mProxy->getPosition() + updatePeriod; mUpdatePeriod = updatePeriod; + sp<AudioRecordThread> t = mAudioRecordThread; + if (t != 0) { + t->wake(); + } return NO_ERROR; } @@ -408,7 +416,7 @@ status_t AudioRecord::getPosition(uint32_t *position) const uint32_t AudioRecord::getInputFramesLost() const { // no need to check mActive, because if inactive this will return 0, which is what we want - return AudioSystem::getInputFramesLost(getInput()); + return AudioSystem::getInputFramesLost(getInputPrivate()); } // ------------------------------------------------------------------------- @@ -416,7 +424,6 @@ uint32_t AudioRecord::getInputFramesLost() const // must be called with mLock held status_t AudioRecord::openRecord_l(size_t epoch) { - status_t status; const sp<IAudioFlinger>& audioFlinger = AudioSystem::get_audio_flinger(); if (audioFlinger == 0) { ALOGE("Could not get audioflinger"); @@ -431,12 +438,16 @@ status_t AudioRecord::openRecord_l(size_t epoch) } // Client can only express a preference for FAST. Server will perform additional tests. - if ((mFlags & AUDIO_INPUT_FLAG_FAST) && !( - // use case: callback transfer mode - (mTransfer == TRANSFER_CALLBACK) && + if ((mFlags & AUDIO_INPUT_FLAG_FAST) && !(( + // either of these use cases: + // use case 1: callback transfer mode + (mTransfer == TRANSFER_CALLBACK) || + // use case 2: obtain/release mode + (mTransfer == TRANSFER_OBTAIN)) && // matching sample rate (mSampleRate == afSampleRate))) { - ALOGW("AUDIO_INPUT_FLAG_FAST denied by client"); + ALOGW("AUDIO_INPUT_FLAG_FAST denied by client; transfer %d, track %u Hz, primary %u Hz", + mTransfer, mSampleRate, afSampleRate); // once denied, do not request again if IAudioRecord is re-created mFlags = (audio_input_flags_t) (mFlags & ~AUDIO_INPUT_FLAG_FAST); } @@ -452,7 +463,8 @@ status_t AudioRecord::openRecord_l(size_t epoch) } audio_io_handle_t input; - status = AudioSystem::getInputForAttr(&mAttributes, &input, (audio_session_t)mSessionId, + status_t status = AudioSystem::getInputForAttr(&mAttributes, &input, + (audio_session_t)mSessionId, mSampleRate, mFormat, mChannelMask, mFlags); if (status != NO_ERROR) { @@ -684,9 +696,9 @@ status_t AudioRecord::obtainBuffer(Buffer* audioBuffer, const struct timespec *r return status; } -void AudioRecord::releaseBuffer(Buffer* audioBuffer) +void AudioRecord::releaseBuffer(const Buffer* audioBuffer) { - // all TRANSFER_* are valid + // FIXME add error checking on mode, by adding an internal version size_t stepCount = audioBuffer->size / mFrameSize; if (stepCount == 0) { @@ -704,7 +716,7 @@ void AudioRecord::releaseBuffer(Buffer* audioBuffer) // the server does not automatically disable recorder on overrun, so no need to restart } -audio_io_handle_t AudioRecord::getInput() const +audio_io_handle_t AudioRecord::getInputPrivate() const { AutoMutex lock(mLock); return mInput; @@ -712,7 +724,7 @@ audio_io_handle_t AudioRecord::getInput() const // ------------------------------------------------------------------------- -ssize_t AudioRecord::read(void* buffer, size_t userSize) +ssize_t AudioRecord::read(void* buffer, size_t userSize, bool blocking) { if (mTransfer != TRANSFER_SYNC) { return INVALID_OPERATION; @@ -731,7 +743,8 @@ ssize_t AudioRecord::read(void* buffer, size_t userSize) while (userSize >= mFrameSize) { audioBuffer.frameCount = userSize / mFrameSize; - status_t err = obtainBuffer(&audioBuffer, &ClientProxy::kForever); + status_t err = obtainBuffer(&audioBuffer, + blocking ? &ClientProxy::kForever : &ClientProxy::kNonBlocking); if (err < 0) { if (read > 0) { break; @@ -863,8 +876,11 @@ nsecs_t AudioRecord::processAudioBuffer() if (!markerReached && position < markerPosition) { minFrames = markerPosition - position; } - if (updatePeriod > 0 && updatePeriod < minFrames) { - minFrames = updatePeriod; + if (updatePeriod > 0) { + uint32_t remaining = newPosition - position; + if (remaining < minFrames) { + minFrames = remaining; + } } // If > 0, poll periodically to recover from a stuck server. A good value is 2. @@ -990,14 +1006,13 @@ status_t AudioRecord::restoreRecord_l(const char *from) { ALOGW("dead IAudioRecord, creating a new one from %s()", from); ++mSequence; - status_t result; // if the new IAudioRecord is created, openRecord_l() will modify the // following member variables: mAudioRecord, mCblkMemory, mCblk, mBufferMemory. // It will also delete the strong references on previous IAudioRecord and IMemory size_t position = mProxy->getPosition(); mNewPosition = position + mUpdatePeriod; - result = openRecord_l(position); + status_t result = openRecord_l(position); if (result == NO_ERROR) { if (mActive) { // callback thread or sync event hasn't changed @@ -1069,8 +1084,8 @@ bool AudioRecord::AudioRecordThread::threadLoop() case NS_NEVER: return false; case NS_WHENEVER: - // FIXME increase poll interval, or make event-driven - ns = 1000000000LL; + // Event driven: call wake() when callback notifications conditions change. + ns = INT64_MAX; // fall through default: LOG_ALWAYS_FATAL_IF(ns < 0, "processAudioBuffer() returned %" PRId64, ns); @@ -1103,6 +1118,17 @@ void AudioRecord::AudioRecordThread::resume() } } +void AudioRecord::AudioRecordThread::wake() +{ + AutoMutex _l(mMyLock); + if (!mPaused && mPausedInt && mPausedNs > 0) { + // audio record is active and internally paused with timeout. + mIgnoreNextPausedInt = true; + mPausedInt = false; + mMyCond.signal(); + } +} + void AudioRecord::AudioRecordThread::pauseInternal(nsecs_t ns) { AutoMutex _l(mMyLock); @@ -1112,4 +1138,4 @@ void AudioRecord::AudioRecordThread::pauseInternal(nsecs_t ns) // ------------------------------------------------------------------------- -}; // namespace android +} // namespace android diff --git a/media/libmedia/AudioSystem.cpp b/media/libmedia/AudioSystem.cpp index 9cae21c..c6b34a7 100644 --- a/media/libmedia/AudioSystem.cpp +++ b/media/libmedia/AudioSystem.cpp @@ -499,8 +499,8 @@ void AudioSystem::AudioFlingerClient::ioConfigChanged(int event, audio_io_handle OutputDescriptor *outputDesc = new OutputDescriptor(*desc); gOutputs.add(ioHandle, outputDesc); - ALOGV("ioConfigChanged() new output samplingRate %u, format %#x channel mask %#x frameCount %zu " - "latency %d", + ALOGV("ioConfigChanged() new output samplingRate %u, format %#x channel mask %#x " + "frameCount %zu latency %d", outputDesc->samplingRate, outputDesc->format, outputDesc->channelMask, outputDesc->frameCount, outputDesc->latency); } break; @@ -523,8 +523,8 @@ void AudioSystem::AudioFlingerClient::ioConfigChanged(int event, audio_io_handle if (param2 == NULL) break; desc = (const OutputDescriptor *)param2; - ALOGV("ioConfigChanged() new config for output %d samplingRate %u, format %#x channel mask %#x " - "frameCount %zu latency %d", + ALOGV("ioConfigChanged() new config for output %d samplingRate %u, format %#x " + "channel mask %#x frameCount %zu latency %d", ioHandle, desc->samplingRate, desc->format, desc->channelMask, desc->frameCount, desc->latency); OutputDescriptor *outputDesc = gOutputs.valueAt(index); @@ -590,18 +590,22 @@ const sp<IAudioPolicyService> AudioSystem::get_audio_policy_service() status_t AudioSystem::setDeviceConnectionState(audio_devices_t device, audio_policy_dev_state_t state, - const char *device_address) + const char *device_address, + const char *device_name) { const sp<IAudioPolicyService>& aps = AudioSystem::get_audio_policy_service(); const char *address = ""; + const char *name = ""; if (aps == 0) return PERMISSION_DENIED; if (device_address != NULL) { address = device_address; } - - return aps->setDeviceConnectionState(device, state, address); + if (device_name != NULL) { + name = device_name; + } + return aps->setDeviceConnectionState(device, state, address, name); } audio_policy_dev_state_t AudioSystem::getDeviceConnectionState(audio_devices_t device, @@ -999,4 +1003,4 @@ void AudioSystem::AudioPolicyServiceClient::onAudioPatchListUpdate() } } -}; // namespace android +} // namespace android diff --git a/media/libmedia/AudioTrack.cpp b/media/libmedia/AudioTrack.cpp index 735db5c..ce30c62 100644 --- a/media/libmedia/AudioTrack.cpp +++ b/media/libmedia/AudioTrack.cpp @@ -33,11 +33,16 @@ #define WAIT_PERIOD_MS 10 #define WAIT_STREAM_END_TIMEOUT_SEC 120 - +static const int kMaxLoopCountNotifications = 32; namespace android { // --------------------------------------------------------------------------- +template <typename T> +const T &min(const T &x, const T &y) { + return x < y ? x : y; +} + static int64_t convertTimespecToUs(const struct timespec &tv) { return tv.tv_sec * 1000000ll + tv.tv_nsec / 1000; @@ -61,12 +66,11 @@ status_t AudioTrack::getMinFrameCount( return BAD_VALUE; } - // FIXME merge with similar code in createTrack_l(), except we're missing - // some information here that is available in createTrack_l(): + // FIXME handle in server, like createTrack_l(), possible missing info: // audio_io_handle_t output // audio_format_t format // audio_channel_mask_t channelMask - // audio_output_flags_t flags + // audio_output_flags_t flags (FAST) uint32_t afSampleRate; status_t status; status = AudioSystem::getOutputSamplingRate(&afSampleRate, streamType); @@ -96,16 +100,16 @@ status_t AudioTrack::getMinFrameCount( minBufCount = 2; } - *frameCount = (sampleRate == 0) ? afFrameCount * minBufCount : - afFrameCount * minBufCount * uint64_t(sampleRate) / afSampleRate; - // The formula above should always produce a non-zero value, but return an error - // in the unlikely event that it does not, as that's part of the API contract. + *frameCount = minBufCount * sourceFramesNeeded(sampleRate, afFrameCount, afSampleRate); + // The formula above should always produce a non-zero value under normal circumstances: + // AudioTrack.SAMPLE_RATE_HZ_MIN <= sampleRate <= AudioTrack.SAMPLE_RATE_HZ_MAX. + // Return error in the unlikely event that it does not, as that's part of the API contract. if (*frameCount == 0) { - ALOGE("AudioTrack::getMinFrameCount failed for streamType %d, sampleRate %d", + ALOGE("AudioTrack::getMinFrameCount failed for streamType %d, sampleRate %u", streamType, sampleRate); return BAD_VALUE; } - ALOGV("getMinFrameCount=%zu: afFrameCount=%zu, minBufCount=%d, afSampleRate=%d, afLatency=%d", + ALOGV("getMinFrameCount=%zu: afFrameCount=%zu, minBufCount=%u, afSampleRate=%u, afLatency=%u", *frameCount, afFrameCount, minBufCount, afSampleRate, afLatency); return NO_ERROR; } @@ -199,8 +203,8 @@ AudioTrack::~AudioTrack() mCblkMemory.clear(); mSharedBuffer.clear(); IPCThreadState::self()->flushCommands(); - ALOGV("~AudioTrack, releasing session id from %d on behalf of %d", - IPCThreadState::self()->getCallingPid(), mClientPid); + ALOGV("~AudioTrack, releasing session id %d from %d on behalf of %d", + mSessionId, IPCThreadState::self()->getCallingPid(), mClientPid); AudioSystem::releaseAudioSessionId(mSessionId, mClientPid); } } @@ -225,9 +229,9 @@ status_t AudioTrack::set( const audio_attributes_t* pAttributes) { ALOGV("set(): streamType %d, sampleRate %u, format %#x, channelMask %#x, frameCount %zu, " - "flags #%x, notificationFrames %u, sessionId %d, transferType %d", + "flags #%x, notificationFrames %u, sessionId %d, transferType %d, uid %d, pid %d", streamType, sampleRate, format, channelMask, frameCount, flags, notificationFrames, - sessionId, transferType); + sessionId, transferType, uid, pid); switch (transferType) { case TRANSFER_DEFAULT: @@ -270,8 +274,6 @@ status_t AudioTrack::set( ALOGV("set() streamType %d frameCount %zu flags %04x", streamType, frameCount, flags); - AutoMutex lock(mLock); - // invariant that mAudioTrack != 0 is true only after set() returns successfully if (mAudioTrack != 0) { ALOGE("Track already in use"); @@ -295,6 +297,9 @@ status_t AudioTrack::set( ALOGV("Building AudioTrack with attributes: usage=%d content=%d flags=0x%x tags=[%s]", mAttributes.usage, mAttributes.content_type, mAttributes.flags, mAttributes.tags); mStreamType = AUDIO_STREAM_DEFAULT; + if ((mAttributes.flags & AUDIO_FLAG_HW_AV_SYNC) != 0) { + flags = (audio_output_flags_t)(flags | AUDIO_OUTPUT_FLAG_HW_AV_SYNC); + } } // these below should probably come from the audioFlinger too... @@ -317,12 +322,6 @@ status_t AudioTrack::set( uint32_t channelCount = audio_channel_count_from_out_mask(channelMask); mChannelCount = channelCount; - // AudioFlinger does not currently support 8-bit data in shared memory - if (format == AUDIO_FORMAT_PCM_8_BIT && sharedBuffer != 0) { - ALOGE("8-bit data in shared memory is not supported"); - return BAD_VALUE; - } - // force direct flag if format is not linear PCM // or offload was requested if ((flags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) @@ -346,12 +345,9 @@ status_t AudioTrack::set( } else { mFrameSize = sizeof(uint8_t); } - mFrameSizeAF = mFrameSize; } else { ALOG_ASSERT(audio_is_linear_pcm(format)); mFrameSize = channelCount * audio_bytes_per_sample(format); - mFrameSizeAF = channelCount * audio_bytes_per_sample( - format == AUDIO_FORMAT_PCM_8_BIT ? AUDIO_FORMAT_PCM_16_BIT : format); // createTrack will return an error if PCM format is not supported by server, // so no need to check for specific PCM formats here } @@ -403,6 +399,7 @@ status_t AudioTrack::set( if (cbf != NULL) { mAudioTrackThread = new AudioTrackThread(*this, threadCanCallJava); mAudioTrackThread->run("AudioTrack", ANDROID_PRIORITY_AUDIO, 0 /*stack*/); + // thread begins in paused state, and will not reference us until start() } // create the IAudioTrack @@ -420,7 +417,10 @@ status_t AudioTrack::set( mStatus = NO_ERROR; mState = STATE_STOPPED; mUserData = user; - mLoopPeriod = 0; + mLoopCount = 0; + mLoopStart = 0; + mLoopEnd = 0; + mLoopCountNotified = 0; mMarkerPosition = 0; mMarkerReached = false; mNewPosition = 0; @@ -531,14 +531,12 @@ void AudioTrack::stop() // the playback head position will reset to 0, so if a marker is set, we need // to activate it again mMarkerReached = false; -#if 0 - // Force flush if a shared buffer is used otherwise audioflinger - // will not stop before end of buffer is reached. - // It may be needed to make sure that we stop playback, likely in case looping is on. + if (mSharedBuffer != 0) { - flush_l(); + // clear buffer position and loop count. + mStaticProxy->setBufferPositionAndLoop(0 /* position */, + 0 /* loopStart */, 0 /* loopEnd */, 0 /* loopCount */); } -#endif sp<AudioTrackThread> t = mAudioTrackThread; if (t != 0) { @@ -669,14 +667,18 @@ void AudioTrack::getAuxEffectSendLevel(float* level) const status_t AudioTrack::setSampleRate(uint32_t rate) { - if (mIsTimed || isOffloadedOrDirect()) { + AutoMutex lock(mLock); + if (rate == mSampleRate) { + return NO_ERROR; + } + if (mIsTimed || isOffloadedOrDirect_l() || (mFlags & AUDIO_OUTPUT_FLAG_FAST)) { return INVALID_OPERATION; } - - AutoMutex lock(mLock); if (mOutput == AUDIO_IO_HANDLE_NONE) { return NO_INIT; } + // NOTE: it is theoretically possible, but highly unlikely, that a device change + // could mean a previously allowed sampling rate is no longer allowed. uint32_t afSamplingRate; if (AudioSystem::getSamplingRate(mOutput, &afSamplingRate) != NO_ERROR) { return NO_INIT; @@ -740,10 +742,15 @@ status_t AudioTrack::setLoop(uint32_t loopStart, uint32_t loopEnd, int loopCount void AudioTrack::setLoop_l(uint32_t loopStart, uint32_t loopEnd, int loopCount) { - // Setting the loop will reset next notification update period (like setPosition). - mNewPosition = updateAndGetPosition_l() + mUpdatePeriod; - mLoopPeriod = loopCount != 0 ? loopEnd - loopStart : 0; + // We do not update the periodic notification point. + // mNewPosition = updateAndGetPosition_l() + mUpdatePeriod; + mLoopCount = loopCount; + mLoopEnd = loopEnd; + mLoopStart = loopStart; + mLoopCountNotified = loopCount; mStaticProxy->setLoop(loopStart, loopEnd, loopCount); + + // Waking the AudioTrackThread is not needed as this cannot be called when active. } status_t AudioTrack::setMarkerPosition(uint32_t marker) @@ -757,6 +764,10 @@ status_t AudioTrack::setMarkerPosition(uint32_t marker) mMarkerPosition = marker; mMarkerReached = false; + sp<AudioTrackThread> t = mAudioTrackThread; + if (t != 0) { + t->wake(); + } return NO_ERROR; } @@ -786,6 +797,10 @@ status_t AudioTrack::setPositionUpdatePeriod(uint32_t updatePeriod) mNewPosition = updateAndGetPosition_l() + updatePeriod; mUpdatePeriod = updatePeriod; + sp<AudioTrackThread> t = mAudioTrackThread; + if (t != 0) { + t->wake(); + } return NO_ERROR; } @@ -823,12 +838,11 @@ status_t AudioTrack::setPosition(uint32_t position) if (mState == STATE_ACTIVE) { return INVALID_OPERATION; } + // After setting the position, use full update period before notification. mNewPosition = updateAndGetPosition_l() + mUpdatePeriod; - mLoopPeriod = 0; - // FIXME Check whether loops and setting position are incompatible in old code. - // If we use setLoop for both purposes we lose the capability to set the position while looping. - mStaticProxy->setLoop(position, mFrameCount, 0); + mStaticProxy->setBufferPosition(position); + // Waking the AudioTrackThread is not needed as this cannot be called when active. return NO_ERROR; } @@ -893,10 +907,18 @@ status_t AudioTrack::reload() return INVALID_OPERATION; } mNewPosition = mUpdatePeriod; - mLoopPeriod = 0; - // FIXME The new code cannot reload while keeping a loop specified. - // Need to check how the old code handled this, and whether it's a significant change. - mStaticProxy->setLoop(0, mFrameCount, 0); + (void) updateAndGetPosition_l(); + mPosition = 0; +#if 0 + // The documentation is not clear on the behavior of reload() and the restoration + // of loop count. Historically we have not restored loop count, start, end, + // but it makes sense if one desires to repeat playing a particular sound. + if (mLoopCount != 0) { + mLoopCountNotified = mLoopCount; + mStaticProxy->setLoop(mLoopStart, mLoopEnd, mLoopCount); + } +#endif + mStaticProxy->setBufferPosition(0); return NO_ERROR; } @@ -945,9 +967,9 @@ status_t AudioTrack::createTrack_l() if (status != NO_ERROR || output == AUDIO_IO_HANDLE_NONE) { - ALOGE("Could not get audio output for stream type %d, usage %d, sample rate %u, format %#x," + ALOGE("Could not get audio output for session %d, stream type %d, usage %d, sample rate %u, format %#x," " channel mask %#x, flags %#x", - streamType, mAttributes.usage, mSampleRate, mFormat, mChannelMask, mFlags); + mSessionId, streamType, mAttributes.usage, mSampleRate, mFormat, mChannelMask, mFlags); return BAD_VALUE; } { @@ -962,6 +984,7 @@ status_t AudioTrack::createTrack_l() ALOGE("getLatency(%d) failed status %d", output, status); goto release; } + ALOGV("createTrack_l() output %d afLatency %u", output, afLatency); size_t afFrameCount; status = AudioSystem::getFrameCount(output, &afFrameCount); @@ -986,23 +1009,23 @@ status_t AudioTrack::createTrack_l() // use case 1: shared buffer (mSharedBuffer != 0) || // use case 2: callback transfer mode - (mTransfer == TRANSFER_CALLBACK)) && + (mTransfer == TRANSFER_CALLBACK) || + // use case 3: obtain/release mode + (mTransfer == TRANSFER_OBTAIN)) && // matching sample rate (mSampleRate == afSampleRate))) { - ALOGW("AUDIO_OUTPUT_FLAG_FAST denied by client"); + ALOGW("AUDIO_OUTPUT_FLAG_FAST denied by client; transfer %d, track %u Hz, output %u Hz", + mTransfer, mSampleRate, afSampleRate); // once denied, do not request again if IAudioTrack is re-created mFlags = (audio_output_flags_t) (mFlags & ~AUDIO_OUTPUT_FLAG_FAST); } - ALOGV("createTrack_l() output %d afLatency %d", output, afLatency); // The client's AudioTrack buffer is divided into n parts for purpose of wakeup by server, where // n = 1 fast track with single buffering; nBuffering is ignored // n = 2 fast track with double buffering - // n = 2 normal track, no sample rate conversion - // n = 3 normal track, with sample rate conversion - // (pessimistic; some non-1:1 conversion ratios don't actually need triple-buffering) - // n > 3 very high latency or very small notification interval; nBuffering is ignored - const uint32_t nBuffering = (mSampleRate == afSampleRate) ? 2 : 3; + // n = 2 normal track, (including those with sample rate conversion) + // n >= 3 very high latency or very small notification interval (unused). + const uint32_t nBuffering = 2; mNotificationFramesAct = mNotificationFramesReq; @@ -1019,12 +1042,12 @@ status_t AudioTrack::createTrack_l() mNotificationFramesAct = frameCount; } } else if (mSharedBuffer != 0) { - - // Ensure that buffer alignment matches channel count - // 8-bit data in shared memory is not currently supported by AudioFlinger - size_t alignment = audio_bytes_per_sample( - mFormat == AUDIO_FORMAT_PCM_8_BIT ? AUDIO_FORMAT_PCM_16_BIT : mFormat); + // FIXME: Ensure client side memory buffers need + // not have additional alignment beyond sample + // (e.g. 16 bit stereo accessed as 32 bit frame). + size_t alignment = audio_bytes_per_sample(mFormat); if (alignment & 1) { + // for AUDIO_FORMAT_PCM_24_BIT_PACKED (not exposed through Java). alignment = 1; } if (mChannelCount > 1) { @@ -1042,40 +1065,10 @@ status_t AudioTrack::createTrack_l() // there's no frameCount parameter. // But when initializing a shared buffer AudioTrack via set(), // there _is_ a frameCount parameter. We silently ignore it. - frameCount = mSharedBuffer->size() / mFrameSizeAF; - - } else if (!(mFlags & AUDIO_OUTPUT_FLAG_FAST)) { - - // FIXME move these calculations and associated checks to server - - // Ensure that buffer depth covers at least audio hardware latency - uint32_t minBufCount = afLatency / ((1000 * afFrameCount)/afSampleRate); - ALOGV("afFrameCount=%zu, minBufCount=%d, afSampleRate=%u, afLatency=%d", - afFrameCount, minBufCount, afSampleRate, afLatency); - if (minBufCount <= nBuffering) { - minBufCount = nBuffering; - } - - size_t minFrameCount = afFrameCount * minBufCount * uint64_t(mSampleRate) / afSampleRate; - ALOGV("minFrameCount: %zu, afFrameCount=%zu, minBufCount=%d, sampleRate=%u, afSampleRate=%u" - ", afLatency=%d", - minFrameCount, afFrameCount, minBufCount, mSampleRate, afSampleRate, afLatency); - - if (frameCount == 0) { - frameCount = minFrameCount; - } else if (frameCount < minFrameCount) { - // not ALOGW because it happens all the time when playing key clicks over A2DP - ALOGV("Minimum buffer size corrected from %zu to %zu", - frameCount, minFrameCount); - frameCount = minFrameCount; - } - // Make sure that application is notified with sufficient margin before underrun - if (mNotificationFramesAct == 0 || mNotificationFramesAct > frameCount/nBuffering) { - mNotificationFramesAct = frameCount/nBuffering; - } - + frameCount = mSharedBuffer->size() / mFrameSize; } else { - // For fast tracks, the frame count calculations and checks are done by server + // For fast and normal streaming tracks, + // the frame count calculations and checks are done by server } IAudioFlinger::track_flags_t trackFlags = IAudioFlinger::TRACK_DEFAULT; @@ -1101,12 +1094,10 @@ status_t AudioTrack::createTrack_l() size_t temp = frameCount; // temp may be replaced by a revised value of frameCount, // but we will still need the original value also + int originalSessionId = mSessionId; sp<IAudioTrack> track = audioFlinger->createTrack(streamType, mSampleRate, - // AudioFlinger only sees 16-bit PCM - mFormat == AUDIO_FORMAT_PCM_8_BIT && - !(mFlags & AUDIO_OUTPUT_FLAG_DIRECT) ? - AUDIO_FORMAT_PCM_16_BIT : mFormat, + mFormat, mChannelMask, &temp, &trackFlags, @@ -1116,6 +1107,8 @@ status_t AudioTrack::createTrack_l() &mSessionId, mClientUid, &status); + ALOGE_IF(originalSessionId != AUDIO_SESSION_ALLOCATE && mSessionId != originalSessionId, + "session ID changed from %d to %d", originalSessionId, mSessionId); if (status != NO_ERROR) { ALOGE("AudioFlinger could not create track, status: %d", status); @@ -1161,23 +1154,10 @@ status_t AudioTrack::createTrack_l() if (trackFlags & IAudioFlinger::TRACK_FAST) { ALOGV("AUDIO_OUTPUT_FLAG_FAST successful; frameCount %zu", frameCount); mAwaitBoost = true; - if (mSharedBuffer == 0) { - // Theoretically double-buffering is not required for fast tracks, - // due to tighter scheduling. But in practice, to accommodate kernels with - // scheduling jitter, and apps with computation jitter, we use double-buffering. - if (mNotificationFramesAct == 0 || mNotificationFramesAct > frameCount/nBuffering) { - mNotificationFramesAct = frameCount/nBuffering; - } - } } else { ALOGV("AUDIO_OUTPUT_FLAG_FAST denied by server; frameCount %zu", frameCount); // once denied, do not request again if IAudioTrack is re-created mFlags = (audio_output_flags_t) (mFlags & ~AUDIO_OUTPUT_FLAG_FAST); - if (mSharedBuffer == 0) { - if (mNotificationFramesAct == 0 || mNotificationFramesAct > frameCount/nBuffering) { - mNotificationFramesAct = frameCount/nBuffering; - } - } } } if (mFlags & AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD) { @@ -1200,6 +1180,16 @@ status_t AudioTrack::createTrack_l() //return NO_INIT; } } + // Make sure that application is notified with sufficient margin before underrun + if (mSharedBuffer == 0 && audio_is_linear_pcm(mFormat)) { + // Theoretically double-buffering is not required for fast tracks, + // due to tighter scheduling. But in practice, to accommodate kernels with + // scheduling jitter, and apps with computation jitter, we use double-buffering + // for fast tracks just like normal streaming tracks. + if (mNotificationFramesAct == 0 || mNotificationFramesAct > frameCount / nBuffering) { + mNotificationFramesAct = frameCount / nBuffering; + } + } // We retain a copy of the I/O handle, but don't own the reference mOutput = output; @@ -1211,9 +1201,13 @@ status_t AudioTrack::createTrack_l() // address space. AudioFlinger::TrackBase::mBuffer is for the server address space. void* buffers; if (mSharedBuffer == 0) { - buffers = (char*)cblk + sizeof(audio_track_cblk_t); + buffers = cblk + 1; } else { buffers = mSharedBuffer->pointer(); + if (buffers == NULL) { + ALOGE("Could not get buffer pointer"); + return NO_INIT; + } } mAudioTrack->attachAuxEffect(mAuxEffectId); @@ -1230,9 +1224,9 @@ status_t AudioTrack::createTrack_l() // update proxy if (mSharedBuffer == 0) { mStaticProxy.clear(); - mProxy = new AudioTrackClientProxy(cblk, buffers, frameCount, mFrameSizeAF); + mProxy = new AudioTrackClientProxy(cblk, buffers, frameCount, mFrameSize); } else { - mStaticProxy = new StaticAudioTrackClientProxy(cblk, buffers, frameCount, mFrameSizeAF); + mStaticProxy = new StaticAudioTrackClientProxy(cblk, buffers, frameCount, mFrameSize); mProxy = mStaticProxy; } @@ -1258,7 +1252,7 @@ release: return status; } -status_t AudioTrack::obtainBuffer(Buffer* audioBuffer, int32_t waitCount) +status_t AudioTrack::obtainBuffer(Buffer* audioBuffer, int32_t waitCount, size_t *nonContig) { if (audioBuffer == NULL) { return BAD_VALUE; @@ -1285,7 +1279,7 @@ status_t AudioTrack::obtainBuffer(Buffer* audioBuffer, int32_t waitCount) ALOGE("%s invalid waitCount %d", __func__, waitCount); requested = NULL; } - return obtainBuffer(audioBuffer, requested); + return obtainBuffer(audioBuffer, requested, NULL /*elapsed*/, nonContig); } status_t AudioTrack::obtainBuffer(Buffer* audioBuffer, const struct timespec *requested, @@ -1352,7 +1346,7 @@ status_t AudioTrack::obtainBuffer(Buffer* audioBuffer, const struct timespec *re } while ((status == DEAD_OBJECT) && (tryCounter-- > 0)); audioBuffer->frameCount = buffer.mFrameCount; - audioBuffer->size = buffer.mFrameCount * mFrameSizeAF; + audioBuffer->size = buffer.mFrameCount * mFrameSize; audioBuffer->raw = buffer.mRaw; if (nonContig != NULL) { *nonContig = buffer.mNonContig; @@ -1360,13 +1354,14 @@ status_t AudioTrack::obtainBuffer(Buffer* audioBuffer, const struct timespec *re return status; } -void AudioTrack::releaseBuffer(Buffer* audioBuffer) +void AudioTrack::releaseBuffer(const Buffer* audioBuffer) { + // FIXME add error checking on mode, by adding an internal version if (mTransfer == TRANSFER_SHARED) { return; } - size_t stepCount = audioBuffer->size / mFrameSizeAF; + size_t stepCount = audioBuffer->size / mFrameSize; if (stepCount == 0) { return; } @@ -1431,15 +1426,8 @@ ssize_t AudioTrack::write(const void* buffer, size_t userSize, bool blocking) return ssize_t(err); } - size_t toWrite; - if (mFormat == AUDIO_FORMAT_PCM_8_BIT && !(mFlags & AUDIO_OUTPUT_FLAG_DIRECT)) { - // Divide capacity by 2 to take expansion into account - toWrite = audioBuffer.size >> 1; - memcpy_to_i16_from_u8(audioBuffer.i16, (const uint8_t *) buffer, toWrite); - } else { - toWrite = audioBuffer.size; - memcpy(audioBuffer.i8, buffer, toWrite); - } + size_t toWrite = audioBuffer.size; + memcpy(audioBuffer.i8, buffer, toWrite); buffer = ((const char *) buffer) + toWrite; userSize -= toWrite; written += toWrite; @@ -1559,9 +1547,8 @@ nsecs_t AudioTrack::processAudioBuffer() // that the upper layers can recreate the track if (!isOffloadedOrDirect_l() || (mSequence == mObservedSequence)) { status_t status = restoreTrack_l("processAudioBuffer"); - mLock.unlock(); - // Run again immediately, but with a new IAudioTrack - return 0; + // after restoration, continue below to make sure that the loop and buffer events + // are notified because they have been cleared from mCblk->mFlags above. } } @@ -1610,7 +1597,6 @@ nsecs_t AudioTrack::processAudioBuffer() } // Cache other fields that will be needed soon - uint32_t loopPeriod = mLoopPeriod; uint32_t sampleRate = mSampleRate; uint32_t notificationFrames = mNotificationFramesAct; if (mRefreshRemaining) { @@ -1622,8 +1608,30 @@ nsecs_t AudioTrack::processAudioBuffer() uint32_t sequence = mSequence; sp<AudioTrackClientProxy> proxy = mProxy; + // Determine the number of new loop callback(s) that will be needed, while locked. + int loopCountNotifications = 0; + uint32_t loopPeriod = 0; // time in frames for next EVENT_LOOP_END or EVENT_BUFFER_END + + if (mLoopCount > 0) { + int loopCount; + size_t bufferPosition; + mStaticProxy->getBufferPositionAndLoopCount(&bufferPosition, &loopCount); + loopPeriod = ((loopCount > 0) ? mLoopEnd : mFrameCount) - bufferPosition; + loopCountNotifications = min(mLoopCountNotified - loopCount, kMaxLoopCountNotifications); + mLoopCountNotified = loopCount; // discard any excess notifications + } else if (mLoopCount < 0) { + // FIXME: We're not accurate with notification count and position with infinite looping + // since loopCount from server side will always return -1 (we could decrement it). + size_t bufferPosition = mStaticProxy->getBufferPosition(); + loopCountNotifications = int((flags & (CBLK_LOOP_CYCLE | CBLK_LOOP_FINAL)) != 0); + loopPeriod = mLoopEnd - bufferPosition; + } else if (/* mLoopCount == 0 && */ mSharedBuffer != 0) { + size_t bufferPosition = mStaticProxy->getBufferPosition(); + loopPeriod = mFrameCount - bufferPosition; + } + // These fields don't need to be cached, because they are assigned only by set(): - // mTransfer, mCbf, mUserData, mFormat, mFrameSize, mFrameSizeAF, mFlags + // mTransfer, mCbf, mUserData, mFormat, mFrameSize, mFlags // mFlags is also assigned by createTrack_l(), but not the bit we care about. mLock.unlock(); @@ -1662,10 +1670,9 @@ nsecs_t AudioTrack::processAudioBuffer() if (newUnderrun) { mCbf(EVENT_UNDERRUN, mUserData, NULL); } - // FIXME we will miss loops if loop cycle was signaled several times since last call - // to processAudioBuffer() - if (flags & (CBLK_LOOP_CYCLE | CBLK_LOOP_FINAL)) { + while (loopCountNotifications > 0) { mCbf(EVENT_LOOP_END, mUserData, NULL); + --loopCountNotifications; } if (flags & CBLK_BUFFER_END) { mCbf(EVENT_BUFFER_END, mUserData, NULL); @@ -1701,10 +1708,11 @@ nsecs_t AudioTrack::processAudioBuffer() minFrames = markerPosition - position; } if (loopPeriod > 0 && loopPeriod < minFrames) { + // loopPeriod is already adjusted for actual position. minFrames = loopPeriod; } - if (updatePeriod > 0 && updatePeriod < minFrames) { - minFrames = updatePeriod; + if (updatePeriod > 0) { + minFrames = min(minFrames, uint32_t(newPosition - position)); } // If > 0, poll periodically to recover from a stuck server. A good value is 2. @@ -1767,13 +1775,6 @@ nsecs_t AudioTrack::processAudioBuffer() } } - // Divide buffer size by 2 to take into account the expansion - // due to 8 to 16 bit conversion: the callback must fill only half - // of the destination buffer - if (mFormat == AUDIO_FORMAT_PCM_8_BIT && !(mFlags & AUDIO_OUTPUT_FLAG_DIRECT)) { - audioBuffer.size >>= 1; - } - size_t reqSize = audioBuffer.size; mCbf(EVENT_MORE_DATA, mUserData, &audioBuffer); size_t writtenSize = audioBuffer.size; @@ -1793,13 +1794,7 @@ nsecs_t AudioTrack::processAudioBuffer() return WAIT_PERIOD_MS * 1000000LL; } - if (mFormat == AUDIO_FORMAT_PCM_8_BIT && !(mFlags & AUDIO_OUTPUT_FLAG_DIRECT)) { - // 8 to 16 bit conversion, note that source and destination are the same address - memcpy_to_i16_from_u8(audioBuffer.i16, (const uint8_t *) audioBuffer.i8, writtenSize); - audioBuffer.size <<= 1; - } - - size_t releasedFrames = audioBuffer.size / mFrameSizeAF; + size_t releasedFrames = writtenSize / mFrameSize; audioBuffer.frameCount = releasedFrames; mRemainingFrames -= releasedFrames; if (misalignment >= releasedFrames) { @@ -1844,7 +1839,6 @@ status_t AudioTrack::restoreTrack_l(const char *from) ALOGW("dead IAudioTrack, %s, creating a new one from %s()", isOffloadedOrDirect_l() ? "Offloaded or Direct" : "PCM", from); ++mSequence; - status_t result; // refresh the audio configuration cache in this process to make sure we get new // output parameters and new IAudioFlinger in createTrack_l() @@ -1856,39 +1850,39 @@ status_t AudioTrack::restoreTrack_l(const char *from) } // save the old static buffer position - size_t bufferPosition = mStaticProxy != NULL ? mStaticProxy->getBufferPosition() : 0; + size_t bufferPosition = 0; + int loopCount = 0; + if (mStaticProxy != 0) { + mStaticProxy->getBufferPositionAndLoopCount(&bufferPosition, &loopCount); + } // If a new IAudioTrack is successfully created, createTrack_l() will modify the // following member variables: mAudioTrack, mCblkMemory and mCblk. // It will also delete the strong references on previous IAudioTrack and IMemory. // If a new IAudioTrack cannot be created, the previous (dead) instance will be left intact. - result = createTrack_l(); + status_t result = createTrack_l(); // take the frames that will be lost by track recreation into account in saved position + // For streaming tracks, this is the amount we obtained from the user/client + // (not the number actually consumed at the server - those are already lost). (void) updateAndGetPosition_l(); - mPosition = mReleased; + if (mStaticProxy == 0) { + mPosition = mReleased; + } if (result == NO_ERROR) { - // continue playback from last known position, but - // don't attempt to restore loop after invalidation; it's difficult and not worthwhile - if (mStaticProxy != NULL) { - mLoopPeriod = 0; - mStaticProxy->setLoop(bufferPosition, mFrameCount, 0); - } - // FIXME How do we simulate the fact that all frames present in the buffer at the time of - // track destruction have been played? This is critical for SoundPool implementation - // This must be broken, and needs to be tested/debugged. -#if 0 - // restore write index and set other indexes to reflect empty buffer status - if (!strcmp(from, "start")) { - // Make sure that a client relying on callback events indicating underrun or - // the actual amount of audio frames played (e.g SoundPool) receives them. - if (mSharedBuffer == 0) { - // restart playback even if buffer is not completely filled. - android_atomic_or(CBLK_FORCEREADY, &mCblk->mFlags); + // Continue playback from last known position and restore loop. + if (mStaticProxy != 0) { + if (loopCount != 0) { + mStaticProxy->setBufferPositionAndLoop(bufferPosition, + mLoopStart, mLoopEnd, loopCount); + } else { + mStaticProxy->setBufferPosition(bufferPosition); + if (bufferPosition == mFrameCount) { + ALOGD("restoring track at end of static buffer"); + } } } -#endif if (mState == STATE_ACTIVE) { result = mAudioTrack->start(); } @@ -2148,8 +2142,8 @@ bool AudioTrack::AudioTrackThread::threadLoop() case NS_NEVER: return false; case NS_WHENEVER: - // FIXME increase poll interval, or make event-driven - ns = 1000000000LL; + // Event driven: call wake() when callback notifications conditions change. + ns = INT64_MAX; // fall through default: LOG_ALWAYS_FATAL_IF(ns < 0, "processAudioBuffer() returned %" PRId64, ns); @@ -2182,6 +2176,17 @@ void AudioTrack::AudioTrackThread::resume() } } +void AudioTrack::AudioTrackThread::wake() +{ + AutoMutex _l(mMyLock); + if (!mPaused && mPausedInt && mPausedNs > 0) { + // audio track is active and internally paused with timeout. + mIgnoreNextPausedInt = true; + mPausedInt = false; + mMyCond.signal(); + } +} + void AudioTrack::AudioTrackThread::pauseInternal(nsecs_t ns) { AutoMutex _l(mMyLock); @@ -2189,4 +2194,4 @@ void AudioTrack::AudioTrackThread::pauseInternal(nsecs_t ns) mPausedNs = ns; } -}; // namespace android +} // namespace android diff --git a/media/libmedia/AudioTrackShared.cpp b/media/libmedia/AudioTrackShared.cpp index ff24475..6d5f1af 100644 --- a/media/libmedia/AudioTrackShared.cpp +++ b/media/libmedia/AudioTrackShared.cpp @@ -28,7 +28,21 @@ namespace android { // used to clamp a value to size_t. TODO: move to another file. template <typename T> size_t clampToSize(T x) { - return x > SIZE_MAX ? SIZE_MAX : x < 0 ? 0 : (size_t) x; + return sizeof(T) > sizeof(size_t) && x > (T) SIZE_MAX ? SIZE_MAX : x < 0 ? 0 : (size_t) x; +} + +// incrementSequence is used to determine the next sequence value +// for the loop and position sequence counters. It should return +// a value between "other" + 1 and "other" + INT32_MAX, the choice of +// which needs to be the "least recently used" sequence value for "self". +// In general, this means (new_self) returned is max(self, other) + 1. + +static uint32_t incrementSequence(uint32_t self, uint32_t other) { + int32_t diff = self - other; + if (diff >= 0 && diff < INT32_MAX) { + return self + 1; // we're already ahead of other. + } + return other + 1; // we're behind, so move just ahead of other. } audio_track_cblk_t::audio_track_cblk_t() @@ -409,7 +423,6 @@ status_t AudioTrackClientProxy::waitStreamEndDone(const struct timespec *request goto end; } // check for obtainBuffer interrupted by client - // check for obtainBuffer interrupted by client if (flags & CBLK_INTERRUPT) { ALOGV("waitStreamEndDone() interrupted by client"); status = -EINTR; @@ -485,8 +498,11 @@ end: StaticAudioTrackClientProxy::StaticAudioTrackClientProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, size_t frameSize) : AudioTrackClientProxy(cblk, buffers, frameCount, frameSize), - mMutator(&cblk->u.mStatic.mSingleStateQueue), mBufferPosition(0) + mMutator(&cblk->u.mStatic.mSingleStateQueue), + mPosLoopObserver(&cblk->u.mStatic.mPosLoopQueue) { + memset(&mState, 0, sizeof(mState)); + memset(&mPosLoop, 0, sizeof(mPosLoop)); } void StaticAudioTrackClientProxy::flush() @@ -501,30 +517,72 @@ void StaticAudioTrackClientProxy::setLoop(size_t loopStart, size_t loopEnd, int // FIXME Should return an error status return; } - StaticAudioTrackState newState; - newState.mLoopStart = (uint32_t) loopStart; - newState.mLoopEnd = (uint32_t) loopEnd; - newState.mLoopCount = loopCount; - size_t bufferPosition; - if (loopCount == 0 || (bufferPosition = getBufferPosition()) >= loopEnd) { - bufferPosition = loopStart; + mState.mLoopStart = (uint32_t) loopStart; + mState.mLoopEnd = (uint32_t) loopEnd; + mState.mLoopCount = loopCount; + mState.mLoopSequence = incrementSequence(mState.mLoopSequence, mState.mPositionSequence); + // set patch-up variables until the mState is acknowledged by the ServerProxy. + // observed buffer position and loop count will freeze until then to give the + // illusion of a synchronous change. + getBufferPositionAndLoopCount(NULL, NULL); + // preserve behavior to restart at mState.mLoopStart if position exceeds mState.mLoopEnd. + if (mState.mLoopCount != 0 && mPosLoop.mBufferPosition >= mState.mLoopEnd) { + mPosLoop.mBufferPosition = mState.mLoopStart; + } + mPosLoop.mLoopCount = mState.mLoopCount; + (void) mMutator.push(mState); +} + +void StaticAudioTrackClientProxy::setBufferPosition(size_t position) +{ + // This can only happen on a 64-bit client + if (position > UINT32_MAX) { + // FIXME Should return an error status + return; + } + mState.mPosition = (uint32_t) position; + mState.mPositionSequence = incrementSequence(mState.mPositionSequence, mState.mLoopSequence); + // set patch-up variables until the mState is acknowledged by the ServerProxy. + // observed buffer position and loop count will freeze until then to give the + // illusion of a synchronous change. + if (mState.mLoopCount > 0) { // only check if loop count is changing + getBufferPositionAndLoopCount(NULL, NULL); // get last position + } + mPosLoop.mBufferPosition = position; + if (position >= mState.mLoopEnd) { + // no ongoing loop is possible if position is greater than loopEnd. + mPosLoop.mLoopCount = 0; } - mBufferPosition = bufferPosition; // snapshot buffer position until loop is acknowledged. - (void) mMutator.push(newState); + (void) mMutator.push(mState); +} + +void StaticAudioTrackClientProxy::setBufferPositionAndLoop(size_t position, size_t loopStart, + size_t loopEnd, int loopCount) +{ + setLoop(loopStart, loopEnd, loopCount); + setBufferPosition(position); } size_t StaticAudioTrackClientProxy::getBufferPosition() { - size_t bufferPosition; - if (mMutator.ack()) { - bufferPosition = (size_t) mCblk->u.mStatic.mBufferPosition; - if (bufferPosition > mFrameCount) { - bufferPosition = mFrameCount; - } - } else { - bufferPosition = mBufferPosition; + getBufferPositionAndLoopCount(NULL, NULL); + return mPosLoop.mBufferPosition; +} + +void StaticAudioTrackClientProxy::getBufferPositionAndLoopCount( + size_t *position, int *loopCount) +{ + if (mMutator.ack() == StaticAudioTrackSingleStateQueue::SSQ_DONE) { + if (mPosLoopObserver.poll(mPosLoop)) { + ; // a valid mPosLoop should be available if ackDone is true. + } + } + if (position != NULL) { + *position = mPosLoop.mBufferPosition; + } + if (loopCount != NULL) { + *loopCount = mPosLoop.mLoopCount; } - return bufferPosition; } // --------------------------------------------------------------------------- @@ -560,7 +618,8 @@ status_t ServerProxy::obtainBuffer(Buffer* buffer, bool ackFlush) ssize_t filled = rear - newFront; // Rather than shutting down on a corrupt flush, just treat it as a full flush if (!(0 <= filled && (size_t) filled <= mFrameCount)) { - ALOGE("mFlush %#x -> %#x, front %#x, rear %#x, mask %#x, newFront %#x, filled %d=%#x", + ALOGE("mFlush %#x -> %#x, front %#x, rear %#x, mask %#x, newFront %#x, " + "filled %d=%#x", mFlush, flush, front, rear, mask, newFront, filled, filled); newFront = rear; } @@ -739,13 +798,12 @@ void AudioTrackServerProxy::tallyUnderrunFrames(uint32_t frameCount) StaticAudioTrackServerProxy::StaticAudioTrackServerProxy(audio_track_cblk_t* cblk, void *buffers, size_t frameCount, size_t frameSize) : AudioTrackServerProxy(cblk, buffers, frameCount, frameSize), - mObserver(&cblk->u.mStatic.mSingleStateQueue), mPosition(0), + mObserver(&cblk->u.mStatic.mSingleStateQueue), + mPosLoopMutator(&cblk->u.mStatic.mPosLoopQueue), mFramesReadySafe(frameCount), mFramesReady(frameCount), mFramesReadyIsCalledByMultipleThreads(false) { - mState.mLoopStart = 0; - mState.mLoopEnd = 0; - mState.mLoopCount = 0; + memset(&mState, 0, sizeof(mState)); } void StaticAudioTrackServerProxy::framesReadyIsCalledByMultipleThreads() @@ -762,55 +820,97 @@ size_t StaticAudioTrackServerProxy::framesReady() return mFramesReadySafe; } -ssize_t StaticAudioTrackServerProxy::pollPosition() +status_t StaticAudioTrackServerProxy::updateStateWithLoop( + StaticAudioTrackState *localState, const StaticAudioTrackState &update) const { - size_t position = mPosition; - StaticAudioTrackState state; - if (mObserver.poll(state)) { + if (localState->mLoopSequence != update.mLoopSequence) { bool valid = false; - size_t loopStart = state.mLoopStart; - size_t loopEnd = state.mLoopEnd; - if (state.mLoopCount == 0) { - if (loopStart > mFrameCount) { - loopStart = mFrameCount; - } - // ignore loopEnd - mPosition = position = loopStart; - mFramesReady = mFrameCount - mPosition; - mState.mLoopCount = 0; + const size_t loopStart = update.mLoopStart; + const size_t loopEnd = update.mLoopEnd; + size_t position = localState->mPosition; + if (update.mLoopCount == 0) { valid = true; - } else if (state.mLoopCount >= -1) { + } else if (update.mLoopCount >= -1) { if (loopStart < loopEnd && loopEnd <= mFrameCount && loopEnd - loopStart >= MIN_LOOP) { // If the current position is greater than the end of the loop // we "wrap" to the loop start. This might cause an audible pop. if (position >= loopEnd) { - mPosition = position = loopStart; - } - if (state.mLoopCount == -1) { - mFramesReady = INT64_MAX; - } else { - // mFramesReady is 64 bits to handle the effective number of frames - // that the static audio track contains, including loops. - // TODO: Later consider fixing overflow, but does not seem needed now - // as will not overflow if loopStart and loopEnd are Java "ints". - mFramesReady = int64_t(state.mLoopCount) * (loopEnd - loopStart) - + mFrameCount - mPosition; + position = loopStart; } - mState = state; valid = true; } } - if (!valid || mPosition > mFrameCount) { + if (!valid || position > mFrameCount) { + return NO_INIT; + } + localState->mPosition = position; + localState->mLoopCount = update.mLoopCount; + localState->mLoopEnd = loopEnd; + localState->mLoopStart = loopStart; + localState->mLoopSequence = update.mLoopSequence; + } + return OK; +} + +status_t StaticAudioTrackServerProxy::updateStateWithPosition( + StaticAudioTrackState *localState, const StaticAudioTrackState &update) const +{ + if (localState->mPositionSequence != update.mPositionSequence) { + if (update.mPosition > mFrameCount) { + return NO_INIT; + } else if (localState->mLoopCount != 0 && update.mPosition >= localState->mLoopEnd) { + localState->mLoopCount = 0; // disable loop count if position is beyond loop end. + } + localState->mPosition = update.mPosition; + localState->mPositionSequence = update.mPositionSequence; + } + return OK; +} + +ssize_t StaticAudioTrackServerProxy::pollPosition() +{ + StaticAudioTrackState state; + if (mObserver.poll(state)) { + StaticAudioTrackState trystate = mState; + bool result; + const int32_t diffSeq = state.mLoopSequence - state.mPositionSequence; + + if (diffSeq < 0) { + result = updateStateWithLoop(&trystate, state) == OK && + updateStateWithPosition(&trystate, state) == OK; + } else { + result = updateStateWithPosition(&trystate, state) == OK && + updateStateWithLoop(&trystate, state) == OK; + } + if (!result) { + mObserver.done(); + // caution: no update occurs so server state will be inconsistent with client state. ALOGE("%s client pushed an invalid state, shutting down", __func__); mIsShutdown = true; return (ssize_t) NO_INIT; } + mState = trystate; + if (mState.mLoopCount == -1) { + mFramesReady = INT64_MAX; + } else if (mState.mLoopCount == 0) { + mFramesReady = mFrameCount - mState.mPosition; + } else if (mState.mLoopCount > 0) { + // TODO: Later consider fixing overflow, but does not seem needed now + // as will not overflow if loopStart and loopEnd are Java "ints". + mFramesReady = int64_t(mState.mLoopCount) * (mState.mLoopEnd - mState.mLoopStart) + + mFrameCount - mState.mPosition; + } mFramesReadySafe = clampToSize(mFramesReady); // This may overflow, but client is not supposed to rely on it - mCblk->u.mStatic.mBufferPosition = (uint32_t) position; + StaticAudioTrackPosLoop posLoop; + + posLoop.mLoopCount = (int32_t) mState.mLoopCount; + posLoop.mBufferPosition = (uint32_t) mState.mPosition; + mPosLoopMutator.push(posLoop); + mObserver.done(); // safe to read mStatic variables. } - return (ssize_t) position; + return (ssize_t) mState.mPosition; } status_t StaticAudioTrackServerProxy::obtainBuffer(Buffer* buffer, bool ackFlush __unused) @@ -849,7 +949,7 @@ status_t StaticAudioTrackServerProxy::obtainBuffer(Buffer* buffer, bool ackFlush } // As mFramesReady is the total remaining frames in the static audio track, // it is always larger or equal to avail. - LOG_ALWAYS_FATAL_IF(mFramesReady < avail); + LOG_ALWAYS_FATAL_IF(mFramesReady < (int64_t) avail); buffer->mNonContig = mFramesReady == INT64_MAX ? SIZE_MAX : clampToSize(mFramesReady - avail); mUnreleased = avail; return NO_ERROR; @@ -858,7 +958,7 @@ status_t StaticAudioTrackServerProxy::obtainBuffer(Buffer* buffer, bool ackFlush void StaticAudioTrackServerProxy::releaseBuffer(Buffer* buffer) { size_t stepCount = buffer->mFrameCount; - LOG_ALWAYS_FATAL_IF(!(stepCount <= mFramesReady)); + LOG_ALWAYS_FATAL_IF(!((int64_t) stepCount <= mFramesReady)); LOG_ALWAYS_FATAL_IF(!(stepCount <= mUnreleased)); if (stepCount == 0) { // prevent accidental re-use of buffer @@ -868,11 +968,12 @@ void StaticAudioTrackServerProxy::releaseBuffer(Buffer* buffer) } mUnreleased -= stepCount; audio_track_cblk_t* cblk = mCblk; - size_t position = mPosition; + size_t position = mState.mPosition; size_t newPosition = position + stepCount; int32_t setFlags = 0; if (!(position <= newPosition && newPosition <= mFrameCount)) { - ALOGW("%s newPosition %zu outside [%zu, %zu]", __func__, newPosition, position, mFrameCount); + ALOGW("%s newPosition %zu outside [%zu, %zu]", __func__, newPosition, position, + mFrameCount); newPosition = mFrameCount; } else if (mState.mLoopCount != 0 && newPosition == mState.mLoopEnd) { newPosition = mState.mLoopStart; @@ -885,7 +986,7 @@ void StaticAudioTrackServerProxy::releaseBuffer(Buffer* buffer) if (newPosition == mFrameCount) { setFlags |= CBLK_BUFFER_END; } - mPosition = newPosition; + mState.mPosition = newPosition; if (mFramesReady != INT64_MAX) { mFramesReady -= stepCount; } @@ -893,7 +994,10 @@ void StaticAudioTrackServerProxy::releaseBuffer(Buffer* buffer) cblk->mServer += stepCount; // This may overflow, but client is not supposed to rely on it - cblk->u.mStatic.mBufferPosition = (uint32_t) newPosition; + StaticAudioTrackPosLoop posLoop; + posLoop.mBufferPosition = mState.mPosition; + posLoop.mLoopCount = mState.mLoopCount; + mPosLoopMutator.push(posLoop); if (setFlags != 0) { (void) android_atomic_or(setFlags, &cblk->mFlags); // this would be a good place to wake a futex diff --git a/media/libmedia/IAudioFlinger.cpp b/media/libmedia/IAudioFlinger.cpp index 8e3b633..6f038ea 100644 --- a/media/libmedia/IAudioFlinger.cpp +++ b/media/libmedia/IAudioFlinger.cpp @@ -1369,4 +1369,4 @@ status_t BnAudioFlinger::onTransact( // ---------------------------------------------------------------------------- -}; // namespace android +} // namespace android diff --git a/media/libmedia/IAudioFlingerClient.cpp b/media/libmedia/IAudioFlingerClient.cpp index 1c299f7..641e6c1 100644 --- a/media/libmedia/IAudioFlingerClient.cpp +++ b/media/libmedia/IAudioFlingerClient.cpp @@ -99,4 +99,4 @@ status_t BnAudioFlingerClient::onTransact( // ---------------------------------------------------------------------------- -}; // namespace android +} // namespace android diff --git a/media/libmedia/IAudioPolicyService.cpp b/media/libmedia/IAudioPolicyService.cpp index cfb28a9..39374d8 100644 --- a/media/libmedia/IAudioPolicyService.cpp +++ b/media/libmedia/IAudioPolicyService.cpp @@ -86,13 +86,15 @@ public: virtual status_t setDeviceConnectionState( audio_devices_t device, audio_policy_dev_state_t state, - const char *device_address) + const char *device_address, + const char *device_name) { Parcel data, reply; data.writeInterfaceToken(IAudioPolicyService::getInterfaceDescriptor()); data.writeInt32(static_cast <uint32_t>(device)); data.writeInt32(static_cast <uint32_t>(state)); data.writeCString(device_address); + data.writeCString(device_name); remote()->transact(SET_DEVICE_CONNECTION_STATE, data, &reply); return static_cast <status_t> (reply.readInt32()); } @@ -728,9 +730,11 @@ status_t BnAudioPolicyService::onTransact( audio_policy_dev_state_t state = static_cast <audio_policy_dev_state_t>(data.readInt32()); const char *device_address = data.readCString(); + const char *device_name = data.readCString(); reply->writeInt32(static_cast<uint32_t> (setDeviceConnectionState(device, state, - device_address))); + device_address, + device_name))); return NO_ERROR; } break; @@ -1224,4 +1228,4 @@ status_t BnAudioPolicyService::onTransact( // ---------------------------------------------------------------------------- -}; // namespace android +} // namespace android diff --git a/media/libmedia/IAudioPolicyServiceClient.cpp b/media/libmedia/IAudioPolicyServiceClient.cpp index e802277..7c65878 100644 --- a/media/libmedia/IAudioPolicyServiceClient.cpp +++ b/media/libmedia/IAudioPolicyServiceClient.cpp @@ -80,4 +80,4 @@ status_t BnAudioPolicyServiceClient::onTransact( // ---------------------------------------------------------------------------- -}; // namespace android +} // namespace android diff --git a/media/libmedia/IAudioRecord.cpp b/media/libmedia/IAudioRecord.cpp index 8a4a383..9d80753 100644 --- a/media/libmedia/IAudioRecord.cpp +++ b/media/libmedia/IAudioRecord.cpp @@ -91,4 +91,4 @@ status_t BnAudioRecord::onTransact( } } -}; // namespace android +} // namespace android diff --git a/media/libmedia/IAudioTrack.cpp b/media/libmedia/IAudioTrack.cpp index df209fd..651cb61 100644 --- a/media/libmedia/IAudioTrack.cpp +++ b/media/libmedia/IAudioTrack.cpp @@ -292,4 +292,4 @@ status_t BnAudioTrack::onTransact( } } -}; // namespace android +} // namespace android diff --git a/media/libmedia/IDrm.cpp b/media/libmedia/IDrm.cpp index b08fa82..714a0b3 100644 --- a/media/libmedia/IDrm.cpp +++ b/media/libmedia/IDrm.cpp @@ -125,7 +125,8 @@ struct BpDrm : public BpInterface<IDrm> { Vector<uint8_t> const &initData, String8 const &mimeType, DrmPlugin::KeyType keyType, KeyedVector<String8, String8> const &optionalParameters, - Vector<uint8_t> &request, String8 &defaultUrl) { + Vector<uint8_t> &request, String8 &defaultUrl, + DrmPlugin::KeyRequestType *keyRequestType) { Parcel data, reply; data.writeInterfaceToken(IDrm::getInterfaceDescriptor()); @@ -143,6 +144,7 @@ struct BpDrm : public BpInterface<IDrm> { readVector(reply, request); defaultUrl = reply.readString8(); + *keyRequestType = static_cast<DrmPlugin::KeyRequestType>(reply.readInt32()); return reply.readInt32(); } @@ -562,13 +564,15 @@ status_t BnDrm::onTransact( Vector<uint8_t> request; String8 defaultUrl; + DrmPlugin::KeyRequestType keyRequestType; + + status_t result = getKeyRequest(sessionId, initData, mimeType, + keyType, optionalParameters, request, defaultUrl, + &keyRequestType); - status_t result = getKeyRequest(sessionId, initData, - mimeType, keyType, - optionalParameters, - request, defaultUrl); writeVector(reply, request); reply->writeString8(defaultUrl); + reply->writeInt32(static_cast<int32_t>(keyRequestType)); reply->writeInt32(result); return OK; } diff --git a/media/libmedia/IDrmClient.cpp b/media/libmedia/IDrmClient.cpp index f50715e..490c6ed 100644 --- a/media/libmedia/IDrmClient.cpp +++ b/media/libmedia/IDrmClient.cpp @@ -78,4 +78,4 @@ status_t BnDrmClient::onTransact( } } -}; // namespace android +} // namespace android diff --git a/media/libmedia/IEffect.cpp b/media/libmedia/IEffect.cpp index c2fff78..eb4b098 100644 --- a/media/libmedia/IEffect.cpp +++ b/media/libmedia/IEffect.cpp @@ -201,4 +201,4 @@ status_t BnEffect::onTransact( // ---------------------------------------------------------------------------- -}; // namespace android +} // namespace android diff --git a/media/libmedia/IEffectClient.cpp b/media/libmedia/IEffectClient.cpp index aef4371..1322e72 100644 --- a/media/libmedia/IEffectClient.cpp +++ b/media/libmedia/IEffectClient.cpp @@ -141,4 +141,4 @@ status_t BnEffectClient::onTransact( // ---------------------------------------------------------------------------- -}; // namespace android +} // namespace android diff --git a/media/libmedia/IMediaCodecList.cpp b/media/libmedia/IMediaCodecList.cpp index bf7c5ca..80020db 100644 --- a/media/libmedia/IMediaCodecList.cpp +++ b/media/libmedia/IMediaCodecList.cpp @@ -160,4 +160,4 @@ status_t BnMediaCodecList::onTransact( // ---------------------------------------------------------------------------- -}; // namespace android +} // namespace android diff --git a/media/libmedia/IMediaDeathNotifier.cpp b/media/libmedia/IMediaDeathNotifier.cpp index 38e9ca0..d4360ea 100644 --- a/media/libmedia/IMediaDeathNotifier.cpp +++ b/media/libmedia/IMediaDeathNotifier.cpp @@ -108,4 +108,4 @@ IMediaDeathNotifier::DeathNotifier::~DeathNotifier() } } -}; // namespace android +} // namespace android diff --git a/media/libmedia/IMediaHTTPConnection.cpp b/media/libmedia/IMediaHTTPConnection.cpp index 7e26ee6..2ff7658 100644 --- a/media/libmedia/IMediaHTTPConnection.cpp +++ b/media/libmedia/IMediaHTTPConnection.cpp @@ -178,5 +178,4 @@ private: IMPLEMENT_META_INTERFACE( MediaHTTPConnection, "android.media.IMediaHTTPConnection"); -} // namespace android - +} // namespace android diff --git a/media/libmedia/IMediaHTTPService.cpp b/media/libmedia/IMediaHTTPService.cpp index 1260582..f30d0f3 100644 --- a/media/libmedia/IMediaHTTPService.cpp +++ b/media/libmedia/IMediaHTTPService.cpp @@ -54,5 +54,4 @@ struct BpMediaHTTPService : public BpInterface<IMediaHTTPService> { IMPLEMENT_META_INTERFACE( MediaHTTPService, "android.media.IMediaHTTPService"); -} // namespace android - +} // namespace android diff --git a/media/libmedia/IMediaLogService.cpp b/media/libmedia/IMediaLogService.cpp index a4af7b7..230749e 100644 --- a/media/libmedia/IMediaLogService.cpp +++ b/media/libmedia/IMediaLogService.cpp @@ -91,4 +91,4 @@ status_t BnMediaLogService::onTransact( // ---------------------------------------------------------------------------- -}; // namespace android +} // namespace android diff --git a/media/libmedia/IMediaMetadataRetriever.cpp b/media/libmedia/IMediaMetadataRetriever.cpp index aa2665a..551cffe 100644 --- a/media/libmedia/IMediaMetadataRetriever.cpp +++ b/media/libmedia/IMediaMetadataRetriever.cpp @@ -297,4 +297,4 @@ status_t BnMediaMetadataRetriever::onTransact( // ---------------------------------------------------------------------------- -}; // namespace android +} // namespace android diff --git a/media/libmedia/IMediaPlayer.cpp b/media/libmedia/IMediaPlayer.cpp index 7f3e5cc..ce3009a 100644 --- a/media/libmedia/IMediaPlayer.cpp +++ b/media/libmedia/IMediaPlayer.cpp @@ -39,6 +39,7 @@ enum { START, STOP, IS_PLAYING, + SET_PLAYBACK_RATE, PAUSE, SEEK_TO, GET_CURRENT_POSITION, @@ -164,6 +165,15 @@ public: return reply.readInt32(); } + status_t setPlaybackRate(float rate) + { + Parcel data, reply; + data.writeInterfaceToken(IMediaPlayer::getInterfaceDescriptor()); + data.writeFloat(rate); + remote()->transact(SET_PLAYBACK_RATE, data, &reply); + return reply.readInt32(); + } + status_t pause() { Parcel data, reply; @@ -426,6 +436,11 @@ status_t BnMediaPlayer::onTransact( reply->writeInt32(ret); return NO_ERROR; } break; + case SET_PLAYBACK_RATE: { + CHECK_INTERFACE(IMediaPlayer, data, reply); + reply->writeInt32(setPlaybackRate(data.readFloat())); + return NO_ERROR; + } break; case PAUSE: { CHECK_INTERFACE(IMediaPlayer, data, reply); reply->writeInt32(pause()); @@ -559,4 +574,4 @@ status_t BnMediaPlayer::onTransact( // ---------------------------------------------------------------------------- -}; // namespace android +} // namespace android diff --git a/media/libmedia/IMediaPlayerClient.cpp b/media/libmedia/IMediaPlayerClient.cpp index a670c96..d608386 100644 --- a/media/libmedia/IMediaPlayerClient.cpp +++ b/media/libmedia/IMediaPlayerClient.cpp @@ -75,4 +75,4 @@ status_t BnMediaPlayerClient::onTransact( } } -}; // namespace android +} // namespace android diff --git a/media/libmedia/IMediaPlayerService.cpp b/media/libmedia/IMediaPlayerService.cpp index feea267..aa7b2e1 100644 --- a/media/libmedia/IMediaPlayerService.cpp +++ b/media/libmedia/IMediaPlayerService.cpp @@ -234,4 +234,4 @@ status_t BnMediaPlayerService::onTransact( // ---------------------------------------------------------------------------- -}; // namespace android +} // namespace android diff --git a/media/libmedia/IMediaRecorder.cpp b/media/libmedia/IMediaRecorder.cpp index a733b68..8ca256c 100644 --- a/media/libmedia/IMediaRecorder.cpp +++ b/media/libmedia/IMediaRecorder.cpp @@ -46,7 +46,6 @@ enum { SET_OUTPUT_FORMAT, SET_VIDEO_ENCODER, SET_AUDIO_ENCODER, - SET_OUTPUT_FILE_PATH, SET_OUTPUT_FILE_FD, SET_VIDEO_SIZE, SET_VIDEO_FRAMERATE, @@ -158,16 +157,6 @@ public: return reply.readInt32(); } - status_t setOutputFile(const char* path) - { - ALOGV("setOutputFile(%s)", path); - Parcel data, reply; - data.writeInterfaceToken(IMediaRecorder::getInterfaceDescriptor()); - data.writeCString(path); - remote()->transact(SET_OUTPUT_FILE_PATH, data, &reply); - return reply.readInt32(); - } - status_t setOutputFile(int fd, int64_t offset, int64_t length) { ALOGV("setOutputFile(%d, %" PRId64 ", %" PRId64 ")", fd, offset, length); Parcel data, reply; @@ -300,7 +289,8 @@ IMPLEMENT_META_INTERFACE(MediaRecorder, "android.media.IMediaRecorder"); // ---------------------------------------------------------------------- status_t BnMediaRecorder::onTransact( - uint32_t code, const Parcel& data, Parcel* reply, uint32_t flags) + uint32_t code, const Parcel& data, Parcel* reply, + uint32_t flags) { switch (code) { case RELEASE: { @@ -390,13 +380,6 @@ status_t BnMediaRecorder::onTransact( return NO_ERROR; } break; - case SET_OUTPUT_FILE_PATH: { - ALOGV("SET_OUTPUT_FILE_PATH"); - CHECK_INTERFACE(IMediaRecorder, data, reply); - const char* path = data.readCString(); - reply->writeInt32(setOutputFile(path)); - return NO_ERROR; - } break; case SET_OUTPUT_FILE_FD: { ALOGV("SET_OUTPUT_FILE_FD"); CHECK_INTERFACE(IMediaRecorder, data, reply); @@ -445,7 +428,8 @@ status_t BnMediaRecorder::onTransact( case SET_PREVIEW_SURFACE: { ALOGV("SET_PREVIEW_SURFACE"); CHECK_INTERFACE(IMediaRecorder, data, reply); - sp<IGraphicBufferProducer> surface = interface_cast<IGraphicBufferProducer>(data.readStrongBinder()); + sp<IGraphicBufferProducer> surface = interface_cast<IGraphicBufferProducer>( + data.readStrongBinder()); reply->writeInt32(setPreviewSurface(surface)); return NO_ERROR; } break; @@ -479,4 +463,4 @@ status_t BnMediaRecorder::onTransact( // ---------------------------------------------------------------------------- -}; // namespace android +} // namespace android diff --git a/media/libmedia/IMediaRecorderClient.cpp b/media/libmedia/IMediaRecorderClient.cpp index e7907e3..6795d23 100644 --- a/media/libmedia/IMediaRecorderClient.cpp +++ b/media/libmedia/IMediaRecorderClient.cpp @@ -67,4 +67,4 @@ status_t BnMediaRecorderClient::onTransact( } } -}; // namespace android +} // namespace android diff --git a/media/libmedia/IRemoteDisplay.cpp b/media/libmedia/IRemoteDisplay.cpp index 1e15434..869d11a 100644 --- a/media/libmedia/IRemoteDisplay.cpp +++ b/media/libmedia/IRemoteDisplay.cpp @@ -91,4 +91,4 @@ status_t BnRemoteDisplay::onTransact( } } -}; // namespace android +} // namespace android diff --git a/media/libmedia/IRemoteDisplayClient.cpp b/media/libmedia/IRemoteDisplayClient.cpp index 9d63bc9..bedeb6c 100644 --- a/media/libmedia/IRemoteDisplayClient.cpp +++ b/media/libmedia/IRemoteDisplayClient.cpp @@ -101,4 +101,4 @@ status_t BnRemoteDisplayClient::onTransact( } } -}; // namespace android +} // namespace android diff --git a/media/libmedia/IResourceManagerClient.cpp b/media/libmedia/IResourceManagerClient.cpp new file mode 100644 index 0000000..6fa56fc --- /dev/null +++ b/media/libmedia/IResourceManagerClient.cpp @@ -0,0 +1,70 @@ +/* +** +** Copyright 2015, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + +#include <utils/RefBase.h> +#include <binder/IInterface.h> +#include <binder/Parcel.h> + +#include <media/IResourceManagerClient.h> + +namespace android { + +enum { + RECLAIM_RESOURCE = IBinder::FIRST_CALL_TRANSACTION, +}; + +class BpResourceManagerClient: public BpInterface<IResourceManagerClient> +{ +public: + BpResourceManagerClient(const sp<IBinder> &impl) + : BpInterface<IResourceManagerClient>(impl) + { + } + + virtual bool reclaimResource() { + Parcel data, reply; + data.writeInterfaceToken(IResourceManagerClient::getInterfaceDescriptor()); + + bool ret = false; + status_t status = remote()->transact(RECLAIM_RESOURCE, data, &reply); + if (status == NO_ERROR) { + ret = (bool)reply.readInt32(); + } + return ret; + } +}; + +IMPLEMENT_META_INTERFACE(ResourceManagerClient, "android.media.IResourceManagerClient"); + +// ---------------------------------------------------------------------- + +status_t BnResourceManagerClient::onTransact( + uint32_t code, const Parcel &data, Parcel *reply, uint32_t flags) +{ + switch (code) { + case RECLAIM_RESOURCE: { + CHECK_INTERFACE(IResourceManagerClient, data, reply); + bool ret = reclaimResource(); + reply->writeInt32(ret); + return NO_ERROR; + } break; + default: + return BBinder::onTransact(code, data, reply, flags); + } +} + +}; // namespace android diff --git a/media/libmedia/IResourceManagerService.cpp b/media/libmedia/IResourceManagerService.cpp new file mode 100644 index 0000000..95a2d1c --- /dev/null +++ b/media/libmedia/IResourceManagerService.cpp @@ -0,0 +1,169 @@ +/* +** +** Copyright 2015, The Android Open Source Project +** +** Licensed under the Apache License, Version 2.0 (the "License"); +** you may not use this file except in compliance with the License. +** You may obtain a copy of the License at +** +** http://www.apache.org/licenses/LICENSE-2.0 +** +** Unless required by applicable law or agreed to in writing, software +** distributed under the License is distributed on an "AS IS" BASIS, +** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +** See the License for the specific language governing permissions and +** limitations under the License. +*/ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "IResourceManagerService" +#include <utils/Log.h> + +#include "media/IResourceManagerService.h" + +#include <binder/Parcel.h> + +#include <stdint.h> +#include <sys/types.h> + +namespace android { + +enum { + CONFIG = IBinder::FIRST_CALL_TRANSACTION, + ADD_RESOURCE, + REMOVE_RESOURCE, + RECLAIM_RESOURCE, +}; + +template <typename T> +static void writeToParcel(Parcel *data, const Vector<T> &items) { + size_t size = items.size(); + size_t sizePosition = data->dataPosition(); + // truncates size, but should be okay for this usecase + data->writeUint32(static_cast<uint32_t>(size)); + for (size_t i = 0; i < size; i++) { + size_t position = data->dataPosition(); + items[i].writeToParcel(data); + } +} + +template <typename T> +static void readFromParcel(const Parcel &data, Vector<T> *items) { + size_t size = (size_t)data.readUint32(); + for (size_t i = 0; i < size; i++) { + T item; + item.readFromParcel(data); + items->add(item); + } +} + +class BpResourceManagerService : public BpInterface<IResourceManagerService> +{ +public: + BpResourceManagerService(const sp<IBinder> &impl) + : BpInterface<IResourceManagerService>(impl) + { + } + + virtual void config(const Vector<MediaResourcePolicy> &policies) { + Parcel data, reply; + data.writeInterfaceToken(IResourceManagerService::getInterfaceDescriptor()); + writeToParcel(&data, policies); + remote()->transact(CONFIG, data, &reply); + } + + virtual void addResource( + int pid, + int64_t clientId, + const sp<IResourceManagerClient> client, + const Vector<MediaResource> &resources) { + Parcel data, reply; + data.writeInterfaceToken(IResourceManagerService::getInterfaceDescriptor()); + data.writeInt32(pid); + data.writeInt64(clientId); + data.writeStrongBinder(IInterface::asBinder(client)); + writeToParcel(&data, resources); + + remote()->transact(ADD_RESOURCE, data, &reply); + } + + virtual void removeResource(int64_t clientId) { + Parcel data, reply; + data.writeInterfaceToken(IResourceManagerService::getInterfaceDescriptor()); + data.writeInt64(clientId); + + remote()->transact(REMOVE_RESOURCE, data, &reply); + } + + virtual bool reclaimResource(int callingPid, const Vector<MediaResource> &resources) { + Parcel data, reply; + data.writeInterfaceToken(IResourceManagerService::getInterfaceDescriptor()); + data.writeInt32(callingPid); + writeToParcel(&data, resources); + + bool ret = false; + status_t status = remote()->transact(RECLAIM_RESOURCE, data, &reply); + if (status == NO_ERROR) { + ret = (bool)reply.readInt32(); + } + return ret; + } +}; + +IMPLEMENT_META_INTERFACE(ResourceManagerService, "android.media.IResourceManagerService"); + +// ---------------------------------------------------------------------- + + +status_t BnResourceManagerService::onTransact( + uint32_t code, const Parcel &data, Parcel *reply, uint32_t flags) +{ + switch (code) { + case CONFIG: { + CHECK_INTERFACE(IResourceManagerService, data, reply); + int pid = data.readInt32(); + sp<IResourceManagerClient> client( + interface_cast<IResourceManagerClient>(data.readStrongBinder())); + Vector<MediaResourcePolicy> policies; + readFromParcel(data, &policies); + config(policies); + return NO_ERROR; + } break; + + case ADD_RESOURCE: { + CHECK_INTERFACE(IResourceManagerService, data, reply); + int pid = data.readInt32(); + int64_t clientId = data.readInt64(); + sp<IResourceManagerClient> client( + interface_cast<IResourceManagerClient>(data.readStrongBinder())); + Vector<MediaResource> resources; + readFromParcel(data, &resources); + addResource(pid, clientId, client, resources); + return NO_ERROR; + } break; + + case REMOVE_RESOURCE: { + CHECK_INTERFACE(IResourceManagerService, data, reply); + int64_t clientId = data.readInt64(); + removeResource(clientId); + return NO_ERROR; + } break; + + case RECLAIM_RESOURCE: { + CHECK_INTERFACE(IResourceManagerService, data, reply); + int callingPid = data.readInt32(); + Vector<MediaResource> resources; + readFromParcel(data, &resources); + bool ret = reclaimResource(callingPid, resources); + reply->writeInt32(ret); + return NO_ERROR; + } break; + + default: + return BBinder::onTransact(code, data, reply, flags); + } +} + +// ---------------------------------------------------------------------------- + +}; // namespace android diff --git a/media/libmedia/JetPlayer.cpp b/media/libmedia/JetPlayer.cpp index 721d8d7..271be0c 100644 --- a/media/libmedia/JetPlayer.cpp +++ b/media/libmedia/JetPlayer.cpp @@ -408,7 +408,8 @@ int JetPlayer::queueSegment(int segmentNum, int libNum, int repeatCount, int tra ALOGV("JetPlayer::queueSegment segmentNum=%d, libNum=%d, repeatCount=%d, transpose=%d", segmentNum, libNum, repeatCount, transpose); Mutex::Autolock lock(mMutex); - return JET_QueueSegment(mEasData, segmentNum, libNum, repeatCount, transpose, muteFlags, userID); + return JET_QueueSegment(mEasData, segmentNum, libNum, repeatCount, transpose, muteFlags, + userID); } //------------------------------------------------------------------------------------------------- @@ -449,7 +450,8 @@ void JetPlayer::dump() void JetPlayer::dumpJetStatus(S_JET_STATUS* pJetStatus) { if (pJetStatus!=NULL) - ALOGV(">> current JET player status: userID=%d segmentRepeatCount=%d numQueuedSegments=%d paused=%d", + ALOGV(">> current JET player status: userID=%d segmentRepeatCount=%d numQueuedSegments=%d " + "paused=%d", pJetStatus->currentUserID, pJetStatus->segmentRepeatCount, pJetStatus->numQueuedSegments, pJetStatus->paused); else diff --git a/media/libmedia/MediaProfiles.cpp b/media/libmedia/MediaProfiles.cpp index e2e6042..47f9258 100644 --- a/media/libmedia/MediaProfiles.cpp +++ b/media/libmedia/MediaProfiles.cpp @@ -163,7 +163,8 @@ MediaProfiles::logVideoEditorCap(const MediaProfiles::VideoEditorCap& cap UNUSED } /*static*/ int -MediaProfiles::findTagForName(const MediaProfiles::NameToTagMap *map, size_t nMappings, const char *name) +MediaProfiles::findTagForName(const MediaProfiles::NameToTagMap *map, size_t nMappings, + const char *name) { int tag = -1; for (size_t i = 0; i < nMappings; ++i) { @@ -295,9 +296,8 @@ MediaProfiles::createAudioEncoderCap(const char **atts) CHECK(codec != -1); MediaProfiles::AudioEncoderCap *cap = - new MediaProfiles::AudioEncoderCap(static_cast<audio_encoder>(codec), atoi(atts[5]), atoi(atts[7]), - atoi(atts[9]), atoi(atts[11]), atoi(atts[13]), - atoi(atts[15])); + new MediaProfiles::AudioEncoderCap(static_cast<audio_encoder>(codec), atoi(atts[5]), + atoi(atts[7]), atoi(atts[9]), atoi(atts[11]), atoi(atts[13]), atoi(atts[15])); logAudioEncoderCap(*cap); return cap; } @@ -330,7 +330,8 @@ MediaProfiles::createCamcorderProfile(int cameraId, const char **atts, Vector<in !strcmp("fileFormat", atts[2]) && !strcmp("duration", atts[4])); - const size_t nProfileMappings = sizeof(sCamcorderQualityNameMap)/sizeof(sCamcorderQualityNameMap[0]); + const size_t nProfileMappings = sizeof(sCamcorderQualityNameMap)/ + sizeof(sCamcorderQualityNameMap[0]); const int quality = findTagForName(sCamcorderQualityNameMap, nProfileMappings, atts[1]); CHECK(quality != -1); @@ -722,16 +723,20 @@ MediaProfiles::createDefaultCamcorderTimeLapse480pProfile(camcorder_quality qual MediaProfiles::createDefaultCamcorderTimeLapseLowProfiles( MediaProfiles::CamcorderProfile **lowTimeLapseProfile, MediaProfiles::CamcorderProfile **lowSpecificTimeLapseProfile) { - *lowTimeLapseProfile = createDefaultCamcorderTimeLapseQcifProfile(CAMCORDER_QUALITY_TIME_LAPSE_LOW); - *lowSpecificTimeLapseProfile = createDefaultCamcorderTimeLapseQcifProfile(CAMCORDER_QUALITY_TIME_LAPSE_QCIF); + *lowTimeLapseProfile = createDefaultCamcorderTimeLapseQcifProfile( + CAMCORDER_QUALITY_TIME_LAPSE_LOW); + *lowSpecificTimeLapseProfile = createDefaultCamcorderTimeLapseQcifProfile( + CAMCORDER_QUALITY_TIME_LAPSE_QCIF); } /*static*/ void MediaProfiles::createDefaultCamcorderTimeLapseHighProfiles( MediaProfiles::CamcorderProfile **highTimeLapseProfile, MediaProfiles::CamcorderProfile **highSpecificTimeLapseProfile) { - *highTimeLapseProfile = createDefaultCamcorderTimeLapse480pProfile(CAMCORDER_QUALITY_TIME_LAPSE_HIGH); - *highSpecificTimeLapseProfile = createDefaultCamcorderTimeLapse480pProfile(CAMCORDER_QUALITY_TIME_LAPSE_480P); + *highTimeLapseProfile = createDefaultCamcorderTimeLapse480pProfile( + CAMCORDER_QUALITY_TIME_LAPSE_HIGH); + *highSpecificTimeLapseProfile = createDefaultCamcorderTimeLapse480pProfile( + CAMCORDER_QUALITY_TIME_LAPSE_480P); } /*static*/ MediaProfiles::CamcorderProfile* @@ -809,7 +814,8 @@ MediaProfiles::createDefaultCamcorderProfiles(MediaProfiles *profiles) // high camcorder time lapse profiles. MediaProfiles::CamcorderProfile *highTimeLapseProfile, *highSpecificTimeLapseProfile; - createDefaultCamcorderTimeLapseHighProfiles(&highTimeLapseProfile, &highSpecificTimeLapseProfile); + createDefaultCamcorderTimeLapseHighProfiles(&highTimeLapseProfile, + &highSpecificTimeLapseProfile); profiles->mCamcorderProfiles.add(highTimeLapseProfile); profiles->mCamcorderProfiles.add(highSpecificTimeLapseProfile); diff --git a/media/libmedia/MediaResource.cpp b/media/libmedia/MediaResource.cpp new file mode 100644 index 0000000..8be01bc --- /dev/null +++ b/media/libmedia/MediaResource.cpp @@ -0,0 +1,65 @@ +/* + * Copyright 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "MediaResource" +#include <utils/Log.h> +#include <media/MediaResource.h> + +namespace android { + +const char kResourceSecureCodec[] = "secure-codec"; +const char kResourceNonSecureCodec[] = "non-secure-codec"; +const char kResourceGraphicMemory[] = "graphic-memory"; + +MediaResource::MediaResource() : mValue(0) {} + +MediaResource::MediaResource(String8 type, uint64_t value) + : mType(type), + mValue(value) {} + +MediaResource::MediaResource(String8 type, String8 subType, uint64_t value) + : mType(type), + mSubType(subType), + mValue(value) {} + +void MediaResource::readFromParcel(const Parcel &parcel) { + mType = parcel.readString8(); + mSubType = parcel.readString8(); + mValue = parcel.readUint64(); +} + +void MediaResource::writeToParcel(Parcel *parcel) const { + parcel->writeString8(mType); + parcel->writeString8(mSubType); + parcel->writeUint64(mValue); +} + +String8 MediaResource::toString() const { + String8 str; + str.appendFormat("%s/%s:%llu", mType.string(), mSubType.string(), mValue); + return str; +} + +bool MediaResource::operator==(const MediaResource &other) const { + return (other.mType == mType) && (other.mSubType == mSubType) && (other.mValue == mValue); +} + +bool MediaResource::operator!=(const MediaResource &other) const { + return !(*this == other); +} + +}; // namespace android diff --git a/media/libmedia/MediaResourcePolicy.cpp b/media/libmedia/MediaResourcePolicy.cpp new file mode 100644 index 0000000..2bb996a --- /dev/null +++ b/media/libmedia/MediaResourcePolicy.cpp @@ -0,0 +1,49 @@ +/* + * Copyright 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "MediaResourcePolicy" +#include <utils/Log.h> +#include <media/MediaResourcePolicy.h> + +namespace android { + +const char kPolicySupportsMultipleSecureCodecs[] = "supports-multiple-secure-codecs"; +const char kPolicySupportsSecureWithNonSecureCodec[] = "supports-secure-with-non-secure-codec"; + +MediaResourcePolicy::MediaResourcePolicy() : mValue(0) {} + +MediaResourcePolicy::MediaResourcePolicy(String8 type, uint64_t value) + : mType(type), + mValue(value) {} + +void MediaResourcePolicy::readFromParcel(const Parcel &parcel) { + mType = parcel.readString8(); + mValue = parcel.readUint64(); +} + +void MediaResourcePolicy::writeToParcel(Parcel *parcel) const { + parcel->writeString8(mType); + parcel->writeUint64(mValue); +} + +String8 MediaResourcePolicy::toString() const { + String8 str; + str.appendFormat("%s:%llu", mType.string(), mValue); + return str; +} + +}; // namespace android diff --git a/media/libmedia/SingleStateQueue.cpp b/media/libmedia/SingleStateQueue.cpp deleted file mode 100644 index c241184..0000000 --- a/media/libmedia/SingleStateQueue.cpp +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (C) 2012 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <new> -#include <cutils/atomic.h> -#include <media/SingleStateQueue.h> - -namespace android { - -template<typename T> SingleStateQueue<T>::Mutator::Mutator(Shared *shared) - : mSequence(0), mShared((Shared *) shared) -{ - // exactly one of Mutator and Observer must initialize, currently it is Observer - //shared->init(); -} - -template<typename T> int32_t SingleStateQueue<T>::Mutator::push(const T& value) -{ - Shared *shared = mShared; - int32_t sequence = mSequence; - sequence++; - android_atomic_acquire_store(sequence, &shared->mSequence); - shared->mValue = value; - sequence++; - android_atomic_release_store(sequence, &shared->mSequence); - mSequence = sequence; - // consider signalling a futex here, if we know that observer is waiting - return sequence; -} - -template<typename T> bool SingleStateQueue<T>::Mutator::ack() -{ - return mShared->mAck - mSequence == 0; -} - -template<typename T> bool SingleStateQueue<T>::Mutator::ack(int32_t sequence) -{ - // this relies on 2's complement rollover to detect an ancient sequence number - return mShared->mAck - sequence >= 0; -} - -template<typename T> SingleStateQueue<T>::Observer::Observer(Shared *shared) - : mSequence(0), mSeed(1), mShared((Shared *) shared) -{ - // exactly one of Mutator and Observer must initialize, currently it is Observer - shared->init(); -} - -template<typename T> bool SingleStateQueue<T>::Observer::poll(T& value) -{ - Shared *shared = mShared; - int32_t before = shared->mSequence; - if (before == mSequence) { - return false; - } - for (int tries = 0; ; ) { - const int MAX_TRIES = 5; - if (before & 1) { - if (++tries >= MAX_TRIES) { - return false; - } - before = shared->mSequence; - } else { - android_memory_barrier(); - T temp = shared->mValue; - int32_t after = android_atomic_release_load(&shared->mSequence); - if (after == before) { - value = temp; - shared->mAck = before; - mSequence = before; - return true; - } - if (++tries >= MAX_TRIES) { - return false; - } - before = after; - } - } -} - -#if 0 -template<typename T> SingleStateQueue<T>::SingleStateQueue(void /*Shared*/ *shared) -{ - ((Shared *) shared)->init(); -} -#endif - -} // namespace android - -// hack for gcc -#ifdef SINGLE_STATE_QUEUE_INSTANTIATIONS -#include SINGLE_STATE_QUEUE_INSTANTIATIONS -#endif diff --git a/media/libmedia/StringArray.cpp b/media/libmedia/StringArray.cpp index 5f5b57a..b2e5907 100644 --- a/media/libmedia/StringArray.cpp +++ b/media/libmedia/StringArray.cpp @@ -16,7 +16,7 @@ // // Sortable array of strings. STL-ish, but STL-free. -// +// #include <stdlib.h> #include <string.h> @@ -110,4 +110,4 @@ void StringArray::setEntry(int idx, const char* str) { } -}; // namespace android +} // namespace android diff --git a/media/libmedia/Visualizer.cpp b/media/libmedia/Visualizer.cpp index f91e3e4..9d69b6a 100644 --- a/media/libmedia/Visualizer.cpp +++ b/media/libmedia/Visualizer.cpp @@ -429,4 +429,4 @@ bool Visualizer::CaptureThread::threadLoop() return false; } -}; // namespace android +} // namespace android diff --git a/media/libmedia/docs/Makefile b/media/libmedia/docs/Makefile new file mode 100644 index 0000000..bddbc9b --- /dev/null +++ b/media/libmedia/docs/Makefile @@ -0,0 +1,2 @@ +paused.png : paused.dot + dot -Tpng < $< > $@ diff --git a/media/libmedia/docs/paused.dot b/media/libmedia/docs/paused.dot new file mode 100644 index 0000000..11e1777 --- /dev/null +++ b/media/libmedia/docs/paused.dot @@ -0,0 +1,85 @@ +digraph paused { +initial [label="INITIAL\n\ +mIgnoreNextPausedInt = false\n\ +mPaused = false\n\ +mPausedInt = false"]; + +resume_body [label="mIgnoreNextPausedInt = true\nif (mPaused || mPausedInt)"]; +resume_paused [label="mPaused = false\nmPausedInt = false\nsignal()"]; +resume_paused -> resume_merged; +resume_merged [label="return"]; + +Application -> ATstop; +ATstop [label="AudioTrack::stop()"]; +ATstop -> pause; +Application -> ATpause; +ATpause [label="AudioTrack::pause()"]; +ATpause -> pause; +ATstart -> resume; +ATstart [label="AudioTrack::start()"]; +destructor [label="~AudioTrack()"]; +destructor -> requestExit; +requestExit [label="AudioTrackThread::requestExit()"]; +requestExit -> resume; +Application -> ATsetMarkerPosition +ATsetMarkerPosition [label="AudioTrack::setMarkerPosition()\n[sets marker variables]"]; +ATsetMarkerPosition -> ATTwake +Application -> ATsetPositionUpdatePeriod +ATsetPositionUpdatePeriod [label="AudioTrack::setPositionUpdatePeriod()\n[sets update period variables]"]; +ATsetPositionUpdatePeriod -> ATTwake +Application -> ATstart; + +resume [label="AudioTrackThread::resume()"]; +resume -> resume_body; + +resume_body -> resume_paused [label="true"]; +resume_body -> resume_merged [label="false"]; + +ATTwake [label="AudioTrackThread::wake()\nif (!mPaused && mPausedInt && mPausedNs > 0)"]; +ATTwake-> ATTWake_wakeable [label="true"]; +ATTWake_wakeable [label="mIgnoreNextPausedInt = true\nmPausedInt = false\nsignal()"]; +ATTwake-> ATTWake_cannotwake [label="false"] +ATTWake_cannotwake [label="ignore"]; + +pause [label="mPaused = true"]; +pause -> return; + +threadLoop [label="AudioTrackThread::threadLoop()\nENTRY"]; +threadLoop -> threadLoop_1; +threadLoop_1 [label="if (mPaused)"]; +threadLoop_1 -> threadLoop_1_true [label="true"]; +threadLoop_1 -> threadLoop_2 [label="false"]; +threadLoop_1_true [label="wait()\nreturn true"]; +threadLoop_2 [label="if (mIgnoreNextPausedInt)"]; +threadLoop_2 -> threadLoop_2_true [label="true"]; +threadLoop_2 -> threadLoop_3 [label="false"]; +threadLoop_2_true [label="mIgnoreNextPausedInt = false\nmPausedInt = false"]; +threadLoop_2_true -> threadLoop_3; +threadLoop_3 [label="if (mPausedInt)"]; +threadLoop_3 -> threadLoop_3_true [label="true"]; +threadLoop_3 -> threadLoop_4 [label="false"]; +threadLoop_3_true [label="wait()\nmPausedInt = false\nreturn true"]; +threadLoop_4 [label="if (exitPending)"]; +threadLoop_4 -> threadLoop_4_true [label="true"]; +threadLoop_4 -> threadLoop_5 [label="false"]; +threadLoop_4_true [label="return false"]; +threadLoop_5 [label="ns = processAudioBuffer()"]; +threadLoop_5 -> threadLoop_6; +threadLoop_6 [label="case ns"]; +threadLoop_6 -> threadLoop_6_0 [label="0"]; +threadLoop_6 -> threadLoop_6_NS_INACTIVE [label="NS_INACTIVE"]; +threadLoop_6 -> threadLoop_6_NS_NEVER [label="NS_NEVER"]; +threadLoop_6 -> threadLoop_6_NS_WHENEVER [label="NS_WHENEVER"]; +threadLoop_6 -> threadLoop_6_default [label="default"]; +threadLoop_6_default [label="if (ns < 0)"]; +threadLoop_6_default -> threadLoop_6_default_true [label="true"]; +threadLoop_6_default -> threadLoop_6_default_false [label="false"]; +threadLoop_6_default_true [label="FATAL"]; +threadLoop_6_default_false [label="pauseInternal(ns) [wake()-able]\nmPausedInternal = true\nmPausedNs = ns\nreturn true"]; +threadLoop_6_0 [label="return true"]; +threadLoop_6_NS_INACTIVE [label="pauseInternal()\nmPausedInternal = true\nmPausedNs = 0\nreturn true"]; +threadLoop_6_NS_NEVER [label="return false"]; +threadLoop_6_NS_WHENEVER [label="ns = 1s"]; +threadLoop_6_NS_WHENEVER -> threadLoop_6_default_false; + +} diff --git a/media/libmedia/mediametadataretriever.cpp b/media/libmedia/mediametadataretriever.cpp index 8e8a1ed..873808a 100644 --- a/media/libmedia/mediametadataretriever.cpp +++ b/media/libmedia/mediametadataretriever.cpp @@ -176,4 +176,4 @@ MediaMetadataRetriever::DeathNotifier::~DeathNotifier() } } -}; // namespace android +} // namespace android diff --git a/media/libmedia/mediaplayer.cpp b/media/libmedia/mediaplayer.cpp index 05c89ed..5dd8c02 100644 --- a/media/libmedia/mediaplayer.cpp +++ b/media/libmedia/mediaplayer.cpp @@ -59,6 +59,7 @@ MediaPlayer::MediaPlayer() mLoop = false; mLeftVolume = mRightVolume = 1.0; mVideoWidth = mVideoHeight = 0; + mPlaybackRate = 1.0; mLockThreadId = 0; mAudioSessionId = AudioSystem::newAudioUniqueId(); AudioSystem::acquireAudioSessionId(mAudioSessionId, -1); @@ -240,7 +241,7 @@ status_t MediaPlayer::setVideoSurfaceTexture( // must call with lock held status_t MediaPlayer::prepareAsync_l() { - if ( (mPlayer != 0) && ( mCurrentState & ( MEDIA_PLAYER_INITIALIZED | MEDIA_PLAYER_STOPPED) ) ) { + if ( (mPlayer != 0) && ( mCurrentState & (MEDIA_PLAYER_INITIALIZED | MEDIA_PLAYER_STOPPED) ) ) { mPlayer->setAudioStreamType(mStreamType); if (mAudioAttributesParcel != NULL) { mPlayer->setParameter(KEY_PARAMETER_AUDIO_ATTRIBUTES, *mAudioAttributesParcel); @@ -378,6 +379,24 @@ bool MediaPlayer::isPlaying() return false; } +status_t MediaPlayer::setPlaybackRate(float rate) +{ + ALOGV("setPlaybackRate: %f", rate); + if (rate <= 0.0) { + return BAD_VALUE; + } + Mutex::Autolock _l(mLock); + if (mPlayer != 0) { + if (mPlaybackRate == rate) { + return NO_ERROR; + } + mPlaybackRate = rate; + return mPlayer->setPlaybackRate(rate); + } + ALOGV("setPlaybackRate: no active player"); + return INVALID_OPERATION; +} + status_t MediaPlayer::getVideoWidth(int *w) { ALOGV("getVideoWidth"); @@ -414,7 +433,8 @@ status_t MediaPlayer::getCurrentPosition(int *msec) status_t MediaPlayer::getDuration_l(int *msec) { ALOGV("getDuration_l"); - bool isValidState = (mCurrentState & (MEDIA_PLAYER_PREPARED | MEDIA_PLAYER_STARTED | MEDIA_PLAYER_PAUSED | MEDIA_PLAYER_STOPPED | MEDIA_PLAYER_PLAYBACK_COMPLETE)); + bool isValidState = (mCurrentState & (MEDIA_PLAYER_PREPARED | MEDIA_PLAYER_STARTED | + MEDIA_PLAYER_PAUSED | MEDIA_PLAYER_STOPPED | MEDIA_PLAYER_PLAYBACK_COMPLETE)); if (mPlayer != 0 && isValidState) { int durationMs; status_t ret = mPlayer->getDuration(&durationMs); @@ -443,7 +463,8 @@ status_t MediaPlayer::getDuration(int *msec) status_t MediaPlayer::seekTo_l(int msec) { ALOGV("seekTo %d", msec); - if ((mPlayer != 0) && ( mCurrentState & ( MEDIA_PLAYER_STARTED | MEDIA_PLAYER_PREPARED | MEDIA_PLAYER_PAUSED | MEDIA_PLAYER_PLAYBACK_COMPLETE) ) ) { + if ((mPlayer != 0) && ( mCurrentState & ( MEDIA_PLAYER_STARTED | MEDIA_PLAYER_PREPARED | + MEDIA_PLAYER_PAUSED | MEDIA_PLAYER_PLAYBACK_COMPLETE) ) ) { if ( msec < 0 ) { ALOGW("Attempt to seek to invalid position: %d", msec); msec = 0; @@ -477,7 +498,8 @@ status_t MediaPlayer::seekTo_l(int msec) return NO_ERROR; } } - ALOGE("Attempt to perform seekTo in wrong state: mPlayer=%p, mCurrentState=%u", mPlayer.get(), mCurrentState); + ALOGE("Attempt to perform seekTo in wrong state: mPlayer=%p, mCurrentState=%u", mPlayer.get(), + mCurrentState); return INVALID_OPERATION; } @@ -855,4 +877,4 @@ status_t MediaPlayer::setNextMediaPlayer(const sp<MediaPlayer>& next) { return mPlayer->setNextPlayer(next == NULL ? NULL : next->mPlayer); } -}; // namespace android +} // namespace android diff --git a/media/libmedia/mediarecorder.cpp b/media/libmedia/mediarecorder.cpp index 1952b86..a2d6e53 100644 --- a/media/libmedia/mediarecorder.cpp +++ b/media/libmedia/mediarecorder.cpp @@ -264,32 +264,6 @@ status_t MediaRecorder::setAudioEncoder(int ae) return ret; } -status_t MediaRecorder::setOutputFile(const char* path) -{ - ALOGV("setOutputFile(%s)", path); - if (mMediaRecorder == NULL) { - ALOGE("media recorder is not initialized yet"); - return INVALID_OPERATION; - } - if (mIsOutputFileSet) { - ALOGE("output file has already been set"); - return INVALID_OPERATION; - } - if (!(mCurrentState & MEDIA_RECORDER_DATASOURCE_CONFIGURED)) { - ALOGE("setOutputFile called in an invalid state(%d)", mCurrentState); - return INVALID_OPERATION; - } - - status_t ret = mMediaRecorder->setOutputFile(path); - if (OK != ret) { - ALOGV("setOutputFile failed: %d", ret); - mCurrentState = MEDIA_RECORDER_ERROR; - return ret; - } - mIsOutputFileSet = true; - return ret; -} - status_t MediaRecorder::setOutputFile(int fd, int64_t offset, int64_t length) { ALOGV("setOutputFile(%d, %" PRId64 ", %" PRId64 ")", fd, offset, length); @@ -706,4 +680,4 @@ void MediaRecorder::died() notify(MEDIA_RECORDER_EVENT_ERROR, MEDIA_ERROR_SERVER_DIED, 0); } -}; // namespace android +} // namespace android diff --git a/media/libmediaplayerservice/Android.mk b/media/libmediaplayerservice/Android.mk index 9d8fe62..4b31715 100644 --- a/media/libmediaplayerservice/Android.mk +++ b/media/libmediaplayerservice/Android.mk @@ -10,6 +10,7 @@ LOCAL_SRC_FILES:= \ ActivityManager.cpp \ Crypto.cpp \ Drm.cpp \ + DrmSessionManager.cpp \ HDCP.cpp \ MediaPlayerFactory.cpp \ MediaPlayerService.cpp \ diff --git a/media/libmediaplayerservice/Drm.cpp b/media/libmediaplayerservice/Drm.cpp index 73f1a2a..49e01d1 100644 --- a/media/libmediaplayerservice/Drm.cpp +++ b/media/libmediaplayerservice/Drm.cpp @@ -23,6 +23,8 @@ #include "Drm.h" +#include "DrmSessionClientInterface.h" +#include "DrmSessionManager.h" #include <media/drm/DrmAPI.h> #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AString.h> @@ -33,6 +35,10 @@ namespace android { +static inline int getCallingPid() { + return IPCThreadState::self()->getCallingPid(); +} + static bool checkPermission(const char* permissionString) { #ifndef HAVE_ANDROID_OS return true; @@ -57,14 +63,41 @@ static bool operator<(const Vector<uint8_t> &lhs, const Vector<uint8_t> &rhs) { return memcmp((void *)lhs.array(), (void *)rhs.array(), rhs.size()) < 0; } +struct DrmSessionClient : public DrmSessionClientInterface { + DrmSessionClient(Drm* drm) : mDrm(drm) {} + + virtual bool reclaimSession(const Vector<uint8_t>& sessionId) { + sp<Drm> drm = mDrm.promote(); + if (drm == NULL) { + return true; + } + status_t err = drm->closeSession(sessionId); + if (err != OK) { + return false; + } + drm->sendEvent(DrmPlugin::kDrmPluginEventSessionReclaimed, 0, &sessionId, NULL); + return true; + } + +protected: + virtual ~DrmSessionClient() {} + +private: + wp<Drm> mDrm; + + DISALLOW_EVIL_CONSTRUCTORS(DrmSessionClient); +}; + Drm::Drm() : mInitCheck(NO_INIT), + mDrmSessionClient(new DrmSessionClient(this)), mListener(NULL), mFactory(NULL), mPlugin(NULL) { } Drm::~Drm() { + DrmSessionManager::Instance()->removeDrm(mDrmSessionClient); delete mPlugin; mPlugin = NULL; closeFactory(); @@ -289,7 +322,18 @@ status_t Drm::openSession(Vector<uint8_t> &sessionId) { return -EINVAL; } - return mPlugin->openSession(sessionId); + status_t err = mPlugin->openSession(sessionId); + if (err == ERROR_DRM_RESOURCE_BUSY) { + bool retry = false; + retry = DrmSessionManager::Instance()->reclaimSession(getCallingPid()); + if (retry) { + err = mPlugin->openSession(sessionId); + } + } + if (err == OK) { + DrmSessionManager::Instance()->addSession(getCallingPid(), mDrmSessionClient, sessionId); + } + return err; } status_t Drm::closeSession(Vector<uint8_t> const &sessionId) { @@ -303,14 +347,19 @@ status_t Drm::closeSession(Vector<uint8_t> const &sessionId) { return -EINVAL; } - return mPlugin->closeSession(sessionId); + status_t err = mPlugin->closeSession(sessionId); + if (err == OK) { + DrmSessionManager::Instance()->removeSession(sessionId); + } + return err; } status_t Drm::getKeyRequest(Vector<uint8_t> const &sessionId, Vector<uint8_t> const &initData, String8 const &mimeType, DrmPlugin::KeyType keyType, KeyedVector<String8, String8> const &optionalParameters, - Vector<uint8_t> &request, String8 &defaultUrl) { + Vector<uint8_t> &request, String8 &defaultUrl, + DrmPlugin::KeyRequestType *keyRequestType) { Mutex::Autolock autoLock(mLock); if (mInitCheck != OK) { @@ -321,8 +370,11 @@ status_t Drm::getKeyRequest(Vector<uint8_t> const &sessionId, return -EINVAL; } + DrmSessionManager::Instance()->useSession(sessionId); + return mPlugin->getKeyRequest(sessionId, initData, mimeType, keyType, - optionalParameters, request, defaultUrl); + optionalParameters, request, defaultUrl, + keyRequestType); } status_t Drm::provideKeyResponse(Vector<uint8_t> const &sessionId, @@ -338,6 +390,8 @@ status_t Drm::provideKeyResponse(Vector<uint8_t> const &sessionId, return -EINVAL; } + DrmSessionManager::Instance()->useSession(sessionId); + return mPlugin->provideKeyResponse(sessionId, response, keySetId); } @@ -367,6 +421,8 @@ status_t Drm::restoreKeys(Vector<uint8_t> const &sessionId, return -EINVAL; } + DrmSessionManager::Instance()->useSession(sessionId); + return mPlugin->restoreKeys(sessionId, keySetId); } @@ -382,6 +438,8 @@ status_t Drm::queryKeyStatus(Vector<uint8_t> const &sessionId, return -EINVAL; } + DrmSessionManager::Instance()->useSession(sessionId); + return mPlugin->queryKeyStatus(sessionId, infoMap); } @@ -561,6 +619,8 @@ status_t Drm::setCipherAlgorithm(Vector<uint8_t> const &sessionId, return -EINVAL; } + DrmSessionManager::Instance()->useSession(sessionId); + return mPlugin->setCipherAlgorithm(sessionId, algorithm); } @@ -576,6 +636,8 @@ status_t Drm::setMacAlgorithm(Vector<uint8_t> const &sessionId, return -EINVAL; } + DrmSessionManager::Instance()->useSession(sessionId); + return mPlugin->setMacAlgorithm(sessionId, algorithm); } @@ -594,6 +656,8 @@ status_t Drm::encrypt(Vector<uint8_t> const &sessionId, return -EINVAL; } + DrmSessionManager::Instance()->useSession(sessionId); + return mPlugin->encrypt(sessionId, keyId, input, iv, output); } @@ -612,6 +676,8 @@ status_t Drm::decrypt(Vector<uint8_t> const &sessionId, return -EINVAL; } + DrmSessionManager::Instance()->useSession(sessionId); + return mPlugin->decrypt(sessionId, keyId, input, iv, output); } @@ -629,6 +695,8 @@ status_t Drm::sign(Vector<uint8_t> const &sessionId, return -EINVAL; } + DrmSessionManager::Instance()->useSession(sessionId); + return mPlugin->sign(sessionId, keyId, message, signature); } @@ -647,6 +715,8 @@ status_t Drm::verify(Vector<uint8_t> const &sessionId, return -EINVAL; } + DrmSessionManager::Instance()->useSession(sessionId); + return mPlugin->verify(sessionId, keyId, message, signature, match); } @@ -669,6 +739,8 @@ status_t Drm::signRSA(Vector<uint8_t> const &sessionId, return -EPERM; } + DrmSessionManager::Instance()->useSession(sessionId); + return mPlugin->signRSA(sessionId, algorithm, message, wrappedKey, signature); } diff --git a/media/libmediaplayerservice/Drm.h b/media/libmediaplayerservice/Drm.h index 0e1eb2c..7e8f246 100644 --- a/media/libmediaplayerservice/Drm.h +++ b/media/libmediaplayerservice/Drm.h @@ -28,6 +28,7 @@ namespace android { struct DrmFactory; struct DrmPlugin; +struct DrmSessionClientInterface; struct Drm : public BnDrm, public IBinder::DeathRecipient, @@ -52,7 +53,8 @@ struct Drm : public BnDrm, Vector<uint8_t> const &initData, String8 const &mimeType, DrmPlugin::KeyType keyType, KeyedVector<String8, String8> const &optionalParameters, - Vector<uint8_t> &request, String8 &defaultUrl); + Vector<uint8_t> &request, String8 &defaultUrl, + DrmPlugin::KeyRequestType *keyRequestType); virtual status_t provideKeyResponse(Vector<uint8_t> const &sessionId, Vector<uint8_t> const &response, @@ -138,6 +140,8 @@ private: status_t mInitCheck; + sp<DrmSessionClientInterface> mDrmSessionClient; + sp<IDrmClient> mListener; mutable Mutex mEventLock; mutable Mutex mNotifyLock; diff --git a/media/libmedia/SingleStateQueueInstantiations.cpp b/media/libmediaplayerservice/DrmSessionClientInterface.h index 0265c8c..17faf08 100644 --- a/media/libmedia/SingleStateQueueInstantiations.cpp +++ b/media/libmediaplayerservice/DrmSessionClientInterface.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012 The Android Open Source Project + * Copyright (C) 2015 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,21 @@ * limitations under the License. */ -#include <media/SingleStateQueue.h> -#include <private/media/StaticAudioTrackState.h> -#include <media/AudioTimestamp.h> +#ifndef DRM_PROXY_INTERFACE_H_ +#define DRM_PROXY_INTERFACE_H_ -// FIXME hack for gcc +#include <utils/RefBase.h> +#include <utils/Vector.h> namespace android { -template class SingleStateQueue<StaticAudioTrackState>; // typedef StaticAudioTrackSingleStateQueue -template class SingleStateQueue<AudioTimestamp>; // typedef AudioTimestampSingleStateQueue +struct DrmSessionClientInterface : public RefBase { + virtual bool reclaimSession(const Vector<uint8_t>& sessionId) = 0; -} +protected: + virtual ~DrmSessionClientInterface() {} +}; + +} // namespace android + +#endif // DRM_PROXY_INTERFACE_H_ diff --git a/media/libmediaplayerservice/DrmSessionManager.cpp b/media/libmediaplayerservice/DrmSessionManager.cpp new file mode 100644 index 0000000..641f881 --- /dev/null +++ b/media/libmediaplayerservice/DrmSessionManager.cpp @@ -0,0 +1,240 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "DrmSessionManager" +#include <utils/Log.h> + +#include "DrmSessionManager.h" + +#include "DrmSessionClientInterface.h" +#include <binder/IPCThreadState.h> +#include <binder/IProcessInfoService.h> +#include <binder/IServiceManager.h> +#include <media/stagefright/ProcessInfo.h> +#include <unistd.h> +#include <utils/String8.h> + +namespace android { + +static String8 GetSessionIdString(const Vector<uint8_t> &sessionId) { + String8 sessionIdStr; + for (size_t i = 0; i < sessionId.size(); ++i) { + sessionIdStr.appendFormat("%u ", sessionId[i]); + } + return sessionIdStr; +} + +bool isEqualSessionId(const Vector<uint8_t> &sessionId1, const Vector<uint8_t> &sessionId2) { + if (sessionId1.size() != sessionId2.size()) { + return false; + } + for (size_t i = 0; i < sessionId1.size(); ++i) { + if (sessionId1[i] != sessionId2[i]) { + return false; + } + } + return true; +} + +sp<DrmSessionManager> DrmSessionManager::Instance() { + static sp<DrmSessionManager> drmSessionManager = new DrmSessionManager(); + return drmSessionManager; +} + +DrmSessionManager::DrmSessionManager() + : mProcessInfo(new ProcessInfo()), + mTime(0) {} + +DrmSessionManager::DrmSessionManager(sp<ProcessInfoInterface> processInfo) + : mProcessInfo(processInfo), + mTime(0) {} + +DrmSessionManager::~DrmSessionManager() {} + +void DrmSessionManager::addSession( + int pid, sp<DrmSessionClientInterface> drm, const Vector<uint8_t> &sessionId) { + ALOGV("addSession(pid %d, drm %p, sessionId %s)", pid, drm.get(), + GetSessionIdString(sessionId).string()); + + Mutex::Autolock lock(mLock); + SessionInfo info; + info.drm = drm; + info.sessionId = sessionId; + info.timeStamp = getTime_l(); + ssize_t index = mSessionMap.indexOfKey(pid); + if (index < 0) { + // new pid + SessionInfos infosForPid; + infosForPid.push_back(info); + mSessionMap.add(pid, infosForPid); + } else { + mSessionMap.editValueAt(index).push_back(info); + } +} + +void DrmSessionManager::useSession(const Vector<uint8_t> &sessionId) { + ALOGV("useSession(%s)", GetSessionIdString(sessionId).string()); + + Mutex::Autolock lock(mLock); + for (size_t i = 0; i < mSessionMap.size(); ++i) { + SessionInfos& infos = mSessionMap.editValueAt(i); + for (size_t j = 0; j < infos.size(); ++j) { + SessionInfo& info = infos.editItemAt(j); + if (isEqualSessionId(sessionId, info.sessionId)) { + info.timeStamp = getTime_l(); + return; + } + } + } +} + +void DrmSessionManager::removeSession(const Vector<uint8_t> &sessionId) { + ALOGV("removeSession(%s)", GetSessionIdString(sessionId).string()); + + Mutex::Autolock lock(mLock); + for (size_t i = 0; i < mSessionMap.size(); ++i) { + SessionInfos& infos = mSessionMap.editValueAt(i); + for (size_t j = 0; j < infos.size(); ++j) { + if (isEqualSessionId(sessionId, infos[j].sessionId)) { + infos.removeAt(j); + return; + } + } + } +} + +void DrmSessionManager::removeDrm(sp<DrmSessionClientInterface> drm) { + ALOGV("removeDrm(%p)", drm.get()); + + Mutex::Autolock lock(mLock); + bool found = false; + for (size_t i = 0; i < mSessionMap.size(); ++i) { + SessionInfos& infos = mSessionMap.editValueAt(i); + for (size_t j = 0; j < infos.size();) { + if (infos[j].drm == drm) { + ALOGV("removed session (%s)", GetSessionIdString(infos[j].sessionId).string()); + j = infos.removeAt(j); + found = true; + } else { + ++j; + } + } + if (found) { + break; + } + } +} + +bool DrmSessionManager::reclaimSession(int callingPid) { + ALOGV("reclaimSession(%d)", callingPid); + + sp<DrmSessionClientInterface> drm; + Vector<uint8_t> sessionId; + int lowestPriorityPid; + int lowestPriority; + { + Mutex::Autolock lock(mLock); + int callingPriority; + if (!mProcessInfo->getPriority(callingPid, &callingPriority)) { + return false; + } + if (!getLowestPriority_l(&lowestPriorityPid, &lowestPriority)) { + return false; + } + if (lowestPriority <= callingPriority) { + return false; + } + + if (!getLeastUsedSession_l(lowestPriorityPid, &drm, &sessionId)) { + return false; + } + } + + if (drm == NULL) { + return false; + } + + ALOGV("reclaim session(%s) opened by pid %d", + GetSessionIdString(sessionId).string(), lowestPriorityPid); + + return drm->reclaimSession(sessionId); +} + +int64_t DrmSessionManager::getTime_l() { + return mTime++; +} + +bool DrmSessionManager::getLowestPriority_l(int* lowestPriorityPid, int* lowestPriority) { + int pid = -1; + int priority = -1; + for (size_t i = 0; i < mSessionMap.size(); ++i) { + if (mSessionMap.valueAt(i).size() == 0) { + // no opened session by this process. + continue; + } + int tempPid = mSessionMap.keyAt(i); + int tempPriority; + if (!mProcessInfo->getPriority(tempPid, &tempPriority)) { + // shouldn't happen. + return false; + } + if (pid == -1) { + pid = tempPid; + priority = tempPriority; + } else { + if (tempPriority > priority) { + pid = tempPid; + priority = tempPriority; + } + } + } + if (pid != -1) { + *lowestPriorityPid = pid; + *lowestPriority = priority; + } + return (pid != -1); +} + +bool DrmSessionManager::getLeastUsedSession_l( + int pid, sp<DrmSessionClientInterface>* drm, Vector<uint8_t>* sessionId) { + ssize_t index = mSessionMap.indexOfKey(pid); + if (index < 0) { + return false; + } + + int leastUsedIndex = -1; + int64_t minTs = LLONG_MAX; + const SessionInfos& infos = mSessionMap.valueAt(index); + for (size_t j = 0; j < infos.size(); ++j) { + if (leastUsedIndex == -1) { + leastUsedIndex = j; + minTs = infos[j].timeStamp; + } else { + if (infos[j].timeStamp < minTs) { + leastUsedIndex = j; + minTs = infos[j].timeStamp; + } + } + } + if (leastUsedIndex != -1) { + *drm = infos[leastUsedIndex].drm; + *sessionId = infos[leastUsedIndex].sessionId; + } + return (leastUsedIndex != -1); +} + +} // namespace android diff --git a/media/libmediaplayerservice/DrmSessionManager.h b/media/libmediaplayerservice/DrmSessionManager.h new file mode 100644 index 0000000..ba5c268 --- /dev/null +++ b/media/libmediaplayerservice/DrmSessionManager.h @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DRM_SESSION_MANAGER_H_ + +#define DRM_SESSION_MANAGER_H_ + +#include <media/stagefright/foundation/ABase.h> +#include <utils/RefBase.h> +#include <utils/KeyedVector.h> +#include <utils/threads.h> +#include <utils/Vector.h> + +namespace android { + +class DrmSessionManagerTest; +struct DrmSessionClientInterface; +struct ProcessInfoInterface; + +bool isEqualSessionId(const Vector<uint8_t> &sessionId1, const Vector<uint8_t> &sessionId2); + +struct SessionInfo { + sp<DrmSessionClientInterface> drm; + Vector<uint8_t> sessionId; + int64_t timeStamp; +}; + +typedef Vector<SessionInfo > SessionInfos; +typedef KeyedVector<int, SessionInfos > PidSessionInfosMap; + +struct DrmSessionManager : public RefBase { + static sp<DrmSessionManager> Instance(); + + DrmSessionManager(); + DrmSessionManager(sp<ProcessInfoInterface> processInfo); + + void addSession(int pid, sp<DrmSessionClientInterface> drm, const Vector<uint8_t>& sessionId); + void useSession(const Vector<uint8_t>& sessionId); + void removeSession(const Vector<uint8_t>& sessionId); + void removeDrm(sp<DrmSessionClientInterface> drm); + bool reclaimSession(int callingPid); + +protected: + virtual ~DrmSessionManager(); + +private: + friend class DrmSessionManagerTest; + + int64_t getTime_l(); + bool getLowestPriority_l(int* lowestPriorityPid, int* lowestPriority); + bool getLeastUsedSession_l( + int pid, sp<DrmSessionClientInterface>* drm, Vector<uint8_t>* sessionId); + + sp<ProcessInfoInterface> mProcessInfo; + mutable Mutex mLock; + PidSessionInfosMap mSessionMap; + int64_t mTime; + + DISALLOW_EVIL_CONSTRUCTORS(DrmSessionManager); +}; + +} // namespace android + +#endif // DRM_SESSION_MANAGER_H_ diff --git a/media/libmediaplayerservice/MediaPlayerService.cpp b/media/libmediaplayerservice/MediaPlayerService.cpp index 694f1a4..f113e21 100644 --- a/media/libmediaplayerservice/MediaPlayerService.cpp +++ b/media/libmediaplayerservice/MediaPlayerService.cpp @@ -290,8 +290,9 @@ MediaPlayerService::MediaPlayerService() const sp<IServiceManager> sm(defaultServiceManager()); if (sm != NULL) { const String16 name("batterystats"); + // use checkService() to avoid blocking if service is not up yet sp<IBatteryStats> batteryStats = - interface_cast<IBatteryStats>(sm->getService(name)); + interface_cast<IBatteryStats>(sm->checkService(name)); if (batteryStats != NULL) { batteryStats->noteResetVideo(); batteryStats->noteResetAudio(); @@ -441,6 +442,9 @@ status_t MediaPlayerService::dump(int fd, const Vector<String16>& args) const size_t SIZE = 256; char buffer[SIZE]; String8 result; + SortedVector< sp<Client> > clients; //to serialise the mutex unlock & client destruction. + SortedVector< sp<MediaRecorderClient> > mediaRecorderClients; + if (checkCallingPermission(String16("android.permission.DUMP")) == false) { snprintf(buffer, SIZE, "Permission Denial: " "can't dump MediaPlayerService from pid=%d, uid=%d\n", @@ -452,6 +456,7 @@ status_t MediaPlayerService::dump(int fd, const Vector<String16>& args) for (int i = 0, n = mClients.size(); i < n; ++i) { sp<Client> c = mClients[i].promote(); if (c != 0) c->dump(fd, args); + clients.add(c); } if (mMediaRecorderClients.size() == 0) { result.append(" No media recorder client\n\n"); @@ -464,6 +469,7 @@ status_t MediaPlayerService::dump(int fd, const Vector<String16>& args) write(fd, result.string(), result.size()); result = "\n"; c->dump(fd, args); + mediaRecorderClients.add(c); } } } @@ -961,6 +967,14 @@ status_t MediaPlayerService::Client::isPlaying(bool* state) return NO_ERROR; } +status_t MediaPlayerService::Client::setPlaybackRate(float rate) +{ + ALOGV("[%d] setPlaybackRate(%f)", mConnId, rate); + sp<MediaPlayerBase> p = getPlayer(); + if (p == 0) return UNKNOWN_ERROR; + return p->setPlaybackRate(rate); +} + status_t MediaPlayerService::Client::getCurrentPosition(int *msec) { ALOGV("getCurrentPosition"); @@ -1664,13 +1678,13 @@ void MediaPlayerService::AudioOutput::switchToNextOutput() { } } -ssize_t MediaPlayerService::AudioOutput::write(const void* buffer, size_t size) +ssize_t MediaPlayerService::AudioOutput::write(const void* buffer, size_t size, bool blocking) { LOG_ALWAYS_FATAL_IF(mCallback != NULL, "Don't call write if supplying a callback."); //ALOGV("write(%p, %u)", buffer, size); if (mTrack != 0) { - ssize_t ret = mTrack->write(buffer, size); + ssize_t ret = mTrack->write(buffer, size, blocking); if (ret >= 0) { mBytesWritten += ret; } diff --git a/media/libmediaplayerservice/MediaPlayerService.h b/media/libmediaplayerservice/MediaPlayerService.h index fad3447..4ce4b81 100644 --- a/media/libmediaplayerservice/MediaPlayerService.h +++ b/media/libmediaplayerservice/MediaPlayerService.h @@ -97,7 +97,7 @@ class MediaPlayerService : public BnMediaPlayerService const audio_offload_info_t *offloadInfo = NULL); virtual status_t start(); - virtual ssize_t write(const void* buffer, size_t size); + virtual ssize_t write(const void* buffer, size_t size, bool blocking = true); virtual void stop(); virtual void flush(); virtual void pause(); @@ -261,6 +261,7 @@ private: virtual status_t stop(); virtual status_t pause(); virtual status_t isPlaying(bool* state); + virtual status_t setPlaybackRate(float rate); virtual status_t seekTo(int msec); virtual status_t getCurrentPosition(int* msec); virtual status_t getDuration(int* msec); diff --git a/media/libmediaplayerservice/MediaRecorderClient.cpp b/media/libmediaplayerservice/MediaRecorderClient.cpp index 194abbb..4d4de9b 100644 --- a/media/libmediaplayerservice/MediaRecorderClient.cpp +++ b/media/libmediaplayerservice/MediaRecorderClient.cpp @@ -154,17 +154,6 @@ status_t MediaRecorderClient::setAudioEncoder(int ae) return mRecorder->setAudioEncoder((audio_encoder)ae); } -status_t MediaRecorderClient::setOutputFile(const char* path) -{ - ALOGV("setOutputFile(%s)", path); - Mutex::Autolock lock(mLock); - if (mRecorder == NULL) { - ALOGE("recorder is not initialized"); - return NO_INIT; - } - return mRecorder->setOutputFile(path); -} - status_t MediaRecorderClient::setOutputFile(int fd, int64_t offset, int64_t length) { ALOGV("setOutputFile(%d, %lld, %lld)", fd, offset, length); diff --git a/media/libmediaplayerservice/MediaRecorderClient.h b/media/libmediaplayerservice/MediaRecorderClient.h index a65ec9f..a444b6c 100644 --- a/media/libmediaplayerservice/MediaRecorderClient.h +++ b/media/libmediaplayerservice/MediaRecorderClient.h @@ -38,7 +38,6 @@ public: virtual status_t setOutputFormat(int of); virtual status_t setVideoEncoder(int ve); virtual status_t setAudioEncoder(int ae); - virtual status_t setOutputFile(const char* path); virtual status_t setOutputFile(int fd, int64_t offset, int64_t length); virtual status_t setVideoSize(int width, int height); diff --git a/media/libmediaplayerservice/StagefrightRecorder.cpp b/media/libmediaplayerservice/StagefrightRecorder.cpp index 86639cb..55763f0 100644 --- a/media/libmediaplayerservice/StagefrightRecorder.cpp +++ b/media/libmediaplayerservice/StagefrightRecorder.cpp @@ -75,6 +75,7 @@ StagefrightRecorder::StagefrightRecorder() mAudioSource(AUDIO_SOURCE_CNT), mVideoSource(VIDEO_SOURCE_LIST_END), mCaptureTimeLapse(false), + mCaptureFps(0.0f), mStarted(false) { ALOGV("Constructor"); @@ -206,7 +207,7 @@ status_t StagefrightRecorder::setVideoSize(int width, int height) { status_t StagefrightRecorder::setVideoFrameRate(int frames_per_second) { ALOGV("setVideoFrameRate: %d", frames_per_second); if ((frames_per_second <= 0 && frames_per_second != -1) || - frames_per_second > 120) { + frames_per_second > kMaxHighSpeedFps) { ALOGE("Invalid video frame rate: %d", frames_per_second); return BAD_VALUE; } @@ -241,14 +242,6 @@ status_t StagefrightRecorder::setPreviewSurface(const sp<IGraphicBufferProducer> return OK; } -status_t StagefrightRecorder::setOutputFile(const char * /* path */) { - ALOGE("setOutputFile(const char*) must not be called"); - // We don't actually support this at all, as the media_server process - // no longer has permissions to create files. - - return -EPERM; -} - status_t StagefrightRecorder::setOutputFile(int fd, int64_t offset, int64_t length) { ALOGV("setOutputFile: %d, %lld, %lld", fd, offset, length); // These don't make any sense, do they? @@ -271,6 +264,31 @@ status_t StagefrightRecorder::setOutputFile(int fd, int64_t offset, int64_t leng return OK; } +// Attempt to parse an float literal optionally surrounded by whitespace, +// returns true on success, false otherwise. +static bool safe_strtof(const char *s, float *val) { + char *end; + + // It is lame, but according to man page, we have to set errno to 0 + // before calling strtof(). + errno = 0; + *val = strtof(s, &end); + + if (end == s || errno == ERANGE) { + return false; + } + + // Skip trailing whitespace + while (isspace(*end)) { + ++end; + } + + // For a successful return, the string must contain nothing but a valid + // float literal optionally surrounded by whitespace. + + return *end == '\0'; +} + // Attempt to parse an int64 literal optionally surrounded by whitespace, // returns true on success, false otherwise. static bool safe_strtoi64(const char *s, int64_t *val) { @@ -554,8 +572,10 @@ status_t StagefrightRecorder::setParamTimeLapseEnable(int32_t timeLapseEnable) { return OK; } -status_t StagefrightRecorder::setParamTimeBetweenTimeLapseFrameCapture(int64_t timeUs) { - ALOGV("setParamTimeBetweenTimeLapseFrameCapture: %lld us", timeUs); +status_t StagefrightRecorder::setParamTimeLapseFps(float fps) { + ALOGV("setParamTimeLapseFps: %.2f", fps); + + int64_t timeUs = (int64_t) (1000000.0 / fps + 0.5f); // Not allowing time more than a day if (timeUs <= 0 || timeUs > 86400*1E6) { @@ -563,6 +583,7 @@ status_t StagefrightRecorder::setParamTimeBetweenTimeLapseFrameCapture(int64_t t return BAD_VALUE; } + mCaptureFps = fps; mTimeBetweenTimeLapseFrameCaptureUs = timeUs; return OK; } @@ -690,11 +711,10 @@ status_t StagefrightRecorder::setParameter( if (safe_strtoi32(value.string(), &timeLapseEnable)) { return setParamTimeLapseEnable(timeLapseEnable); } - } else if (key == "time-between-time-lapse-frame-capture") { - int64_t timeBetweenTimeLapseFrameCaptureUs; - if (safe_strtoi64(value.string(), &timeBetweenTimeLapseFrameCaptureUs)) { - return setParamTimeBetweenTimeLapseFrameCapture( - timeBetweenTimeLapseFrameCaptureUs); + } else if (key == "time-lapse-fps") { + float fps; + if (safe_strtof(value.string(), &fps)) { + return setParamTimeLapseFps(fps); } } else { ALOGE("setParameter: failed to find key %s", key.string()); @@ -1589,10 +1609,11 @@ status_t StagefrightRecorder::setupMPEG4orWEBMRecording() { status_t err = OK; sp<MediaWriter> writer; + sp<MPEG4Writer> mp4writer; if (mOutputFormat == OUTPUT_FORMAT_WEBM) { writer = new WebmWriter(mOutputFd); } else { - writer = new MPEG4Writer(mOutputFd); + writer = mp4writer = new MPEG4Writer(mOutputFd); } if (mVideoSource < VIDEO_SOURCE_LIST_END) { @@ -1625,13 +1646,15 @@ status_t StagefrightRecorder::setupMPEG4orWEBMRecording() { mTotalBitRate += mAudioBitRate; } + if (mCaptureTimeLapse) { + mp4writer->setCaptureRate(mCaptureFps); + } + if (mInterleaveDurationUs > 0) { - reinterpret_cast<MPEG4Writer *>(writer.get())-> - setInterleaveDuration(mInterleaveDurationUs); + mp4writer->setInterleaveDuration(mInterleaveDurationUs); } if (mLongitudex10000 > -3600000 && mLatitudex10000 > -3600000) { - reinterpret_cast<MPEG4Writer *>(writer.get())-> - setGeoData(mLatitudex10000, mLongitudex10000); + mp4writer->setGeoData(mLatitudex10000, mLongitudex10000); } } if (mMaxFileDurationUs != 0) { diff --git a/media/libmediaplayerservice/StagefrightRecorder.h b/media/libmediaplayerservice/StagefrightRecorder.h index 54c38d3..f34c229 100644 --- a/media/libmediaplayerservice/StagefrightRecorder.h +++ b/media/libmediaplayerservice/StagefrightRecorder.h @@ -53,7 +53,6 @@ struct StagefrightRecorder : public MediaRecorderBase { virtual status_t setVideoFrameRate(int frames_per_second); virtual status_t setCamera(const sp<ICamera>& camera, const sp<ICameraRecordingProxy>& proxy); virtual status_t setPreviewSurface(const sp<IGraphicBufferProducer>& surface); - virtual status_t setOutputFile(const char *path); virtual status_t setOutputFile(int fd, int64_t offset, int64_t length); virtual status_t setParameters(const String8& params); virtual status_t setListener(const sp<IMediaRecorderClient>& listener); @@ -110,6 +109,7 @@ private: int32_t mTotalBitRate; bool mCaptureTimeLapse; + float mCaptureFps; int64_t mTimeBetweenTimeLapseFrameCaptureUs; sp<CameraSourceTimeLapse> mCameraSourceTimeLapse; @@ -127,6 +127,8 @@ private: sp<IGraphicBufferProducer> mGraphicBufferProducer; sp<ALooper> mLooper; + static const int kMaxHighSpeedFps = 1000; + status_t prepareInternal(); status_t setupMPEG4orWEBMRecording(); void setupMPEG4orWEBMMetaData(sp<MetaData> *meta); @@ -154,7 +156,7 @@ private: status_t setParamAudioSamplingRate(int32_t sampleRate); status_t setParamAudioTimeScale(int32_t timeScale); status_t setParamTimeLapseEnable(int32_t timeLapseEnable); - status_t setParamTimeBetweenTimeLapseFrameCapture(int64_t timeUs); + status_t setParamTimeLapseFps(float fps); status_t setParamVideoEncodingBitRate(int32_t bitRate); status_t setParamVideoIFramesInterval(int32_t seconds); status_t setParamVideoEncoderProfile(int32_t profile); diff --git a/media/libmediaplayerservice/nuplayer/GenericSource.cpp b/media/libmediaplayerservice/nuplayer/GenericSource.cpp index 1b2fc5e..c234013 100644 --- a/media/libmediaplayerservice/nuplayer/GenericSource.cpp +++ b/media/libmediaplayerservice/nuplayer/GenericSource.cpp @@ -65,12 +65,12 @@ NuPlayer::GenericSource::GenericSource( mUID(uid), mFd(-1), mDrmManagerClient(NULL), - mMetaDataSize(-1ll), mBitrate(-1ll), mPollBufferingGeneration(0), mPendingReadBufferTypes(0), mBuffering(false), - mPrepareBuffering(false) { + mPrepareBuffering(false), + mPrevBufferPercentage(-1) { resetDataSource(); DataSource::RegisterDefaultSniffers(); } @@ -130,23 +130,34 @@ sp<MetaData> NuPlayer::GenericSource::getFileFormatMeta() const { status_t NuPlayer::GenericSource::initFromDataSource() { sp<MediaExtractor> extractor; + String8 mimeType; + float confidence; + sp<AMessage> dummy; + bool isWidevineStreaming = false; CHECK(mDataSource != NULL); if (mIsWidevine) { - String8 mimeType; - float confidence; - sp<AMessage> dummy; - bool success; - - success = SniffWVM(mDataSource, &mimeType, &confidence, &dummy); - if (!success - || strcasecmp( + isWidevineStreaming = SniffWVM( + mDataSource, &mimeType, &confidence, &dummy); + if (!isWidevineStreaming || + strcasecmp( mimeType.string(), MEDIA_MIMETYPE_CONTAINER_WVM)) { ALOGE("unsupported widevine mime: %s", mimeType.string()); return UNKNOWN_ERROR; } + } else if (mIsStreaming) { + if (!mDataSource->sniff(&mimeType, &confidence, &dummy)) { + return UNKNOWN_ERROR; + } + isWidevineStreaming = !strcasecmp( + mimeType.string(), MEDIA_MIMETYPE_CONTAINER_WVM); + } + if (isWidevineStreaming) { + // we don't want cached source for widevine streaming. + mCachedSource.clear(); + mDataSource = mHttpSource; mWVMExtractor = new WVMExtractor(mDataSource); mWVMExtractor->setAdaptiveStreamingMode(true); if (mUIDValid) { @@ -155,7 +166,7 @@ status_t NuPlayer::GenericSource::initFromDataSource() { extractor = mWVMExtractor; } else { extractor = MediaExtractor::Create(mDataSource, - mSniffedMIME.empty() ? NULL: mSniffedMIME.c_str()); + mimeType.isEmpty() ? NULL : mimeType.string()); } if (extractor == NULL) { @@ -181,14 +192,6 @@ status_t NuPlayer::GenericSource::initFromDataSource() { if (mFileMeta->findCString(kKeyMIMEType, &fileMime) && !strncasecmp(fileMime, "video/wvm", 9)) { mIsWidevine = true; - if (!mUri.empty()) { - // streaming, but the app forgot to specify widevine:// url - mWVMExtractor = static_cast<WVMExtractor *>(extractor.get()); - mWVMExtractor->setAdaptiveStreamingMode(true); - if (mUIDValid) { - mWVMExtractor->setUID(mUID); - } - } } } } @@ -332,7 +335,7 @@ void NuPlayer::GenericSource::prepareAsync() { mLooper->registerHandler(this); } - sp<AMessage> msg = new AMessage(kWhatPrepareAsync, id()); + sp<AMessage> msg = new AMessage(kWhatPrepareAsync, this); msg->post(); } @@ -345,6 +348,7 @@ void NuPlayer::GenericSource::onPrepareAsync() { if (!mUri.empty()) { const char* uri = mUri.c_str(); + String8 contentType; mIsWidevine = !strncasecmp(uri, "widevine://", 11); if (!strncasecmp("http://", uri, 7) @@ -359,7 +363,7 @@ void NuPlayer::GenericSource::onPrepareAsync() { } mDataSource = DataSource::CreateFromURI( - mHTTPService, uri, &mUriHeaders, &mContentType, + mHTTPService, uri, &mUriHeaders, &contentType, static_cast<HTTPBase *>(mHttpSource.get())); } else { mIsWidevine = false; @@ -387,20 +391,8 @@ void NuPlayer::GenericSource::onPrepareAsync() { mIsStreaming = (mIsWidevine || mCachedSource != NULL); } - // check initial caching status - status_t err = prefillCacheIfNecessary(); - if (err != OK) { - if (err == -EAGAIN) { - (new AMessage(kWhatPrepareAsync, id()))->post(200000); - } else { - ALOGE("Failed to prefill data cache!"); - notifyPreparedAndCleanup(UNKNOWN_ERROR); - } - return; - } - - // init extrator from data source - err = initFromDataSource(); + // init extractor from data source + status_t err = initFromDataSource(); if (err != OK) { ALOGE("Failed to init from data source!"); @@ -465,9 +457,6 @@ void NuPlayer::GenericSource::finishPrepareAsync() { void NuPlayer::GenericSource::notifyPreparedAndCleanup(status_t err) { if (err != OK) { - mMetaDataSize = -1ll; - mContentType = ""; - mSniffedMIME = ""; mDataSource.clear(); mCachedSource.clear(); mHttpSource.clear(); @@ -478,76 +467,6 @@ void NuPlayer::GenericSource::notifyPreparedAndCleanup(status_t err) { notifyPrepared(err); } -status_t NuPlayer::GenericSource::prefillCacheIfNecessary() { - CHECK(mDataSource != NULL); - - if (mCachedSource == NULL) { - // no prefill if the data source is not cached - return OK; - } - - // We're not doing this for streams that appear to be audio-only - // streams to ensure that even low bandwidth streams start - // playing back fairly instantly. - if (!strncasecmp(mContentType.string(), "audio/", 6)) { - return OK; - } - - // We're going to prefill the cache before trying to instantiate - // the extractor below, as the latter is an operation that otherwise - // could block on the datasource for a significant amount of time. - // During that time we'd be unable to abort the preparation phase - // without this prefill. - - // Initially make sure we have at least 192 KB for the sniff - // to complete without blocking. - static const size_t kMinBytesForSniffing = 192 * 1024; - static const size_t kDefaultMetaSize = 200000; - - status_t finalStatus; - - size_t cachedDataRemaining = - mCachedSource->approxDataRemaining(&finalStatus); - - if (finalStatus != OK || (mMetaDataSize >= 0 - && (off64_t)cachedDataRemaining >= mMetaDataSize)) { - ALOGV("stop caching, status %d, " - "metaDataSize %lld, cachedDataRemaining %zu", - finalStatus, mMetaDataSize, cachedDataRemaining); - return OK; - } - - ALOGV("now cached %zu bytes of data", cachedDataRemaining); - - if (mMetaDataSize < 0 - && cachedDataRemaining >= kMinBytesForSniffing) { - String8 tmp; - float confidence; - sp<AMessage> meta; - if (!mCachedSource->sniff(&tmp, &confidence, &meta)) { - return UNKNOWN_ERROR; - } - - // We successfully identified the file's extractor to - // be, remember this mime type so we don't have to - // sniff it again when we call MediaExtractor::Create() - mSniffedMIME = tmp.string(); - - if (meta == NULL - || !meta->findInt64("meta-data-size", - reinterpret_cast<int64_t*>(&mMetaDataSize))) { - mMetaDataSize = kDefaultMetaSize; - } - - if (mMetaDataSize < 0ll) { - ALOGE("invalid metaDataSize = %lld bytes", mMetaDataSize); - return UNKNOWN_ERROR; - } - } - - return -EAGAIN; -} - void NuPlayer::GenericSource::start() { ALOGI("start"); @@ -563,7 +482,7 @@ void NuPlayer::GenericSource::start() { setDrmPlaybackStatusIfNeeded(Playback::START, getLastReadPosition() / 1000); mStarted = true; - (new AMessage(kWhatStart, id()))->post(); + (new AMessage(kWhatStart, this))->post(); } void NuPlayer::GenericSource::stop() { @@ -572,7 +491,7 @@ void NuPlayer::GenericSource::stop() { mStarted = false; if (mIsWidevine || mIsSecure) { // For widevine or secure sources we need to prevent any further reads. - sp<AMessage> msg = new AMessage(kWhatStopWidevine, id()); + sp<AMessage> msg = new AMessage(kWhatStopWidevine, this); sp<AMessage> response; (void) msg->postAndAwaitResponse(&response); } @@ -589,7 +508,7 @@ void NuPlayer::GenericSource::resume() { setDrmPlaybackStatusIfNeeded(Playback::START, getLastReadPosition() / 1000); mStarted = true; - (new AMessage(kWhatResume, id()))->post(); + (new AMessage(kWhatResume, this))->post(); } void NuPlayer::GenericSource::disconnect() { @@ -616,7 +535,7 @@ status_t NuPlayer::GenericSource::feedMoreTSData() { } void NuPlayer::GenericSource::schedulePollBuffering() { - sp<AMessage> msg = new AMessage(kWhatPollBuffering, id()); + sp<AMessage> msg = new AMessage(kWhatPollBuffering, this); msg->setInt32("generation", mPollBufferingGeneration); msg->post(1000000ll); } @@ -624,6 +543,7 @@ void NuPlayer::GenericSource::schedulePollBuffering() { void NuPlayer::GenericSource::cancelPollBuffering() { mBuffering = false; ++mPollBufferingGeneration; + mPrevBufferPercentage = -1; } void NuPlayer::GenericSource::restartPollBuffering() { @@ -633,7 +553,19 @@ void NuPlayer::GenericSource::restartPollBuffering() { } } -void NuPlayer::GenericSource::notifyBufferingUpdate(int percentage) { +void NuPlayer::GenericSource::notifyBufferingUpdate(int32_t percentage) { + // Buffering percent could go backward as it's estimated from remaining + // data and last access time. This could cause the buffering position + // drawn on media control to jitter slightly. Remember previously reported + // percentage and don't allow it to go backward. + if (percentage < mPrevBufferPercentage) { + percentage = mPrevBufferPercentage; + } else if (percentage > 100) { + percentage = 100; + } + + mPrevBufferPercentage = percentage; + ALOGV("notifyBufferingUpdate: buffering %d%%", percentage); sp<AMessage> msg = dupNotify(); @@ -687,10 +619,10 @@ void NuPlayer::GenericSource::sendCacheStats() { int32_t kbps = 0; status_t err = UNKNOWN_ERROR; - if (mCachedSource != NULL) { - err = mCachedSource->getEstimatedBandwidthKbps(&kbps); - } else if (mWVMExtractor != NULL) { + if (mWVMExtractor != NULL) { err = mWVMExtractor->getEstimatedBandwidthKbps(&kbps); + } else if (mCachedSource != NULL) { + err = mCachedSource->getEstimatedBandwidthKbps(&kbps); } if (err == OK) { @@ -712,7 +644,13 @@ void NuPlayer::GenericSource::onPollBuffering() { int64_t cachedDurationUs = -1ll; ssize_t cachedDataRemaining = -1; - if (mCachedSource != NULL) { + ALOGW_IF(mWVMExtractor != NULL && mCachedSource != NULL, + "WVMExtractor and NuCachedSource both present"); + + if (mWVMExtractor != NULL) { + cachedDurationUs = + mWVMExtractor->getCachedDurationUs(&finalStatus); + } else if (mCachedSource != NULL) { cachedDataRemaining = mCachedSource->approxDataRemaining(&finalStatus); @@ -728,9 +666,6 @@ void NuPlayer::GenericSource::onPollBuffering() { cachedDurationUs = cachedDataRemaining * 8000000ll / bitrate; } } - } else if (mWVMExtractor != NULL) { - cachedDurationUs - = mWVMExtractor->getCachedDurationUs(&finalStatus); } if (finalStatus != OK) { @@ -918,7 +853,7 @@ void NuPlayer::GenericSource::onMessageReceived(const sp<AMessage> &msg) { mVideoTrack.mPackets->clear(); } sp<AMessage> response = new AMessage; - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); response->postReply(replyID); break; @@ -958,7 +893,7 @@ void NuPlayer::GenericSource::fetchTextData( const int64_t oneSecUs = 1000000ll; delayUs -= oneSecUs; } - sp<AMessage> msg2 = new AMessage(sendWhat, id()); + sp<AMessage> msg2 = new AMessage(sendWhat, this); msg2->setInt32("generation", msgGeneration); msg2->post(delayUs < 0 ? 0 : delayUs); } @@ -998,7 +933,7 @@ void NuPlayer::GenericSource::sendTextData( } sp<MetaData> NuPlayer::GenericSource::getFormatMeta(bool audio) { - sp<AMessage> msg = new AMessage(kWhatGetFormat, id()); + sp<AMessage> msg = new AMessage(kWhatGetFormat, this); msg->setInt32("audio", audio); sp<AMessage> response; @@ -1020,7 +955,7 @@ void NuPlayer::GenericSource::onGetFormatMeta(sp<AMessage> msg) const { sp<MetaData> format = doGetFormatMeta(audio); response->setPointer("format", format.get()); - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); response->postReply(replyID); } @@ -1087,7 +1022,7 @@ status_t NuPlayer::GenericSource::dequeueAccessUnit( if (mSubtitleTrack.mSource != NULL && !mSubtitleTrack.mPackets->hasBufferAvailable(&eosResult)) { - sp<AMessage> msg = new AMessage(kWhatFetchSubtitleData, id()); + sp<AMessage> msg = new AMessage(kWhatFetchSubtitleData, this); msg->setInt64("timeUs", timeUs); msg->setInt32("generation", mFetchSubtitleDataGeneration); msg->post(); @@ -1095,7 +1030,7 @@ status_t NuPlayer::GenericSource::dequeueAccessUnit( if (mTimedTextTrack.mSource != NULL && !mTimedTextTrack.mPackets->hasBufferAvailable(&eosResult)) { - sp<AMessage> msg = new AMessage(kWhatFetchTimedTextData, id()); + sp<AMessage> msg = new AMessage(kWhatFetchTimedTextData, this); msg->setInt64("timeUs", timeUs); msg->setInt32("generation", mFetchTimedTextDataGeneration); msg->post(); @@ -1160,7 +1095,7 @@ sp<AMessage> NuPlayer::GenericSource::getTrackInfo(size_t trackIndex) const { } ssize_t NuPlayer::GenericSource::getSelectedTrack(media_track_type type) const { - sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, id()); + sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, this); msg->setInt32("type", type); sp<AMessage> response; @@ -1183,7 +1118,7 @@ void NuPlayer::GenericSource::onGetSelectedTrack(sp<AMessage> msg) const { ssize_t index = doGetSelectedTrack(type); response->setInt32("index", index); - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); response->postReply(replyID); } @@ -1216,7 +1151,7 @@ ssize_t NuPlayer::GenericSource::doGetSelectedTrack(media_track_type type) const status_t NuPlayer::GenericSource::selectTrack(size_t trackIndex, bool select, int64_t timeUs) { ALOGV("%s track: %zu", select ? "select" : "deselect", trackIndex); - sp<AMessage> msg = new AMessage(kWhatSelectTrack, id()); + sp<AMessage> msg = new AMessage(kWhatSelectTrack, this); msg->setInt32("trackIndex", trackIndex); msg->setInt32("select", select); msg->setInt64("timeUs", timeUs); @@ -1241,7 +1176,7 @@ void NuPlayer::GenericSource::onSelectTrack(sp<AMessage> msg) { status_t err = doSelectTrack(trackIndex, select, timeUs); response->setInt32("err", err); - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); response->postReply(replyID); } @@ -1302,7 +1237,7 @@ status_t NuPlayer::GenericSource::doSelectTrack(size_t trackIndex, bool select, status_t eosResult; // ignored if (mSubtitleTrack.mSource != NULL && !mSubtitleTrack.mPackets->hasBufferAvailable(&eosResult)) { - sp<AMessage> msg = new AMessage(kWhatFetchSubtitleData, id()); + sp<AMessage> msg = new AMessage(kWhatFetchSubtitleData, this); msg->setInt64("timeUs", timeUs); msg->setInt32("generation", mFetchSubtitleDataGeneration); msg->post(); @@ -1310,7 +1245,7 @@ status_t NuPlayer::GenericSource::doSelectTrack(size_t trackIndex, bool select, if (mTimedTextTrack.mSource != NULL && !mTimedTextTrack.mPackets->hasBufferAvailable(&eosResult)) { - sp<AMessage> msg = new AMessage(kWhatFetchTimedTextData, id()); + sp<AMessage> msg = new AMessage(kWhatFetchTimedTextData, this); msg->setInt64("timeUs", timeUs); msg->setInt32("generation", mFetchTimedTextDataGeneration); msg->post(); @@ -1324,7 +1259,7 @@ status_t NuPlayer::GenericSource::doSelectTrack(size_t trackIndex, bool select, return OK; } - sp<AMessage> msg = new AMessage(kWhatChangeAVSource, id()); + sp<AMessage> msg = new AMessage(kWhatChangeAVSource, this); msg->setInt32("trackIndex", trackIndex); msg->post(); return OK; @@ -1334,7 +1269,7 @@ status_t NuPlayer::GenericSource::doSelectTrack(size_t trackIndex, bool select, } status_t NuPlayer::GenericSource::seekTo(int64_t seekTimeUs) { - sp<AMessage> msg = new AMessage(kWhatSeek, id()); + sp<AMessage> msg = new AMessage(kWhatSeek, this); msg->setInt64("seekTimeUs", seekTimeUs); sp<AMessage> response; @@ -1354,7 +1289,7 @@ void NuPlayer::GenericSource::onSeek(sp<AMessage> msg) { status_t err = doSeek(seekTimeUs); response->setInt32("err", err); - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); response->postReply(replyID); } @@ -1474,7 +1409,7 @@ void NuPlayer::GenericSource::postReadBuffer(media_track_type trackType) { if ((mPendingReadBufferTypes & (1 << trackType)) == 0) { mPendingReadBufferTypes |= (1 << trackType); - sp<AMessage> msg = new AMessage(kWhatReadBuffer, id()); + sp<AMessage> msg = new AMessage(kWhatReadBuffer, this); msg->setInt32("trackType", trackType); msg->post(); } diff --git a/media/libmediaplayerservice/nuplayer/GenericSource.h b/media/libmediaplayerservice/nuplayer/GenericSource.h index 2d73ea9..862ee5f 100644 --- a/media/libmediaplayerservice/nuplayer/GenericSource.h +++ b/media/libmediaplayerservice/nuplayer/GenericSource.h @@ -140,14 +140,13 @@ private: sp<DecryptHandle> mDecryptHandle; bool mStarted; bool mStopRead; - String8 mContentType; - AString mSniffedMIME; - off64_t mMetaDataSize; int64_t mBitrate; int32_t mPollBufferingGeneration; uint32_t mPendingReadBufferTypes; bool mBuffering; bool mPrepareBuffering; + int32_t mPrevBufferPercentage; + mutable Mutex mReadBufferLock; sp<ALooper> mLooper; @@ -159,8 +158,6 @@ private: int64_t getLastReadPosition(); void setDrmPlaybackStatusIfNeeded(int playbackStatus, int64_t position); - status_t prefillCacheIfNecessary(); - void notifyPreparedAndCleanup(status_t err); void onSecureDecodersInstantiated(status_t err); void finishPrepareAsync(); @@ -204,7 +201,7 @@ private: void cancelPollBuffering(); void restartPollBuffering(); void onPollBuffering(); - void notifyBufferingUpdate(int percentage); + void notifyBufferingUpdate(int32_t percentage); void startBufferingIfNecessary(); void stopBufferingIfNecessary(); void sendCacheStats(); diff --git a/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp b/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp index a26ef9e..0476c9b 100644 --- a/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp +++ b/media/libmediaplayerservice/nuplayer/HTTPLiveSource.cpp @@ -81,7 +81,7 @@ void NuPlayer::HTTPLiveSource::prepareAsync() { mLiveLooper->registerHandler(this); } - sp<AMessage> notify = new AMessage(kWhatSessionNotify, id()); + sp<AMessage> notify = new AMessage(kWhatSessionNotify, this); mLiveSession = new LiveSession( notify, @@ -153,7 +153,7 @@ status_t NuPlayer::HTTPLiveSource::selectTrack(size_t trackIndex, bool select, i if (err == OK) { mFetchSubtitleDataGeneration++; if (select) { - sp<AMessage> msg = new AMessage(kWhatFetchSubtitleData, id()); + sp<AMessage> msg = new AMessage(kWhatFetchSubtitleData, this); msg->setInt32("generation", mFetchSubtitleDataGeneration); msg->post(); } @@ -281,6 +281,34 @@ void NuPlayer::HTTPLiveSource::onSessionNotify(const sp<AMessage> &msg) { break; } + case LiveSession::kWhatBufferingStart: + { + sp<AMessage> notify = dupNotify(); + notify->setInt32("what", kWhatPauseOnBufferingStart); + notify->post(); + break; + } + + case LiveSession::kWhatBufferingEnd: + { + sp<AMessage> notify = dupNotify(); + notify->setInt32("what", kWhatResumeOnBufferingEnd); + notify->post(); + break; + } + + + case LiveSession::kWhatBufferingUpdate: + { + sp<AMessage> notify = dupNotify(); + int32_t percentage; + CHECK(msg->findInt32("percentage", &percentage)); + notify->setInt32("what", kWhatBufferingUpdate); + notify->setInt32("percentage", percentage); + notify->post(); + break; + } + case LiveSession::kWhatError: { break; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp index aeea204..cfa5258 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp @@ -180,6 +180,7 @@ NuPlayer::NuPlayer() mFlushingVideo(NONE), mResumePending(false), mVideoScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW), + mPlaybackRate(1.0), mStarted(false), mPaused(false), mPausedByClient(false) { @@ -199,9 +200,9 @@ void NuPlayer::setDriver(const wp<NuPlayerDriver> &driver) { } void NuPlayer::setDataSourceAsync(const sp<IStreamSource> &source) { - sp<AMessage> msg = new AMessage(kWhatSetDataSource, id()); + sp<AMessage> msg = new AMessage(kWhatSetDataSource, this); - sp<AMessage> notify = new AMessage(kWhatSourceNotify, id()); + sp<AMessage> notify = new AMessage(kWhatSourceNotify, this); msg->setObject("source", new StreamingSource(notify, source)); msg->post(); @@ -229,10 +230,10 @@ void NuPlayer::setDataSourceAsync( const char *url, const KeyedVector<String8, String8> *headers) { - sp<AMessage> msg = new AMessage(kWhatSetDataSource, id()); + sp<AMessage> msg = new AMessage(kWhatSetDataSource, this); size_t len = strlen(url); - sp<AMessage> notify = new AMessage(kWhatSourceNotify, id()); + sp<AMessage> notify = new AMessage(kWhatSourceNotify, this); sp<Source> source; if (IsHTTPLiveURL(url)) { @@ -266,9 +267,9 @@ void NuPlayer::setDataSourceAsync( } void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) { - sp<AMessage> msg = new AMessage(kWhatSetDataSource, id()); + sp<AMessage> msg = new AMessage(kWhatSetDataSource, this); - sp<AMessage> notify = new AMessage(kWhatSourceNotify, id()); + sp<AMessage> notify = new AMessage(kWhatSourceNotify, this); sp<GenericSource> source = new GenericSource(notify, mUIDValid, mUID); @@ -285,12 +286,12 @@ void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) { } void NuPlayer::prepareAsync() { - (new AMessage(kWhatPrepare, id()))->post(); + (new AMessage(kWhatPrepare, this))->post(); } void NuPlayer::setVideoSurfaceTextureAsync( const sp<IGraphicBufferProducer> &bufferProducer) { - sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, id()); + sp<AMessage> msg = new AMessage(kWhatSetVideoNativeWindow, this); if (bufferProducer == NULL) { msg->setObject("native-window", NULL); @@ -305,17 +306,23 @@ void NuPlayer::setVideoSurfaceTextureAsync( } void NuPlayer::setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink) { - sp<AMessage> msg = new AMessage(kWhatSetAudioSink, id()); + sp<AMessage> msg = new AMessage(kWhatSetAudioSink, this); msg->setObject("sink", sink); msg->post(); } void NuPlayer::start() { - (new AMessage(kWhatStart, id()))->post(); + (new AMessage(kWhatStart, this))->post(); +} + +void NuPlayer::setPlaybackRate(float rate) { + sp<AMessage> msg = new AMessage(kWhatSetRate, this); + msg->setFloat("rate", rate); + msg->post(); } void NuPlayer::pause() { - (new AMessage(kWhatPause, id()))->post(); + (new AMessage(kWhatPause, this))->post(); } void NuPlayer::resetAsync() { @@ -329,11 +336,11 @@ void NuPlayer::resetAsync() { mSource->disconnect(); } - (new AMessage(kWhatReset, id()))->post(); + (new AMessage(kWhatReset, this))->post(); } void NuPlayer::seekToAsync(int64_t seekTimeUs, bool needNotify) { - sp<AMessage> msg = new AMessage(kWhatSeek, id()); + sp<AMessage> msg = new AMessage(kWhatSeek, this); msg->setInt64("seekTimeUs", seekTimeUs); msg->setInt32("needNotify", needNotify); msg->post(); @@ -401,7 +408,7 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { case kWhatGetTrackInfo: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); Parcel* reply; @@ -454,7 +461,7 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { sp<AMessage> response = new AMessage; response->setInt32("err", err); - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); response->postReply(replyID); break; @@ -462,7 +469,7 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { case kWhatSelectTrack: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); size_t trackIndex; @@ -604,6 +611,16 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { break; } + case kWhatSetRate: + { + ALOGV("kWhatSetRate"); + CHECK(msg->findFloat("rate", &mPlaybackRate)); + if (mRenderer != NULL) { + mRenderer->setPlaybackRate(mPlaybackRate); + } + break; + } + case kWhatScanSources: { int32_t generation; @@ -1062,15 +1079,17 @@ void NuPlayer::onStart() { flags |= Renderer::FLAG_OFFLOAD_AUDIO; } - sp<AMessage> notify = new AMessage(kWhatRendererNotify, id()); + sp<AMessage> notify = new AMessage(kWhatRendererNotify, this); ++mRendererGeneration; notify->setInt32("generation", mRendererGeneration); mRenderer = new Renderer(mAudioSink, notify, flags); - mRendererLooper = new ALooper; mRendererLooper->setName("NuPlayerRenderer"); mRendererLooper->start(false, false, ANDROID_PRIORITY_AUDIO); mRendererLooper->registerHandler(mRenderer); + if (mPlaybackRate != 1.0) { + mRenderer->setPlaybackRate(mPlaybackRate); + } sp<MetaData> meta = getFileMeta(); int32_t rate; @@ -1176,7 +1195,7 @@ void NuPlayer::postScanSources() { return; } - sp<AMessage> msg = new AMessage(kWhatScanSources, id()); + sp<AMessage> msg = new AMessage(kWhatScanSources, this); msg->setInt32("generation", mScanSourcesGeneration); msg->post(); @@ -1218,7 +1237,7 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<DecoderBase> *decoder) { AString mime; CHECK(format->findString("mime", &mime)); - sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id()); + sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, this); if (mCCDecoder == NULL) { mCCDecoder = new CCDecoder(ccNotify); } @@ -1233,7 +1252,7 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<DecoderBase> *decoder) { } if (audio) { - sp<AMessage> notify = new AMessage(kWhatAudioNotify, id()); + sp<AMessage> notify = new AMessage(kWhatAudioNotify, this); ++mAudioDecoderGeneration; notify->setInt32("generation", mAudioDecoderGeneration); @@ -1243,7 +1262,7 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<DecoderBase> *decoder) { *decoder = new Decoder(notify, mSource, mRenderer); } } else { - sp<AMessage> notify = new AMessage(kWhatVideoNotify, id()); + sp<AMessage> notify = new AMessage(kWhatVideoNotify, this); ++mVideoDecoderGeneration; notify->setInt32("generation", mVideoDecoderGeneration); @@ -1434,7 +1453,7 @@ status_t NuPlayer::setVideoScalingMode(int32_t mode) { } status_t NuPlayer::getTrackInfo(Parcel* reply) const { - sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, id()); + sp<AMessage> msg = new AMessage(kWhatGetTrackInfo, this); msg->setPointer("reply", reply); sp<AMessage> response; @@ -1443,7 +1462,7 @@ status_t NuPlayer::getTrackInfo(Parcel* reply) const { } status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const { - sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, id()); + sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, this); msg->setPointer("reply", reply); msg->setInt32("type", type); @@ -1456,7 +1475,7 @@ status_t NuPlayer::getSelectedTrack(int32_t type, Parcel* reply) const { } status_t NuPlayer::selectTrack(size_t trackIndex, bool select, int64_t timeUs) { - sp<AMessage> msg = new AMessage(kWhatSelectTrack, id()); + sp<AMessage> msg = new AMessage(kWhatSelectTrack, this); msg->setSize("trackIndex", trackIndex); msg->setInt32("select", select); msg->setInt64("timeUs", timeUs); @@ -1499,7 +1518,7 @@ sp<MetaData> NuPlayer::getFileMeta() { } void NuPlayer::schedulePollDuration() { - sp<AMessage> msg = new AMessage(kWhatPollDuration, id()); + sp<AMessage> msg = new AMessage(kWhatPollDuration, this); msg->setInt32("generation", mPollDurationGeneration); msg->post(); } diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h index 30ede1a..2bc20d7 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h @@ -51,6 +51,7 @@ struct NuPlayer : public AHandler { const sp<IGraphicBufferProducer> &bufferProducer); void setAudioSink(const sp<MediaPlayerBase::AudioSink> &sink); + void setPlaybackRate(float rate); void start(); void pause(); @@ -104,6 +105,7 @@ private: kWhatSetVideoNativeWindow = '=NaW', kWhatSetAudioSink = '=AuS', kWhatMoreDataQueued = 'more', + kWhatSetRate = 'setR', kWhatStart = 'strt', kWhatScanSources = 'scan', kWhatVideoNotify = 'vidN', @@ -175,6 +177,7 @@ private: int32_t mVideoScalingMode; + float mPlaybackRate; bool mStarted; // Actual pause state, either as requested by client or due to buffering. diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.cpp index 9229704..cf3e8ad 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.cpp @@ -19,6 +19,7 @@ #include <utils/Log.h> #include <inttypes.h> +#include "avc_utils.h" #include "NuPlayerCCDecoder.h" #include <media/stagefright/foundation/ABitReader.h> @@ -185,17 +186,38 @@ int32_t NuPlayer::CCDecoder::getTrackIndex(size_t channel) const { // returns true if a new CC track is found bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) { - int64_t timeUs; - CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); - sp<ABuffer> sei; if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) { return false; } + int64_t timeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + bool trackAdded = false; - NALBitReader br(sei->data() + 1, sei->size() - 1); + const NALPosition *nal = (NALPosition *) sei->data(); + + for (size_t i = 0; i < sei->size() / sizeof(NALPosition); ++i, ++nal) { + trackAdded |= parseSEINalUnit( + timeUs, accessUnit->data() + nal->nalOffset, nal->nalSize); + } + + return trackAdded; +} + +// returns true if a new CC track is found +bool NuPlayer::CCDecoder::parseSEINalUnit( + int64_t timeUs, const uint8_t *nalStart, size_t nalSize) { + unsigned nalType = nalStart[0] & 0x1f; + + // the buffer should only have SEI in it + if (nalType != 6) { + return false; + } + + bool trackAdded = false; + NALBitReader br(nalStart + 1, nalSize - 1); // sei_message() while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message() uint32_t payload_type = 0; @@ -214,20 +236,25 @@ bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) { // sei_payload() if (payload_type == 4) { - // user_data_registered_itu_t_t35() - - // ATSC A/72: 6.4.2 - uint8_t itu_t_t35_country_code = br.getBits(8); - uint16_t itu_t_t35_provider_code = br.getBits(16); - uint32_t user_identifier = br.getBits(32); - uint8_t user_data_type_code = br.getBits(8); - - payload_size -= 1 + 2 + 4 + 1; + bool isCC = false; + if (payload_size > 1 + 2 + 4 + 1) { + // user_data_registered_itu_t_t35() + + // ATSC A/72: 6.4.2 + uint8_t itu_t_t35_country_code = br.getBits(8); + uint16_t itu_t_t35_provider_code = br.getBits(16); + uint32_t user_identifier = br.getBits(32); + uint8_t user_data_type_code = br.getBits(8); + + payload_size -= 1 + 2 + 4 + 1; + + isCC = itu_t_t35_country_code == 0xB5 + && itu_t_t35_provider_code == 0x0031 + && user_identifier == 'GA94' + && user_data_type_code == 0x3; + } - if (itu_t_t35_country_code == 0xB5 - && itu_t_t35_provider_code == 0x0031 - && user_identifier == 'GA94' - && user_data_type_code == 0x3) { + if (isCC && payload_size > 2) { // MPEG_cc_data() // ATSC A/53 Part 4: 6.2.3.1 br.skipBits(1); //process_em_data_flag @@ -243,7 +270,7 @@ bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) { sp<ABuffer> ccBuf = new ABuffer(cc_count * sizeof(CCData)); ccBuf->setRange(0, 0); - for (size_t i = 0; i < cc_count; i++) { + for (size_t i = 0; i < cc_count && payload_size >= 3; i++) { uint8_t marker = br.getBits(5); CHECK_EQ(marker, 0x1f); @@ -253,6 +280,8 @@ bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) { uint8_t cc_data_1 = br.getBits(8) & 0x7f; uint8_t cc_data_2 = br.getBits(8) & 0x7f; + payload_size -= 3; + if (cc_valid && (cc_type == 0 || cc_type == 1)) { CCData cc(cc_type, cc_data_1, cc_data_2); @@ -269,7 +298,6 @@ bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) { } } } - payload_size -= cc_count * 3; mCCMap.add(timeUs, ccBuf); break; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.h b/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.h index 5e06f4e..77fb0fe 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerCCDecoder.h @@ -49,6 +49,7 @@ private: bool isTrackValid(size_t index) const; int32_t getTrackIndex(size_t channel) const; bool extractFromSEI(const sp<ABuffer> &accessUnit); + bool parseSEINalUnit(int64_t timeUs, const uint8_t *nalStart, size_t nalSize); sp<ABuffer> filterCCBuf(const sp<ABuffer> &ccBuf, size_t index); DISALLOW_EVIL_CONSTRUCTORS(CCDecoder); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp index 5d98d98..04ac699 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp @@ -56,6 +56,7 @@ NuPlayer::Decoder::Decoder( mIsVideoAVC(false), mIsSecure(false), mFormatChangePending(false), + mTimeChangePending(false), mPaused(true), mResumePending(false), mComponentName("decoder") { @@ -121,6 +122,7 @@ void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) { CHECK(mCodec == NULL); mFormatChangePending = false; + mTimeChangePending = false; ++mBufferGeneration; @@ -235,7 +237,7 @@ void NuPlayer::Decoder::onResume(bool notifyComplete) { } } -void NuPlayer::Decoder::onFlush(bool notifyComplete) { +void NuPlayer::Decoder::doFlush(bool notifyComplete) { if (mCCDecoder != NULL) { mCCDecoder->flush(); } @@ -259,13 +261,22 @@ void NuPlayer::Decoder::onFlush(bool notifyComplete) { // we attempt to release the buffers even if flush fails. } releaseAndResetMediaBuffers(); +} - if (notifyComplete) { - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatFlushCompleted); - notify->post(); - mPaused = true; +void NuPlayer::Decoder::onFlush() { + doFlush(true); + + if (isDiscontinuityPending()) { + // This could happen if the client starts seeking/shutdown + // after we queued an EOS for discontinuities. + // We can consider discontinuity handled. + finishHandleDiscontinuity(false /* flushOnTimeChange */); } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatFlushCompleted); + notify->post(); + mPaused = true; } void NuPlayer::Decoder::onShutdown(bool notifyComplete) { @@ -309,16 +320,17 @@ void NuPlayer::Decoder::onShutdown(bool notifyComplete) { } void NuPlayer::Decoder::doRequestBuffers() { - if (mFormatChangePending) { + if (isDiscontinuityPending()) { return; } status_t err = OK; - while (!mDequeuedInputBuffers.empty()) { + while (err == OK && !mDequeuedInputBuffers.empty()) { size_t bufferIx = *mDequeuedInputBuffers.begin(); sp<AMessage> msg = new AMessage(); msg->setSize("buffer-ix", bufferIx); err = fetchInputData(msg); - if (err != OK) { + if (err != OK && err != ERROR_END_OF_STREAM) { + // if EOS, need to queue EOS buffer break; } mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin()); @@ -336,7 +348,7 @@ void NuPlayer::Decoder::doRequestBuffers() { } bool NuPlayer::Decoder::handleAnInputBuffer() { - if (mFormatChangePending) { + if (isDiscontinuityPending()) { return false; } size_t bufferIx = -1; @@ -391,9 +403,6 @@ bool NuPlayer::Decoder::handleAnInputBuffer() { } bool NuPlayer::Decoder::handleAnOutputBuffer() { - if (mFormatChangePending) { - return false; - } size_t bufferIx = -1; size_t offset; size_t size; @@ -474,17 +483,20 @@ bool NuPlayer::Decoder::handleAnOutputBuffer() { buffer->setRange(offset, size); buffer->meta()->clear(); buffer->meta()->setInt64("timeUs", timeUs); - if (flags & MediaCodec::BUFFER_FLAG_EOS) { - buffer->meta()->setInt32("eos", true); - notifyResumeCompleteIfNecessary(); - } + + bool eos = flags & MediaCodec::BUFFER_FLAG_EOS; // we do not expect CODECCONFIG or SYNCFRAME for decoder - sp<AMessage> reply = new AMessage(kWhatRenderBuffer, id()); + sp<AMessage> reply = new AMessage(kWhatRenderBuffer, this); reply->setSize("buffer-ix", bufferIx); reply->setInt32("generation", mBufferGeneration); - if (mSkipRenderingUntilMediaTimeUs >= 0) { + if (eos) { + ALOGI("[%s] saw output EOS", mIsAudio ? "audio" : "video"); + + buffer->meta()->setInt32("eos", true); + reply->setInt32("eos", true); + } else if (mSkipRenderingUntilMediaTimeUs >= 0) { if (timeUs < mSkipRenderingUntilMediaTimeUs) { ALOGV("[%s] dropping buffer at time %lld as requested.", mComponentName.c_str(), (long long)timeUs); @@ -502,7 +514,7 @@ bool NuPlayer::Decoder::handleAnOutputBuffer() { if (mRenderer != NULL) { // send the buffer to renderer. mRenderer->queueBuffer(mIsAudio, buffer, reply); - if (flags & MediaCodec::BUFFER_FLAG_EOS) { + if (eos && !isDiscontinuityPending()) { mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM); } } @@ -533,11 +545,8 @@ void NuPlayer::Decoder::releaseAndResetMediaBuffers() { } void NuPlayer::Decoder::requestCodecNotification() { - if (mFormatChangePending) { - return; - } if (mCodec != NULL) { - sp<AMessage> reply = new AMessage(kWhatCodecNotify, id()); + sp<AMessage> reply = new AMessage(kWhatCodecNotify, this); reply->setInt32("generation", mBufferGeneration); mCodec->requestActivityNotification(reply); } @@ -582,39 +591,31 @@ status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) { formatChange = !seamlessFormatChange; } - if (formatChange || timeChange) { - sp<AMessage> msg = mNotify->dup(); - msg->setInt32("what", kWhatInputDiscontinuity); - msg->setInt32("formatChange", formatChange); - msg->post(); - } - + // For format or time change, return EOS to queue EOS input, + // then wait for EOS on output. if (formatChange /* not seamless */) { - // must change decoder - // return EOS and wait to be killed mFormatChangePending = true; - return ERROR_END_OF_STREAM; + err = ERROR_END_OF_STREAM; } else if (timeChange) { - // need to flush - // TODO: Ideally we shouldn't need a flush upon time - // discontinuity, flushing will cause loss of frames. - // We probably should queue a time change marker to the - // output queue, and handles it in renderer instead. rememberCodecSpecificData(newFormat); - onFlush(false /* notifyComplete */); - err = OK; + mTimeChangePending = true; + err = ERROR_END_OF_STREAM; } else if (seamlessFormatChange) { // reuse existing decoder and don't flush rememberCodecSpecificData(newFormat); - err = OK; + continue; } else { // This stream is unaffected by the discontinuity return -EWOULDBLOCK; } } + // reply should only be returned without a buffer set + // when there is an error (including EOS) + CHECK(err != OK); + reply->setInt32("err", err); - return OK; + return ERROR_END_OF_STREAM; } if (!mIsAudio) { @@ -636,7 +637,7 @@ status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) { #if 0 int64_t mediaTimeUs; CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs)); - ALOGV("feeding %s input buffer at media time %.2f secs", + ALOGV("[%s] feeding input buffer at media time %" PRId64, mIsAudio ? "audio" : "video", mediaTimeUs / 1E6); #endif @@ -696,10 +697,7 @@ bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) { int32_t streamErr = ERROR_END_OF_STREAM; CHECK(msg->findInt32("err", &streamErr) || !hasBuffer); - if (streamErr == OK) { - /* buffers are returned to hold on to */ - return true; - } + CHECK(streamErr != OK); // attempt to queue EOS status_t err = mCodec->queueInputBuffer( @@ -781,6 +779,7 @@ void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) { status_t err; int32_t render; size_t bufferIx; + int32_t eos; CHECK(msg->findSize("buffer-ix", &bufferIx)); if (!mIsAudio) { @@ -805,6 +804,42 @@ void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) { mComponentName.c_str(), err); handleError(err); } + if (msg->findInt32("eos", &eos) && eos + && isDiscontinuityPending()) { + finishHandleDiscontinuity(true /* flushOnTimeChange */); + } +} + +bool NuPlayer::Decoder::isDiscontinuityPending() const { + return mFormatChangePending || mTimeChangePending; +} + +void NuPlayer::Decoder::finishHandleDiscontinuity(bool flushOnTimeChange) { + ALOGV("finishHandleDiscontinuity: format %d, time %d, flush %d", + mFormatChangePending, mTimeChangePending, flushOnTimeChange); + + // If we have format change, pause and wait to be killed; + // If we have time change only, flush and restart fetching. + + if (mFormatChangePending) { + mPaused = true; + } else if (mTimeChangePending) { + if (flushOnTimeChange) { + doFlush(false /*notifyComplete*/); + } + + // restart fetching input + scheduleRequestBuffers(); + } + + // Notify NuPlayer to either shutdown decoder, or rescan sources + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("what", kWhatInputDiscontinuity); + msg->setInt32("formatChange", mFormatChangePending); + msg->post(); + + mFormatChangePending = false; + mTimeChangePending = false; } bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange( diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h index 1bfa94f..4aab2c6 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h @@ -43,7 +43,7 @@ protected: virtual void onSetRenderer(const sp<Renderer> &renderer); virtual void onGetInputBuffers(Vector<sp<ABuffer> > *dstBuffers); virtual void onResume(bool notifyComplete); - virtual void onFlush(bool notifyComplete); + virtual void onFlush(); virtual void onShutdown(bool notifyComplete); virtual void doRequestBuffers(); @@ -81,6 +81,7 @@ private: bool mIsVideoAVC; bool mIsSecure; bool mFormatChangePending; + bool mTimeChangePending; bool mPaused; bool mResumePending; @@ -93,6 +94,7 @@ private: void requestCodecNotification(); bool isStaleReply(const sp<AMessage> &msg); + void doFlush(bool notifyComplete); status_t fetchInputData(sp<AMessage> &reply); bool onInputBufferFetched(const sp<AMessage> &msg); void onRenderBuffer(const sp<AMessage> &msg); @@ -100,6 +102,8 @@ private: bool supportsSeamlessFormatChange(const sp<AMessage> &to) const; bool supportsSeamlessAudioFormatChange(const sp<AMessage> &targetFormat) const; void rememberCodecSpecificData(const sp<AMessage> &format); + bool isDiscontinuityPending() const; + void finishHandleDiscontinuity(bool flushOnTimeChange); void notifyResumeCompleteIfNecessary(); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.cpp index d56fc4d..4636f0a 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.cpp @@ -61,7 +61,7 @@ status_t PostAndAwaitResponse( } void NuPlayer::DecoderBase::configure(const sp<AMessage> &format) { - sp<AMessage> msg = new AMessage(kWhatConfigure, id()); + sp<AMessage> msg = new AMessage(kWhatConfigure, this); msg->setMessage("format", format); msg->post(); } @@ -71,13 +71,13 @@ void NuPlayer::DecoderBase::init() { } void NuPlayer::DecoderBase::setRenderer(const sp<Renderer> &renderer) { - sp<AMessage> msg = new AMessage(kWhatSetRenderer, id()); + sp<AMessage> msg = new AMessage(kWhatSetRenderer, this); msg->setObject("renderer", renderer); msg->post(); } status_t NuPlayer::DecoderBase::getInputBuffers(Vector<sp<ABuffer> > *buffers) const { - sp<AMessage> msg = new AMessage(kWhatGetInputBuffers, id()); + sp<AMessage> msg = new AMessage(kWhatGetInputBuffers, this); msg->setPointer("buffers", buffers); sp<AMessage> response; @@ -85,17 +85,17 @@ status_t NuPlayer::DecoderBase::getInputBuffers(Vector<sp<ABuffer> > *buffers) c } void NuPlayer::DecoderBase::signalFlush() { - (new AMessage(kWhatFlush, id()))->post(); + (new AMessage(kWhatFlush, this))->post(); } void NuPlayer::DecoderBase::signalResume(bool notifyComplete) { - sp<AMessage> msg = new AMessage(kWhatResume, id()); + sp<AMessage> msg = new AMessage(kWhatResume, this); msg->setInt32("notifyComplete", notifyComplete); msg->post(); } void NuPlayer::DecoderBase::initiateShutdown() { - (new AMessage(kWhatShutdown, id()))->post(); + (new AMessage(kWhatShutdown, this))->post(); } void NuPlayer::DecoderBase::onRequestInputBuffers() { @@ -111,7 +111,7 @@ void NuPlayer::DecoderBase::scheduleRequestBuffers() { return; } mRequestInputBuffersPending = true; - sp<AMessage> msg = new AMessage(kWhatRequestInputBuffers, id()); + sp<AMessage> msg = new AMessage(kWhatRequestInputBuffers, this); msg->post(10 * 1000ll); } @@ -136,7 +136,7 @@ void NuPlayer::DecoderBase::onMessageReceived(const sp<AMessage> &msg) { case kWhatGetInputBuffers: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); Vector<sp<ABuffer> > *dstBuffers; @@ -157,7 +157,7 @@ void NuPlayer::DecoderBase::onMessageReceived(const sp<AMessage> &msg) { case kWhatFlush: { - onFlush(true); + onFlush(); break; } diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.h index 6732ff4..97e9269 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderBase.h @@ -65,7 +65,7 @@ protected: virtual void onSetRenderer(const sp<Renderer> &renderer) = 0; virtual void onGetInputBuffers(Vector<sp<ABuffer> > *dstBuffers) = 0; virtual void onResume(bool notifyComplete) = 0; - virtual void onFlush(bool notifyComplete) = 0; + virtual void onFlush() = 0; virtual void onShutdown(bool notifyComplete) = 0; void onRequestInputBuffers(); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp index 9f7f09a..29b4c26 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp @@ -247,7 +247,7 @@ status_t NuPlayer::DecoderPassThrough::fetchInputData(sp<AMessage> &reply) { } if (timeChange) { - onFlush(false /* notifyComplete */); + doFlush(false /* notifyComplete */); err = OK; } else if (formatChange) { // do seamless format change @@ -333,7 +333,7 @@ void NuPlayer::DecoderPassThrough::onInputBufferFetched( return; } - sp<AMessage> reply = new AMessage(kWhatBufferConsumed, id()); + sp<AMessage> reply = new AMessage(kWhatBufferConsumed, this); reply->setInt32("generation", mBufferGeneration); reply->setInt32("size", bufferSize); @@ -364,7 +364,7 @@ void NuPlayer::DecoderPassThrough::onResume(bool notifyComplete) { } } -void NuPlayer::DecoderPassThrough::onFlush(bool notifyComplete) { +void NuPlayer::DecoderPassThrough::doFlush(bool notifyComplete) { ++mBufferGeneration; mSkipRenderingUntilMediaTimeUs = -1; mPendingAudioAccessUnit.clear(); @@ -376,18 +376,21 @@ void NuPlayer::DecoderPassThrough::onFlush(bool notifyComplete) { mRenderer->signalTimeDiscontinuity(); } - if (notifyComplete) { - mPaused = true; - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatFlushCompleted); - notify->post(); - } - mPendingBuffersToDrain = 0; mCachedBytes = 0; mReachedEOS = false; } +void NuPlayer::DecoderPassThrough::onFlush() { + doFlush(true /* notifyComplete */); + + mPaused = true; + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatFlushCompleted); + notify->post(); + +} + void NuPlayer::DecoderPassThrough::onShutdown(bool notifyComplete) { ++mBufferGeneration; mSkipRenderingUntilMediaTimeUs = -1; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h index a6e1faf..173cfbd 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h @@ -43,7 +43,7 @@ protected: virtual void onSetRenderer(const sp<Renderer> &renderer); virtual void onGetInputBuffers(Vector<sp<ABuffer> > *dstBuffers); virtual void onResume(bool notifyComplete); - virtual void onFlush(bool notifyComplete); + virtual void onFlush(); virtual void onShutdown(bool notifyComplete); virtual void doRequestBuffers(); @@ -77,6 +77,7 @@ private: status_t dequeueAccessUnit(sp<ABuffer> *accessUnit); sp<ABuffer> aggregateBuffer(const sp<ABuffer> &accessUnit); status_t fetchInputData(sp<AMessage> &reply); + void doFlush(bool notifyComplete); void onInputBufferFetched(const sp<AMessage> &msg); void onBufferConsumed(int32_t size); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp index bc79fdb..1fa9cef 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp @@ -341,6 +341,11 @@ bool NuPlayerDriver::isPlaying() { return mState == STATE_RUNNING && !mAtEOS; } +status_t NuPlayerDriver::setPlaybackRate(float rate) { + mPlayer->setPlaybackRate(rate); + return OK; +} + status_t NuPlayerDriver::seekTo(int msec) { ALOGD("seekTo(%p) %d ms", this, msec); Mutex::Autolock autoLock(mLock); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h index 5cba7d9..e53abcd 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h @@ -47,6 +47,7 @@ struct NuPlayerDriver : public MediaPlayerInterface { virtual status_t stop(); virtual status_t pause(); virtual bool isPlaying(); + virtual status_t setPlaybackRate(float rate); virtual status_t seekTo(int msec); virtual status_t getCurrentPosition(int *msec); virtual status_t getDuration(int *msec); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp index 25225a8..a2ec51c 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp @@ -25,6 +25,7 @@ #include <media/stagefright/foundation/AMessage.h> #include <media/stagefright/foundation/AUtils.h> #include <media/stagefright/foundation/AWakeLock.h> +#include <media/stagefright/MediaClock.h> #include <media/stagefright/MediaErrors.h> #include <media/stagefright/MetaData.h> #include <media/stagefright/Utils.h> @@ -63,22 +64,19 @@ NuPlayer::Renderer::Renderer( mDrainVideoQueuePending(false), mAudioQueueGeneration(0), mVideoQueueGeneration(0), + mAudioDrainGeneration(0), + mVideoDrainGeneration(0), + mPlaybackRate(1.0), mAudioFirstAnchorTimeMediaUs(-1), mAnchorTimeMediaUs(-1), - mAnchorTimeRealUs(-1), mAnchorNumFramesWritten(-1), - mAnchorMaxMediaUs(-1), mVideoLateByUs(0ll), mHasAudio(false), mHasVideo(false), - mPauseStartedTimeRealUs(-1), - mFlushingAudio(false), - mFlushingVideo(false), mNotifyCompleteAudio(false), mNotifyCompleteVideo(false), mSyncQueues(false), mPaused(false), - mPausePositionMediaTimeUs(-1), mVideoSampleReceived(false), mVideoRenderingStarted(false), mVideoRenderingStartGeneration(0), @@ -90,7 +88,7 @@ NuPlayer::Renderer::Renderer( mTotalBuffersQueued(0), mLastAudioBufferDrained(0), mWakeLock(new AWakeLock()) { - + mMediaClock = new MediaClock; } NuPlayer::Renderer::~Renderer() { @@ -105,7 +103,8 @@ void NuPlayer::Renderer::queueBuffer( bool audio, const sp<ABuffer> &buffer, const sp<AMessage> ¬ifyConsumed) { - sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id()); + sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this); + msg->setInt32("queueGeneration", getQueueGeneration(audio)); msg->setInt32("audio", static_cast<int32_t>(audio)); msg->setBuffer("buffer", buffer); msg->setMessage("notifyConsumed", notifyConsumed); @@ -115,199 +114,108 @@ void NuPlayer::Renderer::queueBuffer( void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) { CHECK_NE(finalResult, (status_t)OK); - sp<AMessage> msg = new AMessage(kWhatQueueEOS, id()); + sp<AMessage> msg = new AMessage(kWhatQueueEOS, this); + msg->setInt32("queueGeneration", getQueueGeneration(audio)); msg->setInt32("audio", static_cast<int32_t>(audio)); msg->setInt32("finalResult", finalResult); msg->post(); } +void NuPlayer::Renderer::setPlaybackRate(float rate) { + sp<AMessage> msg = new AMessage(kWhatSetRate, this); + msg->setFloat("rate", rate); + msg->post(); +} + void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) { { - Mutex::Autolock autoLock(mFlushLock); + Mutex::Autolock autoLock(mLock); if (audio) { mNotifyCompleteAudio |= notifyComplete; - if (mFlushingAudio) { - return; - } - mFlushingAudio = true; + ++mAudioQueueGeneration; + ++mAudioDrainGeneration; } else { mNotifyCompleteVideo |= notifyComplete; - if (mFlushingVideo) { - return; - } - mFlushingVideo = true; + ++mVideoQueueGeneration; + ++mVideoDrainGeneration; } + + clearAnchorTime_l(); + clearAudioFirstAnchorTime_l(); + mVideoLateByUs = 0; + mSyncQueues = false; } - sp<AMessage> msg = new AMessage(kWhatFlush, id()); + sp<AMessage> msg = new AMessage(kWhatFlush, this); msg->setInt32("audio", static_cast<int32_t>(audio)); msg->post(); } void NuPlayer::Renderer::signalTimeDiscontinuity() { - Mutex::Autolock autoLock(mLock); - // CHECK(mAudioQueue.empty()); - // CHECK(mVideoQueue.empty()); - setAudioFirstAnchorTime(-1); - setAnchorTime(-1, -1); - setVideoLateByUs(0); - mSyncQueues = false; -} - -void NuPlayer::Renderer::signalAudioSinkChanged() { - (new AMessage(kWhatAudioSinkChanged, id()))->post(); } void NuPlayer::Renderer::signalDisableOffloadAudio() { - (new AMessage(kWhatDisableOffloadAudio, id()))->post(); + (new AMessage(kWhatDisableOffloadAudio, this))->post(); } void NuPlayer::Renderer::signalEnableOffloadAudio() { - (new AMessage(kWhatEnableOffloadAudio, id()))->post(); + (new AMessage(kWhatEnableOffloadAudio, this))->post(); } void NuPlayer::Renderer::pause() { - (new AMessage(kWhatPause, id()))->post(); + (new AMessage(kWhatPause, this))->post(); } void NuPlayer::Renderer::resume() { - (new AMessage(kWhatResume, id()))->post(); + (new AMessage(kWhatResume, this))->post(); } void NuPlayer::Renderer::setVideoFrameRate(float fps) { - sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, id()); + sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this); msg->setFloat("frame-rate", fps); msg->post(); } -// Called on any threads, except renderer's thread. -status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) { - { - Mutex::Autolock autoLock(mLock); - int64_t currentPositionUs; - if (getCurrentPositionIfPaused_l(¤tPositionUs)) { - *mediaUs = currentPositionUs; - return OK; - } - } - return getCurrentPositionFromAnchor(mediaUs, ALooper::GetNowUs()); -} - -// Called on only renderer's thread. -status_t NuPlayer::Renderer::getCurrentPositionOnLooper(int64_t *mediaUs) { - return getCurrentPositionOnLooper(mediaUs, ALooper::GetNowUs()); -} - -// Called on only renderer's thread. -// Since mPaused and mPausePositionMediaTimeUs are changed only on renderer's -// thread, no need to acquire mLock. -status_t NuPlayer::Renderer::getCurrentPositionOnLooper( - int64_t *mediaUs, int64_t nowUs, bool allowPastQueuedVideo) { - int64_t currentPositionUs; - if (getCurrentPositionIfPaused_l(¤tPositionUs)) { - *mediaUs = currentPositionUs; - return OK; - } - return getCurrentPositionFromAnchor(mediaUs, nowUs, allowPastQueuedVideo); -} - -// Called either with mLock acquired or on renderer's thread. -bool NuPlayer::Renderer::getCurrentPositionIfPaused_l(int64_t *mediaUs) { - if (!mPaused || mPausePositionMediaTimeUs < 0ll) { - return false; - } - *mediaUs = mPausePositionMediaTimeUs; - return true; -} - // Called on any threads. -status_t NuPlayer::Renderer::getCurrentPositionFromAnchor( - int64_t *mediaUs, int64_t nowUs, bool allowPastQueuedVideo) { - Mutex::Autolock autoLock(mTimeLock); - if (!mHasAudio && !mHasVideo) { - return NO_INIT; - } - - if (mAnchorTimeMediaUs < 0) { - return NO_INIT; - } - - int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs; - - if (mPauseStartedTimeRealUs != -1) { - positionUs -= (nowUs - mPauseStartedTimeRealUs); - } - - // limit position to the last queued media time (for video only stream - // position will be discrete as we don't know how long each frame lasts) - if (mAnchorMaxMediaUs >= 0 && !allowPastQueuedVideo) { - if (positionUs > mAnchorMaxMediaUs) { - positionUs = mAnchorMaxMediaUs; - } - } - - if (positionUs < mAudioFirstAnchorTimeMediaUs) { - positionUs = mAudioFirstAnchorTimeMediaUs; - } - - *mediaUs = (positionUs <= 0) ? 0 : positionUs; - return OK; -} - -void NuPlayer::Renderer::setHasMedia(bool audio) { - Mutex::Autolock autoLock(mTimeLock); - if (audio) { - mHasAudio = true; - } else { - mHasVideo = true; - } +status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) { + return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs); } -void NuPlayer::Renderer::setAudioFirstAnchorTime(int64_t mediaUs) { - Mutex::Autolock autoLock(mTimeLock); - mAudioFirstAnchorTimeMediaUs = mediaUs; +void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() { + mAudioFirstAnchorTimeMediaUs = -1; + mMediaClock->setStartingTimeMedia(-1); } -void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded(int64_t mediaUs) { - Mutex::Autolock autoLock(mTimeLock); +void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) { if (mAudioFirstAnchorTimeMediaUs == -1) { mAudioFirstAnchorTimeMediaUs = mediaUs; + mMediaClock->setStartingTimeMedia(mediaUs); } } -void NuPlayer::Renderer::setAnchorTime( - int64_t mediaUs, int64_t realUs, int64_t numFramesWritten, bool resume) { - Mutex::Autolock autoLock(mTimeLock); - mAnchorTimeMediaUs = mediaUs; - mAnchorTimeRealUs = realUs; - mAnchorNumFramesWritten = numFramesWritten; - if (resume) { - mPauseStartedTimeRealUs = -1; - } +void NuPlayer::Renderer::clearAnchorTime_l() { + mMediaClock->clearAnchor(); + mAnchorTimeMediaUs = -1; + mAnchorNumFramesWritten = -1; } void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) { - Mutex::Autolock autoLock(mTimeLock); + Mutex::Autolock autoLock(mLock); mVideoLateByUs = lateUs; } int64_t NuPlayer::Renderer::getVideoLateByUs() { - Mutex::Autolock autoLock(mTimeLock); + Mutex::Autolock autoLock(mLock); return mVideoLateByUs; } -void NuPlayer::Renderer::setPauseStartedTimeRealUs(int64_t realUs) { - Mutex::Autolock autoLock(mTimeLock); - mPauseStartedTimeRealUs = realUs; -} - status_t NuPlayer::Renderer::openAudioSink( const sp<AMessage> &format, bool offloadOnly, bool hasVideo, uint32_t flags, bool *isOffloaded) { - sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, id()); + sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this); msg->setMessage("format", format); msg->setInt32("offload-only", offloadOnly); msg->setInt32("has-video", hasVideo); @@ -328,7 +236,7 @@ status_t NuPlayer::Renderer::openAudioSink( } void NuPlayer::Renderer::closeAudioSink() { - sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, id()); + sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this); sp<AMessage> response; msg->postAndAwaitResponse(&response); @@ -356,7 +264,7 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { response->setInt32("err", err); response->setInt32("offload", offloadingAudio()); - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); response->postReply(replyID); @@ -365,7 +273,7 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { case kWhatCloseAudioSink: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); onCloseAudioSink(); @@ -384,8 +292,8 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { case kWhatDrainAudioQueue: { int32_t generation; - CHECK(msg->findInt32("generation", &generation)); - if (generation != mAudioQueueGeneration) { + CHECK(msg->findInt32("drainGeneration", &generation)); + if (generation != getDrainGeneration(true /* audio */)) { break; } @@ -407,9 +315,7 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { // Let's give it more data after about half that time // has elapsed. - // kWhatDrainAudioQueue is used for non-offloading mode, - // and mLock is used only for offloading mode. Therefore, - // no need to acquire mLock here. + Mutex::Autolock autoLock(mLock); postDrainAudioQueue_l(delayUs / 2); } break; @@ -418,8 +324,8 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { case kWhatDrainVideoQueue: { int32_t generation; - CHECK(msg->findInt32("generation", &generation)); - if (generation != mVideoQueueGeneration) { + CHECK(msg->findInt32("drainGeneration", &generation)); + if (generation != getDrainGeneration(false /* audio */)) { break; } @@ -427,22 +333,20 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { onDrainVideoQueue(); - Mutex::Autolock autoLock(mLock); - postDrainVideoQueue_l(); + postDrainVideoQueue(); break; } case kWhatPostDrainVideoQueue: { int32_t generation; - CHECK(msg->findInt32("generation", &generation)); - if (generation != mVideoQueueGeneration) { + CHECK(msg->findInt32("drainGeneration", &generation)); + if (generation != getDrainGeneration(false /* audio */)) { break; } mDrainVideoQueuePending = false; - Mutex::Autolock autoLock(mLock); - postDrainVideoQueue_l(); + postDrainVideoQueue(); break; } @@ -458,15 +362,19 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { break; } - case kWhatFlush: + case kWhatSetRate: { - onFlush(msg); + CHECK(msg->findFloat("rate", &mPlaybackRate)); + int32_t ratePermille = (int32_t)(0.5f + 1000 * mPlaybackRate); + mPlaybackRate = ratePermille / 1000.0f; + mMediaClock->setPlaybackRate(mPlaybackRate); + mAudioSink->setPlaybackRatePermille(ratePermille); break; } - case kWhatAudioSinkChanged: + case kWhatFlush: { - onAudioSinkChanged(); + onFlush(msg); break; } @@ -511,7 +419,7 @@ void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) { case kWhatAudioOffloadPauseTimeout: { int32_t generation; - CHECK(msg->findInt32("generation", &generation)); + CHECK(msg->findInt32("drainGeneration", &generation)); if (generation != mAudioOffloadPauseTimeoutGeneration) { break; } @@ -538,19 +446,19 @@ void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) { } mDrainAudioQueuePending = true; - sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id()); - msg->setInt32("generation", mAudioQueueGeneration); + sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this); + msg->setInt32("drainGeneration", mAudioDrainGeneration); msg->post(delayUs); } -void NuPlayer::Renderer::prepareForMediaRenderingStart() { - mAudioRenderingStartGeneration = mAudioQueueGeneration; - mVideoRenderingStartGeneration = mVideoQueueGeneration; +void NuPlayer::Renderer::prepareForMediaRenderingStart_l() { + mAudioRenderingStartGeneration = mAudioDrainGeneration; + mVideoRenderingStartGeneration = mVideoDrainGeneration; } -void NuPlayer::Renderer::notifyIfMediaRenderingStarted() { - if (mVideoRenderingStartGeneration == mVideoQueueGeneration && - mAudioRenderingStartGeneration == mAudioQueueGeneration) { +void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() { + if (mVideoRenderingStartGeneration == mVideoDrainGeneration && + mAudioRenderingStartGeneration == mAudioDrainGeneration) { mVideoRenderingStartGeneration = -1; mAudioRenderingStartGeneration = -1; @@ -618,7 +526,7 @@ size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) { int64_t mediaTimeUs; CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6); - setAudioFirstAnchorTimeIfNeeded(mediaTimeUs); + setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs); } size_t copy = entry->mBuffer->size() - entry->mOffset; @@ -638,34 +546,45 @@ size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) { entry = NULL; } sizeCopied += copy; - notifyIfMediaRenderingStarted(); + + notifyIfMediaRenderingStarted_l(); } if (mAudioFirstAnchorTimeMediaUs >= 0) { int64_t nowUs = ALooper::GetNowUs(); - setAnchorTime(mAudioFirstAnchorTimeMediaUs, nowUs - getPlayedOutAudioDurationUs(nowUs)); + int64_t nowMediaUs = + mAudioFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs); + // we don't know how much data we are queueing for offloaded tracks. + mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX); } - // we don't know how much data we are queueing for offloaded tracks - mAnchorMaxMediaUs = -1; - if (hasEOS) { - (new AMessage(kWhatStopAudioSink, id()))->post(); + (new AMessage(kWhatStopAudioSink, this))->post(); } return sizeCopied; } bool NuPlayer::Renderer::onDrainAudioQueue() { + // TODO: This call to getPosition checks if AudioTrack has been created + // in AudioSink before draining audio. If AudioTrack doesn't exist, then + // CHECKs on getPosition will fail. + // We still need to figure out why AudioTrack is not created when + // this function is called. One possible reason could be leftover + // audio. Another possible place is to check whether decoder + // has received INFO_FORMAT_CHANGED as the first buffer since + // AudioSink is opened there, and possible interactions with flush + // immediately after start. Investigate error message + // "vorbis_dsp_synthesis returned -135", along with RTSP. uint32_t numFramesPlayed; if (mAudioSink->getPosition(&numFramesPlayed) != OK) { return false; } +#if 0 ssize_t numFramesAvailableToWrite = mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed); -#if 0 if (numFramesAvailableToWrite == mAudioSink->frameCount()) { ALOGI("audio sink underrun"); } else { @@ -674,10 +593,7 @@ bool NuPlayer::Renderer::onDrainAudioQueue() { } #endif - size_t numBytesAvailableToWrite = - numFramesAvailableToWrite * mAudioSink->frameSize(); - - while (numBytesAvailableToWrite > 0 && !mAudioQueue.empty()) { + while (!mAudioQueue.empty()) { QueueEntry *entry = &*mAudioQueue.begin(); mLastAudioBufferDrained = entry->mBufferOrdinal; @@ -710,14 +626,16 @@ bool NuPlayer::Renderer::onDrainAudioQueue() { } size_t copy = entry->mBuffer->size() - entry->mOffset; - if (copy > numBytesAvailableToWrite) { - copy = numBytesAvailableToWrite; - } - ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset, copy); + ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset, + copy, false /* blocking */); if (written < 0) { // An error in AudioSink write. Perhaps the AudioSink was not properly opened. - ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy); + if (written == WOULD_BLOCK) { + ALOGW("AudioSink write would block when writing %zu bytes", copy); + } else { + ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy); + } break; } @@ -729,73 +647,92 @@ bool NuPlayer::Renderer::onDrainAudioQueue() { entry = NULL; } - numBytesAvailableToWrite -= written; size_t copiedFrames = written / mAudioSink->frameSize(); mNumFramesWritten += copiedFrames; - notifyIfMediaRenderingStarted(); + { + Mutex::Autolock autoLock(mLock); + notifyIfMediaRenderingStarted_l(); + } if (written != (ssize_t)copy) { // A short count was received from AudioSink::write() // - // AudioSink write should block until exactly the number of bytes are delivered. - // But it may return with a short count (without an error) when: + // AudioSink write is called in non-blocking mode. + // It may return with a short count when: // // 1) Size to be copied is not a multiple of the frame size. We consider this fatal. - // 2) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded. + // 2) The data to be copied exceeds the available buffer in AudioSink. + // 3) An error occurs and data has been partially copied to the buffer in AudioSink. + // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded. // (Case 1) // Must be a multiple of the frame size. If it is not a multiple of a frame size, it // needs to fail, as we should not carry over fractional frames between calls. CHECK_EQ(copy % mAudioSink->frameSize(), 0); - // (Case 2) + // (Case 2, 3, 4) // Return early to the caller. // Beware of calling immediately again as this may busy-loop if you are not careful. - ALOGW("AudioSink write short frame count %zd < %zu", written, copy); + ALOGV("AudioSink write short frame count %zd < %zu", written, copy); break; } } - mAnchorMaxMediaUs = - mAnchorTimeMediaUs + - (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL) - * 1000LL * mAudioSink->msecsPerFrame()); + int64_t maxTimeMedia; + { + Mutex::Autolock autoLock(mLock); + maxTimeMedia = + mAnchorTimeMediaUs + + (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL) + * 1000LL * mAudioSink->msecsPerFrame()); + } + mMediaClock->updateMaxTimeMedia(maxTimeMedia); return !mAudioQueue.empty(); } +int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) { + int32_t sampleRate = offloadingAudio() ? + mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate; + // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours. + return (int64_t)((int32_t)numFrames * 1000000LL / sampleRate); +} + +// Calculate duration of pending samples if played at normal rate (i.e., 1.0). int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) { - int64_t writtenAudioDurationUs = - mNumFramesWritten * 1000LL * mAudioSink->msecsPerFrame(); + int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten); return writtenAudioDurationUs - getPlayedOutAudioDurationUs(nowUs); } int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) { - int64_t currentPositionUs; - if (mPaused || getCurrentPositionOnLooper( - ¤tPositionUs, nowUs, true /* allowPastQueuedVideo */) != OK) { - // If failed to get current position, e.g. due to audio clock is not ready, then just - // play out video immediately without delay. + int64_t realUs; + if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) { + // If failed to get current position, e.g. due to audio clock is + // not ready, then just play out video immediately without delay. return nowUs; } - return (mediaTimeUs - currentPositionUs) + nowUs; + return realUs; } void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) { + Mutex::Autolock autoLock(mLock); // TRICKY: vorbis decoder generates multiple frames with the same // timestamp, so only update on the first frame with a given timestamp if (mediaTimeUs == mAnchorTimeMediaUs) { return; } - setAudioFirstAnchorTimeIfNeeded(mediaTimeUs); + setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs); int64_t nowUs = ALooper::GetNowUs(); - setAnchorTime( - mediaTimeUs, nowUs + getPendingAudioPlayoutDurationUs(nowUs), mNumFramesWritten); + int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs); + mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs); + mAnchorNumFramesWritten = mNumFramesWritten; + mAnchorTimeMediaUs = mediaTimeUs; } -void NuPlayer::Renderer::postDrainVideoQueue_l() { +// Called without mLock acquired. +void NuPlayer::Renderer::postDrainVideoQueue() { if (mDrainVideoQueuePending - || mSyncQueues + || getSyncQueues() || (mPaused && mVideoSampleReceived)) { return; } @@ -806,8 +743,8 @@ void NuPlayer::Renderer::postDrainVideoQueue_l() { QueueEntry &entry = *mVideoQueue.begin(); - sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id()); - msg->setInt32("generation", mVideoQueueGeneration); + sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this); + msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */)); if (entry.mBuffer == NULL) { // EOS doesn't carry a timestamp. @@ -827,16 +764,19 @@ void NuPlayer::Renderer::postDrainVideoQueue_l() { int64_t mediaTimeUs; CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); - if (mAnchorTimeMediaUs < 0) { - setAnchorTime(mediaTimeUs, nowUs); - mPausePositionMediaTimeUs = mediaTimeUs; - mAnchorMaxMediaUs = mediaTimeUs; - realTimeUs = nowUs; - } else { - realTimeUs = getRealTimeUs(mediaTimeUs, nowUs); + { + Mutex::Autolock autoLock(mLock); + if (mAnchorTimeMediaUs < 0) { + mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs); + mAnchorTimeMediaUs = mediaTimeUs; + realTimeUs = nowUs; + } else { + realTimeUs = getRealTimeUs(mediaTimeUs, nowUs); + } } if (!mHasAudio) { - mAnchorMaxMediaUs = mediaTimeUs + 100000; // smooth out videos >= 10fps + // smooth out videos >= 10fps + mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000); } // Heuristics to handle situation when media time changed without a @@ -915,16 +855,19 @@ void NuPlayer::Renderer::onDrainVideoQueue() { ALOGV("video late by %lld us (%.2f secs)", mVideoLateByUs, mVideoLateByUs / 1E6); } else { + int64_t mediaUs = 0; + mMediaClock->getMediaTime(realTimeUs, &mediaUs); ALOGV("rendering video at media time %.2f secs", (mFlags & FLAG_REAL_TIME ? realTimeUs : - (realTimeUs + mAnchorTimeMediaUs - mAnchorTimeRealUs)) / 1E6); + mediaUs) / 1E6); } } else { setVideoLateByUs(0); if (!mVideoSampleReceived && !mHasAudio) { // This will ensure that the first frame after a flush won't be used as anchor // when renderer is in paused state, because resume can happen any time after seek. - setAnchorTime(-1, -1); + Mutex::Autolock autoLock(mLock); + clearAnchorTime_l(); } } @@ -941,7 +884,8 @@ void NuPlayer::Renderer::onDrainVideoQueue() { mVideoRenderingStarted = true; notifyVideoRenderingStart(); } - notifyIfMediaRenderingStarted(); + Mutex::Autolock autoLock(mLock); + notifyIfMediaRenderingStarted_l(); } } @@ -960,14 +904,22 @@ void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t del } void NuPlayer::Renderer::notifyAudioOffloadTearDown() { - (new AMessage(kWhatAudioOffloadTearDown, id()))->post(); + (new AMessage(kWhatAudioOffloadTearDown, this))->post(); } void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) { int32_t audio; CHECK(msg->findInt32("audio", &audio)); - setHasMedia(audio); + if (dropBufferIfStale(audio, msg)) { + return; + } + + if (audio) { + mHasAudio = true; + } else { + mHasVideo = true; + } if (mHasVideo) { if (mVideoScheduler == NULL) { @@ -976,10 +928,6 @@ void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) { } } - if (dropBufferWhileFlushing(audio, msg)) { - return; - } - sp<ABuffer> buffer; CHECK(msg->findBuffer("buffer", &buffer)); @@ -993,15 +941,16 @@ void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) { entry.mFinalResult = OK; entry.mBufferOrdinal = ++mTotalBuffersQueued; - Mutex::Autolock autoLock(mLock); if (audio) { + Mutex::Autolock autoLock(mLock); mAudioQueue.push_back(entry); postDrainAudioQueue_l(); } else { mVideoQueue.push_back(entry); - postDrainVideoQueue_l(); + postDrainVideoQueue(); } + Mutex::Autolock autoLock(mLock); if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) { return; } @@ -1050,7 +999,9 @@ void NuPlayer::Renderer::syncQueuesDone_l() { } if (!mVideoQueue.empty()) { - postDrainVideoQueue_l(); + mLock.unlock(); + postDrainVideoQueue(); + mLock.lock(); } } @@ -1058,7 +1009,7 @@ void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) { int32_t audio; CHECK(msg->findInt32("audio", &audio)); - if (dropBufferWhileFlushing(audio, msg)) { + if (dropBufferIfStale(audio, msg)) { return; } @@ -1069,19 +1020,20 @@ void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) { entry.mOffset = 0; entry.mFinalResult = finalResult; - Mutex::Autolock autoLock(mLock); if (audio) { + Mutex::Autolock autoLock(mLock); if (mAudioQueue.empty() && mSyncQueues) { syncQueuesDone_l(); } mAudioQueue.push_back(entry); postDrainAudioQueue_l(); } else { - if (mVideoQueue.empty() && mSyncQueues) { + if (mVideoQueue.empty() && getSyncQueues()) { + Mutex::Autolock autoLock(mLock); syncQueuesDone_l(); } mVideoQueue.push_back(entry); - postDrainVideoQueue_l(); + postDrainVideoQueue(); } } @@ -1090,31 +1042,25 @@ void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { CHECK(msg->findInt32("audio", &audio)); { - Mutex::Autolock autoLock(mFlushLock); + Mutex::Autolock autoLock(mLock); if (audio) { - mFlushingAudio = false; notifyComplete = mNotifyCompleteAudio; mNotifyCompleteAudio = false; } else { - mFlushingVideo = false; notifyComplete = mNotifyCompleteVideo; mNotifyCompleteVideo = false; } - } - // If we're currently syncing the queues, i.e. dropping audio while - // aligning the first audio/video buffer times and only one of the - // two queues has data, we may starve that queue by not requesting - // more buffers from the decoder. If the other source then encounters - // a discontinuity that leads to flushing, we'll never find the - // corresponding discontinuity on the other queue. - // Therefore we'll stop syncing the queues if at least one of them - // is flushed. - { - Mutex::Autolock autoLock(mLock); - syncQueuesDone_l(); - setPauseStartedTimeRealUs(-1); - setAnchorTime(-1, -1); + // If we're currently syncing the queues, i.e. dropping audio while + // aligning the first audio/video buffer times and only one of the + // two queues has data, we may starve that queue by not requesting + // more buffers from the decoder. If the other source then encounters + // a discontinuity that leads to flushing, we'll never find the + // corresponding discontinuity on the other queue. + // Therefore we'll stop syncing the queues if at least one of them + // is flushed. + syncQueuesDone_l(); + clearAnchorTime_l(); } ALOGV("flushing %s", audio ? "audio" : "video"); @@ -1123,11 +1069,11 @@ void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { Mutex::Autolock autoLock(mLock); flushQueue(&mAudioQueue); - ++mAudioQueueGeneration; - prepareForMediaRenderingStart(); + ++mAudioDrainGeneration; + prepareForMediaRenderingStart_l(); if (offloadingAudio()) { - setAudioFirstAnchorTime(-1); + clearAudioFirstAnchorTime_l(); } } @@ -1142,13 +1088,14 @@ void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { flushQueue(&mVideoQueue); mDrainVideoQueuePending = false; - ++mVideoQueueGeneration; if (mVideoScheduler != NULL) { mVideoScheduler->restart(); } - prepareForMediaRenderingStart(); + Mutex::Autolock autoLock(mLock); + ++mVideoDrainGeneration; + prepareForMediaRenderingStart_l(); } mVideoSampleReceived = false; @@ -1178,20 +1125,12 @@ void NuPlayer::Renderer::notifyFlushComplete(bool audio) { notify->post(); } -bool NuPlayer::Renderer::dropBufferWhileFlushing( +bool NuPlayer::Renderer::dropBufferIfStale( bool audio, const sp<AMessage> &msg) { - bool flushing = false; - - { - Mutex::Autolock autoLock(mFlushLock); - if (audio) { - flushing = mFlushingAudio; - } else { - flushing = mFlushingVideo; - } - } + int32_t queueGeneration; + CHECK(msg->findInt32("queueGeneration", &queueGeneration)); - if (!flushing) { + if (queueGeneration == getQueueGeneration(audio)) { return false; } @@ -1209,7 +1148,10 @@ void NuPlayer::Renderer::onAudioSinkChanged() { } CHECK(!mDrainAudioQueuePending); mNumFramesWritten = 0; - mAnchorNumFramesWritten = -1; + { + Mutex::Autolock autoLock(mLock); + mAnchorNumFramesWritten = -1; + } uint32_t written; if (mAudioSink->getFramesWritten(&written) == OK) { mNumFramesWritten = written; @@ -1219,13 +1161,13 @@ void NuPlayer::Renderer::onAudioSinkChanged() { void NuPlayer::Renderer::onDisableOffloadAudio() { Mutex::Autolock autoLock(mLock); mFlags &= ~FLAG_OFFLOAD_AUDIO; - ++mAudioQueueGeneration; + ++mAudioDrainGeneration; } void NuPlayer::Renderer::onEnableOffloadAudio() { Mutex::Autolock autoLock(mLock); mFlags |= FLAG_OFFLOAD_AUDIO; - ++mAudioQueueGeneration; + ++mAudioDrainGeneration; } void NuPlayer::Renderer::onPause() { @@ -1234,25 +1176,13 @@ void NuPlayer::Renderer::onPause() { return; } int64_t currentPositionUs; - int64_t pausePositionMediaTimeUs; - if (getCurrentPositionFromAnchor( - ¤tPositionUs, ALooper::GetNowUs()) == OK) { - pausePositionMediaTimeUs = currentPositionUs; - } else { - // Set paused position to -1 (unavailabe) if we don't have anchor time - // This could happen if client does a seekTo() immediately followed by - // pause(). Renderer will be flushed with anchor time cleared. We don't - // want to leave stale value in mPausePositionMediaTimeUs. - pausePositionMediaTimeUs = -1; - } { Mutex::Autolock autoLock(mLock); - mPausePositionMediaTimeUs = pausePositionMediaTimeUs; - ++mAudioQueueGeneration; - ++mVideoQueueGeneration; - prepareForMediaRenderingStart(); + ++mAudioDrainGeneration; + ++mVideoDrainGeneration; + prepareForMediaRenderingStart_l(); mPaused = true; - setPauseStartedTimeRealUs(ALooper::GetNowUs()); + mMediaClock->setPlaybackRate(0.0); } mDrainAudioQueuePending = false; @@ -1277,21 +1207,18 @@ void NuPlayer::Renderer::onResume() { mAudioSink->start(); } - Mutex::Autolock autoLock(mLock); - mPaused = false; - if (mPauseStartedTimeRealUs != -1) { - int64_t newAnchorRealUs = - mAnchorTimeRealUs + ALooper::GetNowUs() - mPauseStartedTimeRealUs; - setAnchorTime( - mAnchorTimeMediaUs, newAnchorRealUs, mAnchorNumFramesWritten, true /* resume */); - } + { + Mutex::Autolock autoLock(mLock); + mPaused = false; + mMediaClock->setPlaybackRate(mPlaybackRate); - if (!mAudioQueue.empty()) { - postDrainAudioQueue_l(); + if (!mAudioQueue.empty()) { + postDrainAudioQueue_l(); + } } if (!mVideoQueue.empty()) { - postDrainVideoQueue_l(); + postDrainVideoQueue(); } } @@ -1302,6 +1229,21 @@ void NuPlayer::Renderer::onSetVideoFrameRate(float fps) { mVideoScheduler->init(fps); } +int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) { + Mutex::Autolock autoLock(mLock); + return (audio ? mAudioQueueGeneration : mVideoQueueGeneration); +} + +int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) { + Mutex::Autolock autoLock(mLock); + return (audio ? mAudioDrainGeneration : mVideoDrainGeneration); +} + +bool NuPlayer::Renderer::getSyncQueues() { + Mutex::Autolock autoLock(mLock); + return mSyncQueues; +} + // TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs() // as it acquires locks and may query the audio driver. // @@ -1309,6 +1251,7 @@ void NuPlayer::Renderer::onSetVideoFrameRate(float fps) { // accessing getTimestamp() or getPosition() every time a data buffer with // a media time is received. // +// Calculate duration of played samples if played at normal rate (i.e., 1.0). int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) { uint32_t numFramesPlayed; int64_t numFramesPlayedAt; @@ -1346,9 +1289,8 @@ int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) { //ALOGD("getPosition: %d %lld", numFramesPlayed, numFramesPlayedAt); } - // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours. //CHECK_EQ(numFramesPlayed & (1 << 31), 0); // can't be negative until 12.4 hrs, test - int64_t durationUs = (int64_t)((int32_t)numFramesPlayed * 1000LL * mAudioSink->msecsPerFrame()) + int64_t durationUs = getDurationUsIfPlayedAtSampleRate(numFramesPlayed) + nowUs - numFramesPlayedAt; if (durationUs < 0) { // Occurs when numFramesPlayed position is very small and the following: @@ -1373,7 +1315,7 @@ void NuPlayer::Renderer::onAudioOffloadTearDown(AudioOffloadTearDownReason reaso mAudioOffloadTornDown = true; int64_t currentPositionUs; - if (getCurrentPositionOnLooper(¤tPositionUs) != OK) { + if (getCurrentPosition(¤tPositionUs) != OK) { currentPositionUs = 0; } @@ -1390,8 +1332,8 @@ void NuPlayer::Renderer::onAudioOffloadTearDown(AudioOffloadTearDownReason reaso void NuPlayer::Renderer::startAudioOffloadPauseTimeout() { if (offloadingAudio()) { mWakeLock->acquire(); - sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, id()); - msg->setInt32("generation", mAudioOffloadPauseTimeoutGeneration); + sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this); + msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration); msg->post(kOffloadPauseMaxUs); } } @@ -1487,6 +1429,10 @@ status_t NuPlayer::Renderer::onOpenAudioSink( &offloadInfo); if (err == OK) { + if (mPlaybackRate != 1.0) { + mAudioSink->setPlaybackRatePermille( + (int32_t)(mPlaybackRate * 1000 + 0.5f)); + } // If the playback is offloaded to h/w, we pass // the HAL some metadata information. // We don't want to do this for PCM because it @@ -1542,6 +1488,10 @@ status_t NuPlayer::Renderer::onOpenAudioSink( return err; } mCurrentPcmInfo = info; + if (mPlaybackRate != 1.0) { + mAudioSink->setPlaybackRatePermille( + (int32_t)(mPlaybackRate * 1000 + 0.5f)); + } mAudioSink->start(); } if (audioSinkChanged) { diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h index 003d1d0..38843d5 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h @@ -24,6 +24,7 @@ namespace android { struct ABuffer; class AWakeLock; +struct MediaClock; struct VideoFrameScheduler; struct NuPlayer::Renderer : public AHandler { @@ -47,6 +48,8 @@ struct NuPlayer::Renderer : public AHandler { void queueEOS(bool audio, status_t finalResult); + void setPlaybackRate(float rate); + void flush(bool audio, bool notifyComplete); void signalTimeDiscontinuity(); @@ -61,16 +64,8 @@ struct NuPlayer::Renderer : public AHandler { void setVideoFrameRate(float fps); - // Following setters and getters are protected by mTimeLock. status_t getCurrentPosition(int64_t *mediaUs); - void setHasMedia(bool audio); - void setAudioFirstAnchorTime(int64_t mediaUs); - void setAudioFirstAnchorTimeIfNeeded(int64_t mediaUs); - void setAnchorTime( - int64_t mediaUs, int64_t realUs, int64_t numFramesWritten = -1, bool resume = false); - void setVideoLateByUs(int64_t lateUs); int64_t getVideoLateByUs(); - void setPauseStartedTimeRealUs(int64_t realUs); status_t openAudioSink( const sp<AMessage> &format, @@ -107,8 +102,8 @@ private: kWhatPostDrainVideoQueue = 'pDVQ', kWhatQueueBuffer = 'queB', kWhatQueueEOS = 'qEOS', + kWhatSetRate = 'setR', kWhatFlush = 'flus', - kWhatAudioSinkChanged = 'auSC', kWhatPause = 'paus', kWhatResume = 'resm', kWhatOpenAudioSink = 'opnA', @@ -142,26 +137,18 @@ private: bool mDrainVideoQueuePending; int32_t mAudioQueueGeneration; int32_t mVideoQueueGeneration; + int32_t mAudioDrainGeneration; + int32_t mVideoDrainGeneration; - Mutex mTimeLock; - // |mTimeLock| protects the following 7 member vars that are related to time. - // Note: those members are only written on Renderer thread, so reading on Renderer thread - // doesn't need to be protected. Otherwise accessing those members must be protected by - // |mTimeLock|. - // TODO: move those members to a seperated media clock class. + sp<MediaClock> mMediaClock; + float mPlaybackRate; int64_t mAudioFirstAnchorTimeMediaUs; int64_t mAnchorTimeMediaUs; - int64_t mAnchorTimeRealUs; int64_t mAnchorNumFramesWritten; - int64_t mAnchorMaxMediaUs; int64_t mVideoLateByUs; bool mHasAudio; bool mHasVideo; - int64_t mPauseStartedTimeRealUs; - Mutex mFlushLock; // protects the following 2 member vars. - bool mFlushingAudio; - bool mFlushingVideo; bool mNotifyCompleteAudio; bool mNotifyCompleteVideo; @@ -169,7 +156,6 @@ private: // modified on only renderer's thread. bool mPaused; - int64_t mPausePositionMediaTimeUs; bool mVideoSampleReceived; bool mVideoRenderingStarted; @@ -211,14 +197,19 @@ private: int64_t getPlayedOutAudioDurationUs(int64_t nowUs); void postDrainAudioQueue_l(int64_t delayUs = 0); + void clearAnchorTime_l(); + void clearAudioFirstAnchorTime_l(); + void setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs); + void setVideoLateByUs(int64_t lateUs); + void onNewAudioMediaTime(int64_t mediaTimeUs); int64_t getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs); void onDrainVideoQueue(); - void postDrainVideoQueue_l(); + void postDrainVideoQueue(); - void prepareForMediaRenderingStart(); - void notifyIfMediaRenderingStarted(); + void prepareForMediaRenderingStart_l(); + void notifyIfMediaRenderingStarted_l(); void onQueueBuffer(const sp<AMessage> &msg); void onQueueEOS(const sp<AMessage> &msg); @@ -229,6 +220,9 @@ private: void onPause(); void onResume(); void onSetVideoFrameRate(float fps); + int32_t getQueueGeneration(bool audio); + int32_t getDrainGeneration(bool audio); + bool getSyncQueues(); void onAudioOffloadTearDown(AudioOffloadTearDownReason reason); status_t onOpenAudioSink( const sp<AMessage> &format, @@ -245,7 +239,7 @@ private: void notifyAudioOffloadTearDown(); void flushQueue(List<QueueEntry> *queue); - bool dropBufferWhileFlushing(bool audio, const sp<AMessage> &msg); + bool dropBufferIfStale(bool audio, const sp<AMessage> &msg); void syncQueuesDone_l(); bool offloadingAudio() const { return (mFlags & FLAG_OFFLOAD_AUDIO) != 0; } @@ -253,6 +247,8 @@ private: void startAudioOffloadPauseTimeout(); void cancelAudioOffloadPauseTimeout(); + int64_t getDurationUsIfPlayedAtSampleRate(uint32_t numFrames); + DISALLOW_EVIL_CONSTRUCTORS(Renderer); }; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerStreamListener.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerStreamListener.cpp index 885ebe4..f53afbd 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerStreamListener.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerStreamListener.cpp @@ -29,9 +29,9 @@ namespace android { NuPlayer::NuPlayerStreamListener::NuPlayerStreamListener( const sp<IStreamSource> &source, - ALooper::handler_id id) + const sp<AHandler> &targetHandler) : mSource(source), - mTargetID(id), + mTargetHandler(targetHandler), mEOS(false), mSendDataNotification(true) { mSource->setListener(this); @@ -65,8 +65,8 @@ void NuPlayer::NuPlayerStreamListener::queueBuffer(size_t index, size_t size) { if (mSendDataNotification) { mSendDataNotification = false; - if (mTargetID != 0) { - (new AMessage(kWhatMoreDataQueued, mTargetID))->post(); + if (mTargetHandler != NULL) { + (new AMessage(kWhatMoreDataQueued, mTargetHandler))->post(); } } } @@ -86,8 +86,8 @@ void NuPlayer::NuPlayerStreamListener::issueCommand( if (mSendDataNotification) { mSendDataNotification = false; - if (mTargetID != 0) { - (new AMessage(kWhatMoreDataQueued, mTargetID))->post(); + if (mTargetHandler != NULL) { + (new AMessage(kWhatMoreDataQueued, mTargetHandler))->post(); } } } diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerStreamListener.h b/media/libmediaplayerservice/nuplayer/NuPlayerStreamListener.h index 1874d80..2de829b 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerStreamListener.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerStreamListener.h @@ -29,7 +29,7 @@ struct MemoryDealer; struct NuPlayer::NuPlayerStreamListener : public BnStreamListener { NuPlayerStreamListener( const sp<IStreamSource> &source, - ALooper::handler_id targetID); + const sp<AHandler> &targetHandler); virtual void queueBuffer(size_t index, size_t size); @@ -59,7 +59,7 @@ private: Mutex mLock; sp<IStreamSource> mSource; - ALooper::handler_id mTargetID; + sp<AHandler> mTargetHandler; sp<MemoryDealer> mMemoryDealer; Vector<sp<IMemory> > mBuffers; List<QueueEntry> mQueue; diff --git a/media/libmediaplayerservice/nuplayer/RTSPSource.cpp b/media/libmediaplayerservice/nuplayer/RTSPSource.cpp index 0282a9f..5210fc8 100644 --- a/media/libmediaplayerservice/nuplayer/RTSPSource.cpp +++ b/media/libmediaplayerservice/nuplayer/RTSPSource.cpp @@ -87,7 +87,7 @@ void NuPlayer::RTSPSource::prepareAsync() { CHECK(mHandler == NULL); CHECK(mSDPLoader == NULL); - sp<AMessage> notify = new AMessage(kWhatNotify, id()); + sp<AMessage> notify = new AMessage(kWhatNotify, this); CHECK_EQ(mState, (int)DISCONNECTED); mState = CONNECTING; @@ -116,7 +116,7 @@ void NuPlayer::RTSPSource::stop() { if (mLooper == NULL) { return; } - sp<AMessage> msg = new AMessage(kWhatDisconnect, id()); + sp<AMessage> msg = new AMessage(kWhatDisconnect, this); sp<AMessage> dummy; msg->postAndAwaitResponse(&dummy); @@ -292,7 +292,7 @@ status_t NuPlayer::RTSPSource::getDuration(int64_t *durationUs) { } status_t NuPlayer::RTSPSource::seekTo(int64_t seekTimeUs) { - sp<AMessage> msg = new AMessage(kWhatPerformSeek, id()); + sp<AMessage> msg = new AMessage(kWhatPerformSeek, this); msg->setInt32("generation", ++mSeekGeneration); msg->setInt64("timeUs", seekTimeUs); msg->post(200000ll); @@ -311,7 +311,7 @@ void NuPlayer::RTSPSource::performSeek(int64_t seekTimeUs) { void NuPlayer::RTSPSource::onMessageReceived(const sp<AMessage> &msg) { if (msg->what() == kWhatDisconnect) { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); mDisconnectReplyID = replyID; @@ -600,7 +600,7 @@ void NuPlayer::RTSPSource::onSDPLoaded(const sp<AMessage> &msg) { ALOGE("Unable to find url in SDP"); err = UNKNOWN_ERROR; } else { - sp<AMessage> notify = new AMessage(kWhatNotify, id()); + sp<AMessage> notify = new AMessage(kWhatNotify, this); mHandler = new MyHandler(rtspUri.c_str(), notify, mUIDValid, mUID); mLooper->registerHandler(mHandler); diff --git a/media/libmediaplayerservice/nuplayer/RTSPSource.h b/media/libmediaplayerservice/nuplayer/RTSPSource.h index ac3299a..5f2cf33 100644 --- a/media/libmediaplayerservice/nuplayer/RTSPSource.h +++ b/media/libmediaplayerservice/nuplayer/RTSPSource.h @@ -25,6 +25,7 @@ namespace android { struct ALooper; +struct AReplyToken; struct AnotherPacketSource; struct MyHandler; struct SDPLoader; @@ -96,7 +97,7 @@ private: bool mIsSDP; State mState; status_t mFinalResult; - uint32_t mDisconnectReplyID; + sp<AReplyToken> mDisconnectReplyID; Mutex mBufferingLock; bool mBuffering; diff --git a/media/libmediaplayerservice/nuplayer/StreamingSource.cpp b/media/libmediaplayerservice/nuplayer/StreamingSource.cpp index b3f224d..0246b59 100644 --- a/media/libmediaplayerservice/nuplayer/StreamingSource.cpp +++ b/media/libmediaplayerservice/nuplayer/StreamingSource.cpp @@ -63,7 +63,7 @@ void NuPlayer::StreamingSource::prepareAsync() { } void NuPlayer::StreamingSource::start() { - mStreamListener = new NuPlayerStreamListener(mSource, 0); + mStreamListener = new NuPlayerStreamListener(mSource, NULL); uint32_t sourceFlags = mSource->flags(); @@ -163,7 +163,7 @@ status_t NuPlayer::StreamingSource::postReadBuffer() { mBuffering = true; } - (new AMessage(kWhatReadBuffer, id()))->post(); + (new AMessage(kWhatReadBuffer, this))->post(); return OK; } diff --git a/media/libmediaplayerservice/tests/Android.mk b/media/libmediaplayerservice/tests/Android.mk new file mode 100644 index 0000000..7bc78ff --- /dev/null +++ b/media/libmediaplayerservice/tests/Android.mk @@ -0,0 +1,24 @@ +# Build the unit tests. +LOCAL_PATH:= $(call my-dir) +include $(CLEAR_VARS) + +LOCAL_MODULE := DrmSessionManager_test + +LOCAL_MODULE_TAGS := tests + +LOCAL_SRC_FILES := \ + DrmSessionManager_test.cpp \ + +LOCAL_SHARED_LIBRARIES := \ + liblog \ + libmediaplayerservice \ + libutils \ + +LOCAL_C_INCLUDES := \ + frameworks/av/include \ + frameworks/av/media/libmediaplayerservice \ + +LOCAL_32_BIT_ONLY := true + +include $(BUILD_NATIVE_TEST) + diff --git a/media/libmediaplayerservice/tests/DrmSessionManager_test.cpp b/media/libmediaplayerservice/tests/DrmSessionManager_test.cpp new file mode 100644 index 0000000..d3e760b --- /dev/null +++ b/media/libmediaplayerservice/tests/DrmSessionManager_test.cpp @@ -0,0 +1,249 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "DrmSessionManager_test" +#include <utils/Log.h> + +#include <gtest/gtest.h> + +#include "Drm.h" +#include "DrmSessionClientInterface.h" +#include "DrmSessionManager.h" +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/ProcessInfoInterface.h> + +namespace android { + +struct FakeProcessInfo : public ProcessInfoInterface { + FakeProcessInfo() {} + virtual ~FakeProcessInfo() {} + + virtual bool getPriority(int pid, int* priority) { + // For testing, use pid as priority. + // Lower the value higher the priority. + *priority = pid; + return true; + } + +private: + DISALLOW_EVIL_CONSTRUCTORS(FakeProcessInfo); +}; + +struct FakeDrm : public DrmSessionClientInterface { + FakeDrm() {} + virtual ~FakeDrm() {} + + virtual bool reclaimSession(const Vector<uint8_t>& sessionId) { + mReclaimedSessions.push_back(sessionId); + return true; + } + + const Vector<Vector<uint8_t> >& reclaimedSessions() const { + return mReclaimedSessions; + } + +private: + Vector<Vector<uint8_t> > mReclaimedSessions; + + DISALLOW_EVIL_CONSTRUCTORS(FakeDrm); +}; + +static const int kTestPid1 = 30; +static const int kTestPid2 = 20; +static const uint8_t kTestSessionId1[] = {1, 2, 3}; +static const uint8_t kTestSessionId2[] = {4, 5, 6, 7, 8}; +static const uint8_t kTestSessionId3[] = {9, 0}; + +class DrmSessionManagerTest : public ::testing::Test { +public: + DrmSessionManagerTest() + : mDrmSessionManager(new DrmSessionManager(new FakeProcessInfo())), + mTestDrm1(new FakeDrm()), + mTestDrm2(new FakeDrm()) { + GetSessionId(kTestSessionId1, ARRAY_SIZE(kTestSessionId1), &mSessionId1); + GetSessionId(kTestSessionId2, ARRAY_SIZE(kTestSessionId2), &mSessionId2); + GetSessionId(kTestSessionId3, ARRAY_SIZE(kTestSessionId3), &mSessionId3); + } + +protected: + static void GetSessionId(const uint8_t* ids, size_t num, Vector<uint8_t>* sessionId) { + for (size_t i = 0; i < num; ++i) { + sessionId->push_back(ids[i]); + } + } + + static void ExpectEqSessionInfo(const SessionInfo& info, sp<DrmSessionClientInterface> drm, + const Vector<uint8_t>& sessionId, int64_t timeStamp) { + EXPECT_EQ(drm, info.drm); + EXPECT_TRUE(isEqualSessionId(sessionId, info.sessionId)); + EXPECT_EQ(timeStamp, info.timeStamp); + } + + void addSession() { + mDrmSessionManager->addSession(kTestPid1, mTestDrm1, mSessionId1); + mDrmSessionManager->addSession(kTestPid2, mTestDrm2, mSessionId2); + mDrmSessionManager->addSession(kTestPid2, mTestDrm2, mSessionId3); + const PidSessionInfosMap& map = sessionMap(); + EXPECT_EQ(2, map.size()); + ssize_t index1 = map.indexOfKey(kTestPid1); + ASSERT_GE(index1, 0); + const SessionInfos& infos1 = map[index1]; + EXPECT_EQ(1, infos1.size()); + ExpectEqSessionInfo(infos1[0], mTestDrm1, mSessionId1, 0); + + ssize_t index2 = map.indexOfKey(kTestPid2); + ASSERT_GE(index2, 0); + const SessionInfos& infos2 = map[index2]; + EXPECT_EQ(2, infos2.size()); + ExpectEqSessionInfo(infos2[0], mTestDrm2, mSessionId2, 1); + ExpectEqSessionInfo(infos2[1], mTestDrm2, mSessionId3, 2); + } + + const PidSessionInfosMap& sessionMap() { + return mDrmSessionManager->mSessionMap; + } + + void testGetLowestPriority() { + int pid; + int priority; + EXPECT_FALSE(mDrmSessionManager->getLowestPriority_l(&pid, &priority)); + + addSession(); + EXPECT_TRUE(mDrmSessionManager->getLowestPriority_l(&pid, &priority)); + + EXPECT_EQ(kTestPid1, pid); + FakeProcessInfo processInfo; + int priority1; + processInfo.getPriority(kTestPid1, &priority1); + EXPECT_EQ(priority1, priority); + } + + void testGetLeastUsedSession() { + sp<DrmSessionClientInterface> drm; + Vector<uint8_t> sessionId; + EXPECT_FALSE(mDrmSessionManager->getLeastUsedSession_l(kTestPid1, &drm, &sessionId)); + + addSession(); + + EXPECT_TRUE(mDrmSessionManager->getLeastUsedSession_l(kTestPid1, &drm, &sessionId)); + EXPECT_EQ(mTestDrm1, drm); + EXPECT_TRUE(isEqualSessionId(mSessionId1, sessionId)); + + EXPECT_TRUE(mDrmSessionManager->getLeastUsedSession_l(kTestPid2, &drm, &sessionId)); + EXPECT_EQ(mTestDrm2, drm); + EXPECT_TRUE(isEqualSessionId(mSessionId2, sessionId)); + + // mSessionId2 is no longer the least used session. + mDrmSessionManager->useSession(mSessionId2); + EXPECT_TRUE(mDrmSessionManager->getLeastUsedSession_l(kTestPid2, &drm, &sessionId)); + EXPECT_EQ(mTestDrm2, drm); + EXPECT_TRUE(isEqualSessionId(mSessionId3, sessionId)); + } + + sp<DrmSessionManager> mDrmSessionManager; + sp<FakeDrm> mTestDrm1; + sp<FakeDrm> mTestDrm2; + Vector<uint8_t> mSessionId1; + Vector<uint8_t> mSessionId2; + Vector<uint8_t> mSessionId3; +}; + +TEST_F(DrmSessionManagerTest, addSession) { + addSession(); +} + +TEST_F(DrmSessionManagerTest, useSession) { + addSession(); + + mDrmSessionManager->useSession(mSessionId1); + mDrmSessionManager->useSession(mSessionId3); + + const PidSessionInfosMap& map = sessionMap(); + const SessionInfos& infos1 = map.valueFor(kTestPid1); + const SessionInfos& infos2 = map.valueFor(kTestPid2); + ExpectEqSessionInfo(infos1[0], mTestDrm1, mSessionId1, 3); + ExpectEqSessionInfo(infos2[1], mTestDrm2, mSessionId3, 4); +} + +TEST_F(DrmSessionManagerTest, removeSession) { + addSession(); + + mDrmSessionManager->removeSession(mSessionId2); + + const PidSessionInfosMap& map = sessionMap(); + EXPECT_EQ(2, map.size()); + const SessionInfos& infos1 = map.valueFor(kTestPid1); + const SessionInfos& infos2 = map.valueFor(kTestPid2); + EXPECT_EQ(1, infos1.size()); + EXPECT_EQ(1, infos2.size()); + // mSessionId2 has been removed. + ExpectEqSessionInfo(infos2[0], mTestDrm2, mSessionId3, 2); +} + +TEST_F(DrmSessionManagerTest, removeDrm) { + addSession(); + + sp<FakeDrm> drm = new FakeDrm; + const uint8_t ids[] = {123}; + Vector<uint8_t> sessionId; + GetSessionId(ids, ARRAY_SIZE(ids), &sessionId); + mDrmSessionManager->addSession(kTestPid2, drm, sessionId); + + mDrmSessionManager->removeDrm(mTestDrm2); + + const PidSessionInfosMap& map = sessionMap(); + const SessionInfos& infos2 = map.valueFor(kTestPid2); + EXPECT_EQ(1, infos2.size()); + // mTestDrm2 has been removed. + ExpectEqSessionInfo(infos2[0], drm, sessionId, 3); +} + +TEST_F(DrmSessionManagerTest, reclaimSession) { + EXPECT_FALSE(mDrmSessionManager->reclaimSession(kTestPid1)); + addSession(); + + // calling pid priority is too low + EXPECT_FALSE(mDrmSessionManager->reclaimSession(50)); + + EXPECT_TRUE(mDrmSessionManager->reclaimSession(10)); + EXPECT_EQ(1, mTestDrm1->reclaimedSessions().size()); + EXPECT_TRUE(isEqualSessionId(mSessionId1, mTestDrm1->reclaimedSessions()[0])); + + mDrmSessionManager->removeSession(mSessionId1); + + // add a session from a higher priority process. + sp<FakeDrm> drm = new FakeDrm; + const uint8_t ids[] = {1, 3, 5}; + Vector<uint8_t> sessionId; + GetSessionId(ids, ARRAY_SIZE(ids), &sessionId); + mDrmSessionManager->addSession(15, drm, sessionId); + + EXPECT_TRUE(mDrmSessionManager->reclaimSession(18)); + EXPECT_EQ(1, mTestDrm2->reclaimedSessions().size()); + // mSessionId2 is reclaimed. + EXPECT_TRUE(isEqualSessionId(mSessionId2, mTestDrm2->reclaimedSessions()[0])); +} + +TEST_F(DrmSessionManagerTest, getLowestPriority) { + testGetLowestPriority(); +} + +TEST_F(DrmSessionManagerTest, getLeastUsedSession_l) { + testGetLeastUsedSession(); +} + +} // namespace android diff --git a/media/libnbaio/Android.mk b/media/libnbaio/Android.mk index 9707c4a..1353f28 100644 --- a/media/libnbaio/Android.mk +++ b/media/libnbaio/Android.mk @@ -11,7 +11,6 @@ LOCAL_SRC_FILES := \ MonoPipeReader.cpp \ Pipe.cpp \ PipeReader.cpp \ - roundup.c \ SourceAudioBufferProvider.cpp LOCAL_SRC_FILES += NBLog.cpp @@ -27,12 +26,13 @@ LOCAL_SRC_FILES += NBLog.cpp LOCAL_MODULE := libnbaio LOCAL_SHARED_LIBRARIES := \ + libaudioutils \ libbinder \ libcommon_time_client \ libcutils \ libutils \ liblog -LOCAL_STATIC_LIBRARIES += libinstantssq +LOCAL_C_INCLUDES := $(call include-path-for, audio-utils) include $(BUILD_SHARED_LIBRARY) diff --git a/media/libnbaio/MonoPipe.cpp b/media/libnbaio/MonoPipe.cpp index 0b65861..129e9ef 100644 --- a/media/libnbaio/MonoPipe.cpp +++ b/media/libnbaio/MonoPipe.cpp @@ -27,7 +27,7 @@ #include <utils/Trace.h> #include <media/AudioBufferProvider.h> #include <media/nbaio/MonoPipe.h> -#include <media/nbaio/roundup.h> +#include <audio_utils/roundup.h> namespace android { diff --git a/media/libnbaio/MonoPipeReader.cpp b/media/libnbaio/MonoPipeReader.cpp index de82229..e4d3ed8 100644 --- a/media/libnbaio/MonoPipeReader.cpp +++ b/media/libnbaio/MonoPipeReader.cpp @@ -39,7 +39,7 @@ ssize_t MonoPipeReader::availableToRead() return NEGOTIATE; } ssize_t ret = android_atomic_acquire_load(&mPipe->mRear) - mPipe->mFront; - ALOG_ASSERT((0 <= ret) && (ret <= mMaxFrames)); + ALOG_ASSERT((0 <= ret) && ((size_t) ret <= mPipe->mMaxFrames)); return ret; } diff --git a/media/libnbaio/Pipe.cpp b/media/libnbaio/Pipe.cpp index 6e0ec8c..13f211d 100644 --- a/media/libnbaio/Pipe.cpp +++ b/media/libnbaio/Pipe.cpp @@ -21,7 +21,7 @@ #include <cutils/compiler.h> #include <utils/Log.h> #include <media/nbaio/Pipe.h> -#include <media/nbaio/roundup.h> +#include <audio_utils/roundup.h> namespace android { diff --git a/media/libnbaio/roundup.c b/media/libnbaio/roundup.c deleted file mode 100644 index 1d552d1..0000000 --- a/media/libnbaio/roundup.c +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2012 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <media/nbaio/roundup.h> - -unsigned roundup(unsigned v) -{ - // __builtin_clz is undefined for zero input - if (v == 0) { - v = 1; - } - int lz = __builtin_clz((int) v); - unsigned rounded = ((unsigned) 0x80000000) >> lz; - // 0x800000001 and higher are actually rounded _down_ to prevent overflow - if (v > rounded && lz > 0) { - rounded <<= 1; - } - return rounded; -} diff --git a/media/libstagefright/AACWriter.cpp b/media/libstagefright/AACWriter.cpp index 2e41d80..9d90dbd 100644 --- a/media/libstagefright/AACWriter.cpp +++ b/media/libstagefright/AACWriter.cpp @@ -36,33 +36,19 @@ namespace android { -AACWriter::AACWriter(const char *filename) - : mFd(-1), - mInitCheck(NO_INIT), - mStarted(false), - mPaused(false), - mResumed(false), - mChannelCount(-1), - mSampleRate(-1), - mAACProfile(OMX_AUDIO_AACObjectLC) { - - ALOGV("AACWriter Constructor"); - - mFd = open(filename, O_CREAT | O_LARGEFILE | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR); - if (mFd >= 0) { - mInitCheck = OK; - } -} - AACWriter::AACWriter(int fd) : mFd(dup(fd)), mInitCheck(mFd < 0? NO_INIT: OK), mStarted(false), mPaused(false), mResumed(false), + mThread(0), + mEstimatedSizeBytes(0), + mEstimatedDurationUs(0), mChannelCount(-1), mSampleRate(-1), - mAACProfile(OMX_AUDIO_AACObjectLC) { + mAACProfile(OMX_AUDIO_AACObjectLC), + mFrameDurationUs(0) { } AACWriter::~AACWriter() { diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp index d298cb1..97f3e20 100644 --- a/media/libstagefright/ACodec.cpp +++ b/media/libstagefright/ACodec.cpp @@ -419,6 +419,7 @@ ACodec::ACodec() mMetaDataBuffersToSubmit(0), mRepeatFrameDelayUs(-1ll), mMaxPtsGapUs(-1ll), + mMaxFps(-1), mTimePerFrameUs(-1ll), mTimePerCaptureUs(-1ll), mCreateInputBuffersSuspended(false), @@ -451,61 +452,61 @@ void ACodec::setNotificationMessage(const sp<AMessage> &msg) { void ACodec::initiateSetup(const sp<AMessage> &msg) { msg->setWhat(kWhatSetup); - msg->setTarget(id()); + msg->setTarget(this); msg->post(); } void ACodec::signalSetParameters(const sp<AMessage> ¶ms) { - sp<AMessage> msg = new AMessage(kWhatSetParameters, id()); + sp<AMessage> msg = new AMessage(kWhatSetParameters, this); msg->setMessage("params", params); msg->post(); } void ACodec::initiateAllocateComponent(const sp<AMessage> &msg) { msg->setWhat(kWhatAllocateComponent); - msg->setTarget(id()); + msg->setTarget(this); msg->post(); } void ACodec::initiateConfigureComponent(const sp<AMessage> &msg) { msg->setWhat(kWhatConfigureComponent); - msg->setTarget(id()); + msg->setTarget(this); msg->post(); } void ACodec::initiateCreateInputSurface() { - (new AMessage(kWhatCreateInputSurface, id()))->post(); + (new AMessage(kWhatCreateInputSurface, this))->post(); } void ACodec::signalEndOfInputStream() { - (new AMessage(kWhatSignalEndOfInputStream, id()))->post(); + (new AMessage(kWhatSignalEndOfInputStream, this))->post(); } void ACodec::initiateStart() { - (new AMessage(kWhatStart, id()))->post(); + (new AMessage(kWhatStart, this))->post(); } void ACodec::signalFlush() { ALOGV("[%s] signalFlush", mComponentName.c_str()); - (new AMessage(kWhatFlush, id()))->post(); + (new AMessage(kWhatFlush, this))->post(); } void ACodec::signalResume() { - (new AMessage(kWhatResume, id()))->post(); + (new AMessage(kWhatResume, this))->post(); } void ACodec::initiateShutdown(bool keepComponentAllocated) { - sp<AMessage> msg = new AMessage(kWhatShutdown, id()); + sp<AMessage> msg = new AMessage(kWhatShutdown, this); msg->setInt32("keepComponentAllocated", keepComponentAllocated); msg->post(); if (!keepComponentAllocated) { // ensure shutdown completes in 3 seconds - (new AMessage(kWhatReleaseCodecInstance, id()))->post(3000000); + (new AMessage(kWhatReleaseCodecInstance, this))->post(3000000); } } void ACodec::signalRequestIDRFrame() { - (new AMessage(kWhatRequestIDRFrame, id()))->post(); + (new AMessage(kWhatRequestIDRFrame, this))->post(); } // *** NOTE: THE FOLLOWING WORKAROUND WILL BE REMOVED *** @@ -516,7 +517,7 @@ void ACodec::signalRequestIDRFrame() { void ACodec::signalSubmitOutputMetaDataBufferIfEOS_workaround() { if (mPortEOS[kPortIndexInput] && !mPortEOS[kPortIndexOutput] && mMetaDataBuffersToSubmit > 0) { - (new AMessage(kWhatSubmitOutputMetaDataBufferIfEOS, id()))->post(); + (new AMessage(kWhatSubmitOutputMetaDataBufferIfEOS, this))->post(); } } @@ -1259,6 +1260,10 @@ status_t ACodec::configureCodec( mMaxPtsGapUs = -1ll; } + if (!msg->findFloat("max-fps-to-encoder", &mMaxFps)) { + mMaxFps = -1; + } + if (!msg->findInt64("time-lapse", &mTimePerCaptureUs)) { mTimePerCaptureUs = -1ll; } @@ -1675,6 +1680,11 @@ status_t ACodec::configureCodec( err = setMinBufferSize(kPortIndexInput, 8192); // XXX } + int32_t priority; + if (msg->findInt32("priority", &priority)) { + err = setPriority(priority); + } + mBaseOutputFormat = outputFormat; CHECK_EQ(getPortFormat(kPortIndexInput, inputFormat), (status_t)OK); @@ -1685,6 +1695,22 @@ status_t ACodec::configureCodec( return err; } +status_t ACodec::setPriority(int32_t priority) { + if (priority < 0) { + return BAD_VALUE; + } + OMX_PARAM_U32TYPE config; + InitOMXParams(&config); + config.nU32 = (OMX_U32)priority; + status_t temp = mOMX->setConfig( + mNode, (OMX_INDEXTYPE)OMX_IndexConfigPriority, + &config, sizeof(config)); + if (temp != OK) { + ALOGI("codec does not support config priority (err %d)", temp); + } + return OK; +} + status_t ACodec::setMinBufferSize(OMX_U32 portIndex, size_t size) { OMX_PARAM_PORTDEFINITIONTYPE def; InitOMXParams(&def); @@ -4297,7 +4323,7 @@ void ACodec::BaseState::postFillThisBuffer(BufferInfo *info) { info->mData->meta()->clear(); notify->setBuffer("buffer", info->mData); - sp<AMessage> reply = new AMessage(kWhatInputBufferFilled, mCodec->id()); + sp<AMessage> reply = new AMessage(kWhatInputBufferFilled, mCodec); reply->setInt32("buffer-id", info->mBufferID); notify->setMessage("reply", reply); @@ -4557,7 +4583,7 @@ bool ACodec::BaseState::onOMXFillBufferDone( } sp<AMessage> reply = - new AMessage(kWhatOutputBufferDrained, mCodec->id()); + new AMessage(kWhatOutputBufferDrained, mCodec); if (!mCodec->mSentFormat && rangeLength > 0) { mCodec->sendFormatChange(reply); @@ -4833,7 +4859,7 @@ bool ACodec::UninitializedState::onAllocateComponent(const sp<AMessage> &msg) { sp<IOMX> omx = client.interface(); - sp<AMessage> notify = new AMessage(kWhatOMXDied, mCodec->id()); + sp<AMessage> notify = new AMessage(kWhatOMXDied, mCodec); mDeathNotifier = new DeathNotifier(notify); if (IInterface::asBinder(omx)->linkToDeath(mDeathNotifier) != OK) { @@ -4908,7 +4934,7 @@ bool ACodec::UninitializedState::onAllocateComponent(const sp<AMessage> &msg) { return false; } - notify = new AMessage(kWhatOMXMessage, mCodec->id()); + notify = new AMessage(kWhatOMXMessage, mCodec); observer->setNotificationMessage(notify); mCodec->mComponentName = componentName; @@ -5114,6 +5140,21 @@ void ACodec::LoadedState::onCreateInputSurface( } } + if (err == OK && mCodec->mMaxFps > 0) { + err = mCodec->mOMX->setInternalOption( + mCodec->mNode, + kPortIndexInput, + IOMX::INTERNAL_OPTION_MAX_FPS, + &mCodec->mMaxFps, + sizeof(mCodec->mMaxFps)); + + if (err != OK) { + ALOGE("[%s] Unable to configure max fps (err %d)", + mCodec->mComponentName.c_str(), + err); + } + } + if (err == OK && mCodec->mTimePerCaptureUs > 0ll && mCodec->mTimePerFrameUs > 0ll) { int64_t timeLapse[2]; @@ -5984,7 +6025,7 @@ bool ACodec::FlushingState::onOMXEvent( case OMX_EventPortSettingsChanged: { - sp<AMessage> msg = new AMessage(kWhatOMXMessage, mCodec->id()); + sp<AMessage> msg = new AMessage(kWhatOMXMessage, mCodec); msg->setInt32("type", omx_message::EVENT); msg->setInt32("node", mCodec->mNode); msg->setInt32("event", event); diff --git a/media/libstagefright/AMRWriter.cpp b/media/libstagefright/AMRWriter.cpp index 9aa7d95..f53d7f0 100644 --- a/media/libstagefright/AMRWriter.cpp +++ b/media/libstagefright/AMRWriter.cpp @@ -31,19 +31,6 @@ namespace android { -AMRWriter::AMRWriter(const char *filename) - : mFd(-1), - mInitCheck(NO_INIT), - mStarted(false), - mPaused(false), - mResumed(false) { - - mFd = open(filename, O_CREAT | O_LARGEFILE | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR); - if (mFd >= 0) { - mInitCheck = OK; - } -} - AMRWriter::AMRWriter(int fd) : mFd(dup(fd)), mInitCheck(mFd < 0? NO_INIT: OK), diff --git a/media/libstagefright/Android.mk b/media/libstagefright/Android.mk index 2629afc..a2cbdaf 100644 --- a/media/libstagefright/Android.mk +++ b/media/libstagefright/Android.mk @@ -31,11 +31,13 @@ LOCAL_SRC_FILES:= \ MediaAdapter.cpp \ MediaBuffer.cpp \ MediaBufferGroup.cpp \ + MediaClock.cpp \ MediaCodec.cpp \ MediaCodecList.cpp \ MediaCodecSource.cpp \ MediaDefs.cpp \ MediaExtractor.cpp \ + MediaSync.cpp \ MidiExtractor.cpp \ http/MediaHTTP.cpp \ MediaMuxer.cpp \ @@ -46,6 +48,7 @@ LOCAL_SRC_FILES:= \ OMXClient.cpp \ OMXCodec.cpp \ OggExtractor.cpp \ + ProcessInfo.cpp \ SampleIterator.cpp \ SampleTable.cpp \ SkipCutBuffer.cpp \ @@ -101,6 +104,7 @@ LOCAL_STATIC_LIBRARIES := \ libstagefright_color_conversion \ libstagefright_aacenc \ libstagefright_matroska \ + libstagefright_mediafilter \ libstagefright_webm \ libstagefright_timedtext \ libvpx \ @@ -108,13 +112,14 @@ LOCAL_STATIC_LIBRARIES := \ libstagefright_mpeg2ts \ libstagefright_id3 \ libFLAC \ - libmedia_helper + libmedia_helper \ LOCAL_SHARED_LIBRARIES += \ libstagefright_enc_common \ libstagefright_avc_common \ libstagefright_foundation \ - libdl + libdl \ + libRScpp \ LOCAL_CFLAGS += -Wno-multichar diff --git a/media/libstagefright/FileSource.cpp b/media/libstagefright/FileSource.cpp index a7ca3da..f0db76b 100644 --- a/media/libstagefright/FileSource.cpp +++ b/media/libstagefright/FileSource.cpp @@ -14,6 +14,10 @@ * limitations under the License. */ +//#define LOG_NDEBUG 0 +#define LOG_TAG "FileSource" +#include <utils/Log.h> + #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/FileSource.h> #include <sys/types.h> diff --git a/media/libstagefright/HTTPBase.cpp b/media/libstagefright/HTTPBase.cpp index 0c2ff15..77a652a 100644 --- a/media/libstagefright/HTTPBase.cpp +++ b/media/libstagefright/HTTPBase.cpp @@ -75,7 +75,11 @@ void HTTPBase::addBandwidthMeasurement( bool HTTPBase::estimateBandwidth(int32_t *bandwidth_bps) { Mutex::Autolock autoLock(mLock); - if (mNumBandwidthHistoryItems < 2) { + // Do not do bandwidth estimation if we don't have enough samples, or + // total bytes download are too small (<64K). + // Bandwidth estimation from these samples can often shoot up and cause + // unwanted bw adaption behaviors. + if (mNumBandwidthHistoryItems < 2 || mTotalTransferBytes < 65536) { return false; } diff --git a/media/libstagefright/MPEG2TSWriter.cpp b/media/libstagefright/MPEG2TSWriter.cpp index 9856f92..ef07aa0 100644 --- a/media/libstagefright/MPEG2TSWriter.cpp +++ b/media/libstagefright/MPEG2TSWriter.cpp @@ -135,7 +135,7 @@ void MPEG2TSWriter::SourceInfo::start(const sp<AMessage> ¬ify) { mNotify = notify; - (new AMessage(kWhatStart, id()))->post(); + (new AMessage(kWhatStart, this))->post(); } void MPEG2TSWriter::SourceInfo::stop() { @@ -361,7 +361,7 @@ bool MPEG2TSWriter::SourceInfo::flushAACFrames() { } void MPEG2TSWriter::SourceInfo::readMore() { - (new AMessage(kWhatRead, id()))->post(); + (new AMessage(kWhatRead, this))->post(); } void MPEG2TSWriter::SourceInfo::onMessageReceived(const sp<AMessage> &msg) { @@ -480,19 +480,6 @@ MPEG2TSWriter::MPEG2TSWriter(int fd) init(); } -MPEG2TSWriter::MPEG2TSWriter(const char *filename) - : mFile(fopen(filename, "wb")), - mWriteCookie(NULL), - mWriteFunc(NULL), - mStarted(false), - mNumSourcesDone(0), - mNumTSPacketsWritten(0), - mNumTSPacketsBeforeMeta(0), - mPATContinuityCounter(0), - mPMTContinuityCounter(0) { - init(); -} - MPEG2TSWriter::MPEG2TSWriter( void *cookie, ssize_t (*write)(void *cookie, const void *data, size_t size)) @@ -565,7 +552,7 @@ status_t MPEG2TSWriter::start(MetaData * /* param */) { for (size_t i = 0; i < mSources.size(); ++i) { sp<AMessage> notify = - new AMessage(kWhatSourceNotify, mReflector->id()); + new AMessage(kWhatSourceNotify, mReflector); notify->setInt32("source-index", i); diff --git a/media/libstagefright/MPEG4Extractor.cpp b/media/libstagefright/MPEG4Extractor.cpp index 8bf7f63..d0f42cc 100644 --- a/media/libstagefright/MPEG4Extractor.cpp +++ b/media/libstagefright/MPEG4Extractor.cpp @@ -354,6 +354,8 @@ static bool AdjustChannelsAndRate(uint32_t fourcc, uint32_t *channels, uint32_t MPEG4Extractor::MPEG4Extractor(const sp<DataSource> &source) : mMoofOffset(0), + mMoofFound(false), + mMdatFound(false), mDataSource(source), mInitCheck(NO_INIT), mHasVideo(false), @@ -490,7 +492,9 @@ status_t MPEG4Extractor::readMetaData() { off64_t offset = 0; status_t err; - while (true) { + bool sawMoovOrSidx = false; + + while (!(sawMoovOrSidx && (mMdatFound || mMoofFound))) { off64_t orig_offset = offset; err = parseChunk(&offset, 0); @@ -502,23 +506,9 @@ status_t MPEG4Extractor::readMetaData() { ALOGE("did not advance: 0x%lld->0x%lld", orig_offset, offset); err = ERROR_MALFORMED; break; - } else if (err == OK) { - continue; - } - - uint32_t hdr[2]; - if (mDataSource->readAt(offset, hdr, 8) < 8) { - break; + } else if (err == UNKNOWN_ERROR) { + sawMoovOrSidx = true; } - uint32_t chunk_type = ntohl(hdr[1]); - if (chunk_type == FOURCC('m', 'o', 'o', 'f')) { - // store the offset of the first segment - mMoofOffset = offset; - } else if (chunk_type != FOURCC('m', 'd', 'a', 't')) { - // keep parsing until we get to the data - continue; - } - break; } if (mInitCheck == OK) { @@ -864,6 +854,12 @@ status_t MPEG4Extractor::parseChunk(off64_t *offset, int depth) { case FOURCC('s', 'c', 'h', 'i'): case FOURCC('e', 'd', 't', 's'): { + if (chunk_type == FOURCC('m', 'o', 'o', 'f') && !mMoofFound) { + // store the offset of the first segment + mMoofFound = true; + mMoofOffset = *offset; + } + if (chunk_type == FOURCC('s', 't', 'b', 'l')) { ALOGV("sampleTable chunk is %" PRIu64 " bytes long.", chunk_size); @@ -1830,6 +1826,9 @@ status_t MPEG4Extractor::parseChunk(off64_t *offset, int depth) { case FOURCC('m', 'd', 'a', 't'): { ALOGV("mdat chunk, drm: %d", mIsDrm); + + mMdatFound = true; + if (!mIsDrm) { *offset += chunk_size; break; diff --git a/media/libstagefright/MPEG4Writer.cpp b/media/libstagefright/MPEG4Writer.cpp index 9f20b1d..6f6e362 100644 --- a/media/libstagefright/MPEG4Writer.cpp +++ b/media/libstagefright/MPEG4Writer.cpp @@ -29,6 +29,7 @@ #include <utils/Log.h> #include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> #include <media/stagefright/MPEG4Writer.h> #include <media/stagefright/MediaBuffer.h> #include <media/stagefright/MetaData.h> @@ -62,6 +63,14 @@ static const uint8_t kNalUnitTypeSeqParamSet = 0x07; static const uint8_t kNalUnitTypePicParamSet = 0x08; static const int64_t kInitialDelayTimeUs = 700000LL; +static const char kMetaKey_Model[] = "com.android.model"; +static const char kMetaKey_Version[] = "com.android.version"; +static const char kMetaKey_Build[] = "com.android.build"; +static const char kMetaKey_CaptureFps[] = "com.android.capture.fps"; + +/* uncomment to include model and build in meta */ +//#define SHOW_MODEL_BUILD 1 + class MPEG4Writer::Track { public: Track(MPEG4Writer *owner, const sp<MediaSource> &source, size_t trackId); @@ -345,31 +354,6 @@ private: Track &operator=(const Track &); }; -MPEG4Writer::MPEG4Writer(const char *filename) - : mFd(-1), - mInitCheck(NO_INIT), - mIsRealTimeRecording(true), - mUse4ByteNalLength(true), - mUse32BitOffset(true), - mIsFileSizeLimitExplicitlyRequested(false), - mPaused(false), - mStarted(false), - mWriterThreadStarted(false), - mOffset(0), - mMdatOffset(0), - mEstimatedMoovBoxSize(0), - mInterleaveDurationUs(1000000), - mLatitudex10000(0), - mLongitudex10000(0), - mAreGeoTagsAvailable(false), - mStartTimeOffsetMs(-1) { - - mFd = open(filename, O_CREAT | O_LARGEFILE | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR); - if (mFd >= 0) { - mInitCheck = OK; - } -} - MPEG4Writer::MPEG4Writer(int fd) : mFd(dup(fd)), mInitCheck(mFd < 0? NO_INIT: OK), @@ -383,11 +367,14 @@ MPEG4Writer::MPEG4Writer(int fd) mOffset(0), mMdatOffset(0), mEstimatedMoovBoxSize(0), + mMoovExtraSize(0), mInterleaveDurationUs(1000000), mLatitudex10000(0), mLongitudex10000(0), mAreGeoTagsAvailable(false), + mMetaKeys(new AMessage()), mStartTimeOffsetMs(-1) { + addDeviceMeta(); } MPEG4Writer::~MPEG4Writer() { @@ -507,6 +494,34 @@ status_t MPEG4Writer::startTracks(MetaData *params) { return OK; } +void MPEG4Writer::addDeviceMeta() { + // add device info and estimate space in 'moov' + char val[PROPERTY_VALUE_MAX]; + size_t n; + // meta size is estimated by adding up the following: + // - meta header structures, which occur only once (total 66 bytes) + // - size for each key, which consists of a fixed header (32 bytes), + // plus key length and data length. + mMoovExtraSize += 66; + if (property_get("ro.build.version.release", val, NULL) + && (n = strlen(val)) > 0) { + mMetaKeys->setString(kMetaKey_Version, val, n + 1); + mMoovExtraSize += sizeof(kMetaKey_Version) + n + 32; + } +#ifdef SHOW_MODEL_BUILD + if (property_get("ro.product.model", val, NULL) + && (n = strlen(val)) > 0) { + mMetaKeys->setString(kMetaKey_Model, val, n + 1); + mMoovExtraSize += sizeof(kMetaKey_Model) + n + 32; + } + if (property_get("ro.build.display.id", val, NULL) + && (n = strlen(val)) > 0) { + mMetaKeys->setString(kMetaKey_Build, val, n + 1); + mMoovExtraSize += sizeof(kMetaKey_Build) + n + 32; + } +#endif +} + int64_t MPEG4Writer::estimateMoovBoxSize(int32_t bitRate) { // This implementation is highly experimental/heurisitic. // @@ -560,6 +575,9 @@ int64_t MPEG4Writer::estimateMoovBoxSize(int32_t bitRate) { size = MAX_MOOV_BOX_SIZE; } + // Account for the extra stuff (Geo, meta keys, etc.) + size += mMoovExtraSize; + ALOGI("limits: %" PRId64 "/%" PRId64 " bytes/us, bit rate: %d bps and the" " estimated moov size %" PRId64 " bytes", mMaxFileSizeLimitBytes, mMaxFileDurationLimitUs, bitRate, size); @@ -971,6 +989,7 @@ void MPEG4Writer::writeMoovBox(int64_t durationUs) { if (mAreGeoTagsAvailable) { writeUdtaBox(); } + writeMetaBox(); int32_t id = 1; for (List<Track *>::iterator it = mTracks.begin(); it != mTracks.end(); ++it, ++id) { @@ -1140,6 +1159,14 @@ size_t MPEG4Writer::write( return bytes; } +void MPEG4Writer::beginBox(uint32_t id) { + mBoxes.push_back(mWriteMoovBoxToMemory? + mMoovBoxBufferOffset: mOffset); + + writeInt32(0); + writeInt32(id); +} + void MPEG4Writer::beginBox(const char *fourcc) { CHECK_EQ(strlen(fourcc), 4); @@ -1264,6 +1291,18 @@ status_t MPEG4Writer::setGeoData(int latitudex10000, int longitudex10000) { mLatitudex10000 = latitudex10000; mLongitudex10000 = longitudex10000; mAreGeoTagsAvailable = true; + mMoovExtraSize += 30; + return OK; +} + +status_t MPEG4Writer::setCaptureRate(float captureFps) { + if (captureFps <= 0.0f) { + return BAD_VALUE; + } + + mMetaKeys->setFloat(kMetaKey_CaptureFps, captureFps); + mMoovExtraSize += sizeof(kMetaKey_CaptureFps) + 4 + 32; + return OK; } @@ -3095,6 +3134,103 @@ void MPEG4Writer::writeUdtaBox() { endBox(); } +void MPEG4Writer::writeHdlr() { + beginBox("hdlr"); + writeInt32(0); // Version, Flags + writeInt32(0); // Predefined + writeFourcc("mdta"); + writeInt32(0); // Reserved[0] + writeInt32(0); // Reserved[1] + writeInt32(0); // Reserved[2] + writeInt8(0); // Name (empty) + endBox(); +} + +void MPEG4Writer::writeKeys() { + size_t count = mMetaKeys->countEntries(); + + beginBox("keys"); + writeInt32(0); // Version, Flags + writeInt32(count); // Entry_count + for (size_t i = 0; i < count; i++) { + AMessage::Type type; + const char *key = mMetaKeys->getEntryNameAt(i, &type); + size_t n = strlen(key); + writeInt32(n + 8); + writeFourcc("mdta"); + write(key, n); // write without the \0 + } + endBox(); +} + +void MPEG4Writer::writeIlst() { + size_t count = mMetaKeys->countEntries(); + + beginBox("ilst"); + for (size_t i = 0; i < count; i++) { + beginBox(i + 1); // key id (1-based) + beginBox("data"); + AMessage::Type type; + const char *key = mMetaKeys->getEntryNameAt(i, &type); + switch (type) { + case AMessage::kTypeString: + { + AString val; + CHECK(mMetaKeys->findString(key, &val)); + writeInt32(1); // type = UTF8 + writeInt32(0); // default country/language + write(val.c_str(), strlen(val.c_str())); // write without \0 + break; + } + + case AMessage::kTypeFloat: + { + float val; + CHECK(mMetaKeys->findFloat(key, &val)); + writeInt32(23); // type = float32 + writeInt32(0); // default country/language + writeInt32(*reinterpret_cast<int32_t *>(&val)); + break; + } + + case AMessage::kTypeInt32: + { + int32_t val; + CHECK(mMetaKeys->findInt32(key, &val)); + writeInt32(67); // type = signed int32 + writeInt32(0); // default country/language + writeInt32(val); + break; + } + + default: + { + ALOGW("Unsupported key type, writing 0 instead"); + writeInt32(77); // type = unsigned int32 + writeInt32(0); // default country/language + writeInt32(0); + break; + } + } + endBox(); // data + endBox(); // key id + } + endBox(); // ilst +} + +void MPEG4Writer::writeMetaBox() { + size_t count = mMetaKeys->countEntries(); + if (count == 0) { + return; + } + + beginBox("meta"); + writeHdlr(); + writeKeys(); + writeIlst(); + endBox(); +} + /* * Geodata is stored according to ISO-6709 standard. */ diff --git a/media/libstagefright/MediaClock.cpp b/media/libstagefright/MediaClock.cpp new file mode 100644 index 0000000..433f555 --- /dev/null +++ b/media/libstagefright/MediaClock.cpp @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "MediaClock" +#include <utils/Log.h> + +#include <media/stagefright/MediaClock.h> + +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/ALooper.h> + +namespace android { + +MediaClock::MediaClock() + : mAnchorTimeMediaUs(-1), + mAnchorTimeRealUs(-1), + mMaxTimeMediaUs(INT64_MAX), + mStartingTimeMediaUs(-1), + mPlaybackRate(1.0) { +} + +MediaClock::~MediaClock() { +} + +void MediaClock::setStartingTimeMedia(int64_t startingTimeMediaUs) { + Mutex::Autolock autoLock(mLock); + mStartingTimeMediaUs = startingTimeMediaUs; +} + +void MediaClock::clearAnchor() { + Mutex::Autolock autoLock(mLock); + mAnchorTimeMediaUs = -1; + mAnchorTimeRealUs = -1; +} + +void MediaClock::updateAnchor( + int64_t anchorTimeMediaUs, + int64_t anchorTimeRealUs, + int64_t maxTimeMediaUs) { + if (anchorTimeMediaUs < 0 || anchorTimeRealUs < 0) { + ALOGW("reject anchor time since it is negative."); + return; + } + + Mutex::Autolock autoLock(mLock); + int64_t nowUs = ALooper::GetNowUs(); + int64_t nowMediaUs = + anchorTimeMediaUs + (nowUs - anchorTimeRealUs) * (double)mPlaybackRate; + if (nowMediaUs < 0) { + ALOGW("reject anchor time since it leads to negative media time."); + return; + } + mAnchorTimeRealUs = nowUs; + mAnchorTimeMediaUs = nowMediaUs; + mMaxTimeMediaUs = maxTimeMediaUs; +} + +void MediaClock::updateMaxTimeMedia(int64_t maxTimeMediaUs) { + Mutex::Autolock autoLock(mLock); + mMaxTimeMediaUs = maxTimeMediaUs; +} + +void MediaClock::setPlaybackRate(float rate) { + CHECK_GE(rate, 0.0); + Mutex::Autolock autoLock(mLock); + if (mAnchorTimeRealUs == -1) { + mPlaybackRate = rate; + return; + } + + int64_t nowUs = ALooper::GetNowUs(); + mAnchorTimeMediaUs += (nowUs - mAnchorTimeRealUs) * (double)mPlaybackRate; + if (mAnchorTimeMediaUs < 0) { + ALOGW("setRate: anchor time should not be negative, set to 0."); + mAnchorTimeMediaUs = 0; + } + mAnchorTimeRealUs = nowUs; + mPlaybackRate = rate; +} + +status_t MediaClock::getMediaTime( + int64_t realUs, int64_t *outMediaUs, bool allowPastMaxTime) const { + if (outMediaUs == NULL) { + return BAD_VALUE; + } + + Mutex::Autolock autoLock(mLock); + return getMediaTime_l(realUs, outMediaUs, allowPastMaxTime); +} + +status_t MediaClock::getMediaTime_l( + int64_t realUs, int64_t *outMediaUs, bool allowPastMaxTime) const { + if (mAnchorTimeRealUs == -1) { + return NO_INIT; + } + + int64_t mediaUs = mAnchorTimeMediaUs + + (realUs - mAnchorTimeRealUs) * (double)mPlaybackRate; + if (mediaUs > mMaxTimeMediaUs && !allowPastMaxTime) { + mediaUs = mMaxTimeMediaUs; + } + if (mediaUs < mStartingTimeMediaUs) { + mediaUs = mStartingTimeMediaUs; + } + if (mediaUs < 0) { + mediaUs = 0; + } + *outMediaUs = mediaUs; + return OK; +} + +status_t MediaClock::getRealTimeFor( + int64_t targetMediaUs, int64_t *outRealUs) const { + if (outRealUs == NULL) { + return BAD_VALUE; + } + + Mutex::Autolock autoLock(mLock); + if (mPlaybackRate == 0.0) { + return NO_INIT; + } + + int64_t nowUs = ALooper::GetNowUs(); + int64_t nowMediaUs; + status_t status = + getMediaTime_l(nowUs, &nowMediaUs, true /* allowPastMaxTime */); + if (status != OK) { + return status; + } + *outRealUs = (targetMediaUs - nowMediaUs) / (double)mPlaybackRate + nowUs; + return OK; +} + +} // namespace android diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp index 6ca123a..0597f1d 100644 --- a/media/libstagefright/MediaCodec.cpp +++ b/media/libstagefright/MediaCodec.cpp @@ -36,6 +36,7 @@ #include <media/stagefright/MediaCodecList.h> #include <media/stagefright/MediaDefs.h> #include <media/stagefright/MediaErrors.h> +#include <media/stagefright/MediaFilter.h> #include <media/stagefright/MetaData.h> #include <media/stagefright/NativeWindowWrapper.h> #include <private/android_filesystem_config.h> @@ -173,7 +174,7 @@ status_t MediaCodec::PostAndAwaitResponse( } // static -void MediaCodec::PostReplyWithError(int32_t replyID, int32_t err) { +void MediaCodec::PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err) { sp<AMessage> response = new AMessage; response->setInt32("err", err); response->postReply(replyID); @@ -189,7 +190,16 @@ status_t MediaCodec::init(const AString &name, bool nameIsType, bool encoder) { // quickly, violating the OpenMAX specs, until that is remedied // we need to invest in an extra looper to free the main event // queue. - mCodec = new ACodec; + + if (nameIsType || !strncasecmp(name.c_str(), "omx.", 4)) { + mCodec = new ACodec; + } else if (!nameIsType + && !strncasecmp(name.c_str(), "android.filter.", 15)) { + mCodec = new MediaFilter; + } else { + return NAME_NOT_FOUND; + } + bool needDedicatedLooper = false; if (nameIsType && !strncasecmp(name.c_str(), "video/", 6)) { needDedicatedLooper = true; @@ -227,9 +237,9 @@ status_t MediaCodec::init(const AString &name, bool nameIsType, bool encoder) { mLooper->registerHandler(this); - mCodec->setNotificationMessage(new AMessage(kWhatCodecNotify, id())); + mCodec->setNotificationMessage(new AMessage(kWhatCodecNotify, this)); - sp<AMessage> msg = new AMessage(kWhatInit, id()); + sp<AMessage> msg = new AMessage(kWhatInit, this); msg->setString("name", name); msg->setInt32("nameIsType", nameIsType); @@ -242,7 +252,7 @@ status_t MediaCodec::init(const AString &name, bool nameIsType, bool encoder) { } status_t MediaCodec::setCallback(const sp<AMessage> &callback) { - sp<AMessage> msg = new AMessage(kWhatSetCallback, id()); + sp<AMessage> msg = new AMessage(kWhatSetCallback, this); msg->setMessage("callback", callback); sp<AMessage> response; @@ -254,7 +264,7 @@ status_t MediaCodec::configure( const sp<Surface> &nativeWindow, const sp<ICrypto> &crypto, uint32_t flags) { - sp<AMessage> msg = new AMessage(kWhatConfigure, id()); + sp<AMessage> msg = new AMessage(kWhatConfigure, this); msg->setMessage("format", format); msg->setInt32("flags", flags); @@ -288,7 +298,7 @@ status_t MediaCodec::configure( status_t MediaCodec::createInputSurface( sp<IGraphicBufferProducer>* bufferProducer) { - sp<AMessage> msg = new AMessage(kWhatCreateInputSurface, id()); + sp<AMessage> msg = new AMessage(kWhatCreateInputSurface, this); sp<AMessage> response; status_t err = PostAndAwaitResponse(msg, &response); @@ -307,21 +317,21 @@ status_t MediaCodec::createInputSurface( } status_t MediaCodec::start() { - sp<AMessage> msg = new AMessage(kWhatStart, id()); + sp<AMessage> msg = new AMessage(kWhatStart, this); sp<AMessage> response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::stop() { - sp<AMessage> msg = new AMessage(kWhatStop, id()); + sp<AMessage> msg = new AMessage(kWhatStop, this); sp<AMessage> response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::release() { - sp<AMessage> msg = new AMessage(kWhatRelease, id()); + sp<AMessage> msg = new AMessage(kWhatRelease, this); sp<AMessage> response; return PostAndAwaitResponse(msg, &response); @@ -373,7 +383,7 @@ status_t MediaCodec::queueInputBuffer( errorDetailMsg->clear(); } - sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, id()); + sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this); msg->setSize("index", index); msg->setSize("offset", offset); msg->setSize("size", size); @@ -400,7 +410,7 @@ status_t MediaCodec::queueSecureInputBuffer( errorDetailMsg->clear(); } - sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, id()); + sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this); msg->setSize("index", index); msg->setSize("offset", offset); msg->setPointer("subSamples", (void *)subSamples); @@ -419,7 +429,7 @@ status_t MediaCodec::queueSecureInputBuffer( } status_t MediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) { - sp<AMessage> msg = new AMessage(kWhatDequeueInputBuffer, id()); + sp<AMessage> msg = new AMessage(kWhatDequeueInputBuffer, this); msg->setInt64("timeoutUs", timeoutUs); sp<AMessage> response; @@ -440,7 +450,7 @@ status_t MediaCodec::dequeueOutputBuffer( int64_t *presentationTimeUs, uint32_t *flags, int64_t timeoutUs) { - sp<AMessage> msg = new AMessage(kWhatDequeueOutputBuffer, id()); + sp<AMessage> msg = new AMessage(kWhatDequeueOutputBuffer, this); msg->setInt64("timeoutUs", timeoutUs); sp<AMessage> response; @@ -459,7 +469,7 @@ status_t MediaCodec::dequeueOutputBuffer( } status_t MediaCodec::renderOutputBufferAndRelease(size_t index) { - sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, id()); + sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this); msg->setSize("index", index); msg->setInt32("render", true); @@ -468,7 +478,7 @@ status_t MediaCodec::renderOutputBufferAndRelease(size_t index) { } status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestampNs) { - sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, id()); + sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this); msg->setSize("index", index); msg->setInt32("render", true); msg->setInt64("timestampNs", timestampNs); @@ -478,7 +488,7 @@ status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestam } status_t MediaCodec::releaseOutputBuffer(size_t index) { - sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, id()); + sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this); msg->setSize("index", index); sp<AMessage> response; @@ -486,14 +496,14 @@ status_t MediaCodec::releaseOutputBuffer(size_t index) { } status_t MediaCodec::signalEndOfInputStream() { - sp<AMessage> msg = new AMessage(kWhatSignalEndOfInputStream, id()); + sp<AMessage> msg = new AMessage(kWhatSignalEndOfInputStream, this); sp<AMessage> response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::getOutputFormat(sp<AMessage> *format) const { - sp<AMessage> msg = new AMessage(kWhatGetOutputFormat, id()); + sp<AMessage> msg = new AMessage(kWhatGetOutputFormat, this); sp<AMessage> response; status_t err; @@ -507,7 +517,7 @@ status_t MediaCodec::getOutputFormat(sp<AMessage> *format) const { } status_t MediaCodec::getInputFormat(sp<AMessage> *format) const { - sp<AMessage> msg = new AMessage(kWhatGetInputFormat, id()); + sp<AMessage> msg = new AMessage(kWhatGetInputFormat, this); sp<AMessage> response; status_t err; @@ -521,7 +531,7 @@ status_t MediaCodec::getInputFormat(sp<AMessage> *format) const { } status_t MediaCodec::getName(AString *name) const { - sp<AMessage> msg = new AMessage(kWhatGetName, id()); + sp<AMessage> msg = new AMessage(kWhatGetName, this); sp<AMessage> response; status_t err; @@ -535,7 +545,7 @@ status_t MediaCodec::getName(AString *name) const { } status_t MediaCodec::getInputBuffers(Vector<sp<ABuffer> > *buffers) const { - sp<AMessage> msg = new AMessage(kWhatGetBuffers, id()); + sp<AMessage> msg = new AMessage(kWhatGetBuffers, this); msg->setInt32("portIndex", kPortIndexInput); msg->setPointer("buffers", buffers); @@ -544,7 +554,7 @@ status_t MediaCodec::getInputBuffers(Vector<sp<ABuffer> > *buffers) const { } status_t MediaCodec::getOutputBuffers(Vector<sp<ABuffer> > *buffers) const { - sp<AMessage> msg = new AMessage(kWhatGetBuffers, id()); + sp<AMessage> msg = new AMessage(kWhatGetBuffers, this); msg->setInt32("portIndex", kPortIndexOutput); msg->setPointer("buffers", buffers); @@ -602,20 +612,20 @@ status_t MediaCodec::getBufferAndFormat( } status_t MediaCodec::flush() { - sp<AMessage> msg = new AMessage(kWhatFlush, id()); + sp<AMessage> msg = new AMessage(kWhatFlush, this); sp<AMessage> response; return PostAndAwaitResponse(msg, &response); } status_t MediaCodec::requestIDRFrame() { - (new AMessage(kWhatRequestIDRFrame, id()))->post(); + (new AMessage(kWhatRequestIDRFrame, this))->post(); return OK; } void MediaCodec::requestActivityNotification(const sp<AMessage> ¬ify) { - sp<AMessage> msg = new AMessage(kWhatRequestActivityNotification, id()); + sp<AMessage> msg = new AMessage(kWhatRequestActivityNotification, this); msg->setMessage("notify", notify); msg->post(); } @@ -640,7 +650,7 @@ void MediaCodec::cancelPendingDequeueOperations() { } } -bool MediaCodec::handleDequeueInputBuffer(uint32_t replyID, bool newRequest) { +bool MediaCodec::handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest) { if (!isExecuting() || (mFlags & kFlagIsAsync) || (newRequest && (mFlags & kFlagDequeueInputPending))) { PostReplyWithError(replyID, INVALID_OPERATION); @@ -664,7 +674,7 @@ bool MediaCodec::handleDequeueInputBuffer(uint32_t replyID, bool newRequest) { return true; } -bool MediaCodec::handleDequeueOutputBuffer(uint32_t replyID, bool newRequest) { +bool MediaCodec::handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest) { sp<AMessage> response = new AMessage; if (!isExecuting() || (mFlags & kFlagIsAsync) @@ -1188,7 +1198,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { case kWhatInit: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (mState != UNINITIALIZED) { @@ -1224,7 +1234,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { case kWhatSetCallback: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (mState == UNINITIALIZED @@ -1256,7 +1266,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { case kWhatConfigure: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (mState != INITIALIZED) { @@ -1313,7 +1323,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { case kWhatCreateInputSurface: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); // Must be configured, but can't have been started yet. @@ -1329,7 +1339,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { case kWhatStart: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (mState == FLUSHED) { @@ -1355,7 +1365,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { State targetState = (msg->what() == kWhatStop) ? INITIALIZED : UNINITIALIZED; - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (!((mFlags & kFlagIsComponentAllocated) && targetState == UNINITIALIZED) // See 1 @@ -1403,7 +1413,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { case kWhatDequeueInputBuffer: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (mFlags & kFlagIsAsync) { @@ -1435,7 +1445,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { if (timeoutUs > 0ll) { sp<AMessage> timeoutMsg = - new AMessage(kWhatDequeueInputTimedOut, id()); + new AMessage(kWhatDequeueInputTimedOut, this); timeoutMsg->setInt32( "generation", ++mDequeueInputTimeoutGeneration); timeoutMsg->post(timeoutUs); @@ -1464,7 +1474,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { case kWhatQueueInputBuffer: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (!isExecuting()) { @@ -1483,7 +1493,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { case kWhatDequeueOutputBuffer: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (mFlags & kFlagIsAsync) { @@ -1509,7 +1519,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { if (timeoutUs > 0ll) { sp<AMessage> timeoutMsg = - new AMessage(kWhatDequeueOutputTimedOut, id()); + new AMessage(kWhatDequeueOutputTimedOut, this); timeoutMsg->setInt32( "generation", ++mDequeueOutputTimeoutGeneration); timeoutMsg->post(timeoutUs); @@ -1538,7 +1548,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { case kWhatReleaseOutputBuffer: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (!isExecuting()) { @@ -1557,7 +1567,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { case kWhatSignalEndOfInputStream: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (!isExecuting()) { @@ -1575,7 +1585,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { case kWhatGetBuffers: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (!isExecuting() || (mFlags & kFlagIsAsync)) { @@ -1609,7 +1619,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { case kWhatFlush: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (!isExecuting()) { @@ -1635,7 +1645,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { sp<AMessage> format = (msg->what() == kWhatGetOutputFormat ? mOutputFormat : mInputFormat); - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if ((mState != CONFIGURED && mState != STARTING && @@ -1672,7 +1682,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { case kWhatGetName: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (mComponentName.empty()) { @@ -1688,7 +1698,7 @@ void MediaCodec::onMessageReceived(const sp<AMessage> &msg) { case kWhatSetParameters: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); sp<AMessage> params; @@ -1742,7 +1752,7 @@ status_t MediaCodec::queueCSDInputBuffer(size_t bufferIndex) { AString errorDetailMsg; - sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, id()); + sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this); msg->setSize("index", bufferIndex); msg->setSize("offset", 0); msg->setSize("size", csd->size()); @@ -2197,7 +2207,7 @@ void MediaCodec::postActivityNotificationIfPossible() { } status_t MediaCodec::setParameters(const sp<AMessage> ¶ms) { - sp<AMessage> msg = new AMessage(kWhatSetParameters, id()); + sp<AMessage> msg = new AMessage(kWhatSetParameters, this); msg->setMessage("params", params); sp<AMessage> response; diff --git a/media/libstagefright/MediaCodecSource.cpp b/media/libstagefright/MediaCodecSource.cpp index c26e909..b6fa810 100644 --- a/media/libstagefright/MediaCodecSource.cpp +++ b/media/libstagefright/MediaCodecSource.cpp @@ -121,7 +121,7 @@ status_t MediaCodecSource::Puller::start(const sp<MetaData> &meta, mLooper->registerHandler(this); mNotify = notify; - sp<AMessage> msg = new AMessage(kWhatStart, id()); + sp<AMessage> msg = new AMessage(kWhatStart, this); msg->setObject("meta", meta); return postSynchronouslyAndReturnError(msg); } @@ -137,19 +137,19 @@ void MediaCodecSource::Puller::stop() { mSource->stop(); ALOGV("source (%s) stopped", mIsAudio ? "audio" : "video"); - (new AMessage(kWhatStop, id()))->post(); + (new AMessage(kWhatStop, this))->post(); } void MediaCodecSource::Puller::pause() { - (new AMessage(kWhatPause, id()))->post(); + (new AMessage(kWhatPause, this))->post(); } void MediaCodecSource::Puller::resume() { - (new AMessage(kWhatResume, id()))->post(); + (new AMessage(kWhatResume, this))->post(); } void MediaCodecSource::Puller::schedulePull() { - sp<AMessage> msg = new AMessage(kWhatPull, id()); + sp<AMessage> msg = new AMessage(kWhatPull, this); msg->setInt32("generation", mPullGeneration); msg->post(); } @@ -182,7 +182,7 @@ void MediaCodecSource::Puller::onMessageReceived(const sp<AMessage> &msg) { sp<AMessage> response = new AMessage; response->setInt32("err", err); - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); response->postReply(replyID); break; @@ -269,13 +269,13 @@ sp<MediaCodecSource> MediaCodecSource::Create( } status_t MediaCodecSource::start(MetaData* params) { - sp<AMessage> msg = new AMessage(kWhatStart, mReflector->id()); + sp<AMessage> msg = new AMessage(kWhatStart, mReflector); msg->setObject("meta", params); return postSynchronouslyAndReturnError(msg); } status_t MediaCodecSource::stop() { - sp<AMessage> msg = new AMessage(kWhatStop, mReflector->id()); + sp<AMessage> msg = new AMessage(kWhatStop, mReflector); status_t err = postSynchronouslyAndReturnError(msg); // mPuller->stop() needs to be done outside MediaCodecSource's looper, @@ -294,7 +294,7 @@ status_t MediaCodecSource::stop() { } status_t MediaCodecSource::pause() { - (new AMessage(kWhatPause, mReflector->id()))->post(); + (new AMessage(kWhatPause, mReflector))->post(); return OK; } @@ -422,8 +422,7 @@ status_t MediaCodecSource::initEncoder() { } } - mEncoderActivityNotify = new AMessage( - kWhatEncoderActivity, mReflector->id()); + mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, mReflector); mEncoder->setCallback(mEncoderActivityNotify); err = mEncoder->start(); @@ -492,7 +491,7 @@ void MediaCodecSource::signalEOS(status_t err) { if (mStopping && mEncoderReachedEOS) { ALOGI("encoder (%s) stopped", mIsVideo ? "video" : "audio"); // posting reply to everyone that's waiting - List<uint32_t>::iterator it; + List<sp<AReplyToken>>::iterator it; for (it = mStopReplyIDQueue.begin(); it != mStopReplyIDQueue.end(); it++) { (new AMessage)->postReply(*it); @@ -620,8 +619,7 @@ status_t MediaCodecSource::onStart(MetaData *params) { resume(startTimeUs); } else { CHECK(mPuller != NULL); - sp<AMessage> notify = new AMessage( - kWhatPullerNotify, mReflector->id()); + sp<AMessage> notify = new AMessage(kWhatPullerNotify, mReflector); err = mPuller->start(params, notify); if (err != OK) { return err; @@ -768,7 +766,7 @@ void MediaCodecSource::onMessageReceived(const sp<AMessage> &msg) { } case kWhatStart: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); sp<RefBase> obj; @@ -784,7 +782,7 @@ void MediaCodecSource::onMessageReceived(const sp<AMessage> &msg) { { ALOGI("encoder (%s) stopping", mIsVideo ? "video" : "audio"); - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); if (mEncoderReachedEOS) { diff --git a/media/libstagefright/MediaMuxer.cpp b/media/libstagefright/MediaMuxer.cpp index c7c6f34..b13877d 100644 --- a/media/libstagefright/MediaMuxer.cpp +++ b/media/libstagefright/MediaMuxer.cpp @@ -38,21 +38,6 @@ namespace android { -MediaMuxer::MediaMuxer(const char *path, OutputFormat format) - : mFormat(format), - mState(UNINITIALIZED) { - if (format == OUTPUT_FORMAT_MPEG_4) { - mWriter = new MPEG4Writer(path); - } else if (format == OUTPUT_FORMAT_WEBM) { - mWriter = new WebmWriter(path); - } - - if (mWriter != NULL) { - mFileMeta = new MetaData; - mState = INITIALIZED; - } -} - MediaMuxer::MediaMuxer(int fd, OutputFormat format) : mFormat(format), mState(UNINITIALIZED) { diff --git a/media/libstagefright/MediaSync.cpp b/media/libstagefright/MediaSync.cpp new file mode 100644 index 0000000..7b6c7d9 --- /dev/null +++ b/media/libstagefright/MediaSync.cpp @@ -0,0 +1,541 @@ +/* + * Copyright 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "MediaSync" +#include <inttypes.h> + +#include <gui/BufferQueue.h> +#include <gui/IGraphicBufferConsumer.h> +#include <gui/IGraphicBufferProducer.h> + +#include <media/AudioTrack.h> +#include <media/stagefright/MediaClock.h> +#include <media/stagefright/MediaSync.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/ALooper.h> +#include <media/stagefright/foundation/AMessage.h> + +#include <ui/GraphicBuffer.h> + +// Maximum late time allowed for a video frame to be rendered. When a video +// frame arrives later than this number, it will be discarded without rendering. +static const int64_t kMaxAllowedVideoLateTimeUs = 40000ll; + +namespace android { + +// static +sp<MediaSync> MediaSync::create() { + sp<MediaSync> sync = new MediaSync(); + sync->mLooper->registerHandler(sync); + return sync; +} + +MediaSync::MediaSync() + : mIsAbandoned(false), + mMutex(), + mReleaseCondition(), + mNumOutstandingBuffers(0), + mNativeSampleRateInHz(0), + mNumFramesWritten(0), + mHasAudio(false), + mNextBufferItemMediaUs(-1), + mPlaybackRate(0.0) { + mMediaClock = new MediaClock; + + mLooper = new ALooper; + mLooper->setName("MediaSync"); + mLooper->start(false, false, ANDROID_PRIORITY_AUDIO); +} + +MediaSync::~MediaSync() { + if (mInput != NULL) { + mInput->consumerDisconnect(); + } + if (mOutput != NULL) { + mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); + } + + if (mLooper != NULL) { + mLooper->unregisterHandler(id()); + mLooper->stop(); + } +} + +status_t MediaSync::configureSurface(const sp<IGraphicBufferProducer> &output) { + Mutex::Autolock lock(mMutex); + + // TODO: support suface change. + if (mOutput != NULL) { + ALOGE("configureSurface: output surface has already been configured."); + return INVALID_OPERATION; + } + + if (output != NULL) { + IGraphicBufferProducer::QueueBufferOutput queueBufferOutput; + sp<OutputListener> listener(new OutputListener(this)); + IInterface::asBinder(output)->linkToDeath(listener); + status_t status = + output->connect(listener, + NATIVE_WINDOW_API_MEDIA, + true /* producerControlledByApp */, + &queueBufferOutput); + if (status != NO_ERROR) { + ALOGE("configureSurface: failed to connect (%d)", status); + return status; + } + + mOutput = output; + } + + return NO_ERROR; +} + +// |audioTrack| is used only for querying information. +status_t MediaSync::configureAudioTrack( + const sp<AudioTrack> &audioTrack, uint32_t nativeSampleRateInHz) { + Mutex::Autolock lock(mMutex); + + // TODO: support audio track change. + if (mAudioTrack != NULL) { + ALOGE("configureAudioTrack: audioTrack has already been configured."); + return INVALID_OPERATION; + } + + mAudioTrack = audioTrack; + mNativeSampleRateInHz = nativeSampleRateInHz; + + return NO_ERROR; +} + +status_t MediaSync::createInputSurface( + sp<IGraphicBufferProducer> *outBufferProducer) { + if (outBufferProducer == NULL) { + return BAD_VALUE; + } + + Mutex::Autolock lock(mMutex); + + if (mOutput == NULL) { + return NO_INIT; + } + + if (mInput != NULL) { + return INVALID_OPERATION; + } + + sp<IGraphicBufferProducer> bufferProducer; + sp<IGraphicBufferConsumer> bufferConsumer; + BufferQueue::createBufferQueue(&bufferProducer, &bufferConsumer); + + sp<InputListener> listener(new InputListener(this)); + IInterface::asBinder(bufferConsumer)->linkToDeath(listener); + status_t status = + bufferConsumer->consumerConnect(listener, false /* controlledByApp */); + if (status == NO_ERROR) { + bufferConsumer->setConsumerName(String8("MediaSync")); + *outBufferProducer = bufferProducer; + mInput = bufferConsumer; + } + return status; +} + +status_t MediaSync::setPlaybackRate(float rate) { + if (rate < 0.0) { + return BAD_VALUE; + } + + Mutex::Autolock lock(mMutex); + + if (rate > mPlaybackRate) { + mNextBufferItemMediaUs = -1; + } + mPlaybackRate = rate; + mMediaClock->setPlaybackRate(rate); + onDrainVideo_l(); + + return OK; +} + +sp<const MediaClock> MediaSync::getMediaClock() { + return mMediaClock; +} + +status_t MediaSync::updateQueuedAudioData( + size_t sizeInBytes, int64_t presentationTimeUs) { + if (sizeInBytes == 0) { + return OK; + } + + Mutex::Autolock lock(mMutex); + + if (mAudioTrack == NULL) { + ALOGW("updateQueuedAudioData: audioTrack has NOT been configured."); + return INVALID_OPERATION; + } + + int64_t numFrames = sizeInBytes / mAudioTrack->frameSize(); + int64_t maxMediaTimeUs = presentationTimeUs + + getDurationIfPlayedAtNativeSampleRate_l(numFrames); + mNumFramesWritten += numFrames; + + int64_t nowUs = ALooper::GetNowUs(); + int64_t nowMediaUs = maxMediaTimeUs + - getDurationIfPlayedAtNativeSampleRate_l(mNumFramesWritten) + + getPlayedOutAudioDurationMedia_l(nowUs); + + int64_t oldRealTime = -1; + if (mNextBufferItemMediaUs != -1) { + oldRealTime = getRealTime(mNextBufferItemMediaUs, nowUs); + } + + mMediaClock->updateAnchor(nowMediaUs, nowUs, maxMediaTimeUs); + mHasAudio = true; + + if (oldRealTime != -1) { + int64_t newRealTime = getRealTime(mNextBufferItemMediaUs, nowUs); + if (newRealTime < oldRealTime) { + mNextBufferItemMediaUs = -1; + onDrainVideo_l(); + } + } + + return OK; +} + +void MediaSync::setName(const AString &name) { + Mutex::Autolock lock(mMutex); + mInput->setConsumerName(String8(name.c_str())); +} + +int64_t MediaSync::getRealTime(int64_t mediaTimeUs, int64_t nowUs) { + int64_t realUs; + if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) { + // If failed to get current position, e.g. due to audio clock is + // not ready, then just play out video immediately without delay. + return nowUs; + } + return realUs; +} + +int64_t MediaSync::getDurationIfPlayedAtNativeSampleRate_l(int64_t numFrames) { + return (numFrames * 1000000LL / mNativeSampleRateInHz); +} + +int64_t MediaSync::getPlayedOutAudioDurationMedia_l(int64_t nowUs) { + CHECK(mAudioTrack != NULL); + + uint32_t numFramesPlayed; + int64_t numFramesPlayedAt; + AudioTimestamp ts; + static const int64_t kStaleTimestamp100ms = 100000; + + status_t res = mAudioTrack->getTimestamp(ts); + if (res == OK) { + // case 1: mixing audio tracks. + numFramesPlayed = ts.mPosition; + numFramesPlayedAt = + ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000; + const int64_t timestampAge = nowUs - numFramesPlayedAt; + if (timestampAge > kStaleTimestamp100ms) { + // This is an audio FIXME. + // getTimestamp returns a timestamp which may come from audio + // mixing threads. After pausing, the MixerThread may go idle, + // thus the mTime estimate may become stale. Assuming that the + // MixerThread runs 20ms, with FastMixer at 5ms, the max latency + // should be about 25ms with an average around 12ms (to be + // verified). For safety we use 100ms. + ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) " + "numFramesPlayedAt(%lld)", + (long long)nowUs, (long long)numFramesPlayedAt); + numFramesPlayedAt = nowUs - kStaleTimestamp100ms; + } + //ALOGD("getTimestamp: OK %d %lld", + // numFramesPlayed, (long long)numFramesPlayedAt); + } else if (res == WOULD_BLOCK) { + // case 2: transitory state on start of a new track + numFramesPlayed = 0; + numFramesPlayedAt = nowUs; + //ALOGD("getTimestamp: WOULD_BLOCK %d %lld", + // numFramesPlayed, (long long)numFramesPlayedAt); + } else { + // case 3: transitory at new track or audio fast tracks. + res = mAudioTrack->getPosition(&numFramesPlayed); + CHECK_EQ(res, (status_t)OK); + numFramesPlayedAt = nowUs; + numFramesPlayedAt += 1000LL * mAudioTrack->latency() / 2; /* XXX */ + //ALOGD("getPosition: %d %lld", numFramesPlayed, numFramesPlayedAt); + } + + //can't be negative until 12.4 hrs, test. + //CHECK_EQ(numFramesPlayed & (1 << 31), 0); + int64_t durationUs = + getDurationIfPlayedAtNativeSampleRate_l(numFramesPlayed) + + nowUs - numFramesPlayedAt; + if (durationUs < 0) { + // Occurs when numFramesPlayed position is very small and the following: + // (1) In case 1, the time nowUs is computed before getTimestamp() is + // called and numFramesPlayedAt is greater than nowUs by time more + // than numFramesPlayed. + // (2) In case 3, using getPosition and adding mAudioTrack->latency() + // to numFramesPlayedAt, by a time amount greater than + // numFramesPlayed. + // + // Both of these are transitory conditions. + ALOGV("getPlayedOutAudioDurationMedia_l: negative duration %lld " + "set to zero", (long long)durationUs); + durationUs = 0; + } + ALOGV("getPlayedOutAudioDurationMedia_l(%lld) nowUs(%lld) frames(%u) " + "framesAt(%lld)", + (long long)durationUs, (long long)nowUs, numFramesPlayed, + (long long)numFramesPlayedAt); + return durationUs; +} + +void MediaSync::onDrainVideo_l() { + if (!isPlaying()) { + return; + } + + int64_t nowUs = ALooper::GetNowUs(); + + while (!mBufferItems.empty()) { + BufferItem *bufferItem = &*mBufferItems.begin(); + int64_t itemMediaUs = bufferItem->mTimestamp / 1000; + int64_t itemRealUs = getRealTime(itemMediaUs, nowUs); + if (itemRealUs <= nowUs) { + if (mHasAudio) { + if (nowUs - itemRealUs <= kMaxAllowedVideoLateTimeUs) { + renderOneBufferItem_l(*bufferItem); + } else { + // too late. + returnBufferToInput_l( + bufferItem->mGraphicBuffer, bufferItem->mFence); + } + } else { + // always render video buffer in video-only mode. + renderOneBufferItem_l(*bufferItem); + + // smooth out videos >= 10fps + mMediaClock->updateAnchor( + itemMediaUs, nowUs, itemMediaUs + 100000); + } + + mBufferItems.erase(mBufferItems.begin()); + + if (mBufferItems.empty()) { + mNextBufferItemMediaUs = -1; + } + } else { + if (mNextBufferItemMediaUs == -1 + || mNextBufferItemMediaUs != itemMediaUs) { + sp<AMessage> msg = new AMessage(kWhatDrainVideo, this); + msg->post(itemRealUs - nowUs); + } + break; + } + } +} + +void MediaSync::onFrameAvailableFromInput() { + Mutex::Autolock lock(mMutex); + + // If there are too many outstanding buffers, wait until a buffer is + // released back to the input in onBufferReleased. + while (mNumOutstandingBuffers >= MAX_OUTSTANDING_BUFFERS) { + mReleaseCondition.wait(mMutex); + + // If the sync is abandoned while we are waiting, the release + // condition variable will be broadcast, and we should just return + // without attempting to do anything more (since the input queue will + // also be abandoned). + if (mIsAbandoned) { + return; + } + } + ++mNumOutstandingBuffers; + + // Acquire and detach the buffer from the input. + BufferItem bufferItem; + status_t status = mInput->acquireBuffer(&bufferItem, 0 /* presentWhen */); + if (status != NO_ERROR) { + ALOGE("acquiring buffer from input failed (%d)", status); + return; + } + + ALOGV("acquired buffer %#llx from input", (long long)bufferItem.mGraphicBuffer->getId()); + + status = mInput->detachBuffer(bufferItem.mBuf); + if (status != NO_ERROR) { + ALOGE("detaching buffer from input failed (%d)", status); + if (status == NO_INIT) { + // If the input has been abandoned, move on. + onAbandoned_l(true /* isInput */); + } + return; + } + + mBufferItems.push_back(bufferItem); + onDrainVideo_l(); +} + +void MediaSync::renderOneBufferItem_l( const BufferItem &bufferItem) { + IGraphicBufferProducer::QueueBufferInput queueInput( + bufferItem.mTimestamp, + bufferItem.mIsAutoTimestamp, + bufferItem.mDataSpace, + bufferItem.mCrop, + static_cast<int32_t>(bufferItem.mScalingMode), + bufferItem.mTransform, + bufferItem.mIsDroppable, + bufferItem.mFence); + + // Attach and queue the buffer to the output. + int slot; + status_t status = mOutput->attachBuffer(&slot, bufferItem.mGraphicBuffer); + ALOGE_IF(status != NO_ERROR, "attaching buffer to output failed (%d)", status); + if (status == NO_ERROR) { + IGraphicBufferProducer::QueueBufferOutput queueOutput; + status = mOutput->queueBuffer(slot, queueInput, &queueOutput); + ALOGE_IF(status != NO_ERROR, "queueing buffer to output failed (%d)", status); + } + + if (status != NO_ERROR) { + returnBufferToInput_l(bufferItem.mGraphicBuffer, bufferItem.mFence); + if (status == NO_INIT) { + // If the output has been abandoned, move on. + onAbandoned_l(false /* isInput */); + } + return; + } + + ALOGV("queued buffer %#llx to output", (long long)bufferItem.mGraphicBuffer->getId()); +} + +void MediaSync::onBufferReleasedByOutput() { + Mutex::Autolock lock(mMutex); + + sp<GraphicBuffer> buffer; + sp<Fence> fence; + status_t status = mOutput->detachNextBuffer(&buffer, &fence); + ALOGE_IF(status != NO_ERROR, "detaching buffer from output failed (%d)", status); + + if (status == NO_INIT) { + // If the output has been abandoned, we can't do anything else, + // since buffer is invalid. + onAbandoned_l(false /* isInput */); + return; + } + + ALOGV("detached buffer %#llx from output", (long long)buffer->getId()); + + // If we've been abandoned, we can't return the buffer to the input, so just + // move on. + if (mIsAbandoned) { + return; + } + + returnBufferToInput_l(buffer, fence); +} + +void MediaSync::returnBufferToInput_l( + const sp<GraphicBuffer> &buffer, const sp<Fence> &fence) { + // Attach and release the buffer back to the input. + int consumerSlot; + status_t status = mInput->attachBuffer(&consumerSlot, buffer); + ALOGE_IF(status != NO_ERROR, "attaching buffer to input failed (%d)", status); + if (status == NO_ERROR) { + status = mInput->releaseBuffer(consumerSlot, 0 /* frameNumber */, + EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, fence); + ALOGE_IF(status != NO_ERROR, "releasing buffer to input failed (%d)", status); + } + + if (status != NO_ERROR) { + // TODO: do we need to try to return this buffer later? + return; + } + + ALOGV("released buffer %#llx to input", (long long)buffer->getId()); + + // Notify any waiting onFrameAvailable calls. + --mNumOutstandingBuffers; + mReleaseCondition.signal(); +} + +void MediaSync::onAbandoned_l(bool isInput) { + ALOGE("the %s has abandoned me", (isInput ? "input" : "output")); + if (!mIsAbandoned) { + if (isInput) { + mOutput->disconnect(NATIVE_WINDOW_API_MEDIA); + } else { + mInput->consumerDisconnect(); + } + mIsAbandoned = true; + } + mReleaseCondition.broadcast(); +} + +void MediaSync::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatDrainVideo: + { + Mutex::Autolock lock(mMutex); + onDrainVideo_l(); + break; + } + + default: + TRESPASS(); + break; + } +} + +MediaSync::InputListener::InputListener(const sp<MediaSync> &sync) + : mSync(sync) {} + +MediaSync::InputListener::~InputListener() {} + +void MediaSync::InputListener::onFrameAvailable(const BufferItem &/* item */) { + mSync->onFrameAvailableFromInput(); +} + +// We don't care about sideband streams, since we won't relay them. +void MediaSync::InputListener::onSidebandStreamChanged() { + ALOGE("onSidebandStreamChanged: got sideband stream unexpectedly."); +} + + +void MediaSync::InputListener::binderDied(const wp<IBinder> &/* who */) { + Mutex::Autolock lock(mSync->mMutex); + mSync->onAbandoned_l(true /* isInput */); +} + +MediaSync::OutputListener::OutputListener(const sp<MediaSync> &sync) + : mSync(sync) {} + +MediaSync::OutputListener::~OutputListener() {} + +void MediaSync::OutputListener::onBufferReleased() { + mSync->onBufferReleasedByOutput(); +} + +void MediaSync::OutputListener::binderDied(const wp<IBinder> &/* who */) { + Mutex::Autolock lock(mSync->mMutex); + mSync->onAbandoned_l(false /* isInput */); +} + +} // namespace android diff --git a/media/libstagefright/NuCachedSource2.cpp b/media/libstagefright/NuCachedSource2.cpp index 7d7d631..8d70e50 100644 --- a/media/libstagefright/NuCachedSource2.cpp +++ b/media/libstagefright/NuCachedSource2.cpp @@ -226,7 +226,7 @@ NuCachedSource2::NuCachedSource2( mLooper->start(false /* runOnCallingThread */, true /* canCallJava */); Mutex::Autolock autoLock(mLock); - (new AMessage(kWhatFetchMore, mReflector->id()))->post(); + (new AMessage(kWhatFetchMore, mReflector))->post(); } NuCachedSource2::~NuCachedSource2() { @@ -433,7 +433,7 @@ void NuCachedSource2::onFetch() { delayUs = 100000ll; } - (new AMessage(kWhatFetchMore, mReflector->id()))->post(delayUs); + (new AMessage(kWhatFetchMore, mReflector))->post(delayUs); } void NuCachedSource2::onRead(const sp<AMessage> &msg) { @@ -522,7 +522,7 @@ ssize_t NuCachedSource2::readAt(off64_t offset, void *data, size_t size) { return size; } - sp<AMessage> msg = new AMessage(kWhatRead, mReflector->id()); + sp<AMessage> msg = new AMessage(kWhatRead, mReflector); msg->setInt64("offset", offset); msg->setPointer("data", data); msg->setSize("size", size); diff --git a/media/libstagefright/ProcessInfo.cpp b/media/libstagefright/ProcessInfo.cpp new file mode 100644 index 0000000..b4172b3 --- /dev/null +++ b/media/libstagefright/ProcessInfo.cpp @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "ProcessInfo" +#include <utils/Log.h> + +#include <media/stagefright/ProcessInfo.h> + +#include <binder/IProcessInfoService.h> +#include <binder/IServiceManager.h> + +namespace android { + +ProcessInfo::ProcessInfo() {} + +bool ProcessInfo::getPriority(int pid, int* priority) { + sp<IBinder> binder = defaultServiceManager()->getService(String16("processinfo")); + sp<IProcessInfoService> service = interface_cast<IProcessInfoService>(binder); + + size_t length = 1; + int32_t states; + status_t err = service->getProcessStatesFromPids(length, &pid, &states); + if (err != OK) { + ALOGE("getProcessStatesFromPids failed"); + return false; + } + ALOGV("pid %d states %d", pid, states); + if (states < 0) { + return false; + } + + // Use process state as the priority. Lower the value, higher the priority. + *priority = states; + return true; +} + +ProcessInfo::~ProcessInfo() {} + +} // namespace android diff --git a/media/libstagefright/SurfaceMediaSource.cpp b/media/libstagefright/SurfaceMediaSource.cpp index 530383b..e8abf48 100644 --- a/media/libstagefright/SurfaceMediaSource.cpp +++ b/media/libstagefright/SurfaceMediaSource.cpp @@ -26,6 +26,7 @@ #include <media/hardware/MetadataBufferType.h> #include <ui/GraphicBuffer.h> +#include <gui/BufferItem.h> #include <gui/ISurfaceComposer.h> #include <gui/IGraphicBufferAlloc.h> #include <OMX_Component.h> @@ -290,7 +291,7 @@ status_t SurfaceMediaSource::read( // TODO: mCurrentSlot can be made a bufferstate since there // can be more than one "current" slots. - BufferQueue::BufferItem item; + BufferItem item; // If the recording has started and the queue is empty, then just // wait here till the frames come in from the client side while (mStarted) { diff --git a/media/libstagefright/Utils.cpp b/media/libstagefright/Utils.cpp index b3a79a0..c0be136 100644 --- a/media/libstagefright/Utils.cpp +++ b/media/libstagefright/Utils.cpp @@ -344,6 +344,28 @@ status_t convertMetaDataToMessage( buffer->meta()->setInt32("csd", true); buffer->meta()->setInt64("timeUs", 0); msg->setBuffer("csd-0", buffer); + + if (!meta->findData(kKeyOpusCodecDelay, &type, &data, &size)) { + return -EINVAL; + } + + buffer = new ABuffer(size); + memcpy(buffer->data(), data, size); + + buffer->meta()->setInt32("csd", true); + buffer->meta()->setInt64("timeUs", 0); + msg->setBuffer("csd-1", buffer); + + if (!meta->findData(kKeyOpusSeekPreRoll, &type, &data, &size)) { + return -EINVAL; + } + + buffer = new ABuffer(size); + memcpy(buffer->data(), data, size); + + buffer->meta()->setInt32("csd", true); + buffer->meta()->setInt64("timeUs", 0); + msg->setBuffer("csd-2", buffer); } *format = msg; diff --git a/media/libstagefright/avc_utils.cpp b/media/libstagefright/avc_utils.cpp index 5ec3438..8ef2dca 100644 --- a/media/libstagefright/avc_utils.cpp +++ b/media/libstagefright/avc_utils.cpp @@ -26,6 +26,7 @@ #include <media/stagefright/MediaDefs.h> #include <media/stagefright/MediaErrors.h> #include <media/stagefright/MetaData.h> +#include <utils/misc.h> namespace android { @@ -186,17 +187,31 @@ void FindAVCDimensions( if (aspect_ratio_idc == 255 /* extendedSAR */) { sar_width = br.getBits(16); sar_height = br.getBits(16); - } else if (aspect_ratio_idc > 0 && aspect_ratio_idc < 14) { - static const int32_t kFixedSARWidth[] = { - 1, 12, 10, 16, 40, 24, 20, 32, 80, 18, 15, 64, 160 + } else { + static const struct { unsigned width, height; } kFixedSARs[] = { + { 0, 0 }, // Invalid + { 1, 1 }, + { 12, 11 }, + { 10, 11 }, + { 16, 11 }, + { 40, 33 }, + { 24, 11 }, + { 20, 11 }, + { 32, 11 }, + { 80, 33 }, + { 18, 11 }, + { 15, 11 }, + { 64, 33 }, + { 160, 99 }, + { 4, 3 }, + { 3, 2 }, + { 2, 1 }, }; - static const int32_t kFixedSARHeight[] = { - 1, 11, 11, 11, 33, 11, 11, 11, 33, 11, 11, 33, 99 - }; - - sar_width = kFixedSARWidth[aspect_ratio_idc - 1]; - sar_height = kFixedSARHeight[aspect_ratio_idc - 1]; + if (aspect_ratio_idc > 0 && aspect_ratio_idc < NELEM(kFixedSARs)) { + sar_width = kFixedSARs[aspect_ratio_idc].width; + sar_height = kFixedSARs[aspect_ratio_idc].height; + } } } diff --git a/media/libstagefright/codecs/aacdec/SoftAAC2.cpp b/media/libstagefright/codecs/aacdec/SoftAAC2.cpp index 495bad0..10937ec 100644 --- a/media/libstagefright/codecs/aacdec/SoftAAC2.cpp +++ b/media/libstagefright/codecs/aacdec/SoftAAC2.cpp @@ -623,7 +623,7 @@ void SoftAAC2::onQueueFilled(OMX_U32 /* portIndex */) { } else { int64_t currentTime = mBufferTimestamps.top(); currentTime += mStreamInfo->aacSamplesPerFrame * - 1000000ll / mStreamInfo->sampleRate; + 1000000ll / mStreamInfo->aacSampleRate; mBufferTimestamps.add(currentTime); } } else { @@ -874,7 +874,7 @@ void SoftAAC2::onQueueFilled(OMX_U32 /* portIndex */) { // adjust/interpolate next time stamp *currentBufLeft -= decodedSize; *nextTimeStamp += mStreamInfo->aacSamplesPerFrame * - 1000000ll / mStreamInfo->sampleRate; + 1000000ll / mStreamInfo->aacSampleRate; ALOGV("adjusted nextTimeStamp/size to %lld/%d", (long long) *nextTimeStamp, *currentBufLeft); } else { @@ -975,6 +975,7 @@ void SoftAAC2::onPortFlushCompleted(OMX_U32 portIndex) { mBufferSizes.clear(); mDecodedSizes.clear(); mLastInHeader = NULL; + mEndOfInput = false; } else { int avail; while ((avail = outputDelayRingBufferSamplesAvailable()) > 0) { @@ -989,6 +990,7 @@ void SoftAAC2::onPortFlushCompleted(OMX_U32 portIndex) { mOutputBufferCount++; } mOutputDelayRingBufferReadPos = mOutputDelayRingBufferWritePos; + mEndOfOutput = false; } } diff --git a/media/libstagefright/codecs/on2/dec/SoftVPX.cpp b/media/libstagefright/codecs/on2/dec/SoftVPX.cpp index 8a95643..6e6a78a 100644 --- a/media/libstagefright/codecs/on2/dec/SoftVPX.cpp +++ b/media/libstagefright/codecs/on2/dec/SoftVPX.cpp @@ -38,7 +38,10 @@ SoftVPX::SoftVPX( NULL /* profileLevels */, 0 /* numProfileLevels */, 320 /* width */, 240 /* height */, callbacks, appData, component), mMode(codingType == OMX_VIDEO_CodingVP8 ? MODE_VP8 : MODE_VP9), + mEOSStatus(INPUT_DATA_AVAILABLE), mCtx(NULL), + mFrameParallelMode(false), + mTimeStampIdx(0), mImg(NULL) { // arbitrary from avc/hevc as vpx does not specify a min compression ratio const size_t kMinCompressionRatio = mMode == MODE_VP8 ? 2 : 4; @@ -51,9 +54,7 @@ SoftVPX::SoftVPX( } SoftVPX::~SoftVPX() { - vpx_codec_destroy((vpx_codec_ctx_t *)mCtx); - delete (vpx_codec_ctx_t *)mCtx; - mCtx = NULL; + destroyDecoder(); } static int GetCPUCoreCount() { @@ -73,12 +74,19 @@ status_t SoftVPX::initDecoder() { mCtx = new vpx_codec_ctx_t; vpx_codec_err_t vpx_err; vpx_codec_dec_cfg_t cfg; + vpx_codec_flags_t flags; memset(&cfg, 0, sizeof(vpx_codec_dec_cfg_t)); + memset(&flags, 0, sizeof(vpx_codec_flags_t)); cfg.threads = GetCPUCoreCount(); + + if (mFrameParallelMode) { + flags |= VPX_CODEC_USE_FRAME_THREADING; + } + if ((vpx_err = vpx_codec_dec_init( (vpx_codec_ctx_t *)mCtx, mMode == MODE_VP8 ? &vpx_codec_vp8_dx_algo : &vpx_codec_vp9_dx_algo, - &cfg, 0))) { + &cfg, flags))) { ALOGE("on2 decoder failed to initialize. (%d)", vpx_err); return UNKNOWN_ERROR; } @@ -86,86 +94,155 @@ status_t SoftVPX::initDecoder() { return OK; } +status_t SoftVPX::destroyDecoder() { + vpx_codec_destroy((vpx_codec_ctx_t *)mCtx); + delete (vpx_codec_ctx_t *)mCtx; + mCtx = NULL; + return OK; +} + +bool SoftVPX::outputBuffers(bool flushDecoder, bool display, bool eos, bool *portWillReset) { + List<BufferInfo *> &inQueue = getPortQueue(0); + List<BufferInfo *> &outQueue = getPortQueue(1); + BufferInfo *outInfo = NULL; + OMX_BUFFERHEADERTYPE *outHeader = NULL; + vpx_codec_iter_t iter = NULL; + + if (flushDecoder && mFrameParallelMode) { + // Flush decoder by passing NULL data ptr and 0 size. + // Ideally, this should never fail. + if (vpx_codec_decode((vpx_codec_ctx_t *)mCtx, NULL, 0, NULL, 0)) { + ALOGE("Failed to flush on2 decoder."); + return false; + } + } + + if (!display) { + if (!flushDecoder) { + ALOGE("Invalid operation."); + return false; + } + // Drop all the decoded frames in decoder. + while ((mImg = vpx_codec_get_frame((vpx_codec_ctx_t *)mCtx, &iter))) { + } + return true; + } + + while (!outQueue.empty()) { + if (mImg == NULL) { + mImg = vpx_codec_get_frame((vpx_codec_ctx_t *)mCtx, &iter); + if (mImg == NULL) { + break; + } + } + uint32_t width = mImg->d_w; + uint32_t height = mImg->d_h; + outInfo = *outQueue.begin(); + outHeader = outInfo->mHeader; + CHECK_EQ(mImg->fmt, IMG_FMT_I420); + handlePortSettingsChange(portWillReset, width, height); + if (*portWillReset) { + return true; + } + + outHeader->nOffset = 0; + outHeader->nFilledLen = (width * height * 3) / 2; + outHeader->nFlags = 0; + outHeader->nTimeStamp = *(OMX_TICKS *)mImg->user_priv; + + uint8_t *dst = outHeader->pBuffer; + const uint8_t *srcY = (const uint8_t *)mImg->planes[PLANE_Y]; + const uint8_t *srcU = (const uint8_t *)mImg->planes[PLANE_U]; + const uint8_t *srcV = (const uint8_t *)mImg->planes[PLANE_V]; + size_t srcYStride = mImg->stride[PLANE_Y]; + size_t srcUStride = mImg->stride[PLANE_U]; + size_t srcVStride = mImg->stride[PLANE_V]; + copyYV12FrameToOutputBuffer(dst, srcY, srcU, srcV, srcYStride, srcUStride, srcVStride); + + mImg = NULL; + outInfo->mOwnedByUs = false; + outQueue.erase(outQueue.begin()); + outInfo = NULL; + notifyFillBufferDone(outHeader); + outHeader = NULL; + } + + if (!eos) { + return true; + } + + if (!outQueue.empty()) { + outInfo = *outQueue.begin(); + outQueue.erase(outQueue.begin()); + outHeader = outInfo->mHeader; + outHeader->nTimeStamp = 0; + outHeader->nFilledLen = 0; + outHeader->nFlags = OMX_BUFFERFLAG_EOS; + outInfo->mOwnedByUs = false; + notifyFillBufferDone(outHeader); + mEOSStatus = OUTPUT_FRAMES_FLUSHED; + } + return true; +} + void SoftVPX::onQueueFilled(OMX_U32 /* portIndex */) { - if (mOutputPortSettingsChange != NONE) { + if (mOutputPortSettingsChange != NONE || mEOSStatus == OUTPUT_FRAMES_FLUSHED) { return; } List<BufferInfo *> &inQueue = getPortQueue(0); List<BufferInfo *> &outQueue = getPortQueue(1); bool EOSseen = false; + vpx_codec_err_t err; + bool portWillReset = false; + + while ((mEOSStatus == INPUT_EOS_SEEN || !inQueue.empty()) + && !outQueue.empty()) { + // Output the pending frames that left from last port reset or decoder flush. + if (mEOSStatus == INPUT_EOS_SEEN || mImg != NULL) { + if (!outputBuffers( + mEOSStatus == INPUT_EOS_SEEN, true /* display */, + mEOSStatus == INPUT_EOS_SEEN, &portWillReset)) { + ALOGE("on2 decoder failed to output frame."); + notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); + return; + } + if (portWillReset || mEOSStatus == OUTPUT_FRAMES_FLUSHED || + mEOSStatus == INPUT_EOS_SEEN) { + return; + } + } - while (!inQueue.empty() && !outQueue.empty()) { BufferInfo *inInfo = *inQueue.begin(); OMX_BUFFERHEADERTYPE *inHeader = inInfo->mHeader; + mTimeStamps[mTimeStampIdx] = inHeader->nTimeStamp; BufferInfo *outInfo = *outQueue.begin(); OMX_BUFFERHEADERTYPE *outHeader = outInfo->mHeader; - if (inHeader->nFlags & OMX_BUFFERFLAG_EOS) { + mEOSStatus = INPUT_EOS_SEEN; EOSseen = true; - if (inHeader->nFilledLen == 0) { - inQueue.erase(inQueue.begin()); - inInfo->mOwnedByUs = false; - notifyEmptyBufferDone(inHeader); - - outHeader->nFilledLen = 0; - outHeader->nFlags = OMX_BUFFERFLAG_EOS; - - outQueue.erase(outQueue.begin()); - outInfo->mOwnedByUs = false; - notifyFillBufferDone(outHeader); - return; - } } - if (mImg == NULL) { - if (vpx_codec_decode( - (vpx_codec_ctx_t *)mCtx, - inHeader->pBuffer + inHeader->nOffset, - inHeader->nFilledLen, - NULL, - 0)) { - ALOGE("on2 decoder failed to decode frame."); - - notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); - return; - } - vpx_codec_iter_t iter = NULL; - mImg = vpx_codec_get_frame((vpx_codec_ctx_t *)mCtx, &iter); + if (inHeader->nFilledLen > 0 && + vpx_codec_decode((vpx_codec_ctx_t *)mCtx, + inHeader->pBuffer + inHeader->nOffset, + inHeader->nFilledLen, + &mTimeStamps[mTimeStampIdx], 0)) { + ALOGE("on2 decoder failed to decode frame."); + notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); + return; } + mTimeStampIdx = (mTimeStampIdx + 1) % kNumBuffers; - if (mImg != NULL) { - CHECK_EQ(mImg->fmt, IMG_FMT_I420); - - uint32_t width = mImg->d_w; - uint32_t height = mImg->d_h; - bool portWillReset = false; - handlePortSettingsChange(&portWillReset, width, height); - if (portWillReset) { - return; - } - - outHeader->nOffset = 0; - outHeader->nFilledLen = (width * height * 3) / 2; - outHeader->nFlags = EOSseen ? OMX_BUFFERFLAG_EOS : 0; - outHeader->nTimeStamp = inHeader->nTimeStamp; - - uint8_t *dst = outHeader->pBuffer; - const uint8_t *srcY = (const uint8_t *)mImg->planes[PLANE_Y]; - const uint8_t *srcU = (const uint8_t *)mImg->planes[PLANE_U]; - const uint8_t *srcV = (const uint8_t *)mImg->planes[PLANE_V]; - size_t srcYStride = mImg->stride[PLANE_Y]; - size_t srcUStride = mImg->stride[PLANE_U]; - size_t srcVStride = mImg->stride[PLANE_V]; - copyYV12FrameToOutputBuffer(dst, srcY, srcU, srcV, srcYStride, srcUStride, srcVStride); - - mImg = NULL; - outInfo->mOwnedByUs = false; - outQueue.erase(outQueue.begin()); - outInfo = NULL; - notifyFillBufferDone(outHeader); - outHeader = NULL; + if (!outputBuffers( + EOSseen /* flushDecoder */, true /* display */, EOSseen, &portWillReset)) { + ALOGE("on2 decoder failed to output frame."); + notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); + return; + } + if (portWillReset) { + return; } inInfo->mOwnedByUs = false; @@ -176,6 +253,30 @@ void SoftVPX::onQueueFilled(OMX_U32 /* portIndex */) { } } +void SoftVPX::onPortFlushCompleted(OMX_U32 portIndex) { + if (portIndex == kInputPortIndex) { + bool portWillReset = false; + if (!outputBuffers( + true /* flushDecoder */, false /* display */, false /* eos */, &portWillReset)) { + ALOGE("Failed to flush decoder."); + notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); + return; + } + mEOSStatus = INPUT_DATA_AVAILABLE; + } +} + +void SoftVPX::onReset() { + bool portWillReset = false; + if (!outputBuffers( + true /* flushDecoder */, false /* display */, false /* eos */, &portWillReset)) { + ALOGW("Failed to flush decoder. Try to hard reset decoder"); + destroyDecoder(); + initDecoder(); + } + mEOSStatus = INPUT_DATA_AVAILABLE; +} + } // namespace android android::SoftOMXComponent *createSoftOMXComponent( diff --git a/media/libstagefright/codecs/on2/dec/SoftVPX.h b/media/libstagefright/codecs/on2/dec/SoftVPX.h index 8f68693..8ccbae2 100644 --- a/media/libstagefright/codecs/on2/dec/SoftVPX.h +++ b/media/libstagefright/codecs/on2/dec/SoftVPX.h @@ -38,6 +38,8 @@ protected: virtual ~SoftVPX(); virtual void onQueueFilled(OMX_U32 portIndex); + virtual void onPortFlushCompleted(OMX_U32 portIndex); + virtual void onReset(); private: enum { @@ -49,11 +51,21 @@ private: MODE_VP9 } mMode; - void *mCtx; + enum { + INPUT_DATA_AVAILABLE, // VPX component is ready to decode data. + INPUT_EOS_SEEN, // VPX component saw EOS and is flushing On2 decoder. + OUTPUT_FRAMES_FLUSHED // VPX component finished flushing On2 decoder. + } mEOSStatus; + void *mCtx; + bool mFrameParallelMode; // Frame parallel is only supported by VP9 decoder. + OMX_TICKS mTimeStamps[kNumBuffers]; + uint8_t mTimeStampIdx; vpx_image_t *mImg; status_t initDecoder(); + status_t destroyDecoder(); + bool outputBuffers(bool flushDecoder, bool display, bool eos, bool *portWillReset); DISALLOW_EVIL_CONSTRUCTORS(SoftVPX); }; diff --git a/media/libstagefright/codecs/opus/dec/SoftOpus.cpp b/media/libstagefright/codecs/opus/dec/SoftOpus.cpp index b8084ae..6322dc2 100644 --- a/media/libstagefright/codecs/opus/dec/SoftOpus.cpp +++ b/media/libstagefright/codecs/opus/dec/SoftOpus.cpp @@ -345,9 +345,15 @@ void SoftOpus::onQueueFilled(OMX_U32 portIndex) { } uint8_t channel_mapping[kMaxChannels] = {0}; - memcpy(&channel_mapping, - kDefaultOpusChannelLayout, - kMaxChannelsWithDefaultLayout); + if (mHeader->channels <= kMaxChannelsWithDefaultLayout) { + memcpy(&channel_mapping, + kDefaultOpusChannelLayout, + kMaxChannelsWithDefaultLayout); + } else { + memcpy(&channel_mapping, + mHeader->stream_map, + mHeader->channels); + } int status = OPUS_INVALID_STATE; mDecoder = opus_multistream_decoder_create(kRate, diff --git a/media/libstagefright/colorconversion/SoftwareRenderer.cpp b/media/libstagefright/colorconversion/SoftwareRenderer.cpp index 4e75250..21da707 100644 --- a/media/libstagefright/colorconversion/SoftwareRenderer.cpp +++ b/media/libstagefright/colorconversion/SoftwareRenderer.cpp @@ -98,33 +98,49 @@ void SoftwareRenderer::resetFormatIfChanged(const sp<AMessage> &format) { mCropWidth = mCropRight - mCropLeft + 1; mCropHeight = mCropBottom - mCropTop + 1; - int halFormat; - size_t bufWidth, bufHeight; - - switch (mColorFormat) { - case OMX_COLOR_FormatYUV420Planar: - case OMX_TI_COLOR_FormatYUV420PackedSemiPlanar: - case OMX_COLOR_FormatYUV420SemiPlanar: - { - if (!runningInEmulator()) { + // by default convert everything to RGB565 + int halFormat = HAL_PIXEL_FORMAT_RGB_565; + size_t bufWidth = mCropWidth; + size_t bufHeight = mCropHeight; + + // hardware has YUV12 and RGBA8888 support, so convert known formats + if (!runningInEmulator()) { + switch (mColorFormat) { + case OMX_COLOR_FormatYUV420Planar: + case OMX_COLOR_FormatYUV420SemiPlanar: + case OMX_TI_COLOR_FormatYUV420PackedSemiPlanar: + { halFormat = HAL_PIXEL_FORMAT_YV12; bufWidth = (mCropWidth + 1) & ~1; bufHeight = (mCropHeight + 1) & ~1; break; } - - // fall through. + case OMX_COLOR_Format24bitRGB888: + { + halFormat = HAL_PIXEL_FORMAT_RGB_888; + bufWidth = (mCropWidth + 1) & ~1; + bufHeight = (mCropHeight + 1) & ~1; + break; + } + case OMX_COLOR_Format32bitARGB8888: + case OMX_COLOR_Format32BitRGBA8888: + { + halFormat = HAL_PIXEL_FORMAT_RGBA_8888; + bufWidth = (mCropWidth + 1) & ~1; + bufHeight = (mCropHeight + 1) & ~1; + break; + } + default: + { + break; + } } + } - default: - halFormat = HAL_PIXEL_FORMAT_RGB_565; - bufWidth = mCropWidth; - bufHeight = mCropHeight; - - mConverter = new ColorConverter( - mColorFormat, OMX_COLOR_Format16bitRGB565); - CHECK(mConverter->isValid()); - break; + if (halFormat == HAL_PIXEL_FORMAT_RGB_565) { + mConverter = new ColorConverter( + mColorFormat, OMX_COLOR_Format16bitRGB565); + CHECK(mConverter->isValid()); } CHECK(mNativeWindow != NULL); @@ -201,6 +217,8 @@ void SoftwareRenderer::render( CHECK_EQ(0, mapper.lock( buf->handle, GRALLOC_USAGE_SW_WRITE_OFTEN, bounds, &dst)); + // TODO move the other conversions also into ColorConverter, and + // fix cropping issues (when mCropLeft/Top != 0 or mWidth != mCropWidth) if (mConverter) { mConverter->convert( data, @@ -211,7 +229,8 @@ void SoftwareRenderer::render( 0, 0, mCropWidth - 1, mCropHeight - 1); } else if (mColorFormat == OMX_COLOR_FormatYUV420Planar) { const uint8_t *src_y = (const uint8_t *)data; - const uint8_t *src_u = (const uint8_t *)data + mWidth * mHeight; + const uint8_t *src_u = + (const uint8_t *)data + mWidth * mHeight; const uint8_t *src_v = src_u + (mWidth / 2 * mHeight / 2); uint8_t *dst_y = (uint8_t *)dst; @@ -239,11 +258,9 @@ void SoftwareRenderer::render( } } else if (mColorFormat == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar || mColorFormat == OMX_COLOR_FormatYUV420SemiPlanar) { - const uint8_t *src_y = - (const uint8_t *)data; - - const uint8_t *src_uv = - (const uint8_t *)data + mWidth * (mHeight - mCropTop / 2); + const uint8_t *src_y = (const uint8_t *)data; + const uint8_t *src_uv = (const uint8_t *)data + + mWidth * (mHeight - mCropTop / 2); uint8_t *dst_y = (uint8_t *)dst; @@ -271,6 +288,38 @@ void SoftwareRenderer::render( dst_u += dst_c_stride; dst_v += dst_c_stride; } + } else if (mColorFormat == OMX_COLOR_Format24bitRGB888) { + uint8_t* srcPtr = (uint8_t*)data; + uint8_t* dstPtr = (uint8_t*)dst; + + for (size_t y = 0; y < (size_t)mCropHeight; ++y) { + memcpy(dstPtr, srcPtr, mCropWidth * 3); + srcPtr += mWidth * 3; + dstPtr += buf->stride * 3; + } + } else if (mColorFormat == OMX_COLOR_Format32bitARGB8888) { + uint8_t *srcPtr, *dstPtr; + + for (size_t y = 0; y < (size_t)mCropHeight; ++y) { + srcPtr = (uint8_t*)data + mWidth * 4 * y; + dstPtr = (uint8_t*)dst + buf->stride * 4 * y; + for (size_t x = 0; x < (size_t)mCropWidth; ++x) { + uint8_t a = *srcPtr++; + for (size_t i = 0; i < 3; ++i) { // copy RGB + *dstPtr++ = *srcPtr++; + } + *dstPtr++ = a; // alpha last (ARGB to RGBA) + } + } + } else if (mColorFormat == OMX_COLOR_Format32BitRGBA8888) { + uint8_t* srcPtr = (uint8_t*)data; + uint8_t* dstPtr = (uint8_t*)dst; + + for (size_t y = 0; y < (size_t)mCropHeight; ++y) { + memcpy(dstPtr, srcPtr, mCropWidth * 4); + srcPtr += mWidth * 4; + dstPtr += buf->stride * 4; + } } else { LOG_ALWAYS_FATAL("bad color format %#x", mColorFormat); } diff --git a/media/libstagefright/filters/Android.mk b/media/libstagefright/filters/Android.mk new file mode 100644 index 0000000..36ab444 --- /dev/null +++ b/media/libstagefright/filters/Android.mk @@ -0,0 +1,27 @@ +LOCAL_PATH:= $(call my-dir) +include $(CLEAR_VARS) + +LOCAL_SRC_FILES := \ + ColorConvert.cpp \ + GraphicBufferListener.cpp \ + IntrinsicBlurFilter.cpp \ + MediaFilter.cpp \ + RSFilter.cpp \ + SaturationFilter.cpp \ + saturationARGB.rs \ + SimpleFilter.cpp \ + ZeroFilter.cpp + +LOCAL_C_INCLUDES := \ + $(TOP)/frameworks/native/include/media/openmax \ + $(TOP)/frameworks/rs/cpp \ + $(TOP)/frameworks/rs \ + +intermediates := $(call intermediates-dir-for,STATIC_LIBRARIES,libRS,TARGET,) +LOCAL_C_INCLUDES += $(intermediates) + +LOCAL_CFLAGS += -Wno-multichar + +LOCAL_MODULE:= libstagefright_mediafilter + +include $(BUILD_STATIC_LIBRARY) diff --git a/media/libstagefright/filters/ColorConvert.cpp b/media/libstagefright/filters/ColorConvert.cpp new file mode 100644 index 0000000..a5039f9 --- /dev/null +++ b/media/libstagefright/filters/ColorConvert.cpp @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ColorConvert.h" + +#ifndef max +#define max(a,b) ((a) > (b) ? (a) : (b)) +#endif +#ifndef min +#define min(a,b) ((a) < (b) ? (a) : (b)) +#endif + +namespace android { + +void YUVToRGB( + int32_t y, int32_t u, int32_t v, + int32_t* r, int32_t* g, int32_t* b) { + y -= 16; + u -= 128; + v -= 128; + + *b = 1192 * y + 2066 * u; + *g = 1192 * y - 833 * v - 400 * u; + *r = 1192 * y + 1634 * v; + + *r = min(262143, max(0, *r)); + *g = min(262143, max(0, *g)); + *b = min(262143, max(0, *b)); + + *r >>= 10; + *g >>= 10; + *b >>= 10; +} + +void convertYUV420spToARGB( + uint8_t *pY, uint8_t *pUV, int32_t width, int32_t height, + uint8_t *dest) { + const int32_t bytes_per_pixel = 2; + + for (int32_t i = 0; i < height; i++) { + for (int32_t j = 0; j < width; j++) { + int32_t y = *(pY + i * width + j); + int32_t u = *(pUV + (i/2) * width + bytes_per_pixel * (j/2)); + int32_t v = *(pUV + (i/2) * width + bytes_per_pixel * (j/2) + 1); + + int32_t r, g, b; + YUVToRGB(y, u, v, &r, &g, &b); + + *dest++ = 0xFF; + *dest++ = r; + *dest++ = g; + *dest++ = b; + } + } +} + +void convertYUV420spToRGB888( + uint8_t *pY, uint8_t *pUV, int32_t width, int32_t height, + uint8_t *dest) { + const int32_t bytes_per_pixel = 2; + + for (int32_t i = 0; i < height; i++) { + for (int32_t j = 0; j < width; j++) { + int32_t y = *(pY + i * width + j); + int32_t u = *(pUV + (i/2) * width + bytes_per_pixel * (j/2)); + int32_t v = *(pUV + (i/2) * width + bytes_per_pixel * (j/2) + 1); + + int32_t r, g, b; + YUVToRGB(y, u, v, &r, &g, &b); + + *dest++ = r; + *dest++ = g; + *dest++ = b; + } + } +} + +// HACK - not even slightly optimized +// TODO: remove when RGBA support is added to SoftwareRenderer +void convertRGBAToARGB( + uint8_t *src, int32_t width, int32_t height, uint32_t stride, + uint8_t *dest) { + for (size_t i = 0; i < height; ++i) { + for (size_t j = 0; j < width; ++j) { + uint8_t r = *src++; + uint8_t g = *src++; + uint8_t b = *src++; + uint8_t a = *src++; + *dest++ = a; + *dest++ = r; + *dest++ = g; + *dest++ = b; + } + src += (stride - width) * 4; + } +} + +} // namespace android diff --git a/media/libstagefright/filters/ColorConvert.h b/media/libstagefright/filters/ColorConvert.h new file mode 100644 index 0000000..13faa02 --- /dev/null +++ b/media/libstagefright/filters/ColorConvert.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COLOR_CONVERT_H_ +#define COLOR_CONVERT_H_ + +#include <inttypes.h> + +namespace android { + +void YUVToRGB( + int32_t y, int32_t u, int32_t v, + int32_t* r, int32_t* g, int32_t* b); + +void convertYUV420spToARGB( + uint8_t *pY, uint8_t *pUV, int32_t width, int32_t height, + uint8_t *dest); + +void convertYUV420spToRGB888( + uint8_t *pY, uint8_t *pUV, int32_t width, int32_t height, + uint8_t *dest); + +// TODO: remove when RGBA support is added to SoftwareRenderer +void convertRGBAToARGB( + uint8_t *src, int32_t width, int32_t height, uint32_t stride, + uint8_t *dest); + +} // namespace android + +#endif // COLOR_CONVERT_H_ diff --git a/media/libstagefright/filters/GraphicBufferListener.cpp b/media/libstagefright/filters/GraphicBufferListener.cpp new file mode 100644 index 0000000..66374ba --- /dev/null +++ b/media/libstagefright/filters/GraphicBufferListener.cpp @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "GraphicBufferListener" + +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/MediaErrors.h> + +#include <gui/BufferItem.h> + +#include "GraphicBufferListener.h" + +namespace android { + +status_t GraphicBufferListener::init( + const sp<AMessage> ¬ify, + size_t bufferWidth, size_t bufferHeight, size_t bufferCount) { + mNotify = notify; + + String8 name("GraphicBufferListener"); + BufferQueue::createBufferQueue(&mProducer, &mConsumer); + mConsumer->setConsumerName(name); + mConsumer->setDefaultBufferSize(bufferWidth, bufferHeight); + mConsumer->setConsumerUsageBits(GRALLOC_USAGE_SW_READ_OFTEN); + + status_t err = mConsumer->setMaxAcquiredBufferCount(bufferCount); + if (err != NO_ERROR) { + ALOGE("Unable to set BQ max acquired buffer count to %u: %d", + bufferCount, err); + return err; + } + + wp<BufferQueue::ConsumerListener> listener = + static_cast<BufferQueue::ConsumerListener*>(this); + sp<BufferQueue::ProxyConsumerListener> proxy = + new BufferQueue::ProxyConsumerListener(listener); + + err = mConsumer->consumerConnect(proxy, false); + if (err != NO_ERROR) { + ALOGE("Error connecting to BufferQueue: %s (%d)", + strerror(-err), err); + return err; + } + + ALOGV("init() successful."); + + return OK; +} + +void GraphicBufferListener::onFrameAvailable(const BufferItem& /* item */) { + ALOGV("onFrameAvailable() called"); + + { + Mutex::Autolock autoLock(mMutex); + mNumFramesAvailable++; + } + + sp<AMessage> notify = mNotify->dup(); + mNotify->setWhat(kWhatFrameAvailable); + mNotify->post(); +} + +void GraphicBufferListener::onBuffersReleased() { + ALOGV("onBuffersReleased() called"); + // nothing to do +} + +void GraphicBufferListener::onSidebandStreamChanged() { + ALOGW("GraphicBufferListener cannot consume sideband streams."); + // nothing to do +} + +BufferItem GraphicBufferListener::getBufferItem() { + BufferItem item; + + { + Mutex::Autolock autoLock(mMutex); + if (mNumFramesAvailable <= 0) { + ALOGE("getBuffer() called with no frames available"); + return item; + } + mNumFramesAvailable--; + } + + status_t err = mConsumer->acquireBuffer(&item, 0); + if (err == BufferQueue::NO_BUFFER_AVAILABLE) { + // shouldn't happen, since we track num frames available + ALOGE("frame was not available"); + item.mBuf = -1; + return item; + } else if (err != OK) { + ALOGE("acquireBuffer returned err=%d", err); + item.mBuf = -1; + return item; + } + + // Wait for it to become available. + err = item.mFence->waitForever("GraphicBufferListener::getBufferItem"); + if (err != OK) { + ALOGW("failed to wait for buffer fence: %d", err); + // keep going + } + + // If this is the first time we're seeing this buffer, add it to our + // slot table. + if (item.mGraphicBuffer != NULL) { + ALOGV("setting mBufferSlot %d", item.mBuf); + mBufferSlot[item.mBuf] = item.mGraphicBuffer; + } + + return item; +} + +sp<GraphicBuffer> GraphicBufferListener::getBuffer(BufferItem item) { + sp<GraphicBuffer> buf; + if (item.mBuf < 0 || item.mBuf >= BufferQueue::NUM_BUFFER_SLOTS) { + ALOGE("getBuffer() received invalid BufferItem: mBuf==%d", item.mBuf); + return buf; + } + + buf = mBufferSlot[item.mBuf]; + CHECK(buf.get() != NULL); + + return buf; +} + +status_t GraphicBufferListener::releaseBuffer(BufferItem item) { + if (item.mBuf < 0 || item.mBuf >= BufferQueue::NUM_BUFFER_SLOTS) { + ALOGE("getBuffer() received invalid BufferItem: mBuf==%d", item.mBuf); + return ERROR_OUT_OF_RANGE; + } + + mConsumer->releaseBuffer(item.mBuf, item.mFrameNumber, + EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE); + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/GraphicBufferListener.h b/media/libstagefright/filters/GraphicBufferListener.h new file mode 100644 index 0000000..586bf65 --- /dev/null +++ b/media/libstagefright/filters/GraphicBufferListener.h @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GRAPHIC_BUFFER_LISTENER_H_ +#define GRAPHIC_BUFFER_LISTENER_H_ + +#include <gui/BufferQueue.h> + +namespace android { + +struct AMessage; + +struct GraphicBufferListener : public BufferQueue::ConsumerListener { +public: + GraphicBufferListener() {}; + + status_t init( + const sp<AMessage> ¬ify, + size_t bufferWidth, size_t bufferHeight, size_t bufferCount); + + virtual void onFrameAvailable(const BufferItem& item); + virtual void onBuffersReleased(); + virtual void onSidebandStreamChanged(); + + // Returns the handle to the producer side of the BufferQueue. Buffers + // queued on this will be received by GraphicBufferListener. + sp<IGraphicBufferProducer> getIGraphicBufferProducer() const { + return mProducer; + } + + BufferItem getBufferItem(); + sp<GraphicBuffer> getBuffer(BufferItem item); + status_t releaseBuffer(BufferItem item); + + enum { + kWhatFrameAvailable = 'frav', + }; + +private: + sp<AMessage> mNotify; + size_t mNumFramesAvailable; + + mutable Mutex mMutex; + + // Our BufferQueue interfaces. mProducer is passed to the producer through + // getIGraphicBufferProducer, and mConsumer is used internally to retrieve + // the buffers queued by the producer. + sp<IGraphicBufferProducer> mProducer; + sp<IGraphicBufferConsumer> mConsumer; + + // Cache of GraphicBuffers from the buffer queue. + sp<GraphicBuffer> mBufferSlot[BufferQueue::NUM_BUFFER_SLOTS]; +}; + +} // namespace android + +#endif // GRAPHIC_BUFFER_LISTENER_H diff --git a/media/libstagefright/filters/IntrinsicBlurFilter.cpp b/media/libstagefright/filters/IntrinsicBlurFilter.cpp new file mode 100644 index 0000000..cbcf699 --- /dev/null +++ b/media/libstagefright/filters/IntrinsicBlurFilter.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "IntrinsicBlurFilter" + +#include <utils/Log.h> + +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include "IntrinsicBlurFilter.h" + +namespace android { + +status_t IntrinsicBlurFilter::configure(const sp<AMessage> &msg) { + status_t err = SimpleFilter::configure(msg); + if (err != OK) { + return err; + } + + if (!msg->findString("cacheDir", &mCacheDir)) { + ALOGE("Failed to find cache directory in config message."); + return NAME_NOT_FOUND; + } + + return OK; +} + +status_t IntrinsicBlurFilter::start() { + // TODO: use a single RS context object for entire application + mRS = new RSC::RS(); + + if (!mRS->init(mCacheDir.c_str())) { + ALOGE("Failed to initialize RenderScript context."); + return NO_INIT; + } + + // 32-bit elements for ARGB8888 + RSC::sp<const RSC::Element> e = RSC::Element::U8_4(mRS); + + RSC::Type::Builder tb(mRS, e); + tb.setX(mWidth); + tb.setY(mHeight); + RSC::sp<const RSC::Type> t = tb.create(); + + mAllocIn = RSC::Allocation::createTyped(mRS, t); + mAllocOut = RSC::Allocation::createTyped(mRS, t); + + mBlur = RSC::ScriptIntrinsicBlur::create(mRS, e); + mBlur->setRadius(mBlurRadius); + mBlur->setInput(mAllocIn); + + return OK; +} + +void IntrinsicBlurFilter::reset() { + mBlur.clear(); + mAllocOut.clear(); + mAllocIn.clear(); + mRS.clear(); +} + +status_t IntrinsicBlurFilter::setParameters(const sp<AMessage> &msg) { + sp<AMessage> params; + CHECK(msg->findMessage("params", ¶ms)); + + float blurRadius; + if (params->findFloat("blur-radius", &blurRadius)) { + mBlurRadius = blurRadius; + } + + return OK; +} + +status_t IntrinsicBlurFilter::processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer) { + mAllocIn->copy1DRangeFrom(0, mWidth * mHeight, srcBuffer->data()); + mBlur->forEach(mAllocOut); + mAllocOut->copy1DRangeTo(0, mWidth * mHeight, outBuffer->data()); + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/IntrinsicBlurFilter.h b/media/libstagefright/filters/IntrinsicBlurFilter.h new file mode 100644 index 0000000..4707ab7 --- /dev/null +++ b/media/libstagefright/filters/IntrinsicBlurFilter.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef INTRINSIC_BLUR_FILTER_H_ +#define INTRINSIC_BLUR_FILTER_H_ + +#include "RenderScript.h" +#include "SimpleFilter.h" + +namespace android { + +struct IntrinsicBlurFilter : public SimpleFilter { +public: + IntrinsicBlurFilter() : mBlurRadius(1.f) {}; + + virtual status_t configure(const sp<AMessage> &msg); + virtual status_t start(); + virtual void reset(); + virtual status_t setParameters(const sp<AMessage> &msg); + virtual status_t processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer); + +protected: + virtual ~IntrinsicBlurFilter() {}; + +private: + AString mCacheDir; + RSC::sp<RSC::RS> mRS; + RSC::sp<RSC::Allocation> mAllocIn; + RSC::sp<RSC::Allocation> mAllocOut; + RSC::sp<RSC::ScriptIntrinsicBlur> mBlur; + float mBlurRadius; +}; + +} // namespace android + +#endif // INTRINSIC_BLUR_FILTER_H_ diff --git a/media/libstagefright/filters/MediaFilter.cpp b/media/libstagefright/filters/MediaFilter.cpp new file mode 100644 index 0000000..0a09575 --- /dev/null +++ b/media/libstagefright/filters/MediaFilter.cpp @@ -0,0 +1,818 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "MediaFilter" + +#include <inttypes.h> +#include <utils/Trace.h> + +#include <binder/MemoryDealer.h> + +#include <media/stagefright/BufferProducerWrapper.h> +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include <media/stagefright/MediaDefs.h> +#include <media/stagefright/MediaErrors.h> +#include <media/stagefright/MediaFilter.h> + +#include <gui/BufferItem.h> + +#include "ColorConvert.h" +#include "GraphicBufferListener.h" +#include "IntrinsicBlurFilter.h" +#include "RSFilter.h" +#include "SaturationFilter.h" +#include "ZeroFilter.h" + +namespace android { + +// parameter: number of input and output buffers +static const size_t kBufferCountActual = 4; + +MediaFilter::MediaFilter() + : mState(UNINITIALIZED), + mGeneration(0), + mGraphicBufferListener(NULL) { +} + +MediaFilter::~MediaFilter() { +} + +//////////////////// PUBLIC FUNCTIONS ////////////////////////////////////////// + +void MediaFilter::setNotificationMessage(const sp<AMessage> &msg) { + mNotify = msg; +} + +void MediaFilter::initiateAllocateComponent(const sp<AMessage> &msg) { + msg->setWhat(kWhatAllocateComponent); + msg->setTarget(this); + msg->post(); +} + +void MediaFilter::initiateConfigureComponent(const sp<AMessage> &msg) { + msg->setWhat(kWhatConfigureComponent); + msg->setTarget(this); + msg->post(); +} + +void MediaFilter::initiateCreateInputSurface() { + (new AMessage(kWhatCreateInputSurface, this))->post(); +} + +void MediaFilter::initiateStart() { + (new AMessage(kWhatStart, this))->post(); +} + +void MediaFilter::initiateShutdown(bool keepComponentAllocated) { + sp<AMessage> msg = new AMessage(kWhatShutdown, this); + msg->setInt32("keepComponentAllocated", keepComponentAllocated); + msg->post(); +} + +void MediaFilter::signalFlush() { + (new AMessage(kWhatFlush, this))->post(); +} + +void MediaFilter::signalResume() { + (new AMessage(kWhatResume, this))->post(); +} + +// nothing to do +void MediaFilter::signalRequestIDRFrame() { + return; +} + +void MediaFilter::signalSetParameters(const sp<AMessage> ¶ms) { + sp<AMessage> msg = new AMessage(kWhatSetParameters, this); + msg->setMessage("params", params); + msg->post(); +} + +void MediaFilter::signalEndOfInputStream() { + (new AMessage(kWhatSignalEndOfInputStream, this))->post(); +} + +void MediaFilter::onMessageReceived(const sp<AMessage> &msg) { + switch (msg->what()) { + case kWhatAllocateComponent: + { + onAllocateComponent(msg); + break; + } + case kWhatConfigureComponent: + { + onConfigureComponent(msg); + break; + } + case kWhatStart: + { + onStart(); + break; + } + case kWhatProcessBuffers: + { + processBuffers(); + break; + } + case kWhatInputBufferFilled: + { + onInputBufferFilled(msg); + break; + } + case kWhatOutputBufferDrained: + { + onOutputBufferDrained(msg); + break; + } + case kWhatShutdown: + { + onShutdown(msg); + break; + } + case kWhatFlush: + { + onFlush(); + break; + } + case kWhatResume: + { + // nothing to do + break; + } + case kWhatSetParameters: + { + onSetParameters(msg); + break; + } + case kWhatCreateInputSurface: + { + onCreateInputSurface(); + break; + } + case GraphicBufferListener::kWhatFrameAvailable: + { + onInputFrameAvailable(); + break; + } + case kWhatSignalEndOfInputStream: + { + onSignalEndOfInputStream(); + break; + } + default: + { + ALOGE("Message not handled:\n%s", msg->debugString().c_str()); + break; + } + } +} + +//////////////////// PORT DESCRIPTION ////////////////////////////////////////// + +MediaFilter::PortDescription::PortDescription() { +} + +void MediaFilter::PortDescription::addBuffer( + IOMX::buffer_id id, const sp<ABuffer> &buffer) { + mBufferIDs.push_back(id); + mBuffers.push_back(buffer); +} + +size_t MediaFilter::PortDescription::countBuffers() { + return mBufferIDs.size(); +} + +IOMX::buffer_id MediaFilter::PortDescription::bufferIDAt(size_t index) const { + return mBufferIDs.itemAt(index); +} + +sp<ABuffer> MediaFilter::PortDescription::bufferAt(size_t index) const { + return mBuffers.itemAt(index); +} + +//////////////////// HELPER FUNCTIONS ////////////////////////////////////////// + +void MediaFilter::signalProcessBuffers() { + (new AMessage(kWhatProcessBuffers, this))->post(); +} + +void MediaFilter::signalError(status_t error) { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatError); + notify->setInt32("err", error); + notify->post(); +} + +status_t MediaFilter::allocateBuffersOnPort(OMX_U32 portIndex) { + CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput); + const bool isInput = portIndex == kPortIndexInput; + const size_t bufferSize = isInput ? mMaxInputSize : mMaxOutputSize; + + CHECK(mDealer[portIndex] == NULL); + CHECK(mBuffers[portIndex].isEmpty()); + + ALOGV("Allocating %zu buffers of size %zu on %s port", + kBufferCountActual, bufferSize, + isInput ? "input" : "output"); + + size_t totalSize = kBufferCountActual * bufferSize; + + mDealer[portIndex] = new MemoryDealer(totalSize, "MediaFilter"); + + for (size_t i = 0; i < kBufferCountActual; ++i) { + sp<IMemory> mem = mDealer[portIndex]->allocate(bufferSize); + CHECK(mem.get() != NULL); + + BufferInfo info; + info.mStatus = BufferInfo::OWNED_BY_US; + info.mBufferID = i; + info.mGeneration = mGeneration; + info.mOutputFlags = 0; + info.mData = new ABuffer(mem->pointer(), bufferSize); + info.mData->meta()->setInt64("timeUs", 0); + + mBuffers[portIndex].push_back(info); + + if (!isInput) { + mAvailableOutputBuffers.push( + &mBuffers[portIndex].editItemAt(i)); + } + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatBuffersAllocated); + + notify->setInt32("portIndex", portIndex); + + sp<PortDescription> desc = new PortDescription; + + for (size_t i = 0; i < mBuffers[portIndex].size(); ++i) { + const BufferInfo &info = mBuffers[portIndex][i]; + + desc->addBuffer(info.mBufferID, info.mData); + } + + notify->setObject("portDesc", desc); + notify->post(); + + return OK; +} + +MediaFilter::BufferInfo* MediaFilter::findBufferByID( + uint32_t portIndex, IOMX::buffer_id bufferID, + ssize_t *index) { + for (size_t i = 0; i < mBuffers[portIndex].size(); ++i) { + BufferInfo *info = &mBuffers[portIndex].editItemAt(i); + + if (info->mBufferID == bufferID) { + if (index != NULL) { + *index = i; + } + return info; + } + } + + TRESPASS(); + + return NULL; +} + +void MediaFilter::postFillThisBuffer(BufferInfo *info) { + ALOGV("postFillThisBuffer on buffer %d", info->mBufferID); + if (mPortEOS[kPortIndexInput]) { + return; + } + + CHECK_EQ((int)info->mStatus, (int)BufferInfo::OWNED_BY_US); + + info->mGeneration = mGeneration; + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatFillThisBuffer); + notify->setInt32("buffer-id", info->mBufferID); + + info->mData->meta()->clear(); + notify->setBuffer("buffer", info->mData); + + sp<AMessage> reply = new AMessage(kWhatInputBufferFilled, this); + reply->setInt32("buffer-id", info->mBufferID); + + notify->setMessage("reply", reply); + + info->mStatus = BufferInfo::OWNED_BY_UPSTREAM; + notify->post(); +} + +void MediaFilter::postDrainThisBuffer(BufferInfo *info) { + CHECK_EQ((int)info->mStatus, (int)BufferInfo::OWNED_BY_US); + + info->mGeneration = mGeneration; + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatDrainThisBuffer); + notify->setInt32("buffer-id", info->mBufferID); + notify->setInt32("flags", info->mOutputFlags); + notify->setBuffer("buffer", info->mData); + + sp<AMessage> reply = new AMessage(kWhatOutputBufferDrained, this); + reply->setInt32("buffer-id", info->mBufferID); + + notify->setMessage("reply", reply); + + notify->post(); + + info->mStatus = BufferInfo::OWNED_BY_UPSTREAM; +} + +void MediaFilter::postEOS() { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatEOS); + notify->setInt32("err", ERROR_END_OF_STREAM); + notify->post(); + + ALOGV("Sent kWhatEOS."); +} + +void MediaFilter::sendFormatChange() { + sp<AMessage> notify = mNotify->dup(); + + notify->setInt32("what", kWhatOutputFormatChanged); + + AString mime; + CHECK(mOutputFormat->findString("mime", &mime)); + notify->setString("mime", mime.c_str()); + + notify->setInt32("stride", mStride); + notify->setInt32("slice-height", mSliceHeight); + notify->setInt32("color-format", mColorFormatOut); + notify->setRect("crop", 0, 0, mStride - 1, mSliceHeight - 1); + notify->setInt32("width", mWidth); + notify->setInt32("height", mHeight); + + notify->post(); +} + +void MediaFilter::requestFillEmptyInput() { + if (mPortEOS[kPortIndexInput]) { + return; + } + + for (size_t i = 0; i < mBuffers[kPortIndexInput].size(); ++i) { + BufferInfo *info = &mBuffers[kPortIndexInput].editItemAt(i); + + if (info->mStatus == BufferInfo::OWNED_BY_US) { + postFillThisBuffer(info); + } + } +} + +void MediaFilter::processBuffers() { + if (mAvailableInputBuffers.empty() || mAvailableOutputBuffers.empty()) { + ALOGV("Skipping process (buffers unavailable)"); + return; + } + + if (mPortEOS[kPortIndexOutput]) { + // TODO notify caller of queueInput error when it is supported + // in MediaCodec + ALOGW("Tried to process a buffer after EOS."); + return; + } + + BufferInfo *inputInfo = mAvailableInputBuffers[0]; + mAvailableInputBuffers.removeAt(0); + BufferInfo *outputInfo = mAvailableOutputBuffers[0]; + mAvailableOutputBuffers.removeAt(0); + + status_t err; + err = mFilter->processBuffers(inputInfo->mData, outputInfo->mData); + if (err != (status_t)OK) { + outputInfo->mData->meta()->setInt32("err", err); + } + + int64_t timeUs; + CHECK(inputInfo->mData->meta()->findInt64("timeUs", &timeUs)); + outputInfo->mData->meta()->setInt64("timeUs", timeUs); + outputInfo->mOutputFlags = 0; + int32_t eos = 0; + if (inputInfo->mData->meta()->findInt32("eos", &eos) && eos != 0) { + outputInfo->mOutputFlags |= OMX_BUFFERFLAG_EOS; + mPortEOS[kPortIndexOutput] = true; + outputInfo->mData->meta()->setInt32("eos", eos); + postEOS(); + ALOGV("Output stream saw EOS."); + } + + ALOGV("Processed input buffer %u [%zu], output buffer %u [%zu]", + inputInfo->mBufferID, inputInfo->mData->size(), + outputInfo->mBufferID, outputInfo->mData->size()); + + if (mGraphicBufferListener != NULL) { + delete inputInfo; + } else { + postFillThisBuffer(inputInfo); + } + postDrainThisBuffer(outputInfo); + + // prevent any corner case where buffers could get stuck in queue + signalProcessBuffers(); +} + +void MediaFilter::onAllocateComponent(const sp<AMessage> &msg) { + CHECK_EQ(mState, UNINITIALIZED); + + CHECK(msg->findString("componentName", &mComponentName)); + const char* name = mComponentName.c_str(); + if (!strcasecmp(name, "android.filter.zerofilter")) { + mFilter = new ZeroFilter; + } else if (!strcasecmp(name, "android.filter.saturation")) { + mFilter = new SaturationFilter; + } else if (!strcasecmp(name, "android.filter.intrinsicblur")) { + mFilter = new IntrinsicBlurFilter; + } else if (!strcasecmp(name, "android.filter.RenderScript")) { + mFilter = new RSFilter; + } else { + ALOGE("Unrecognized filter name: %s", name); + signalError(NAME_NOT_FOUND); + return; + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatComponentAllocated); + // HACK - need "OMX.google" to use MediaCodec's software renderer + notify->setString("componentName", "OMX.google.MediaFilter"); + notify->post(); + mState = INITIALIZED; + ALOGV("Handled kWhatAllocateComponent."); +} + +void MediaFilter::onConfigureComponent(const sp<AMessage> &msg) { + // TODO: generalize to allow audio filters as well as video + + CHECK_EQ(mState, INITIALIZED); + + // get params - at least mime, width & height + AString mime; + CHECK(msg->findString("mime", &mime)); + if (strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_RAW)) { + ALOGE("Bad mime: %s", mime.c_str()); + signalError(BAD_VALUE); + return; + } + + CHECK(msg->findInt32("width", &mWidth)); + CHECK(msg->findInt32("height", &mHeight)); + if (!msg->findInt32("stride", &mStride)) { + mStride = mWidth; + } + if (!msg->findInt32("slice-height", &mSliceHeight)) { + mSliceHeight = mHeight; + } + + mMaxInputSize = mWidth * mHeight * 4; // room for ARGB8888 + int32_t maxInputSize; + if (msg->findInt32("max-input-size", &maxInputSize) + && (size_t)maxInputSize > mMaxInputSize) { + mMaxInputSize = maxInputSize; + } + + if (!msg->findInt32("color-format", &mColorFormatIn)) { + // default to OMX_COLOR_Format32bitARGB8888 + mColorFormatIn = OMX_COLOR_Format32bitARGB8888; + msg->setInt32("color-format", mColorFormatIn); + } + mColorFormatOut = mColorFormatIn; + + mMaxOutputSize = mWidth * mHeight * 4; // room for ARGB8888 + + AString cacheDir; + if (!msg->findString("cacheDir", &cacheDir)) { + ALOGE("Failed to find cache directory in config message."); + signalError(NAME_NOT_FOUND); + return; + } + + status_t err; + err = mFilter->configure(msg); + if (err != (status_t)OK) { + ALOGE("Failed to configure filter component, err %d", err); + signalError(err); + return; + } + + mInputFormat = new AMessage(); + mInputFormat->setString("mime", mime.c_str()); + mInputFormat->setInt32("stride", mStride); + mInputFormat->setInt32("slice-height", mSliceHeight); + mInputFormat->setInt32("color-format", mColorFormatIn); + mInputFormat->setRect("crop", 0, 0, mStride, mSliceHeight); + mInputFormat->setInt32("width", mWidth); + mInputFormat->setInt32("height", mHeight); + + mOutputFormat = new AMessage(); + mOutputFormat->setString("mime", mime.c_str()); + mOutputFormat->setInt32("stride", mStride); + mOutputFormat->setInt32("slice-height", mSliceHeight); + mOutputFormat->setInt32("color-format", mColorFormatOut); + mOutputFormat->setRect("crop", 0, 0, mStride, mSliceHeight); + mOutputFormat->setInt32("width", mWidth); + mOutputFormat->setInt32("height", mHeight); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatComponentConfigured); + notify->setString("componentName", "MediaFilter"); + notify->setMessage("input-format", mInputFormat); + notify->setMessage("output-format", mOutputFormat); + notify->post(); + mState = CONFIGURED; + ALOGV("Handled kWhatConfigureComponent."); + + sendFormatChange(); +} + +void MediaFilter::onStart() { + CHECK_EQ(mState, CONFIGURED); + + allocateBuffersOnPort(kPortIndexInput); + + allocateBuffersOnPort(kPortIndexOutput); + + status_t err = mFilter->start(); + if (err != (status_t)OK) { + ALOGE("Failed to start filter component, err %d", err); + signalError(err); + return; + } + + mPortEOS[kPortIndexInput] = false; + mPortEOS[kPortIndexOutput] = false; + mInputEOSResult = OK; + mState = STARTED; + + requestFillEmptyInput(); + ALOGV("Handled kWhatStart."); +} + +void MediaFilter::onInputBufferFilled(const sp<AMessage> &msg) { + IOMX::buffer_id bufferID; + CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID)); + BufferInfo *info = findBufferByID(kPortIndexInput, bufferID); + + if (mState != STARTED) { + // we're not running, so we'll just keep that buffer... + info->mStatus = BufferInfo::OWNED_BY_US; + return; + } + + if (info->mGeneration != mGeneration) { + ALOGV("Caught a stale input buffer [ID %d]", bufferID); + // buffer is stale (taken before a flush/shutdown) - repost it + CHECK_EQ(info->mStatus, BufferInfo::OWNED_BY_US); + postFillThisBuffer(info); + return; + } + + CHECK_EQ(info->mStatus, BufferInfo::OWNED_BY_UPSTREAM); + info->mStatus = BufferInfo::OWNED_BY_US; + + sp<ABuffer> buffer; + int32_t err = OK; + bool eos = false; + + if (!msg->findBuffer("buffer", &buffer)) { + // these are unfilled buffers returned by client + CHECK(msg->findInt32("err", &err)); + + if (err == OK) { + // buffers with no errors are returned on MediaCodec.flush + ALOGV("saw unfilled buffer (MediaCodec.flush)"); + postFillThisBuffer(info); + return; + } else { + ALOGV("saw error %d instead of an input buffer", err); + eos = true; + } + + buffer.clear(); + } + + int32_t isCSD; + if (buffer != NULL && buffer->meta()->findInt32("csd", &isCSD) + && isCSD != 0) { + // ignore codec-specific data buffers + ALOGW("MediaFilter received a codec-specific data buffer"); + postFillThisBuffer(info); + return; + } + + int32_t tmp; + if (buffer != NULL && buffer->meta()->findInt32("eos", &tmp) && tmp) { + eos = true; + err = ERROR_END_OF_STREAM; + } + + mAvailableInputBuffers.push_back(info); + processBuffers(); + + if (eos) { + mPortEOS[kPortIndexInput] = true; + mInputEOSResult = err; + } + + ALOGV("Handled kWhatInputBufferFilled. [ID %u]", bufferID); +} + +void MediaFilter::onOutputBufferDrained(const sp<AMessage> &msg) { + IOMX::buffer_id bufferID; + CHECK(msg->findInt32("buffer-id", (int32_t*)&bufferID)); + BufferInfo *info = findBufferByID(kPortIndexOutput, bufferID); + + if (mState != STARTED) { + // we're not running, so we'll just keep that buffer... + info->mStatus = BufferInfo::OWNED_BY_US; + return; + } + + if (info->mGeneration != mGeneration) { + ALOGV("Caught a stale output buffer [ID %d]", bufferID); + // buffer is stale (taken before a flush/shutdown) - keep it + CHECK_EQ(info->mStatus, BufferInfo::OWNED_BY_US); + return; + } + + CHECK_EQ(info->mStatus, BufferInfo::OWNED_BY_UPSTREAM); + info->mStatus = BufferInfo::OWNED_BY_US; + + mAvailableOutputBuffers.push_back(info); + + processBuffers(); + + ALOGV("Handled kWhatOutputBufferDrained. [ID %u]", + bufferID); +} + +void MediaFilter::onShutdown(const sp<AMessage> &msg) { + mGeneration++; + + if (mState != UNINITIALIZED) { + mFilter->reset(); + } + + int32_t keepComponentAllocated; + CHECK(msg->findInt32("keepComponentAllocated", &keepComponentAllocated)); + if (!keepComponentAllocated || mState == UNINITIALIZED) { + mState = UNINITIALIZED; + } else { + mState = INITIALIZED; + } + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatShutdownCompleted); + notify->post(); +} + +void MediaFilter::onFlush() { + mGeneration++; + + mAvailableInputBuffers.clear(); + for (size_t i = 0; i < mBuffers[kPortIndexInput].size(); ++i) { + BufferInfo *info = &mBuffers[kPortIndexInput].editItemAt(i); + info->mStatus = BufferInfo::OWNED_BY_US; + } + mAvailableOutputBuffers.clear(); + for (size_t i = 0; i < mBuffers[kPortIndexOutput].size(); ++i) { + BufferInfo *info = &mBuffers[kPortIndexOutput].editItemAt(i); + info->mStatus = BufferInfo::OWNED_BY_US; + mAvailableOutputBuffers.push_back(info); + } + + mPortEOS[kPortIndexInput] = false; + mPortEOS[kPortIndexOutput] = false; + mInputEOSResult = OK; + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatFlushCompleted); + notify->post(); + ALOGV("Posted kWhatFlushCompleted"); + + // MediaCodec returns all input buffers after flush, so in + // onInputBufferFilled we call postFillThisBuffer on them +} + +void MediaFilter::onSetParameters(const sp<AMessage> &msg) { + CHECK(mState != STARTED); + + status_t err = mFilter->setParameters(msg); + if (err != (status_t)OK) { + ALOGE("setParameters returned err %d", err); + } +} + +void MediaFilter::onCreateInputSurface() { + CHECK(mState == CONFIGURED); + + mGraphicBufferListener = new GraphicBufferListener; + + sp<AMessage> notify = new AMessage(); + notify->setTarget(this); + status_t err = mGraphicBufferListener->init( + notify, mStride, mSliceHeight, kBufferCountActual); + + if (err != OK) { + ALOGE("Failed to init mGraphicBufferListener: %d", err); + signalError(err); + return; + } + + sp<AMessage> reply = mNotify->dup(); + reply->setInt32("what", CodecBase::kWhatInputSurfaceCreated); + reply->setObject( + "input-surface", + new BufferProducerWrapper( + mGraphicBufferListener->getIGraphicBufferProducer())); + reply->post(); +} + +void MediaFilter::onInputFrameAvailable() { + BufferItem item = mGraphicBufferListener->getBufferItem(); + sp<GraphicBuffer> buf = mGraphicBufferListener->getBuffer(item); + + // get pointer to graphic buffer + void* bufPtr; + buf->lock(GraphicBuffer::USAGE_SW_READ_OFTEN, &bufPtr); + + // HACK - there is no OMX_COLOR_FORMATTYPE value for RGBA, so the format + // conversion is hardcoded until we add this. + // TODO: check input format and convert only if necessary + // copy RGBA graphic buffer into temporary ARGB input buffer + BufferInfo *inputInfo = new BufferInfo; + inputInfo->mData = new ABuffer(buf->getWidth() * buf->getHeight() * 4); + ALOGV("Copying surface data into temp buffer."); + convertRGBAToARGB( + (uint8_t*)bufPtr, buf->getWidth(), buf->getHeight(), + buf->getStride(), inputInfo->mData->data()); + inputInfo->mBufferID = item.mBuf; + inputInfo->mGeneration = mGeneration; + inputInfo->mOutputFlags = 0; + inputInfo->mStatus = BufferInfo::OWNED_BY_US; + inputInfo->mData->meta()->setInt64("timeUs", item.mTimestamp / 1000); + + mAvailableInputBuffers.push_back(inputInfo); + + mGraphicBufferListener->releaseBuffer(item); + + signalProcessBuffers(); +} + +void MediaFilter::onSignalEndOfInputStream() { + // if using input surface, need to send an EOS output buffer + if (mGraphicBufferListener != NULL) { + Vector<BufferInfo> *outputBufs = &mBuffers[kPortIndexOutput]; + BufferInfo* eosBuf; + bool foundBuf = false; + for (size_t i = 0; i < kBufferCountActual; i++) { + eosBuf = &outputBufs->editItemAt(i); + if (eosBuf->mStatus == BufferInfo::OWNED_BY_US) { + foundBuf = true; + break; + } + } + + if (!foundBuf) { + ALOGE("onSignalEndOfInputStream failed to find an output buffer"); + return; + } + + eosBuf->mOutputFlags = OMX_BUFFERFLAG_EOS; + eosBuf->mGeneration = mGeneration; + eosBuf->mData->setRange(0, 0); + postDrainThisBuffer(eosBuf); + ALOGV("Posted EOS on output buffer %zu", eosBuf->mBufferID); + } + + mPortEOS[kPortIndexOutput] = true; + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", CodecBase::kWhatSignaledInputEOS); + notify->post(); + + ALOGV("Output stream saw EOS."); +} + +} // namespace android diff --git a/media/libstagefright/filters/RSFilter.cpp b/media/libstagefright/filters/RSFilter.cpp new file mode 100644 index 0000000..b569945 --- /dev/null +++ b/media/libstagefright/filters/RSFilter.cpp @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "RSFilter" + +#include <utils/Log.h> + +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include "RSFilter.h" + +namespace android { + +RSFilter::RSFilter() { + +} + +RSFilter::~RSFilter() { + +} + +status_t RSFilter::configure(const sp<AMessage> &msg) { + status_t err = SimpleFilter::configure(msg); + if (err != OK) { + return err; + } + + if (!msg->findString("cacheDir", &mCacheDir)) { + ALOGE("Failed to find cache directory in config message."); + return NAME_NOT_FOUND; + } + + sp<RenderScriptWrapper> wrapper; + if (!msg->findObject("rs-wrapper", (sp<RefBase>*)&wrapper)) { + ALOGE("Failed to find RenderScriptWrapper in config message."); + return NAME_NOT_FOUND; + } + + mRS = wrapper->mContext; + mCallback = wrapper->mCallback; + + return OK; +} + +status_t RSFilter::start() { + // 32-bit elements for ARGB8888 + RSC::sp<const RSC::Element> e = RSC::Element::U8_4(mRS); + + RSC::Type::Builder tb(mRS, e); + tb.setX(mWidth); + tb.setY(mHeight); + RSC::sp<const RSC::Type> t = tb.create(); + + mAllocIn = RSC::Allocation::createTyped(mRS, t); + mAllocOut = RSC::Allocation::createTyped(mRS, t); + + return OK; +} + +void RSFilter::reset() { + mCallback.clear(); + mAllocOut.clear(); + mAllocIn.clear(); + mRS.clear(); +} + +status_t RSFilter::setParameters(const sp<AMessage> &msg) { + return mCallback->handleSetParameters(msg); +} + +status_t RSFilter::processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer) { + mAllocIn->copy1DRangeFrom(0, mWidth * mHeight, srcBuffer->data()); + mCallback->processBuffers(mAllocIn.get(), mAllocOut.get()); + mAllocOut->copy1DRangeTo(0, mWidth * mHeight, outBuffer->data()); + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/RSFilter.h b/media/libstagefright/filters/RSFilter.h new file mode 100644 index 0000000..c5b5074 --- /dev/null +++ b/media/libstagefright/filters/RSFilter.h @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RS_FILTER_H_ +#define RS_FILTER_H_ + +#include <media/stagefright/RenderScriptWrapper.h> +#include <RenderScript.h> + +#include "SimpleFilter.h" + +namespace android { + +struct AString; + +struct RSFilter : public SimpleFilter { +public: + RSFilter(); + + virtual status_t configure(const sp<AMessage> &msg); + virtual status_t start(); + virtual void reset(); + virtual status_t setParameters(const sp<AMessage> &msg); + virtual status_t processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer); + +protected: + virtual ~RSFilter(); + +private: + AString mCacheDir; + sp<RenderScriptWrapper::RSFilterCallback> mCallback; + RSC::sp<RSC::RS> mRS; + RSC::sp<RSC::Allocation> mAllocIn; + RSC::sp<RSC::Allocation> mAllocOut; +}; + +} // namespace android + +#endif // RS_FILTER_H_ diff --git a/media/libstagefright/filters/SaturationFilter.cpp b/media/libstagefright/filters/SaturationFilter.cpp new file mode 100644 index 0000000..ba5f75a --- /dev/null +++ b/media/libstagefright/filters/SaturationFilter.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "SaturationFilter" + +#include <utils/Log.h> + +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include "SaturationFilter.h" + +namespace android { + +status_t SaturationFilter::configure(const sp<AMessage> &msg) { + status_t err = SimpleFilter::configure(msg); + if (err != OK) { + return err; + } + + if (!msg->findString("cacheDir", &mCacheDir)) { + ALOGE("Failed to find cache directory in config message."); + return NAME_NOT_FOUND; + } + + return OK; +} + +status_t SaturationFilter::start() { + // TODO: use a single RS context object for entire application + mRS = new RSC::RS(); + + if (!mRS->init(mCacheDir.c_str())) { + ALOGE("Failed to initialize RenderScript context."); + return NO_INIT; + } + + // 32-bit elements for ARGB8888 + RSC::sp<const RSC::Element> e = RSC::Element::U8_4(mRS); + + RSC::Type::Builder tb(mRS, e); + tb.setX(mWidth); + tb.setY(mHeight); + RSC::sp<const RSC::Type> t = tb.create(); + + mAllocIn = RSC::Allocation::createTyped(mRS, t); + mAllocOut = RSC::Allocation::createTyped(mRS, t); + + mScript = new ScriptC_saturationARGB(mRS); + + mScript->set_gSaturation(mSaturation); + + return OK; +} + +void SaturationFilter::reset() { + mScript.clear(); + mAllocOut.clear(); + mAllocIn.clear(); + mRS.clear(); +} + +status_t SaturationFilter::setParameters(const sp<AMessage> &msg) { + sp<AMessage> params; + CHECK(msg->findMessage("params", ¶ms)); + + float saturation; + if (params->findFloat("saturation", &saturation)) { + mSaturation = saturation; + } + + return OK; +} + +status_t SaturationFilter::processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer) { + mAllocIn->copy1DRangeFrom(0, mWidth * mHeight, srcBuffer->data()); + mScript->forEach_root(mAllocIn, mAllocOut); + mAllocOut->copy1DRangeTo(0, mWidth * mHeight, outBuffer->data()); + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/SaturationFilter.h b/media/libstagefright/filters/SaturationFilter.h new file mode 100644 index 0000000..0545021 --- /dev/null +++ b/media/libstagefright/filters/SaturationFilter.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SATURATION_FILTER_H_ +#define SATURATION_FILTER_H_ + +#include <RenderScript.h> + +#include "ScriptC_saturationARGB.h" +#include "SimpleFilter.h" + +namespace android { + +struct SaturationFilter : public SimpleFilter { +public: + SaturationFilter() : mSaturation(1.f) {}; + + virtual status_t configure(const sp<AMessage> &msg); + virtual status_t start(); + virtual void reset(); + virtual status_t setParameters(const sp<AMessage> &msg); + virtual status_t processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer); + +protected: + virtual ~SaturationFilter() {}; + +private: + AString mCacheDir; + RSC::sp<RSC::RS> mRS; + RSC::sp<RSC::Allocation> mAllocIn; + RSC::sp<RSC::Allocation> mAllocOut; + RSC::sp<ScriptC_saturationARGB> mScript; + float mSaturation; +}; + +} // namespace android + +#endif // SATURATION_FILTER_H_ diff --git a/media/libstagefright/filters/SimpleFilter.cpp b/media/libstagefright/filters/SimpleFilter.cpp new file mode 100644 index 0000000..6c1ca2c --- /dev/null +++ b/media/libstagefright/filters/SimpleFilter.cpp @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include "SimpleFilter.h" + +namespace android { + +status_t SimpleFilter::configure(const sp<AMessage> &msg) { + CHECK(msg->findInt32("width", &mWidth)); + CHECK(msg->findInt32("height", &mHeight)); + if (!msg->findInt32("stride", &mStride)) { + mStride = mWidth; + } + if (!msg->findInt32("slice-height", &mSliceHeight)) { + mSliceHeight = mHeight; + } + CHECK(msg->findInt32("color-format", &mColorFormatIn)); + mColorFormatOut = mColorFormatIn; + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/SimpleFilter.h b/media/libstagefright/filters/SimpleFilter.h new file mode 100644 index 0000000..4cd37ef --- /dev/null +++ b/media/libstagefright/filters/SimpleFilter.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SIMPLE_FILTER_H_ +#define SIMPLE_FILTER_H_ + +#include <stdint.h> +#include <utils/Errors.h> +#include <utils/RefBase.h> + +struct ABuffer; +struct AMessage; + +namespace android { + +struct SimpleFilter : public RefBase { +public: + SimpleFilter() : mWidth(0), mHeight(0), mStride(0), mSliceHeight(0), + mColorFormatIn(0), mColorFormatOut(0) {}; + + virtual status_t configure(const sp<AMessage> &msg); + + virtual status_t start() = 0; + virtual void reset() = 0; + virtual status_t setParameters(const sp<AMessage> &msg) = 0; + virtual status_t processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer) = 0; + +protected: + int32_t mWidth, mHeight; + int32_t mStride, mSliceHeight; + int32_t mColorFormatIn, mColorFormatOut; + + virtual ~SimpleFilter() {}; +}; + +} // namespace android + +#endif // SIMPLE_FILTER_H_ diff --git a/media/libstagefright/filters/ZeroFilter.cpp b/media/libstagefright/filters/ZeroFilter.cpp new file mode 100644 index 0000000..3f1243c --- /dev/null +++ b/media/libstagefright/filters/ZeroFilter.cpp @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "ZeroFilter" + +#include <media/stagefright/foundation/ABuffer.h> +#include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AMessage.h> + +#include "ZeroFilter.h" + +namespace android { + +status_t ZeroFilter::setParameters(const sp<AMessage> &msg) { + sp<AMessage> params; + CHECK(msg->findMessage("params", ¶ms)); + + int32_t invert; + if (params->findInt32("invert", &invert)) { + mInvertData = (invert != 0); + } + + return OK; +} + +status_t ZeroFilter::processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer) { + // assuming identical input & output buffers, since we're a copy filter + if (mInvertData) { + uint32_t* src = (uint32_t*)srcBuffer->data(); + uint32_t* dest = (uint32_t*)outBuffer->data(); + for (size_t i = 0; i < srcBuffer->size() / 4; ++i) { + *(dest++) = *(src++) ^ 0xFFFFFFFF; + } + } else { + memcpy(outBuffer->data(), srcBuffer->data(), srcBuffer->size()); + } + outBuffer->setRange(0, srcBuffer->size()); + + return OK; +} + +} // namespace android diff --git a/media/libstagefright/filters/ZeroFilter.h b/media/libstagefright/filters/ZeroFilter.h new file mode 100644 index 0000000..bd34dfb --- /dev/null +++ b/media/libstagefright/filters/ZeroFilter.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ZERO_FILTER_H_ +#define ZERO_FILTER_H_ + +#include "SimpleFilter.h" + +namespace android { + +struct ZeroFilter : public SimpleFilter { +public: + ZeroFilter() : mInvertData(false) {}; + + virtual status_t start() { return OK; }; + virtual void reset() {}; + virtual status_t setParameters(const sp<AMessage> &msg); + virtual status_t processBuffers( + const sp<ABuffer> &srcBuffer, const sp<ABuffer> &outBuffer); + +protected: + virtual ~ZeroFilter() {}; + +private: + bool mInvertData; +}; + +} // namespace android + +#endif // ZERO_FILTER_H_ diff --git a/media/libstagefright/filters/saturation.rs b/media/libstagefright/filters/saturation.rs new file mode 100644 index 0000000..2c867ac --- /dev/null +++ b/media/libstagefright/filters/saturation.rs @@ -0,0 +1,40 @@ +// Sample script for RGB888 support (compare to saturationARGB.rs) +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma version(1) +#pragma rs java_package_name(com.android.rs.cppbasic) +#pragma rs_fp_relaxed + +const static float3 gMonoMult = {0.299f, 0.587f, 0.114f}; + +// global variables (parameters accessible to application code) +float gSaturation = 1.0f; + +void root(const uchar3 *v_in, uchar3 *v_out) { + // scale 0-255 uchar to 0-1.0 float + float3 in = {v_in->r * 0.003921569f, v_in->g * 0.003921569f, + v_in->b * 0.003921569f}; + + // apply saturation filter + float3 result = dot(in, gMonoMult); + result = mix(result, in, gSaturation); + + // convert to uchar, copied from rsPackColorTo8888 + v_out->x = (uchar)clamp((result.r * 255.f + 0.5f), 0.f, 255.f); + v_out->y = (uchar)clamp((result.g * 255.f + 0.5f), 0.f, 255.f); + v_out->z = (uchar)clamp((result.b * 255.f + 0.5f), 0.f, 255.f); +} diff --git a/media/libstagefright/filters/saturationARGB.rs b/media/libstagefright/filters/saturationARGB.rs new file mode 100644 index 0000000..1de9dd8 --- /dev/null +++ b/media/libstagefright/filters/saturationARGB.rs @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma version(1) +#pragma rs java_package_name(com.android.rs.cppbasic) +#pragma rs_fp_relaxed + +const static float3 gMonoMult = {0.299f, 0.587f, 0.114f}; + +// global variables (parameters accessible to application code) +float gSaturation = 1.0f; + +void root(const uchar4 *v_in, uchar4 *v_out) { + v_out->x = v_in->x; // don't modify A + + // get RGB, scale 0-255 uchar to 0-1.0 float + float3 rgb = {v_in->y * 0.003921569f, v_in->z * 0.003921569f, + v_in->w * 0.003921569f}; + + // apply saturation filter + float3 result = dot(rgb, gMonoMult); + result = mix(result, rgb, gSaturation); + + v_out->y = (uchar)clamp((result.r * 255.f + 0.5f), 0.f, 255.f); + v_out->z = (uchar)clamp((result.g * 255.f + 0.5f), 0.f, 255.f); + v_out->w = (uchar)clamp((result.b * 255.f + 0.5f), 0.f, 255.f); +} diff --git a/media/libstagefright/foundation/AHandler.cpp b/media/libstagefright/foundation/AHandler.cpp index bd5f7e9..7dbbe54 100644 --- a/media/libstagefright/foundation/AHandler.cpp +++ b/media/libstagefright/foundation/AHandler.cpp @@ -19,15 +19,23 @@ #include <utils/Log.h> #include <media/stagefright/foundation/AHandler.h> - -#include <media/stagefright/foundation/ALooperRoster.h> +#include <media/stagefright/foundation/AMessage.h> namespace android { -sp<ALooper> AHandler::looper() { - extern ALooperRoster gLooperRoster; +void AHandler::deliverMessage(const sp<AMessage> &msg) { + onMessageReceived(msg); + mMessageCounter++; - return gLooperRoster.findLooper(id()); + if (mVerboseStats) { + uint32_t what = msg->what(); + ssize_t idx = mMessages.indexOfKey(what); + if (idx < 0) { + mMessages.add(what, 1); + } else { + mMessages.editValueAt(idx)++; + } + } } } // namespace android diff --git a/media/libstagefright/foundation/ALooper.cpp b/media/libstagefright/foundation/ALooper.cpp index 88b1c92..90b5f68 100644 --- a/media/libstagefright/foundation/ALooper.cpp +++ b/media/libstagefright/foundation/ALooper.cpp @@ -16,6 +16,9 @@ //#define LOG_NDEBUG 0 #define LOG_TAG "ALooper" + +#include <media/stagefright/foundation/ADebug.h> + #include <utils/Log.h> #include <sys/time.h> @@ -210,7 +213,7 @@ bool ALooper::loop() { mEventQueue.erase(mEventQueue.begin()); } - gLooperRoster.deliverMessage(event.mMessage); + event.mMessage->deliver(); // NOTE: It's important to note that at this point our "ALooper" object // may no longer exist (its final reference may have gone away while @@ -220,4 +223,29 @@ bool ALooper::loop() { return true; } +// to be called by AMessage::postAndAwaitResponse only +sp<AReplyToken> ALooper::createReplyToken() { + return new AReplyToken(this); +} + +// to be called by AMessage::postAndAwaitResponse only +status_t ALooper::awaitResponse(const sp<AReplyToken> &replyToken, sp<AMessage> *response) { + // return status in case we want to handle an interrupted wait + Mutex::Autolock autoLock(mRepliesLock); + CHECK(replyToken != NULL); + while (!replyToken->retrieveReply(response)) { + mRepliesCondition.wait(mRepliesLock); + } + return OK; +} + +status_t ALooper::postReply(const sp<AReplyToken> &replyToken, const sp<AMessage> &reply) { + Mutex::Autolock autoLock(mRepliesLock); + status_t err = replyToken->setReply(reply); + if (err == OK) { + mRepliesCondition.broadcast(); + } + return err; +} + } // namespace android diff --git a/media/libstagefright/foundation/ALooperRoster.cpp b/media/libstagefright/foundation/ALooperRoster.cpp index 2d57aee..473ce1b 100644 --- a/media/libstagefright/foundation/ALooperRoster.cpp +++ b/media/libstagefright/foundation/ALooperRoster.cpp @@ -30,8 +30,7 @@ namespace android { static bool verboseStats = false; ALooperRoster::ALooperRoster() - : mNextHandlerID(1), - mNextReplyID(1) { + : mNextHandlerID(1) { } ALooper::handler_id ALooperRoster::registerHandler( @@ -49,7 +48,7 @@ ALooper::handler_id ALooperRoster::registerHandler( ALooper::handler_id handlerID = mNextHandlerID++; mHandlers.add(handlerID, info); - handler->setID(handlerID); + handler->setID(handlerID, looper); return handlerID; } @@ -68,7 +67,7 @@ void ALooperRoster::unregisterHandler(ALooper::handler_id handlerID) { sp<AHandler> handler = info.mHandler.promote(); if (handler != NULL) { - handler->setID(0); + handler->setID(0, NULL); } mHandlers.removeItemsAt(index); @@ -100,116 +99,6 @@ void ALooperRoster::unregisterStaleHandlers() { } } -status_t ALooperRoster::postMessage( - const sp<AMessage> &msg, int64_t delayUs) { - - sp<ALooper> looper = findLooper(msg->target()); - - if (looper == NULL) { - return -ENOENT; - } - looper->post(msg, delayUs); - return OK; -} - -void ALooperRoster::deliverMessage(const sp<AMessage> &msg) { - sp<AHandler> handler; - - { - Mutex::Autolock autoLock(mLock); - - ssize_t index = mHandlers.indexOfKey(msg->target()); - - if (index < 0) { - ALOGW("failed to deliver message. Target handler not registered."); - return; - } - - const HandlerInfo &info = mHandlers.valueAt(index); - handler = info.mHandler.promote(); - - if (handler == NULL) { - ALOGW("failed to deliver message. " - "Target handler %d registered, but object gone.", - msg->target()); - - mHandlers.removeItemsAt(index); - return; - } - } - - handler->onMessageReceived(msg); - handler->mMessageCounter++; - - if (verboseStats) { - uint32_t what = msg->what(); - ssize_t idx = handler->mMessages.indexOfKey(what); - if (idx < 0) { - handler->mMessages.add(what, 1); - } else { - handler->mMessages.editValueAt(idx)++; - } - } -} - -sp<ALooper> ALooperRoster::findLooper(ALooper::handler_id handlerID) { - Mutex::Autolock autoLock(mLock); - - ssize_t index = mHandlers.indexOfKey(handlerID); - - if (index < 0) { - return NULL; - } - - sp<ALooper> looper = mHandlers.valueAt(index).mLooper.promote(); - - if (looper == NULL) { - mHandlers.removeItemsAt(index); - return NULL; - } - - return looper; -} - -status_t ALooperRoster::postAndAwaitResponse( - const sp<AMessage> &msg, sp<AMessage> *response) { - sp<ALooper> looper = findLooper(msg->target()); - - if (looper == NULL) { - ALOGW("failed to post message. " - "Target handler %d still registered, but object gone.", - msg->target()); - response->clear(); - return -ENOENT; - } - - Mutex::Autolock autoLock(mLock); - - uint32_t replyID = mNextReplyID++; - - msg->setInt32("replyID", replyID); - - looper->post(msg, 0 /* delayUs */); - - ssize_t index; - while ((index = mReplies.indexOfKey(replyID)) < 0) { - mRepliesCondition.wait(mLock); - } - - *response = mReplies.valueAt(index); - mReplies.removeItemsAt(index); - - return OK; -} - -void ALooperRoster::postReply(uint32_t replyID, const sp<AMessage> &reply) { - Mutex::Autolock autoLock(mLock); - - CHECK(mReplies.indexOfKey(replyID) < 0); - mReplies.add(replyID, reply); - mRepliesCondition.broadcast(); -} - static void makeFourCC(uint32_t fourcc, char *s) { s[0] = (fourcc >> 24) & 0xff; if (s[0]) { @@ -225,7 +114,7 @@ static void makeFourCC(uint32_t fourcc, char *s) { void ALooperRoster::dump(int fd, const Vector<String16>& args) { bool clear = false; bool oldVerbose = verboseStats; - for (size_t i = 0;i < args.size(); i++) { + for (size_t i = 0; i < args.size(); i++) { if (args[i] == String16("-c")) { clear = true; } else if (args[i] == String16("-von")) { @@ -241,22 +130,23 @@ void ALooperRoster::dump(int fd, const Vector<String16>& args) { Mutex::Autolock autoLock(mLock); size_t n = mHandlers.size(); - s.appendFormat(" %zd registered handlers:\n", n); + s.appendFormat(" %zu registered handlers:\n", n); for (size_t i = 0; i < n; i++) { - s.appendFormat(" %zd: ", i); + s.appendFormat(" %d: ", mHandlers.keyAt(i)); HandlerInfo &info = mHandlers.editValueAt(i); sp<ALooper> looper = info.mLooper.promote(); if (looper != NULL) { - s.append(looper->mName.c_str()); + s.append(looper->getName()); sp<AHandler> handler = info.mHandler.promote(); if (handler != NULL) { + handler->mVerboseStats = verboseStats; s.appendFormat(": %u messages processed", handler->mMessageCounter); if (verboseStats) { for (size_t j = 0; j < handler->mMessages.size(); j++) { char fourcc[15]; makeFourCC(handler->mMessages.keyAt(j), fourcc); - s.appendFormat("\n %s: %d", + s.appendFormat("\n %s: %u", fourcc, handler->mMessages.valueAt(j)); } diff --git a/media/libstagefright/foundation/AMessage.cpp b/media/libstagefright/foundation/AMessage.cpp index 1f46bc9..e549ff6 100644 --- a/media/libstagefright/foundation/AMessage.cpp +++ b/media/libstagefright/foundation/AMessage.cpp @@ -27,6 +27,7 @@ #include "ABuffer.h" #include "ADebug.h" #include "ALooperRoster.h" +#include "AHandler.h" #include "AString.h" #include <binder/Parcel.h> @@ -36,10 +37,27 @@ namespace android { extern ALooperRoster gLooperRoster; -AMessage::AMessage(uint32_t what, ALooper::handler_id target) +status_t AReplyToken::setReply(const sp<AMessage> &reply) { + if (mReplied) { + ALOGE("trying to post a duplicate reply"); + return -EBUSY; + } + CHECK(mReply == NULL); + mReply = reply; + mReplied = true; + return OK; +} + +AMessage::AMessage(void) + : mWhat(0), + mTarget(0), + mNumItems(0) { +} + +AMessage::AMessage(uint32_t what, const sp<const AHandler> &handler) : mWhat(what), - mTarget(target), mNumItems(0) { + setTarget(handler); } AMessage::~AMessage() { @@ -54,12 +72,16 @@ uint32_t AMessage::what() const { return mWhat; } -void AMessage::setTarget(ALooper::handler_id handlerID) { - mTarget = handlerID; -} - -ALooper::handler_id AMessage::target() const { - return mTarget; +void AMessage::setTarget(const sp<const AHandler> &handler) { + if (handler == NULL) { + mTarget = 0; + mHandler.clear(); + mLooper.clear(); + } else { + mTarget = handler->id(); + mHandler = handler->getHandler(); + mLooper = handler->getLooper(); + } } void AMessage::clear() { @@ -322,33 +344,76 @@ bool AMessage::findRect( return true; } -void AMessage::post(int64_t delayUs) { - gLooperRoster.postMessage(this, delayUs); +void AMessage::deliver() { + sp<AHandler> handler = mHandler.promote(); + if (handler == NULL) { + ALOGW("failed to deliver message as target handler %d is gone.", mTarget); + return; + } + + handler->deliverMessage(this); +} + +status_t AMessage::post(int64_t delayUs) { + sp<ALooper> looper = mLooper.promote(); + if (looper == NULL) { + ALOGW("failed to post message as target looper for handler %d is gone.", mTarget); + return -ENOENT; + } + + looper->post(this, delayUs); + return OK; } status_t AMessage::postAndAwaitResponse(sp<AMessage> *response) { - return gLooperRoster.postAndAwaitResponse(this, response); + sp<ALooper> looper = mLooper.promote(); + if (looper == NULL) { + ALOGW("failed to post message as target looper for handler %d is gone.", mTarget); + return -ENOENT; + } + + sp<AReplyToken> token = looper->createReplyToken(); + if (token == NULL) { + ALOGE("failed to create reply token"); + return -ENOMEM; + } + setObject("replyID", token); + + looper->post(this, 0 /* delayUs */); + return looper->awaitResponse(token, response); } -void AMessage::postReply(uint32_t replyID) { - gLooperRoster.postReply(replyID, this); +status_t AMessage::postReply(const sp<AReplyToken> &replyToken) { + if (replyToken == NULL) { + ALOGW("failed to post reply to a NULL token"); + return -ENOENT; + } + sp<ALooper> looper = replyToken->getLooper(); + if (looper == NULL) { + ALOGW("failed to post reply as target looper is gone."); + return -ENOENT; + } + return looper->postReply(replyToken, this); } -bool AMessage::senderAwaitsResponse(uint32_t *replyID) const { - int32_t tmp; - bool found = findInt32("replyID", &tmp); +bool AMessage::senderAwaitsResponse(sp<AReplyToken> *replyToken) { + sp<RefBase> tmp; + bool found = findObject("replyID", &tmp); if (!found) { return false; } - *replyID = static_cast<uint32_t>(tmp); + *replyToken = static_cast<AReplyToken *>(tmp.get()); + tmp.clear(); + setObject("replyID", tmp); + // TODO: delete Object instead of setting it to NULL - return true; + return *replyToken != NULL; } sp<AMessage> AMessage::dup() const { - sp<AMessage> msg = new AMessage(mWhat, mTarget); + sp<AMessage> msg = new AMessage(mWhat, mHandler.promote()); msg->mNumItems = mNumItems; #ifdef DUMP_STATS @@ -532,7 +597,8 @@ AString AMessage::debugString(int32_t indent) const { // static sp<AMessage> AMessage::FromParcel(const Parcel &parcel) { int32_t what = parcel.readInt32(); - sp<AMessage> msg = new AMessage(what); + sp<AMessage> msg = new AMessage(); + msg->setWhat(what); msg->mNumItems = static_cast<size_t>(parcel.readInt32()); for (size_t i = 0; i < msg->mNumItems; ++i) { diff --git a/media/libstagefright/httplive/LiveSession.cpp b/media/libstagefright/httplive/LiveSession.cpp index d0f3bc2..118c174 100644 --- a/media/libstagefright/httplive/LiveSession.cpp +++ b/media/libstagefright/httplive/LiveSession.cpp @@ -33,6 +33,7 @@ #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> +#include <media/stagefright/foundation/AUtils.h> #include <media/stagefright/DataSource.h> #include <media/stagefright/FileSource.h> #include <media/stagefright/MediaErrors.h> @@ -49,8 +50,95 @@ namespace android { -// Number of recently-read bytes to use for bandwidth estimation -const size_t LiveSession::kBandwidthHistoryBytes = 200 * 1024; +// static +// Bandwidth Switch Mark Defaults +const int64_t LiveSession::kUpSwitchMarkUs = 25000000ll; +const int64_t LiveSession::kDownSwitchMarkUs = 18000000ll; +const int64_t LiveSession::kUpSwitchMarginUs = 5000000ll; + +// Buffer Prepare/Ready/Underflow Marks +const int64_t LiveSession::kReadyMarkUs = 5000000ll; +const int64_t LiveSession::kPrepareMarkUs = 1500000ll; +const int64_t LiveSession::kUnderflowMarkUs = 1000000ll; + +struct LiveSession::BandwidthEstimator : public RefBase { + BandwidthEstimator(); + + void addBandwidthMeasurement(size_t numBytes, int64_t delayUs); + bool estimateBandwidth(int32_t *bandwidth); + +private: + // Bandwidth estimation parameters + static const int32_t kMaxBandwidthHistoryItems = 20; + static const int64_t kMaxBandwidthHistoryWindowUs = 3000000ll; // 3 sec + + struct BandwidthEntry { + int64_t mDelayUs; + size_t mNumBytes; + }; + + Mutex mLock; + List<BandwidthEntry> mBandwidthHistory; + int64_t mTotalTransferTimeUs; + size_t mTotalTransferBytes; + + DISALLOW_EVIL_CONSTRUCTORS(BandwidthEstimator); +}; + +LiveSession::BandwidthEstimator::BandwidthEstimator() : + mTotalTransferTimeUs(0), + mTotalTransferBytes(0) { +} + +void LiveSession::BandwidthEstimator::addBandwidthMeasurement( + size_t numBytes, int64_t delayUs) { + AutoMutex autoLock(mLock); + + BandwidthEntry entry; + entry.mDelayUs = delayUs; + entry.mNumBytes = numBytes; + mTotalTransferTimeUs += delayUs; + mTotalTransferBytes += numBytes; + mBandwidthHistory.push_back(entry); + + // trim old samples, keeping at least kMaxBandwidthHistoryItems samples, + // and total transfer time at least kMaxBandwidthHistoryWindowUs. + while (mBandwidthHistory.size() > kMaxBandwidthHistoryItems) { + List<BandwidthEntry>::iterator it = mBandwidthHistory.begin(); + if (mTotalTransferTimeUs - it->mDelayUs < kMaxBandwidthHistoryWindowUs) { + break; + } + mTotalTransferTimeUs -= it->mDelayUs; + mTotalTransferBytes -= it->mNumBytes; + mBandwidthHistory.erase(mBandwidthHistory.begin()); + } +} + +bool LiveSession::BandwidthEstimator::estimateBandwidth(int32_t *bandwidthBps) { + AutoMutex autoLock(mLock); + + if (mBandwidthHistory.size() < 2) { + return false; + } + + *bandwidthBps = ((double)mTotalTransferBytes * 8E6 / mTotalTransferTimeUs); + return true; +} + +//static +const char *LiveSession::getKeyForStream(StreamType type) { + switch (type) { + case STREAMTYPE_VIDEO: + return "timeUsVideo"; + case STREAMTYPE_AUDIO: + return "timeUsAudio"; + case STREAMTYPE_SUBTITLES: + return "timeUsSubtitle"; + default: + TRESPASS(); + } + return NULL; +} LiveSession::LiveSession( const sp<AMessage> ¬ify, uint32_t flags, @@ -58,146 +146,68 @@ LiveSession::LiveSession( : mNotify(notify), mFlags(flags), mHTTPService(httpService), + mBuffering(false), mInPreparationPhase(true), + mPollBufferingGeneration(0), + mPrevBufferPercentage(-1), mHTTPDataSource(new MediaHTTP(mHTTPService->makeHTTPConnection())), mCurBandwidthIndex(-1), + mOrigBandwidthIndex(-1), + mLastBandwidthBps(-1ll), + mBandwidthEstimator(new BandwidthEstimator()), mStreamMask(0), mNewStreamMask(0), mSwapMask(0), - mCheckBandwidthGeneration(0), mSwitchGeneration(0), mSubtitleGeneration(0), mLastDequeuedTimeUs(0ll), mRealTimeBaseUs(0ll), mReconfigurationInProgress(false), mSwitchInProgress(false), - mDisconnectReplyID(0), - mSeekReplyID(0), + mUpSwitchMark(kUpSwitchMarkUs), + mDownSwitchMark(kDownSwitchMarkUs), + mUpSwitchMargin(kUpSwitchMarginUs), mFirstTimeUsValid(false), mFirstTimeUs(0), mLastSeekTimeUs(0) { - mStreams[kAudioIndex] = StreamItem("audio"); mStreams[kVideoIndex] = StreamItem("video"); mStreams[kSubtitleIndex] = StreamItem("subtitles"); for (size_t i = 0; i < kMaxStreams; ++i) { - mDiscontinuities.add(indexToType(i), new AnotherPacketSource(NULL /* meta */)); mPacketSources.add(indexToType(i), new AnotherPacketSource(NULL /* meta */)); mPacketSources2.add(indexToType(i), new AnotherPacketSource(NULL /* meta */)); - mBuffering[i] = false; - } - - size_t numHistoryItems = kBandwidthHistoryBytes / - PlaylistFetcher::kDownloadBlockSize + 1; - if (numHistoryItems < 5) { - numHistoryItems = 5; } - mHTTPDataSource->setBandwidthHistorySize(numHistoryItems); } LiveSession::~LiveSession() { -} - -sp<ABuffer> LiveSession::createFormatChangeBuffer(bool swap) { - ABuffer *discontinuity = new ABuffer(0); - discontinuity->meta()->setInt32("discontinuity", ATSParser::DISCONTINUITY_FORMATCHANGE); - discontinuity->meta()->setInt32("swapPacketSource", swap); - discontinuity->meta()->setInt32("switchGeneration", mSwitchGeneration); - discontinuity->meta()->setInt64("timeUs", -1); - return discontinuity; -} - -void LiveSession::swapPacketSource(StreamType stream) { - sp<AnotherPacketSource> &aps = mPacketSources.editValueFor(stream); - sp<AnotherPacketSource> &aps2 = mPacketSources2.editValueFor(stream); - sp<AnotherPacketSource> tmp = aps; - aps = aps2; - aps2 = tmp; - aps2->clear(); + if (mFetcherLooper != NULL) { + mFetcherLooper->stop(); + } } status_t LiveSession::dequeueAccessUnit( StreamType stream, sp<ABuffer> *accessUnit) { - if (!(mStreamMask & stream)) { - // return -EWOULDBLOCK to avoid halting the decoder - // when switching between audio/video and audio only. - return -EWOULDBLOCK; - } - - status_t finalResult; - sp<AnotherPacketSource> discontinuityQueue = mDiscontinuities.valueFor(stream); - if (discontinuityQueue->hasBufferAvailable(&finalResult)) { - discontinuityQueue->dequeueAccessUnit(accessUnit); - // seeking, track switching - sp<AMessage> extra; - int64_t timeUs; - if ((*accessUnit)->meta()->findMessage("extra", &extra) - && extra != NULL - && extra->findInt64("timeUs", &timeUs)) { - // seeking only - mLastSeekTimeUs = timeUs; - mDiscontinuityOffsetTimesUs.clear(); - mDiscontinuityAbsStartTimesUs.clear(); - } - return INFO_DISCONTINUITY; - } - + status_t finalResult = OK; sp<AnotherPacketSource> packetSource = mPacketSources.valueFor(stream); ssize_t idx = typeToIndex(stream); - if (!packetSource->hasBufferAvailable(&finalResult)) { + // Do not let client pull data if we don't have data packets yet. + // We might only have a format discontinuity queued without data. + // When NuPlayerDecoder dequeues the format discontinuity, it will + // immediately try to getFormat. If we return NULL, NuPlayerDecoder + // thinks it can do seamless change, so will not shutdown decoder. + // When the actual format arrives, it can't handle it and get stuck. + if (!packetSource->hasDataBufferAvailable(&finalResult)) { if (finalResult == OK) { - mBuffering[idx] = true; return -EAGAIN; } else { return finalResult; } } - int32_t targetDuration = 0; - sp<AMessage> meta = packetSource->getLatestEnqueuedMeta(); - if (meta != NULL) { - meta->findInt32("targetDuration", &targetDuration); - } - - int64_t targetDurationUs = targetDuration * 1000000ll; - if (targetDurationUs == 0 || - targetDurationUs > PlaylistFetcher::kMinBufferedDurationUs) { - // Fetchers limit buffering to - // min(3 * targetDuration, kMinBufferedDurationUs) - targetDurationUs = PlaylistFetcher::kMinBufferedDurationUs; - } - - if (mBuffering[idx]) { - if (mSwitchInProgress - || packetSource->isFinished(0) - || packetSource->getEstimatedDurationUs() > targetDurationUs) { - mBuffering[idx] = false; - } - } - - if (mBuffering[idx]) { - return -EAGAIN; - } - - // wait for counterpart - sp<AnotherPacketSource> otherSource; - uint32_t mask = mNewStreamMask & mStreamMask; - uint32_t fetchersMask = 0; - for (size_t i = 0; i < mFetcherInfos.size(); ++i) { - uint32_t fetcherMask = mFetcherInfos.valueAt(i).mFetcher->getStreamTypeMask(); - fetchersMask |= fetcherMask; - } - mask &= fetchersMask; - if (stream == STREAMTYPE_AUDIO && (mask & STREAMTYPE_VIDEO)) { - otherSource = mPacketSources.valueFor(STREAMTYPE_VIDEO); - } else if (stream == STREAMTYPE_VIDEO && (mask & STREAMTYPE_AUDIO)) { - otherSource = mPacketSources.valueFor(STREAMTYPE_AUDIO); - } - if (otherSource != NULL && !otherSource->hasBufferAvailable(&finalResult)) { - return finalResult == OK ? -EAGAIN : finalResult; - } + // Let the client dequeue as long as we have buffers available + // Do not make pause/resume decisions here. status_t err = packetSource->dequeueAccessUnit(accessUnit); @@ -235,42 +245,6 @@ status_t LiveSession::dequeueAccessUnit( streamStr, type, extra == NULL ? "NULL" : extra->debugString().c_str()); - - int32_t swap; - if ((*accessUnit)->meta()->findInt32("swapPacketSource", &swap) && swap) { - int32_t switchGeneration; - CHECK((*accessUnit)->meta()->findInt32("switchGeneration", &switchGeneration)); - { - Mutex::Autolock lock(mSwapMutex); - if (switchGeneration == mSwitchGeneration) { - swapPacketSource(stream); - sp<AMessage> msg = new AMessage(kWhatSwapped, id()); - msg->setInt32("stream", stream); - msg->setInt32("switchGeneration", switchGeneration); - msg->post(); - } - } - } else { - size_t seq = strm.mCurDiscontinuitySeq; - int64_t offsetTimeUs; - if (mDiscontinuityOffsetTimesUs.indexOfKey(seq) >= 0) { - offsetTimeUs = mDiscontinuityOffsetTimesUs.valueFor(seq); - } else { - offsetTimeUs = 0; - } - - seq += 1; - if (mDiscontinuityAbsStartTimesUs.indexOfKey(strm.mCurDiscontinuitySeq) >= 0) { - int64_t firstTimeUs; - firstTimeUs = mDiscontinuityAbsStartTimesUs.valueFor(strm.mCurDiscontinuitySeq); - offsetTimeUs += strm.mLastDequeuedTimeUs - firstTimeUs; - offsetTimeUs += strm.mLastSampleDurationUs; - } else { - offsetTimeUs += strm.mLastSampleDurationUs; - } - - mDiscontinuityOffsetTimesUs.add(seq, offsetTimeUs); - } } else if (err == OK) { if (stream == STREAMTYPE_AUDIO || stream == STREAMTYPE_VIDEO) { @@ -278,7 +252,26 @@ status_t LiveSession::dequeueAccessUnit( int32_t discontinuitySeq = 0; CHECK((*accessUnit)->meta()->findInt64("timeUs", &timeUs)); (*accessUnit)->meta()->findInt32("discontinuitySeq", &discontinuitySeq); - strm.mCurDiscontinuitySeq = discontinuitySeq; + if (discontinuitySeq > (int32_t) strm.mCurDiscontinuitySeq) { + int64_t offsetTimeUs; + if (mDiscontinuityOffsetTimesUs.indexOfKey(strm.mCurDiscontinuitySeq) >= 0) { + offsetTimeUs = mDiscontinuityOffsetTimesUs.valueFor(strm.mCurDiscontinuitySeq); + } else { + offsetTimeUs = 0; + } + + if (mDiscontinuityAbsStartTimesUs.indexOfKey(strm.mCurDiscontinuitySeq) >= 0) { + int64_t firstTimeUs; + firstTimeUs = mDiscontinuityAbsStartTimesUs.valueFor(strm.mCurDiscontinuitySeq); + offsetTimeUs += strm.mLastDequeuedTimeUs - firstTimeUs; + offsetTimeUs += strm.mLastSampleDurationUs; + } else { + offsetTimeUs += strm.mLastSampleDurationUs; + } + + mDiscontinuityOffsetTimesUs.add(discontinuitySeq, offsetTimeUs); + strm.mCurDiscontinuitySeq = discontinuitySeq; + } int32_t discard = 0; int64_t firstTimeUs; @@ -331,7 +324,6 @@ status_t LiveSession::dequeueAccessUnit( } status_t LiveSession::getStreamFormat(StreamType stream, sp<AMessage> *format) { - // No swapPacketSource race condition; called from the same thread as dequeueAccessUnit. if (!(mStreamMask & stream)) { return UNKNOWN_ERROR; } @@ -344,12 +336,21 @@ status_t LiveSession::getStreamFormat(StreamType stream, sp<AMessage> *format) { return -EAGAIN; } + if (stream == STREAMTYPE_AUDIO) { + // set AAC input buffer size to 32K bytes (256kbps x 1sec) + meta->setInt32(kKeyMaxInputSize, 32 * 1024); + } + return convertMetaDataToMessage(meta, format); } +sp<HTTPBase> LiveSession::getHTTPDataSource() { + return new MediaHTTP(mHTTPService->makeHTTPConnection()); +} + void LiveSession::connectAsync( const char *url, const KeyedVector<String8, String8> *headers) { - sp<AMessage> msg = new AMessage(kWhatConnect, id()); + sp<AMessage> msg = new AMessage(kWhatConnect, this); msg->setString("url", url); if (headers != NULL) { @@ -362,7 +363,7 @@ void LiveSession::connectAsync( } status_t LiveSession::disconnect() { - sp<AMessage> msg = new AMessage(kWhatDisconnect, id()); + sp<AMessage> msg = new AMessage(kWhatDisconnect, this); sp<AMessage> response; status_t err = msg->postAndAwaitResponse(&response); @@ -371,7 +372,7 @@ status_t LiveSession::disconnect() { } status_t LiveSession::seekTo(int64_t timeUs) { - sp<AMessage> msg = new AMessage(kWhatSeek, id()); + sp<AMessage> msg = new AMessage(kWhatSeek, this); msg->setInt64("timeUs", timeUs); sp<AMessage> response; @@ -380,6 +381,102 @@ status_t LiveSession::seekTo(int64_t timeUs) { return err; } +bool LiveSession::checkSwitchProgress( + sp<AMessage> &stopParams, int64_t delayUs, bool *needResumeUntil) { + AString newUri; + CHECK(stopParams->findString("uri", &newUri)); + + *needResumeUntil = false; + sp<AMessage> firstNewMeta[kMaxStreams]; + for (size_t i = 0; i < kMaxStreams; ++i) { + StreamType stream = indexToType(i); + if (!(mSwapMask & mNewStreamMask & stream) + || (mStreams[i].mNewUri != newUri)) { + continue; + } + if (stream == STREAMTYPE_SUBTITLES) { + continue; + } + sp<AnotherPacketSource> &source = mPacketSources.editValueAt(i); + + // First, get latest dequeued meta, which is where the decoder is at. + // (when upswitching, we take the meta after a certain delay, so that + // the decoder is left with some cushion) + sp<AMessage> lastDequeueMeta, lastEnqueueMeta; + if (delayUs > 0) { + lastDequeueMeta = source->getMetaAfterLastDequeued(delayUs); + } else { + lastDequeueMeta = source->getLatestDequeuedMeta(); + } + // Then, trim off packets at beginning of mPacketSources2 that's before + // the latest dequeued time. These samples are definitely too late. + int64_t lastTimeUs, startTimeUs; + int32_t lastSeq, startSeq; + if (lastDequeueMeta != NULL) { + CHECK(lastDequeueMeta->findInt64("timeUs", &lastTimeUs)); + CHECK(lastDequeueMeta->findInt32("discontinuitySeq", &lastSeq)); + firstNewMeta[i] = mPacketSources2.editValueAt(i) + ->trimBuffersBeforeTimeUs(lastSeq, lastTimeUs); + } + // Now firstNewMeta[i] is the first sample after the trim. + // If it's NULL, we failed because dequeue already past all samples + // in mPacketSource2, we have to try again. + if (firstNewMeta[i] == NULL) { + ALOGV("[%s] dequeue time (%d, %lld) past start time", + stream == STREAMTYPE_AUDIO ? "audio" : "video", + lastSeq, (long long) lastTimeUs); + return false; + } + + // Otherwise, we check if mPacketSources2 overlaps with what old fetcher + // already fetched, and see if we need to resumeUntil + lastEnqueueMeta = source->getLatestEnqueuedMeta(); + // lastEnqueueMeta == NULL means old fetcher stopped at a discontinuity + // boundary, no need to resume as the content will look different anyways + if (lastEnqueueMeta != NULL) { + CHECK(lastEnqueueMeta->findInt64("timeUs", &lastTimeUs)); + CHECK(lastEnqueueMeta->findInt32("discontinuitySeq", &lastSeq)); + CHECK(firstNewMeta[i]->findInt64("timeUs", &startTimeUs)); + CHECK(firstNewMeta[i]->findInt32("discontinuitySeq", &startSeq)); + + // no need to resume old fetcher if new fetcher started in different + // discontinuity sequence, as the content will look different. + *needResumeUntil |= + (startSeq == lastSeq + && startTimeUs - lastTimeUs > 100000ll); + + // update the stopTime for resumeUntil, as we might have removed some + // packets from the head in mPacketSource2 + stopParams->setInt64(getKeyForStream(stream), startTimeUs); + } + } + + // if we're here, it means dequeue progress hasn't passed some samples in + // mPacketSource2, we can trim off the excess in mPacketSource. + // (old fetcher might still need to resumeUntil the start time of new fetcher) + for (size_t i = 0; i < kMaxStreams; ++i) { + StreamType stream = indexToType(i); + if (!(mSwapMask & mNewStreamMask & stream) + || (newUri != mStreams[i].mNewUri)) { + continue; + } + if (stream == STREAMTYPE_SUBTITLES) { + continue; + } + int64_t startTimeUs; + int32_t startSeq; + CHECK(firstNewMeta[i] != NULL); + CHECK(firstNewMeta[i]->findInt64("timeUs", &startTimeUs)); + CHECK(firstNewMeta[i]->findInt32("discontinuitySeq", &startSeq)); + mPacketSources.valueFor(stream)->trimBuffersAfterTimeUs(startSeq, startTimeUs); + } + + // no resumeUntil if already underflow + *needResumeUntil &= !mBuffering; + + return true; +} + void LiveSession::onMessageReceived(const sp<AMessage> &msg) { switch (msg->what()) { case kWhatConnect: @@ -402,7 +499,7 @@ void LiveSession::onMessageReceived(const sp<AMessage> &msg) { case kWhatSeek: { - uint32_t seekReplyID; + sp<AReplyToken> seekReplyID; CHECK(msg->senderAwaitsResponse(&seekReplyID)); mSeekReplyID = seekReplyID; mSeekReply = new AMessage; @@ -426,16 +523,25 @@ void LiveSession::onMessageReceived(const sp<AMessage> &msg) { case PlaylistFetcher::kWhatPaused: case PlaylistFetcher::kWhatStopped: { - if (what == PlaylistFetcher::kWhatStopped) { - AString uri; - CHECK(msg->findString("uri", &uri)); - if (mFetcherInfos.removeItem(uri) < 0) { - // ignore duplicated kWhatStopped messages. - break; - } + AString uri; + CHECK(msg->findString("uri", &uri)); + ssize_t index = mFetcherInfos.indexOfKey(uri); + if (index < 0) { + // ignore msgs from fetchers that's already gone + break; + } - if (mSwitchInProgress) { - tryToFinishBandwidthSwitch(); + if (what == PlaylistFetcher::kWhatStopped) { + mFetcherLooper->unregisterHandler( + mFetcherInfos[index].mFetcher->id()); + mFetcherInfos.removeItemsAt(index); + } else if (what == PlaylistFetcher::kWhatPaused) { + int32_t seekMode; + CHECK(msg->findInt32("seekMode", &seekMode)); + for (size_t i = 0; i < kMaxStreams; ++i) { + if (mStreams[i].mUri == uri) { + mStreams[i].mSeekMode = (SeekMode) seekMode; + } } } @@ -443,14 +549,6 @@ void LiveSession::onMessageReceived(const sp<AMessage> &msg) { CHECK_GT(mContinuationCounter, 0); if (--mContinuationCounter == 0) { mContinuation->post(); - - if (mSeekReplyID != 0) { - CHECK(mSeekReply != NULL); - mSeekReply->setInt32("err", OK); - mSeekReply->postReply(mSeekReplyID); - mSeekReplyID = 0; - mSeekReply.clear(); - } } } break; @@ -464,8 +562,21 @@ void LiveSession::onMessageReceived(const sp<AMessage> &msg) { int64_t durationUs; CHECK(msg->findInt64("durationUs", &durationUs)); - FetcherInfo *info = &mFetcherInfos.editValueFor(uri); - info->mDurationUs = durationUs; + ssize_t index = mFetcherInfos.indexOfKey(uri); + if (index >= 0) { + FetcherInfo *info = &mFetcherInfos.editValueFor(uri); + info->mDurationUs = durationUs; + } + break; + } + + case PlaylistFetcher::kWhatTargetDurationUpdate: + { + int64_t targetDurationUs; + CHECK(msg->findInt64("targetDurationUs", &targetDurationUs)); + mUpSwitchMark = min(kUpSwitchMarkUs, targetDurationUs * 3); + mDownSwitchMark = min(kDownSwitchMarkUs, targetDurationUs * 9 / 4); + mUpSwitchMargin = min(kUpSwitchMarginUs, targetDurationUs); break; } @@ -506,38 +617,23 @@ void LiveSession::onMessageReceived(const sp<AMessage> &msg) { mPacketSources.valueFor( STREAMTYPE_SUBTITLES)->signalEOS(err); - sp<AMessage> notify = mNotify->dup(); - notify->setInt32("what", kWhatError); - notify->setInt32("err", err); - notify->post(); + postError(err); break; } - case PlaylistFetcher::kWhatTemporarilyDoneFetching: + case PlaylistFetcher::kWhatStopReached: { + ALOGV("kWhatStopReached"); + AString uri; CHECK(msg->findString("uri", &uri)); - if (mFetcherInfos.indexOfKey(uri) < 0) { - ALOGE("couldn't find uri"); + ssize_t index = mFetcherInfos.indexOfKey(uri); + if (index < 0) { break; } - FetcherInfo *info = &mFetcherInfos.editValueFor(uri); - info->mIsPrepared = true; - - if (mInPreparationPhase) { - bool allFetchersPrepared = true; - for (size_t i = 0; i < mFetcherInfos.size(); ++i) { - if (!mFetcherInfos.valueAt(i).mIsPrepared) { - allFetchersPrepared = false; - break; - } - } - if (allFetchersPrepared) { - postPrepared(OK); - } - } + tryToFinishBandwidthSwitch(uri); break; } @@ -550,15 +646,69 @@ void LiveSession::onMessageReceived(const sp<AMessage> &msg) { break; } - // Resume fetcher for the original variant; the resumed fetcher should - // continue until the timestamps found in msg, which is stored by the - // new fetcher to indicate where the new variant has started buffering. - for (size_t i = 0; i < mFetcherInfos.size(); i++) { - const FetcherInfo info = mFetcherInfos.valueAt(i); - if (info.mToBeRemoved) { - info.mFetcher->resumeUntilAsync(msg); + AString uri; + CHECK(msg->findString("uri", &uri)); + + // mark new fetcher mToBeResumed + ssize_t index = mFetcherInfos.indexOfKey(uri); + if (index >= 0) { + mFetcherInfos.editValueAt(index).mToBeResumed = true; + } + + // temporarily disable packet sources to be swapped to prevent + // NuPlayerDecoder from dequeuing while we check progress + for (size_t i = 0; i < mPacketSources.size(); ++i) { + if ((mSwapMask & mPacketSources.keyAt(i)) + && uri == mStreams[i].mNewUri) { + mPacketSources.editValueAt(i)->enable(false); + } + } + bool switchUp = (mCurBandwidthIndex > mOrigBandwidthIndex); + // If switching up, require a cushion bigger than kUnderflowMark + // to avoid buffering immediately after the switch. + // (If we don't have that cushion we'd rather cancel and try again.) + int64_t delayUs = switchUp ? (kUnderflowMarkUs + 1000000ll) : 0; + bool needResumeUntil = false; + sp<AMessage> stopParams = msg; + if (checkSwitchProgress(stopParams, delayUs, &needResumeUntil)) { + // playback time hasn't passed startAt time + if (!needResumeUntil) { + for (size_t i = 0; i < kMaxStreams; ++i) { + if ((mSwapMask & indexToType(i)) + && uri == mStreams[i].mNewUri) { + // have to make a copy of mStreams[i].mUri because + // tryToFinishBandwidthSwitch is modifying mStreams[] + AString oldURI = mStreams[i].mUri; + tryToFinishBandwidthSwitch(oldURI); + break; + } + } + } else { + // startAt time is after last enqueue time + // Resume fetcher for the original variant; the resumed fetcher should + // continue until the timestamps found in msg, which is stored by the + // new fetcher to indicate where the new variant has started buffering. + for (size_t i = 0; i < mFetcherInfos.size(); i++) { + const FetcherInfo &info = mFetcherInfos.valueAt(i); + if (info.mToBeRemoved) { + info.mFetcher->resumeUntilAsync(stopParams); + } + } + } + } else { + // playback time passed startAt time + if (switchUp) { + // if switching up, cancel and retry if condition satisfies again + cancelBandwidthSwitch(true /* resume */); + } else { + resumeFetcher(uri, mSwapMask, -1, true /* newUri */); } } + // re-enable all packet sources + for (size_t i = 0; i < mPacketSources.size(); ++i) { + mPacketSources.editValueAt(i)->enable(true); + } + break; } @@ -569,19 +719,6 @@ void LiveSession::onMessageReceived(const sp<AMessage> &msg) { break; } - case kWhatCheckBandwidth: - { - int32_t generation; - CHECK(msg->findInt32("generation", &generation)); - - if (generation != mCheckBandwidthGeneration) { - break; - } - - onCheckBandwidth(msg); - break; - } - case kWhatChangeConfiguration: { onChangeConfiguration(msg); @@ -606,21 +743,13 @@ void LiveSession::onMessageReceived(const sp<AMessage> &msg) { break; } - case kWhatSwapped: + case kWhatPollBuffering: { - onSwapped(msg); - break; - } - - case kWhatCheckSwitchDown: - { - onCheckSwitchDown(); - break; - } - - case kWhatSwitchDown: - { - onSwitchDown(); + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + if (generation == mPollBufferingGeneration) { + onPollBuffering(); + } break; } @@ -691,6 +820,14 @@ void LiveSession::onConnect(const sp<AMessage> &msg) { return; } + // create looper for fetchers + if (mFetcherLooper == NULL) { + mFetcherLooper = new ALooper(); + + mFetcherLooper->setName("Fetcher"); + mFetcherLooper->start(false, false); + } + // We trust the content provider to make a reasonable choice of preferred // initial bandwidth by listing it first in the variant playlist. // At startup we really don't have a good estimate on the available @@ -742,22 +879,20 @@ void LiveSession::onConnect(const sp<AMessage> &msg) { } void LiveSession::finishDisconnect() { + ALOGV("finishDisconnect"); + // No reconfiguration is currently pending, make sure none will trigger // during disconnection either. - cancelCheckBandwidthEvent(); - - // Protect mPacketSources from a swapPacketSource race condition through disconnect. - // (finishDisconnect, onFinishDisconnect2) cancelBandwidthSwitch(); - // cancel switch down monitor - mSwitchDownMonitor.clear(); + // cancel buffer polling + cancelPollBuffering(); for (size_t i = 0; i < mFetcherInfos.size(); ++i) { mFetcherInfos.valueAt(i).mFetcher->stopAsync(); } - sp<AMessage> msg = new AMessage(kWhatFinishDisconnect2, id()); + sp<AMessage> msg = new AMessage(kWhatFinishDisconnect2, this); mContinuationCounter = mFetcherInfos.size(); mContinuation = msg; @@ -780,7 +915,7 @@ void LiveSession::onFinishDisconnect2() { response->setInt32("err", OK); response->postReply(mDisconnectReplyID); - mDisconnectReplyID = 0; + mDisconnectReplyID.clear(); } sp<PlaylistFetcher> LiveSession::addFetcher(const char *uri) { @@ -790,16 +925,16 @@ sp<PlaylistFetcher> LiveSession::addFetcher(const char *uri) { return NULL; } - sp<AMessage> notify = new AMessage(kWhatFetcherNotify, id()); + sp<AMessage> notify = new AMessage(kWhatFetcherNotify, this); notify->setString("uri", uri); notify->setInt32("switchGeneration", mSwitchGeneration); FetcherInfo info; info.mFetcher = new PlaylistFetcher(notify, this, uri, mSubtitleGeneration); info.mDurationUs = -1ll; - info.mIsPrepared = false; info.mToBeRemoved = false; - looper()->registerHandler(info.mFetcher); + info.mToBeResumed = false; + mFetcherLooper->registerHandler(info.mFetcher); mFetcherInfos.add(uri, info); @@ -827,14 +962,15 @@ ssize_t LiveSession::fetchFile( int64_t range_offset, int64_t range_length, uint32_t block_size, /* download block size */ sp<DataSource> *source, /* to return and reuse source */ - String8 *actualUrl) { + String8 *actualUrl, + bool forceConnectHTTP /* force connect HTTP when resuing source */) { off64_t size; sp<DataSource> temp_source; if (source == NULL) { source = &temp_source; } - if (*source == NULL) { + if (*source == NULL || forceConnectHTTP) { if (!strncasecmp(url, "file://", 7)) { *source = new FileSource(url + 7); } else if (strncasecmp(url, "http://", 7) @@ -853,13 +989,18 @@ ssize_t LiveSession::fetchFile( ? "" : AStringPrintf("%lld", range_offset + range_length - 1).c_str()).c_str())); } - status_t err = mHTTPDataSource->connect(url, &headers); + + HTTPBase* httpDataSource = + (*source == NULL) ? mHTTPDataSource.get() : (HTTPBase*)source->get(); + status_t err = httpDataSource->connect(url, &headers); if (err != OK) { return err; } - *source = mHTTPDataSource; + if (*source == NULL) { + *source = mHTTPDataSource; + } } } @@ -995,8 +1136,108 @@ static double uniformRand() { } #endif -size_t LiveSession::getBandwidthIndex() { - if (mBandwidthItems.size() == 0) { +bool LiveSession::resumeFetcher( + const AString &uri, uint32_t streamMask, int64_t timeUs, bool newUri) { + ssize_t index = mFetcherInfos.indexOfKey(uri); + if (index < 0) { + ALOGE("did not find fetcher for uri: %s", uri.c_str()); + return false; + } + + bool resume = false; + sp<AnotherPacketSource> sources[kMaxStreams]; + for (size_t i = 0; i < kMaxStreams; ++i) { + if ((streamMask & indexToType(i)) + && ((!newUri && uri == mStreams[i].mUri) + || (newUri && uri == mStreams[i].mNewUri))) { + resume = true; + if (newUri) { + sources[i] = mPacketSources2.valueFor(indexToType(i)); + sources[i]->clear(); + } else { + sources[i] = mPacketSources.valueFor(indexToType(i)); + } + } + } + + if (resume) { + ALOGV("resuming fetcher %s, timeUs %lld", uri.c_str(), (long long)timeUs); + SeekMode seekMode = newUri ? kSeekModeNextSample : kSeekModeExactPosition; + mFetcherInfos.editValueAt(index).mFetcher->startAsync( + sources[kAudioIndex], + sources[kVideoIndex], + sources[kSubtitleIndex], + timeUs, -1, -1, seekMode); + } + + return resume; +} + +float LiveSession::getAbortThreshold( + ssize_t currentBWIndex, ssize_t targetBWIndex) const { + float abortThreshold = -1.0f; + if (currentBWIndex > 0 && targetBWIndex < currentBWIndex) { + /* + If we're switching down, we need to decide whether to + + 1) finish last segment of high-bandwidth variant, or + 2) abort last segment of high-bandwidth variant, and fetch an + overlapping portion from low-bandwidth variant. + + Here we try to maximize the amount of buffer left when the + switch point is met. Given the following parameters: + + B: our current buffering level in seconds + T: target duration in seconds + X: sample duration in seconds remain to fetch in last segment + bw0: bandwidth of old variant (as specified in playlist) + bw1: bandwidth of new variant (as specified in playlist) + bw: measured bandwidth available + + If we choose 1), when switch happens at the end of current + segment, our buffering will be + B + X - X * bw0 / bw + + If we choose 2), when switch happens where we aborted current + segment, our buffering will be + B - (T - X) * bw1 / bw + + We should only choose 1) if + X/T < bw1 / (bw1 + bw0 - bw) + */ + + // Taking the measured current bandwidth at 50% face value only, + // as our bandwidth estimation is a lagging indicator. Being + // conservative on this, we prefer switching to lower bandwidth + // unless we're really confident finishing up the last segment + // of higher bandwidth will be fast. + CHECK(mLastBandwidthBps >= 0); + abortThreshold = + (float)mBandwidthItems.itemAt(targetBWIndex).mBandwidth + / ((float)mBandwidthItems.itemAt(targetBWIndex).mBandwidth + + (float)mBandwidthItems.itemAt(currentBWIndex).mBandwidth + - (float)mLastBandwidthBps * 0.5f); + if (abortThreshold < 0.0f) { + abortThreshold = -1.0f; // do not abort + } + ALOGV("Switching Down: bps %ld => %ld, measured %d, abort ratio %.2f", + mBandwidthItems.itemAt(currentBWIndex).mBandwidth, + mBandwidthItems.itemAt(targetBWIndex).mBandwidth, + mLastBandwidthBps, + abortThreshold); + } + return abortThreshold; +} + +void LiveSession::addBandwidthMeasurement(size_t numBytes, int64_t delayUs) { + mBandwidthEstimator->addBandwidthMeasurement(numBytes, delayUs); +} + +size_t LiveSession::getBandwidthIndex(int32_t bandwidthBps) { + if (mBandwidthItems.size() < 2) { + // shouldn't be here if we only have 1 bandwidth, check + // logic to get rid of redundant bandwidth polling + ALOGW("getBandwidthIndex() called for single bandwidth playlist!"); return 0; } @@ -1014,15 +1255,6 @@ size_t LiveSession::getBandwidthIndex() { } if (index < 0) { - int32_t bandwidthBps; - if (mHTTPDataSource != NULL - && mHTTPDataSource->estimateBandwidth(&bandwidthBps)) { - ALOGV("bandwidth estimated at %.2f kbps", bandwidthBps / 1024.0f); - } else { - ALOGV("no bandwidth estimate."); - return 0; // Pick the lowest bandwidth stream by default. - } - char value[PROPERTY_VALUE_MAX]; if (property_get("media.httplive.max-bw", value, NULL)) { char *end; @@ -1039,15 +1271,9 @@ size_t LiveSession::getBandwidthIndex() { index = mBandwidthItems.size() - 1; while (index > 0) { - // consider only 80% of the available bandwidth, but if we are switching up, - // be even more conservative (70%) to avoid overestimating and immediately - // switching back. - size_t adjustedBandwidthBps = bandwidthBps; - if (index > mCurBandwidthIndex) { - adjustedBandwidthBps = adjustedBandwidthBps * 7 / 10; - } else { - adjustedBandwidthBps = adjustedBandwidthBps * 8 / 10; - } + // be conservative (70%) to avoid overestimating and immediately + // switching down again. + size_t adjustedBandwidthBps = bandwidthBps * 7 / 10; if (mBandwidthItems.itemAt(index).mBandwidth <= adjustedBandwidthBps) { break; } @@ -1130,7 +1356,7 @@ status_t LiveSession::onSeek(const sp<AMessage> &msg) { CHECK(msg->findInt64("timeUs", &timeUs)); if (!mReconfigurationInProgress) { - changeConfiguration(timeUs, mCurBandwidthIndex); + changeConfiguration(timeUs); return OK; } else { return -EWOULDBLOCK; @@ -1185,8 +1411,7 @@ status_t LiveSession::selectTrack(size_t index, bool select) { ++mSubtitleGeneration; status_t err = mPlaylist->selectTrack(index, select); if (err == OK) { - sp<AMessage> msg = new AMessage(kWhatChangeConfiguration, id()); - msg->setInt32("bandwidthIndex", mCurBandwidthIndex); + sp<AMessage> msg = new AMessage(kWhatChangeConfiguration, this); msg->setInt32("pickTrack", select); msg->post(); } @@ -1201,35 +1426,18 @@ ssize_t LiveSession::getSelectedTrack(media_track_type type) const { } } -bool LiveSession::canSwitchUp() { - // Allow upwards bandwidth switch when a stream has buffered at least 10 seconds. - status_t err = OK; - for (size_t i = 0; i < mPacketSources.size(); ++i) { - sp<AnotherPacketSource> source = mPacketSources.valueAt(i); - int64_t dur = source->getBufferedDurationUs(&err); - if (err == OK && dur > 10000000) { - return true; - } - } - return false; -} - void LiveSession::changeConfiguration( - int64_t timeUs, size_t bandwidthIndex, bool pickTrack) { - // Protect mPacketSources from a swapPacketSource race condition through reconfiguration. - // (changeConfiguration, onChangeConfiguration2, onChangeConfiguration3). + int64_t timeUs, ssize_t bandwidthIndex, bool pickTrack) { cancelBandwidthSwitch(); CHECK(!mReconfigurationInProgress); mReconfigurationInProgress = true; - - mCurBandwidthIndex = bandwidthIndex; - - ALOGV("changeConfiguration => timeUs:%" PRId64 " us, bwIndex:%zu, pickTrack:%d", - timeUs, bandwidthIndex, pickTrack); - - CHECK_LT(bandwidthIndex, mBandwidthItems.size()); - const BandwidthItem &item = mBandwidthItems.itemAt(bandwidthIndex); + if (bandwidthIndex >= 0) { + mOrigBandwidthIndex = mCurBandwidthIndex; + mCurBandwidthIndex = bandwidthIndex; + } + CHECK_LT(mCurBandwidthIndex, mBandwidthItems.size()); + const BandwidthItem &item = mBandwidthItems.itemAt(mCurBandwidthIndex); uint32_t streamMask = 0; // streams that should be fetched by the new fetcher uint32_t resumeMask = 0; // streams that should be fetched by the original fetcher @@ -1244,38 +1452,55 @@ void LiveSession::changeConfiguration( // Step 1, stop and discard fetchers that are no longer needed. // Pause those that we'll reuse. for (size_t i = 0; i < mFetcherInfos.size(); ++i) { + // skip fetchers that are marked mToBeRemoved, + // these are done and can't be reused + if (mFetcherInfos[i].mToBeRemoved) { + continue; + } + const AString &uri = mFetcherInfos.keyAt(i); bool discardFetcher = true; - // If we're seeking all current fetchers are discarded. if (timeUs < 0ll) { // delay fetcher removal if not picking tracks discardFetcher = pickTrack; + } - for (size_t j = 0; j < kMaxStreams; ++j) { - StreamType type = indexToType(j); - if ((streamMask & type) && uri == URIs[j]) { - resumeMask |= type; - streamMask &= ~type; - discardFetcher = false; - } + for (size_t j = 0; j < kMaxStreams; ++j) { + StreamType type = indexToType(j); + if ((streamMask & type) && uri == URIs[j]) { + resumeMask |= type; + streamMask &= ~type; + discardFetcher = false; } } if (discardFetcher) { mFetcherInfos.valueAt(i).mFetcher->stopAsync(); } else { - mFetcherInfos.valueAt(i).mFetcher->pauseAsync(); + float threshold = -1.0f; // always finish fetching by default + if (timeUs >= 0ll) { + // seeking, no need to finish fetching + threshold = 0.0f; + } else if (!pickTrack) { + // adapting, abort if remaining of current segment is over threshold + threshold = getAbortThreshold( + mOrigBandwidthIndex, mCurBandwidthIndex); + } + + ALOGV("Pausing with threshold %.3f", threshold); + + mFetcherInfos.valueAt(i).mFetcher->pauseAsync(threshold); } } sp<AMessage> msg; if (timeUs < 0ll) { // skip onChangeConfiguration2 (decoder destruction) if not seeking. - msg = new AMessage(kWhatChangeConfiguration3, id()); + msg = new AMessage(kWhatChangeConfiguration3, this); } else { - msg = new AMessage(kWhatChangeConfiguration2, id()); + msg = new AMessage(kWhatChangeConfiguration2, this); } msg->setInt32("streamMask", streamMask); msg->setInt32("resumeMask", resumeMask); @@ -1296,23 +1521,14 @@ void LiveSession::changeConfiguration( if (mContinuationCounter == 0) { msg->post(); - - if (mSeekReplyID != 0) { - CHECK(mSeekReply != NULL); - mSeekReply->setInt32("err", OK); - mSeekReply->postReply(mSeekReplyID); - mSeekReplyID = 0; - mSeekReply.clear(); - } } } void LiveSession::onChangeConfiguration(const sp<AMessage> &msg) { if (!mReconfigurationInProgress) { - int32_t pickTrack = 0, bandwidthIndex = mCurBandwidthIndex; + int32_t pickTrack = 0; msg->findInt32("pickTrack", &pickTrack); - msg->findInt32("bandwidthIndex", &bandwidthIndex); - changeConfiguration(-1ll /* timeUs */, bandwidthIndex, pickTrack); + changeConfiguration(-1ll /* timeUs */, -1, pickTrack); } else { msg->post(1000000ll); // retry in 1 sec } @@ -1323,13 +1539,42 @@ void LiveSession::onChangeConfiguration2(const sp<AMessage> &msg) { // All fetchers are either suspended or have been removed now. + // If we're seeking, clear all packet sources before we report + // seek complete, to prevent decoder from pulling stale data. + int64_t timeUs; + CHECK(msg->findInt64("timeUs", &timeUs)); + + if (timeUs >= 0) { + mLastSeekTimeUs = timeUs; + + for (size_t i = 0; i < mPacketSources.size(); i++) { + mPacketSources.editValueAt(i)->clear(); + } + + for (size_t i = 0; i < kMaxStreams; ++i) { + mStreams[i].mCurDiscontinuitySeq = 0; + } + + mDiscontinuityOffsetTimesUs.clear(); + mDiscontinuityAbsStartTimesUs.clear(); + + if (mSeekReplyID != NULL) { + CHECK(mSeekReply != NULL); + mSeekReply->setInt32("err", OK); + mSeekReply->postReply(mSeekReplyID); + mSeekReplyID.clear(); + mSeekReply.clear(); + } + + // restart buffer polling after seek becauese previous + // buffering position is no longer valid. + restartPollBuffering(); + } + uint32_t streamMask, resumeMask; CHECK(msg->findInt32("streamMask", (int32_t *)&streamMask)); CHECK(msg->findInt32("resumeMask", (int32_t *)&resumeMask)); - // currently onChangeConfiguration2 is only called for seeking; - // remove the following CHECK if using it else where. - CHECK_EQ(resumeMask, 0); streamMask |= resumeMask; AString URIs[kMaxStreams]; @@ -1341,17 +1586,25 @@ void LiveSession::onChangeConfiguration2(const sp<AMessage> &msg) { } } - // Determine which decoders to shutdown on the player side, - // a decoder has to be shutdown if either - // 1) its streamtype was active before but now longer isn't. - // or - // 2) its streamtype was already active and still is but the URI - // has changed. uint32_t changedMask = 0; for (size_t i = 0; i < kMaxStreams && i != kSubtitleIndex; ++i) { - if (((mStreamMask & streamMask & indexToType(i)) - && !(URIs[i] == mStreams[i].mUri)) - || (mStreamMask & ~streamMask & indexToType(i))) { + // stream URI could change even if onChangeConfiguration2 is only + // used for seek. Seek could happen during a bw switch, in this + // case bw switch will be cancelled, but the seekTo position will + // fetch from the new URI. + if ((mStreamMask & streamMask & indexToType(i)) + && !mStreams[i].mUri.empty() + && !(URIs[i] == mStreams[i].mUri)) { + ALOGV("stream %zu changed: oldURI %s, newURI %s", i, + mStreams[i].mUri.c_str(), URIs[i].c_str()); + sp<AnotherPacketSource> source = mPacketSources.valueFor(indexToType(i)); + source->queueDiscontinuity( + ATSParser::DISCONTINUITY_FORMATCHANGE, NULL, true); + } + // Determine which decoders to shutdown on the player side, + // a decoder has to be shutdown if its streamtype was active + // before but now longer isn't. + if ((mStreamMask & ~streamMask & indexToType(i))) { changedMask |= indexToType(i); } } @@ -1372,7 +1625,7 @@ void LiveSession::onChangeConfiguration2(const sp<AMessage> &msg) { notify->setInt32("changedMask", changedMask); msg->setWhat(kWhatChangeConfiguration3); - msg->setTarget(id()); + msg->setTarget(this); notify->setMessage("reply", msg); notify->post(); @@ -1390,12 +1643,14 @@ void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { int64_t timeUs; int32_t pickTrack; bool switching = false; + bool finishSwitching = false; CHECK(msg->findInt64("timeUs", &timeUs)); CHECK(msg->findInt32("pickTrack", &pickTrack)); if (timeUs < 0ll) { if (!pickTrack) { switching = true; + finishSwitching = (streamMask == 0); } mRealTimeBaseUs = ALooper::GetNowUs() - mLastDequeuedTimeUs; } else { @@ -1423,36 +1678,12 @@ void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { ALOGV("resuming fetchers for mask 0x%08x", resumeMask); for (size_t i = 0; i < mFetcherInfos.size(); ++i) { const AString &uri = mFetcherInfos.keyAt(i); - - sp<AnotherPacketSource> sources[kMaxStreams]; - for (size_t j = 0; j < kMaxStreams; ++j) { - if ((resumeMask & indexToType(j)) && uri == mStreams[j].mUri) { - sources[j] = mPacketSources.valueFor(indexToType(j)); - - if (j != kSubtitleIndex) { - ALOGV("queueing dummy discontinuity for stream type %d", indexToType(j)); - sp<AnotherPacketSource> discontinuityQueue; - discontinuityQueue = mDiscontinuities.valueFor(indexToType(j)); - discontinuityQueue->queueDiscontinuity( - ATSParser::DISCONTINUITY_NONE, - NULL, - true); - } - } - } - - FetcherInfo &info = mFetcherInfos.editValueAt(i); - if (sources[kAudioIndex] != NULL || sources[kVideoIndex] != NULL - || sources[kSubtitleIndex] != NULL) { - info.mFetcher->startAsync( - sources[kAudioIndex], sources[kVideoIndex], sources[kSubtitleIndex]); - } else { - info.mToBeRemoved = true; + if (!resumeFetcher(uri, resumeMask, timeUs)) { + mFetcherInfos.editValueAt(i).mToBeRemoved = true; } } // streamMask now only contains the types that need a new fetcher created. - if (streamMask != 0) { ALOGV("creating new fetchers for mask 0x%08x", streamMask); } @@ -1473,6 +1704,7 @@ void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { int64_t startTimeUs = -1; int64_t segmentStartTimeUs = -1ll; int32_t discontinuitySeq = -1; + SeekMode seekMode = kSeekModeExactPosition; sp<AnotherPacketSource> sources[kMaxStreams]; if (i == kSubtitleIndex) { @@ -1486,42 +1718,62 @@ void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { sources[j] = mPacketSources.valueFor(indexToType(j)); if (timeUs >= 0) { - sources[j]->clear(); startTimeUs = timeUs; - - sp<AnotherPacketSource> discontinuityQueue; - sp<AMessage> extra = new AMessage; - extra->setInt64("timeUs", timeUs); - discontinuityQueue = mDiscontinuities.valueFor(indexToType(j)); - discontinuityQueue->queueDiscontinuity( - ATSParser::DISCONTINUITY_TIME, extra, true); } else { int32_t type; sp<AMessage> meta; if (pickTrack) { // selecting + + // FIXME: + // This should only apply to the track that's being picked, we + // need a bitmask to indicate that. + // + // It's possible that selectTrack() gets called during a bandwidth + // switch, and we needed to fetch a new variant. The new fetcher + // should start from where old fetcher left off, not where decoder + // is dequeueing at. + meta = sources[j]->getLatestDequeuedMeta(); } else { // adapting meta = sources[j]->getLatestEnqueuedMeta(); + if (meta != NULL && mCurBandwidthIndex > mOrigBandwidthIndex) { + // switching up + meta = sources[j]->getMetaAfterLastDequeued(mUpSwitchMargin); + } } - if (meta != NULL && !meta->findInt32("discontinuity", &type)) { + if (j != kSubtitleIndex + && meta != NULL + && !meta->findInt32("discontinuity", &type)) { int64_t tmpUs; int64_t tmpSegmentUs; + int32_t seq; CHECK(meta->findInt64("timeUs", &tmpUs)); CHECK(meta->findInt64("segmentStartTimeUs", &tmpSegmentUs)); - if (startTimeUs < 0 || tmpSegmentUs < segmentStartTimeUs) { + CHECK(meta->findInt32("discontinuitySeq", &seq)); + // If we're switching and looking for next sample or segment, set the target + // segment start time to tmpSegmentUs + tmpDurationUs / 2, which is + // the middle point of the segment where the last sample was. + // This is needed if segments of the two variants are not perfectly + // aligned. (If the corresponding segment in new variant starts slightly + // later than that in the old variant, we still want the switching to + // start in the next one, not the current one) + if (mStreams[j].mSeekMode == kSeekModeNextSample + || mStreams[j].mSeekMode == kSeekModeNextSegment) { + int64_t tmpDurationUs; + CHECK(meta->findInt64("segmentDurationUs", &tmpDurationUs)); + tmpSegmentUs += tmpDurationUs / 2; + } + if (startTimeUs < 0 || seq > discontinuitySeq + || (seq == discontinuitySeq + && (tmpSegmentUs > segmentStartTimeUs + || (tmpSegmentUs == segmentStartTimeUs + && tmpUs > startTimeUs)))) { startTimeUs = tmpUs; segmentStartTimeUs = tmpSegmentUs; - } else if (tmpSegmentUs == segmentStartTimeUs && tmpUs < startTimeUs) { - startTimeUs = tmpUs; - } - - int32_t seq; - CHECK(meta->findInt32("discontinuitySeq", &seq)); - if (discontinuitySeq < 0 || seq < discontinuitySeq) { discontinuitySeq = seq; } } @@ -1532,17 +1784,26 @@ void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { if (j == kSubtitleIndex) { break; } - sp<AnotherPacketSource> discontinuityQueue; - discontinuityQueue = mDiscontinuities.valueFor(indexToType(j)); - discontinuityQueue->queueDiscontinuity( - ATSParser::DISCONTINUITY_FORMATCHANGE, NULL, true); + + ALOGV("stream[%zu]: queue format change", j); + + sources[j]->queueDiscontinuity( + ATSParser::DISCONTINUITY_FORMAT_ONLY, NULL, true); } else { // adapting, queue discontinuities after resume sources[j] = mPacketSources2.valueFor(indexToType(j)); sources[j]->clear(); uint32_t extraStreams = mNewStreamMask & (~mStreamMask); if (extraStreams & indexToType(j)) { - sources[j]->queueAccessUnit(createFormatChangeBuffer(/*swap*/ false)); + sources[j]->queueDiscontinuity( + ATSParser::DISCONTINUITY_FORMAT_ONLY, NULL, true); + } + // the new fetcher might be providing streams that used to be + // provided by two different fetchers, if one of the fetcher + // paused in the middle while the other somehow paused in next + // seg, we have to start from next seg. + if (seekMode < mStreams[j].mSeekMode) { + seekMode = mStreams[j].mSeekMode; } } } @@ -1558,47 +1819,88 @@ void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { startTimeUs < 0 ? mLastSeekTimeUs : startTimeUs, segmentStartTimeUs, discontinuitySeq, - switching); + seekMode); } // All fetchers have now been started, the configuration change // has completed. - cancelCheckBandwidthEvent(); - scheduleCheckBandwidthEvent(); - ALOGV("XXX configuration change completed."); mReconfigurationInProgress = false; if (switching) { mSwitchInProgress = true; + + if (finishSwitching) { + // Switch is finished now, no new fetchers are created. + // This path is hit when old variant had video and audio from + // two separate fetchers, while new variant has audio only, + // which reuses the previous audio fetcher. + for (size_t i = 0; i < kMaxStreams; ++i) { + if (mSwapMask & indexToType(i)) { + tryToFinishBandwidthSwitch(mStreams[i].mUri); + } + } + } } else { mStreamMask = mNewStreamMask; + mOrigBandwidthIndex = mCurBandwidthIndex; } - if (mDisconnectReplyID != 0) { + if (mDisconnectReplyID != NULL) { finishDisconnect(); } } -void LiveSession::onSwapped(const sp<AMessage> &msg) { - int32_t switchGeneration; - CHECK(msg->findInt32("switchGeneration", &switchGeneration)); - if (switchGeneration != mSwitchGeneration) { +void LiveSession::swapPacketSource(StreamType stream) { + ALOGV("swapPacketSource: stream = %d", stream); + + // transfer packets from source2 to source + sp<AnotherPacketSource> &aps = mPacketSources.editValueFor(stream); + sp<AnotherPacketSource> &aps2 = mPacketSources2.editValueFor(stream); + + // queue discontinuity in mPacketSource + aps->queueDiscontinuity(ATSParser::DISCONTINUITY_FORMAT_ONLY, NULL, false); + + // queue packets in mPacketSource2 to mPacketSource + status_t finalResult = OK; + sp<ABuffer> accessUnit; + while (aps2->hasBufferAvailable(&finalResult) && finalResult == OK && + OK == aps2->dequeueAccessUnit(&accessUnit)) { + aps->queueAccessUnit(accessUnit); + } + aps2->clear(); +} + +void LiveSession::tryToFinishBandwidthSwitch(const AString &oldUri) { + if (!mSwitchInProgress) { return; } - int32_t stream; - CHECK(msg->findInt32("stream", &stream)); + ssize_t index = mFetcherInfos.indexOfKey(oldUri); + if (index < 0 || !mFetcherInfos[index].mToBeRemoved) { + return; + } - ssize_t idx = typeToIndex(stream); - CHECK(idx >= 0); - if ((mNewStreamMask & stream) && mStreams[idx].mNewUri.empty()) { - ALOGW("swapping stream type %d %s to empty stream", stream, mStreams[idx].mUri.c_str()); + // Swap packet source of streams provided by old variant + for (size_t idx = 0; idx < kMaxStreams; idx++) { + StreamType stream = indexToType(idx); + if ((mSwapMask & stream) && (oldUri == mStreams[idx].mUri)) { + swapPacketSource(stream); + + if ((mNewStreamMask & stream) && mStreams[idx].mNewUri.empty()) { + ALOGW("swapping stream type %d %s to empty stream", + stream, mStreams[idx].mUri.c_str()); + } + mStreams[idx].mUri = mStreams[idx].mNewUri; + mStreams[idx].mNewUri.clear(); + + mSwapMask &= ~stream; + } } - mStreams[idx].mUri = mStreams[idx].mNewUri; - mStreams[idx].mNewUri.clear(); - mSwapMask &= ~stream; + mFetcherInfos.editValueAt(index).mFetcher->stopAsync(false /* clear */); + + ALOGV("tryToFinishBandwidthSwitch: mSwapMask=%x", mSwapMask); if (mSwapMask != 0) { return; } @@ -1606,155 +1908,299 @@ void LiveSession::onSwapped(const sp<AMessage> &msg) { // Check if new variant contains extra streams. uint32_t extraStreams = mNewStreamMask & (~mStreamMask); while (extraStreams) { - StreamType extraStream = (StreamType) (extraStreams & ~(extraStreams - 1)); - swapPacketSource(extraStream); - extraStreams &= ~extraStream; + StreamType stream = (StreamType) (extraStreams & ~(extraStreams - 1)); + extraStreams &= ~stream; + + swapPacketSource(stream); - idx = typeToIndex(extraStream); + ssize_t idx = typeToIndex(stream); CHECK(idx >= 0); if (mStreams[idx].mNewUri.empty()) { ALOGW("swapping extra stream type %d %s to empty stream", - extraStream, mStreams[idx].mUri.c_str()); + stream, mStreams[idx].mUri.c_str()); } mStreams[idx].mUri = mStreams[idx].mNewUri; mStreams[idx].mNewUri.clear(); } - tryToFinishBandwidthSwitch(); -} - -void LiveSession::onCheckSwitchDown() { - if (mSwitchDownMonitor == NULL) { - return; - } - - if (mSwitchInProgress || mReconfigurationInProgress) { - ALOGV("Switch/Reconfig in progress, defer switch down"); - mSwitchDownMonitor->post(1000000ll); - return; + // Restart new fetcher (it was paused after the first 47k block) + // and let it fetch into mPacketSources (not mPacketSources2) + for (size_t i = 0; i < mFetcherInfos.size(); ++i) { + FetcherInfo &info = mFetcherInfos.editValueAt(i); + if (info.mToBeResumed) { + resumeFetcher(mFetcherInfos.keyAt(i), mNewStreamMask); + info.mToBeResumed = false; + } } - for (size_t i = 0; i < kMaxStreams; ++i) { - int32_t targetDuration; - sp<AnotherPacketSource> packetSource = mPacketSources.valueFor(indexToType(i)); - sp<AMessage> meta = packetSource->getLatestDequeuedMeta(); + ALOGI("#### Finished Bandwidth Switch: %zd => %zd", + mOrigBandwidthIndex, mCurBandwidthIndex); - if (meta != NULL && meta->findInt32("targetDuration", &targetDuration) ) { - int64_t bufferedDurationUs = packetSource->getEstimatedDurationUs(); - int64_t targetDurationUs = targetDuration * 1000000ll; + mStreamMask = mNewStreamMask; + mSwitchInProgress = false; + mOrigBandwidthIndex = mCurBandwidthIndex; - if (bufferedDurationUs < targetDurationUs / 3) { - (new AMessage(kWhatSwitchDown, id()))->post(); - break; - } - } - } - mSwitchDownMonitor->post(1000000ll); + restartPollBuffering(); } -void LiveSession::onSwitchDown() { - if (mReconfigurationInProgress || mSwitchInProgress || mCurBandwidthIndex == 0) { - return; - } +void LiveSession::schedulePollBuffering() { + sp<AMessage> msg = new AMessage(kWhatPollBuffering, this); + msg->setInt32("generation", mPollBufferingGeneration); + msg->post(1000000ll); +} - ssize_t bandwidthIndex = getBandwidthIndex(); - if (bandwidthIndex < mCurBandwidthIndex) { - changeConfiguration(-1, bandwidthIndex, false); - return; - } +void LiveSession::cancelPollBuffering() { + ++mPollBufferingGeneration; + mPrevBufferPercentage = -1; +} +void LiveSession::restartPollBuffering() { + cancelPollBuffering(); + onPollBuffering(); } -// Mark switch done when: -// 1. all old buffers are swapped out -void LiveSession::tryToFinishBandwidthSwitch() { - if (!mSwitchInProgress) { - return; - } +void LiveSession::onPollBuffering() { + ALOGV("onPollBuffering: mSwitchInProgress %d, mReconfigurationInProgress %d, " + "mInPreparationPhase %d, mCurBandwidthIndex %zd, mStreamMask 0x%x", + mSwitchInProgress, mReconfigurationInProgress, + mInPreparationPhase, mCurBandwidthIndex, mStreamMask); - bool needToRemoveFetchers = false; - for (size_t i = 0; i < mFetcherInfos.size(); ++i) { - if (mFetcherInfos.valueAt(i).mToBeRemoved) { - needToRemoveFetchers = true; - break; + bool underflow, ready, down, up; + if (checkBuffering(underflow, ready, down, up)) { + if (mInPreparationPhase && ready) { + postPrepared(OK); } - } - if (!needToRemoveFetchers && mSwapMask == 0) { - ALOGI("mSwitchInProgress = false"); - mStreamMask = mNewStreamMask; - mSwitchInProgress = false; - } -} + // don't switch before we report prepared + if (!mInPreparationPhase) { + if (ready) { + stopBufferingIfNecessary(); + } else if (underflow) { + startBufferingIfNecessary(); + } + switchBandwidthIfNeeded(up, down); + } -void LiveSession::scheduleCheckBandwidthEvent() { - sp<AMessage> msg = new AMessage(kWhatCheckBandwidth, id()); - msg->setInt32("generation", mCheckBandwidthGeneration); - msg->post(10000000ll); -} + } -void LiveSession::cancelCheckBandwidthEvent() { - ++mCheckBandwidthGeneration; + schedulePollBuffering(); } -void LiveSession::cancelBandwidthSwitch() { - Mutex::Autolock lock(mSwapMutex); - mSwitchGeneration++; - mSwitchInProgress = false; - mSwapMask = 0; +void LiveSession::cancelBandwidthSwitch(bool resume) { + ALOGV("cancelBandwidthSwitch: mSwitchGen(%d)++, orig %zd, cur %zd", + mSwitchGeneration, mOrigBandwidthIndex, mCurBandwidthIndex); + if (!mSwitchInProgress) { + return; + } for (size_t i = 0; i < mFetcherInfos.size(); ++i) { FetcherInfo& info = mFetcherInfos.editValueAt(i); if (info.mToBeRemoved) { info.mToBeRemoved = false; + if (resume) { + resumeFetcher(mFetcherInfos.keyAt(i), mSwapMask); + } } } for (size_t i = 0; i < kMaxStreams; ++i) { - if (!mStreams[i].mNewUri.empty()) { - ssize_t j = mFetcherInfos.indexOfKey(mStreams[i].mNewUri); - if (j < 0) { - mStreams[i].mNewUri.clear(); + AString newUri = mStreams[i].mNewUri; + if (!newUri.empty()) { + // clear all mNewUri matching this newUri + for (size_t j = i; j < kMaxStreams; ++j) { + if (mStreams[j].mNewUri == newUri) { + mStreams[j].mNewUri.clear(); + } + } + ALOGV("stopping newUri = %s", newUri.c_str()); + ssize_t index = mFetcherInfos.indexOfKey(newUri); + if (index < 0) { + ALOGE("did not find fetcher for newUri: %s", newUri.c_str()); continue; } - - const FetcherInfo &info = mFetcherInfos.valueAt(j); + FetcherInfo &info = mFetcherInfos.editValueAt(index); + info.mToBeRemoved = true; info.mFetcher->stopAsync(); - mFetcherInfos.removeItemsAt(j); - mStreams[i].mNewUri.clear(); } } + + ALOGI("#### Canceled Bandwidth Switch: %zd => %zd", + mCurBandwidthIndex, mOrigBandwidthIndex); + + mSwitchGeneration++; + mSwitchInProgress = false; + mCurBandwidthIndex = mOrigBandwidthIndex; + mSwapMask = 0; } -bool LiveSession::canSwitchBandwidthTo(size_t bandwidthIndex) { - if (mReconfigurationInProgress || mSwitchInProgress) { +bool LiveSession::checkBuffering( + bool &underflow, bool &ready, bool &down, bool &up) { + underflow = ready = down = up = false; + + if (mReconfigurationInProgress) { + ALOGV("Switch/Reconfig in progress, defer buffer polling"); return false; } - if (mCurBandwidthIndex < 0) { - return true; + size_t activeCount, underflowCount, readyCount, downCount, upCount; + activeCount = underflowCount = readyCount = downCount = upCount =0; + int32_t minBufferPercent = -1; + int64_t durationUs; + if (getDuration(&durationUs) != OK) { + durationUs = -1; } + for (size_t i = 0; i < mPacketSources.size(); ++i) { + // we don't check subtitles for buffering level + if (!(mStreamMask & mPacketSources.keyAt(i) + & (STREAMTYPE_AUDIO | STREAMTYPE_VIDEO))) { + continue; + } + // ignore streams that never had any packet queued. + // (it's possible that the variant only has audio or video) + sp<AMessage> meta = mPacketSources[i]->getLatestEnqueuedMeta(); + if (meta == NULL) { + continue; + } - if (bandwidthIndex == (size_t)mCurBandwidthIndex) { - return false; - } else if (bandwidthIndex > (size_t)mCurBandwidthIndex) { - return canSwitchUp(); - } else { + int64_t bufferedDurationUs = + mPacketSources[i]->getEstimatedDurationUs(); + ALOGV("source[%zu]: buffered %lld us", i, (long long)bufferedDurationUs); + if (durationUs >= 0) { + int32_t percent; + if (mPacketSources[i]->isFinished(0 /* duration */)) { + percent = 100; + } else { + percent = (int32_t)(100.0 * (mLastDequeuedTimeUs + bufferedDurationUs) / durationUs); + } + if (minBufferPercent < 0 || percent < minBufferPercent) { + minBufferPercent = percent; + } + } + + ++activeCount; + int64_t readyMark = mInPreparationPhase ? kPrepareMarkUs : kReadyMarkUs; + if (bufferedDurationUs > readyMark + || mPacketSources[i]->isFinished(0)) { + ++readyCount; + } + if (!mPacketSources[i]->isFinished(0)) { + if (bufferedDurationUs < kUnderflowMarkUs) { + ++underflowCount; + } + if (bufferedDurationUs > mUpSwitchMark) { + ++upCount; + } else if (bufferedDurationUs < mDownSwitchMark) { + ++downCount; + } + } + } + + if (minBufferPercent >= 0) { + notifyBufferingUpdate(minBufferPercent); + } + + if (activeCount > 0) { + up = (upCount == activeCount); + down = (downCount > 0); + ready = (readyCount == activeCount); + underflow = (underflowCount > 0); return true; } + + return false; +} + +void LiveSession::startBufferingIfNecessary() { + ALOGV("startBufferingIfNecessary: mInPreparationPhase=%d, mBuffering=%d", + mInPreparationPhase, mBuffering); + if (!mBuffering) { + mBuffering = true; + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatBufferingStart); + notify->post(); + } } -void LiveSession::onCheckBandwidth(const sp<AMessage> &msg) { - size_t bandwidthIndex = getBandwidthIndex(); - if (canSwitchBandwidthTo(bandwidthIndex)) { - changeConfiguration(-1ll /* timeUs */, bandwidthIndex); +void LiveSession::stopBufferingIfNecessary() { + ALOGV("stopBufferingIfNecessary: mInPreparationPhase=%d, mBuffering=%d", + mInPreparationPhase, mBuffering); + + if (mBuffering) { + mBuffering = false; + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatBufferingEnd); + notify->post(); + } +} + +void LiveSession::notifyBufferingUpdate(int32_t percentage) { + if (percentage < mPrevBufferPercentage) { + percentage = mPrevBufferPercentage; + } else if (percentage > 100) { + percentage = 100; + } + + mPrevBufferPercentage = percentage; + + ALOGV("notifyBufferingUpdate: percentage=%d%%", percentage); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatBufferingUpdate); + notify->setInt32("percentage", percentage); + notify->post(); +} + +void LiveSession::switchBandwidthIfNeeded(bool bufferHigh, bool bufferLow) { + // no need to check bandwidth if we only have 1 bandwidth settings + if (mSwitchInProgress || mBandwidthItems.size() < 2) { + return; + } + + int32_t bandwidthBps; + if (mBandwidthEstimator->estimateBandwidth(&bandwidthBps)) { + ALOGV("bandwidth estimated at %.2f kbps", bandwidthBps / 1024.0f); + mLastBandwidthBps = bandwidthBps; } else { - // Come back and check again 10 seconds later in case there is nothing to do now. - // If we DO change configuration, once that completes it'll schedule a new - // check bandwidth event with an incremented mCheckBandwidthGeneration. - msg->post(10000000ll); + ALOGV("no bandwidth estimate."); + return; } + + int32_t curBandwidth = mBandwidthItems.itemAt(mCurBandwidthIndex).mBandwidth; + bool bandwidthLow = bandwidthBps < (int32_t)curBandwidth * 8 / 10; + bool bandwidthHigh = bandwidthBps > (int32_t)curBandwidth * 12 / 10; + + if ((bufferHigh && bandwidthHigh) || (bufferLow && bandwidthLow)) { + ssize_t bandwidthIndex = getBandwidthIndex(bandwidthBps); + + if (bandwidthIndex == mCurBandwidthIndex + || (bufferHigh && bandwidthIndex < mCurBandwidthIndex) + || (bufferLow && bandwidthIndex > mCurBandwidthIndex)) { + return; + } + + ALOGI("#### Starting Bandwidth Switch: %zd => %zd", + mCurBandwidthIndex, bandwidthIndex); + changeConfiguration(-1, bandwidthIndex, false); + } +} + +void LiveSession::postError(status_t err) { + // if we reached EOS, notify buffering of 100% + if (err == ERROR_END_OF_STREAM) { + notifyBufferingUpdate(100); + } + // we'll stop buffer polling now, before that notify + // stop buffering to stop the spinning icon + stopBufferingIfNecessary(); + cancelPollBuffering(); + + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatError); + notify->setInt32("err", err); + notify->post(); } void LiveSession::postPrepared(status_t err) { @@ -1764,6 +2210,8 @@ void LiveSession::postPrepared(status_t err) { if (err == OK || err == ERROR_END_OF_STREAM) { notify->setInt32("what", kWhatPrepared); } else { + cancelPollBuffering(); + notify->setInt32("what", kWhatPreparationFailed); notify->setInt32("err", err); } @@ -1771,10 +2219,8 @@ void LiveSession::postPrepared(status_t err) { notify->post(); mInPreparationPhase = false; - - mSwitchDownMonitor = new AMessage(kWhatCheckSwitchDown, id()); - mSwitchDownMonitor->post(); } + } // namespace android diff --git a/media/libstagefright/httplive/LiveSession.h b/media/libstagefright/httplive/LiveSession.h index 2d3a25a..3d62cab 100644 --- a/media/libstagefright/httplive/LiveSession.h +++ b/media/libstagefright/httplive/LiveSession.h @@ -26,6 +26,7 @@ namespace android { struct ABuffer; +struct AReplyToken; struct AnotherPacketSource; struct DataSource; struct HTTPBase; @@ -39,10 +40,6 @@ struct LiveSession : public AHandler { // Don't log any URLs. kFlagIncognito = 1, }; - LiveSession( - const sp<AMessage> ¬ify, - uint32_t flags, - const sp<IMediaHTTPService> &httpService); enum StreamIndex { kAudioIndex = 0, @@ -56,10 +53,24 @@ struct LiveSession : public AHandler { STREAMTYPE_VIDEO = 1 << kVideoIndex, STREAMTYPE_SUBTITLES = 1 << kSubtitleIndex, }; + + enum SeekMode { + kSeekModeExactPosition = 0, // used for seeking + kSeekModeNextSample = 1, // used for seamless switching + kSeekModeNextSegment = 2, // used for seamless switching + }; + + LiveSession( + const sp<AMessage> ¬ify, + uint32_t flags, + const sp<IMediaHTTPService> &httpService); + status_t dequeueAccessUnit(StreamType stream, sp<ABuffer> *accessUnit); status_t getStreamFormat(StreamType stream, sp<AMessage> *format); + sp<HTTPBase> getHTTPDataSource(); + void connectAsync( const char *url, const KeyedVector<String8, String8> *headers = NULL); @@ -78,18 +89,18 @@ struct LiveSession : public AHandler { bool isSeekable() const; bool hasDynamicDuration() const; + static const char *getKeyForStream(StreamType type); + enum { kWhatStreamsChanged, kWhatError, kWhatPrepared, kWhatPreparationFailed, + kWhatBufferingStart, + kWhatBufferingEnd, + kWhatBufferingUpdate, }; - // create a format-change discontinuity - // - // swap: - // whether is format-change discontinuity should trigger a buffer swap - sp<ABuffer> createFormatChangeBuffer(bool swap = true); protected: virtual ~LiveSession(); @@ -103,18 +114,24 @@ private: kWhatDisconnect = 'disc', kWhatSeek = 'seek', kWhatFetcherNotify = 'notf', - kWhatCheckBandwidth = 'bndw', kWhatChangeConfiguration = 'chC0', kWhatChangeConfiguration2 = 'chC2', kWhatChangeConfiguration3 = 'chC3', kWhatFinishDisconnect2 = 'fin2', - kWhatSwapped = 'swap', - kWhatCheckSwitchDown = 'ckSD', - kWhatSwitchDown = 'sDwn', + kWhatPollBuffering = 'poll', }; - static const size_t kBandwidthHistoryBytes; + // Bandwidth Switch Mark Defaults + static const int64_t kUpSwitchMarkUs; + static const int64_t kDownSwitchMarkUs; + static const int64_t kUpSwitchMarginUs; + + // Buffer Prepare/Ready/Underflow Marks + static const int64_t kReadyMarkUs; + static const int64_t kPrepareMarkUs; + static const int64_t kUnderflowMarkUs; + struct BandwidthEstimator; struct BandwidthItem { size_t mPlaylistIndex; unsigned long mBandwidth; @@ -123,23 +140,22 @@ private: struct FetcherInfo { sp<PlaylistFetcher> mFetcher; int64_t mDurationUs; - bool mIsPrepared; bool mToBeRemoved; + bool mToBeResumed; }; struct StreamItem { const char *mType; AString mUri, mNewUri; + SeekMode mSeekMode; size_t mCurDiscontinuitySeq; int64_t mLastDequeuedTimeUs; int64_t mLastSampleDurationUs; StreamItem() - : mType(""), - mCurDiscontinuitySeq(0), - mLastDequeuedTimeUs(0), - mLastSampleDurationUs(0) {} + : StreamItem("") {} StreamItem(const char *type) : mType(type), + mSeekMode(kSeekModeExactPosition), mCurDiscontinuitySeq(0), mLastDequeuedTimeUs(0), mLastSampleDurationUs(0) {} @@ -155,8 +171,10 @@ private: uint32_t mFlags; sp<IMediaHTTPService> mHTTPService; + bool mBuffering; bool mInPreparationPhase; - bool mBuffering[kMaxStreams]; + int32_t mPollBufferingGeneration; + int32_t mPrevBufferPercentage; sp<HTTPBase> mHTTPDataSource; KeyedVector<String8, String8> mExtraHeaders; @@ -165,9 +183,13 @@ private: Vector<BandwidthItem> mBandwidthItems; ssize_t mCurBandwidthIndex; + ssize_t mOrigBandwidthIndex; + int32_t mLastBandwidthBps; + sp<BandwidthEstimator> mBandwidthEstimator; sp<M3UParser> mPlaylist; + sp<ALooper> mFetcherLooper; KeyedVector<AString, FetcherInfo> mFetcherInfos; uint32_t mStreamMask; @@ -180,17 +202,10 @@ private: // we use this to track reconfiguration progress. uint32_t mSwapMask; - KeyedVector<StreamType, sp<AnotherPacketSource> > mDiscontinuities; KeyedVector<StreamType, sp<AnotherPacketSource> > mPacketSources; // A second set of packet sources that buffer content for the variant we're switching to. KeyedVector<StreamType, sp<AnotherPacketSource> > mPacketSources2; - // A mutex used to serialize two sets of events: - // * the swapping of packet sources in dequeueAccessUnit on the player thread, AND - // * a forced bandwidth switch termination in cancelSwitch on the live looper. - Mutex mSwapMutex; - - int32_t mCheckBandwidthGeneration; int32_t mSwitchGeneration; int32_t mSubtitleGeneration; @@ -203,13 +218,16 @@ private: bool mReconfigurationInProgress; bool mSwitchInProgress; - uint32_t mDisconnectReplyID; - uint32_t mSeekReplyID; + int64_t mUpSwitchMark; + int64_t mDownSwitchMark; + int64_t mUpSwitchMargin; + + sp<AReplyToken> mDisconnectReplyID; + sp<AReplyToken> mSeekReplyID; bool mFirstTimeUsValid; int64_t mFirstTimeUs; int64_t mLastSeekTimeUs; - sp<AMessage> mSwitchDownMonitor; KeyedVector<size_t, int64_t> mDiscontinuityAbsStartTimesUs; KeyedVector<size_t, int64_t> mDiscontinuityOffsetTimesUs; @@ -238,12 +256,21 @@ private: uint32_t block_size = 0, /* reuse DataSource if doing partial fetch */ sp<DataSource> *source = NULL, - String8 *actualUrl = NULL); + String8 *actualUrl = NULL, + /* force connect http even when resuing DataSource */ + bool forceConnectHTTP = false); sp<M3UParser> fetchPlaylist( const char *url, uint8_t *curPlaylistHash, bool *unchanged); - size_t getBandwidthIndex(); + bool resumeFetcher( + const AString &uri, uint32_t streamMask, + int64_t timeUs = -1ll, bool newUri = false); + + float getAbortThreshold( + ssize_t currentBWIndex, ssize_t targetBWIndex) const; + void addBandwidthMeasurement(size_t numBytes, int64_t delayUs); + size_t getBandwidthIndex(int32_t bandwidthBps); int64_t latestMediaSegmentStartTimeUs(); static int SortByBandwidth(const BandwidthItem *, const BandwidthItem *); @@ -251,32 +278,32 @@ private: static ssize_t typeToIndex(int32_t type); void changeConfiguration( - int64_t timeUs, size_t bandwidthIndex, bool pickTrack = false); + int64_t timeUs, ssize_t bwIndex = -1, bool pickTrack = false); void onChangeConfiguration(const sp<AMessage> &msg); void onChangeConfiguration2(const sp<AMessage> &msg); void onChangeConfiguration3(const sp<AMessage> &msg); - void onSwapped(const sp<AMessage> &msg); - void onCheckSwitchDown(); - void onSwitchDown(); - void tryToFinishBandwidthSwitch(); - - void scheduleCheckBandwidthEvent(); - void cancelCheckBandwidthEvent(); - - // cancelBandwidthSwitch is atomic wrt swapPacketSource; call it to prevent packet sources - // from being swapped out on stale discontinuities while manipulating - // mPacketSources/mPacketSources2. - void cancelBandwidthSwitch(); - bool canSwitchBandwidthTo(size_t bandwidthIndex); - void onCheckBandwidth(const sp<AMessage> &msg); + void swapPacketSource(StreamType stream); + void tryToFinishBandwidthSwitch(const AString &oldUri); + void cancelBandwidthSwitch(bool resume = false); + bool checkSwitchProgress( + sp<AMessage> &msg, int64_t delayUs, bool *needResumeUntil); + + void switchBandwidthIfNeeded(bool bufferHigh, bool bufferLow); + + void schedulePollBuffering(); + void cancelPollBuffering(); + void restartPollBuffering(); + void onPollBuffering(); + bool checkBuffering(bool &underflow, bool &ready, bool &down, bool &up); + void startBufferingIfNecessary(); + void stopBufferingIfNecessary(); + void notifyBufferingUpdate(int32_t percentage); void finishDisconnect(); void postPrepared(status_t err); - - void swapPacketSource(StreamType stream); - bool canSwitchUp(); + void postError(status_t err); DISALLOW_EVIL_CONSTRUCTORS(LiveSession); }; diff --git a/media/libstagefright/httplive/M3UParser.cpp b/media/libstagefright/httplive/M3UParser.cpp index 997b694..3c5d7cf 100644 --- a/media/libstagefright/httplive/M3UParser.cpp +++ b/media/libstagefright/httplive/M3UParser.cpp @@ -251,6 +251,7 @@ M3UParser::M3UParser( mIsComplete(false), mIsEvent(false), mDiscontinuitySeq(0), + mDiscontinuityCount(0), mSelectedIndex(-1) { mInitCheck = parse(data, size); } @@ -582,6 +583,7 @@ status_t M3UParser::parse(const void *_data, size_t size) { itemMeta = new AMessage; } itemMeta->setInt32("discontinuity", true); + ++mDiscontinuityCount; } else if (line.startsWith("#EXT-X-STREAM-INF")) { if (mMeta != NULL) { return ERROR_MALFORMED; @@ -609,6 +611,9 @@ status_t M3UParser::parse(const void *_data, size_t size) { } else if (line.startsWith("#EXT-X-MEDIA")) { err = parseMedia(line); } else if (line.startsWith("#EXT-X-DISCONTINUITY-SEQUENCE")) { + if (mIsVariantPlaylist) { + return ERROR_MALFORMED; + } size_t seq; err = parseDiscontinuitySequence(line, &seq); if (err == OK) { @@ -628,6 +633,7 @@ status_t M3UParser::parse(const void *_data, size_t size) { || !itemMeta->findInt64("durationUs", &durationUs)) { return ERROR_MALFORMED; } + itemMeta->setInt32("discontinuity-sequence", mDiscontinuitySeq + mDiscontinuityCount); } mItems.push(); diff --git a/media/libstagefright/httplive/M3UParser.h b/media/libstagefright/httplive/M3UParser.h index 1cad060..d475683 100644 --- a/media/libstagefright/httplive/M3UParser.h +++ b/media/libstagefright/httplive/M3UParser.h @@ -70,6 +70,7 @@ private: bool mIsComplete; bool mIsEvent; size_t mDiscontinuitySeq; + int32_t mDiscontinuityCount; sp<AMessage> mMeta; Vector<Item> mItems; diff --git a/media/libstagefright/httplive/PlaylistFetcher.cpp b/media/libstagefright/httplive/PlaylistFetcher.cpp index 1227600..3ace396 100644 --- a/media/libstagefright/httplive/PlaylistFetcher.cpp +++ b/media/libstagefright/httplive/PlaylistFetcher.cpp @@ -33,6 +33,7 @@ #include <media/stagefright/foundation/ABitReader.h> #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> +#include <media/stagefright/foundation/AUtils.h> #include <media/stagefright/foundation/hexdump.h> #include <media/stagefright/FileSource.h> #include <media/stagefright/MediaDefs.h> @@ -47,19 +48,105 @@ namespace android { // static -const int64_t PlaylistFetcher::kMinBufferedDurationUs = 10000000ll; +const int64_t PlaylistFetcher::kMinBufferedDurationUs = 30000000ll; const int64_t PlaylistFetcher::kMaxMonitorDelayUs = 3000000ll; +const int64_t PlaylistFetcher::kFetcherResumeThreshold = 100000ll; // LCM of 188 (size of a TS packet) & 1k works well const int32_t PlaylistFetcher::kDownloadBlockSize = 47 * 1024; const int32_t PlaylistFetcher::kNumSkipFrames = 5; +struct PlaylistFetcher::DownloadState : public RefBase { + DownloadState(); + void resetState(); + bool hasSavedState() const; + void restoreState( + AString &uri, + sp<AMessage> &itemMeta, + sp<ABuffer> &buffer, + sp<ABuffer> &tsBuffer, + int32_t &firstSeqNumberInPlaylist, + int32_t &lastSeqNumberInPlaylist); + void saveState( + AString &uri, + sp<AMessage> &itemMeta, + sp<ABuffer> &buffer, + sp<ABuffer> &tsBuffer, + int32_t &firstSeqNumberInPlaylist, + int32_t &lastSeqNumberInPlaylist); + +private: + bool mHasSavedState; + AString mUri; + sp<AMessage> mItemMeta; + sp<ABuffer> mBuffer; + sp<ABuffer> mTsBuffer; + int32_t mFirstSeqNumberInPlaylist; + int32_t mLastSeqNumberInPlaylist; +}; + +PlaylistFetcher::DownloadState::DownloadState() { + resetState(); +} + +bool PlaylistFetcher::DownloadState::hasSavedState() const { + return mHasSavedState; +} + +void PlaylistFetcher::DownloadState::resetState() { + mHasSavedState = false; + + mUri.clear(); + mItemMeta = NULL; + mBuffer = NULL; + mTsBuffer = NULL; + mFirstSeqNumberInPlaylist = 0; + mLastSeqNumberInPlaylist = 0; +} + +void PlaylistFetcher::DownloadState::restoreState( + AString &uri, + sp<AMessage> &itemMeta, + sp<ABuffer> &buffer, + sp<ABuffer> &tsBuffer, + int32_t &firstSeqNumberInPlaylist, + int32_t &lastSeqNumberInPlaylist) { + if (!mHasSavedState) { + return; + } + + uri = mUri; + itemMeta = mItemMeta; + buffer = mBuffer; + tsBuffer = mTsBuffer; + firstSeqNumberInPlaylist = mFirstSeqNumberInPlaylist; + lastSeqNumberInPlaylist = mLastSeqNumberInPlaylist; + + resetState(); +} + +void PlaylistFetcher::DownloadState::saveState( + AString &uri, + sp<AMessage> &itemMeta, + sp<ABuffer> &buffer, + sp<ABuffer> &tsBuffer, + int32_t &firstSeqNumberInPlaylist, + int32_t &lastSeqNumberInPlaylist) { + mHasSavedState = true; + + mUri = uri; + mItemMeta = itemMeta; + mBuffer = buffer; + mTsBuffer = tsBuffer; + mFirstSeqNumberInPlaylist = firstSeqNumberInPlaylist; + mLastSeqNumberInPlaylist = lastSeqNumberInPlaylist; +} + PlaylistFetcher::PlaylistFetcher( const sp<AMessage> ¬ify, const sp<LiveSession> &session, const char *uri, int32_t subtitleGeneration) : mNotify(notify), - mStartTimeUsNotify(notify->dup()), mSession(session), mURI(uri), mStreamTypeMask(0), @@ -71,18 +158,21 @@ PlaylistFetcher::PlaylistFetcher( mSeqNumber(-1), mNumRetries(0), mStartup(true), - mAdaptive(false), - mPrepared(false), + mIDRFound(false), + mSeekMode(LiveSession::kSeekModeExactPosition), + mTimeChangeSignaled(false), mNextPTSTimeUs(-1ll), mMonitorQueueGeneration(0), mSubtitleGeneration(subtitleGeneration), + mLastDiscontinuitySeq(-1ll), mRefreshState(INITIAL_MINIMUM_RELOAD_DELAY), mFirstPTSValid(false), - mAbsoluteTimeAnchorUs(0ll), - mVideoBuffer(new AnotherPacketSource(NULL)) { + mFirstTimeUs(-1ll), + mVideoBuffer(new AnotherPacketSource(NULL)), + mThresholdRatio(-1.0f), + mDownloadState(new DownloadState()) { memset(mPlaylistHash, 0, sizeof(mPlaylistHash)); - mStartTimeUsNotify->setInt32("what", kWhatStartedAt); - mStartTimeUsNotify->setInt32("streamMask", 0); + mHTTPDataSource = mSession->getHTTPDataSource(); } PlaylistFetcher::~PlaylistFetcher() { @@ -119,6 +209,32 @@ int64_t PlaylistFetcher::getSegmentStartTimeUs(int32_t seqNumber) const { return segmentStartUs; } +int64_t PlaylistFetcher::getSegmentDurationUs(int32_t seqNumber) const { + CHECK(mPlaylist != NULL); + + int32_t firstSeqNumberInPlaylist; + if (mPlaylist->meta() == NULL || !mPlaylist->meta()->findInt32( + "media-sequence", &firstSeqNumberInPlaylist)) { + firstSeqNumberInPlaylist = 0; + } + + int32_t lastSeqNumberInPlaylist = + firstSeqNumberInPlaylist + (int32_t)mPlaylist->size() - 1; + + CHECK_GE(seqNumber, firstSeqNumberInPlaylist); + CHECK_LE(seqNumber, lastSeqNumberInPlaylist); + + int32_t index = seqNumber - firstSeqNumberInPlaylist; + sp<AMessage> itemMeta; + CHECK(mPlaylist->itemAt( + index, NULL /* uri */, &itemMeta)); + + int64_t itemDurationUs; + CHECK(itemMeta->findInt64("durationUs", &itemDurationUs)); + + return itemDurationUs; +} + int64_t PlaylistFetcher::delayUsToRefreshPlaylist() const { int64_t nowUs = ALooper::GetNowUs(); @@ -325,7 +441,7 @@ void PlaylistFetcher::postMonitorQueue(int64_t delayUs, int64_t minDelayUs) { ALOGV("Need to refresh playlist in %" PRId64 , maxDelayUs); delayUs = maxDelayUs; } - sp<AMessage> msg = new AMessage(kWhatMonitorQueue, id()); + sp<AMessage> msg = new AMessage(kWhatMonitorQueue, this); msg->setInt32("generation", mMonitorQueueGeneration); msg->post(delayUs); } @@ -334,6 +450,14 @@ void PlaylistFetcher::cancelMonitorQueue() { ++mMonitorQueueGeneration; } +void PlaylistFetcher::setStoppingThreshold(float thresholdRatio) { + AutoMutex _l(mThresholdLock); + if (mStreamTypeMask == LiveSession::STREAMTYPE_SUBTITLES) { + return; + } + mThresholdRatio = thresholdRatio; +} + void PlaylistFetcher::startAsync( const sp<AnotherPacketSource> &audioSource, const sp<AnotherPacketSource> &videoSource, @@ -341,8 +465,8 @@ void PlaylistFetcher::startAsync( int64_t startTimeUs, int64_t segmentStartTimeUs, int32_t startDiscontinuitySeq, - bool adaptive) { - sp<AMessage> msg = new AMessage(kWhatStart, id()); + LiveSession::SeekMode seekMode) { + sp<AMessage> msg = new AMessage(kWhatStart, this); uint32_t streamTypeMask = 0ul; @@ -365,22 +489,27 @@ void PlaylistFetcher::startAsync( msg->setInt64("startTimeUs", startTimeUs); msg->setInt64("segmentStartTimeUs", segmentStartTimeUs); msg->setInt32("startDiscontinuitySeq", startDiscontinuitySeq); - msg->setInt32("adaptive", adaptive); + msg->setInt32("seekMode", seekMode); msg->post(); } -void PlaylistFetcher::pauseAsync() { - (new AMessage(kWhatPause, id()))->post(); +void PlaylistFetcher::pauseAsync(float thresholdRatio) { + if (thresholdRatio >= 0.0f) { + setStoppingThreshold(thresholdRatio); + } + (new AMessage(kWhatPause, this))->post(); } void PlaylistFetcher::stopAsync(bool clear) { - sp<AMessage> msg = new AMessage(kWhatStop, id()); + setStoppingThreshold(0.0f); + + sp<AMessage> msg = new AMessage(kWhatStop, this); msg->setInt32("clear", clear); msg->post(); } void PlaylistFetcher::resumeUntilAsync(const sp<AMessage> ¶ms) { - AMessage* msg = new AMessage(kWhatResumeUntil, id()); + AMessage* msg = new AMessage(kWhatResumeUntil, this); msg->setMessage("params", params); msg->post(); } @@ -404,6 +533,10 @@ void PlaylistFetcher::onMessageReceived(const sp<AMessage> &msg) { sp<AMessage> notify = mNotify->dup(); notify->setInt32("what", kWhatPaused); + notify->setInt32("seekMode", + mDownloadState->hasSavedState() + ? LiveSession::kSeekModeNextSample + : LiveSession::kSeekModeNextSegment); notify->post(); break; } @@ -450,6 +583,10 @@ void PlaylistFetcher::onMessageReceived(const sp<AMessage> &msg) { status_t PlaylistFetcher::onStart(const sp<AMessage> &msg) { mPacketSources.clear(); + mStopParams.clear(); + mStartTimeUsNotify = mNotify->dup(); + mStartTimeUsNotify->setInt32("what", kWhatStartedAt); + mStartTimeUsNotify->setString("uri", mURI); uint32_t streamTypeMask; CHECK(msg->findInt32("streamTypeMask", (int32_t *)&streamTypeMask)); @@ -457,11 +594,11 @@ status_t PlaylistFetcher::onStart(const sp<AMessage> &msg) { int64_t startTimeUs; int64_t segmentStartTimeUs; int32_t startDiscontinuitySeq; - int32_t adaptive; + int32_t seekMode; CHECK(msg->findInt64("startTimeUs", &startTimeUs)); CHECK(msg->findInt64("segmentStartTimeUs", &segmentStartTimeUs)); CHECK(msg->findInt32("startDiscontinuitySeq", &startDiscontinuitySeq)); - CHECK(msg->findInt32("adaptive", &adaptive)); + CHECK(msg->findInt32("seekMode", &seekMode)); if (streamTypeMask & LiveSession::STREAMTYPE_AUDIO) { void *ptr; @@ -493,14 +630,26 @@ status_t PlaylistFetcher::onStart(const sp<AMessage> &msg) { mStreamTypeMask = streamTypeMask; mSegmentStartTimeUs = segmentStartTimeUs; - mDiscontinuitySeq = startDiscontinuitySeq; + + if (startDiscontinuitySeq >= 0) { + mDiscontinuitySeq = startDiscontinuitySeq; + } + + mRefreshState = INITIAL_MINIMUM_RELOAD_DELAY; + mSeekMode = (LiveSession::SeekMode) seekMode; + + if (startTimeUs >= 0 || mSeekMode == LiveSession::kSeekModeNextSample) { + mStartup = true; + mIDRFound = false; + mVideoBuffer->clear(); + } if (startTimeUs >= 0) { mStartTimeUs = startTimeUs; + mFirstPTSValid = false; mSeqNumber = -1; - mStartup = true; - mPrepared = false; - mAdaptive = adaptive; + mTimeChangeSignaled = false; + mDownloadState->resetState(); } postMonitorQueue(); @@ -510,6 +659,9 @@ status_t PlaylistFetcher::onStart(const sp<AMessage> &msg) { void PlaylistFetcher::onPause() { cancelMonitorQueue(); + mLastDiscontinuitySeq = mDiscontinuitySeq; + + setStoppingThreshold(-1.0f); } void PlaylistFetcher::onStop(const sp<AMessage> &msg) { @@ -524,8 +676,11 @@ void PlaylistFetcher::onStop(const sp<AMessage> &msg) { } } + mDownloadState->resetState(); mPacketSources.clear(); mStreamTypeMask = 0; + + setStoppingThreshold(-1.0f); } // Resume until we have reached the boundary timestamps listed in `msg`; when @@ -535,30 +690,21 @@ status_t PlaylistFetcher::onResumeUntil(const sp<AMessage> &msg) { sp<AMessage> params; CHECK(msg->findMessage("params", ¶ms)); - bool stop = false; + size_t stopCount = 0; for (size_t i = 0; i < mPacketSources.size(); i++) { sp<AnotherPacketSource> packetSource = mPacketSources.valueAt(i); - const char *stopKey; - int streamType = mPacketSources.keyAt(i); - switch (streamType) { - case LiveSession::STREAMTYPE_VIDEO: - stopKey = "timeUsVideo"; - break; - - case LiveSession::STREAMTYPE_AUDIO: - stopKey = "timeUsAudio"; - break; + LiveSession::StreamType streamType = mPacketSources.keyAt(i); - case LiveSession::STREAMTYPE_SUBTITLES: - stopKey = "timeUsSubtitle"; - break; - - default: - TRESPASS(); + if (streamType == LiveSession::STREAMTYPE_SUBTITLES) { + // the subtitle track can always be stopped + ++stopCount; + continue; } - // Don't resume if we would stop within a resume threshold. + const char *stopKey = LiveSession::getKeyForStream(streamType); + + // check if this stream has too little data left to be resumed int32_t discontinuitySeq; int64_t latestTimeUs = 0, stopTimeUs = 0; sp<AMessage> latestMeta = packetSource->getLatestEnqueuedMeta(); @@ -567,25 +713,29 @@ status_t PlaylistFetcher::onResumeUntil(const sp<AMessage> &msg) { && discontinuitySeq == mDiscontinuitySeq && latestMeta->findInt64("timeUs", &latestTimeUs) && params->findInt64(stopKey, &stopTimeUs) - && stopTimeUs - latestTimeUs < resumeThreshold(latestMeta)) { - stop = true; + && stopTimeUs - latestTimeUs < kFetcherResumeThreshold) { + ++stopCount; } } - if (stop) { - for (size_t i = 0; i < mPacketSources.size(); i++) { - mPacketSources.valueAt(i)->queueAccessUnit(mSession->createFormatChangeBuffer()); - } - stopAsync(/* clear = */ false); + // Don't resume if all streams are within a resume threshold + if (stopCount == mPacketSources.size()) { + notifyStopReached(); return OK; } mStopParams = params; - postMonitorQueue(); + onDownloadNext(); return OK; } +void PlaylistFetcher::notifyStopReached() { + sp<AMessage> notify = mNotify->dup(); + notify->setInt32("what", kWhatStopReached); + notify->post(); +} + void PlaylistFetcher::notifyError(status_t err) { sp<AMessage> notify = mNotify->dup(); notify->setInt32("what", kWhatError); @@ -605,7 +755,12 @@ void PlaylistFetcher::queueDiscontinuity( void PlaylistFetcher::onMonitorQueue() { bool downloadMore = false; - refreshPlaylist(); + + // in the middle of an unfinished download, delay + // playlist refresh as it'll change seq numbers + if (!mDownloadState->hasSavedState()) { + refreshPlaylist(); + } int32_t targetDurationSecs; int64_t targetDurationUs = kMinBufferedDurationUs; @@ -619,27 +774,23 @@ void PlaylistFetcher::onMonitorQueue() { targetDurationUs = targetDurationSecs * 1000000ll; } - // buffer at least 3 times the target duration, or up to 10 seconds - int64_t durationToBufferUs = targetDurationUs * 3; - if (durationToBufferUs > kMinBufferedDurationUs) { - durationToBufferUs = kMinBufferedDurationUs; - } - int64_t bufferedDurationUs = 0ll; - status_t finalResult = NOT_ENOUGH_DATA; + status_t finalResult = OK; if (mStreamTypeMask == LiveSession::STREAMTYPE_SUBTITLES) { sp<AnotherPacketSource> packetSource = mPacketSources.valueFor(LiveSession::STREAMTYPE_SUBTITLES); bufferedDurationUs = packetSource->getBufferedDurationUs(&finalResult); - finalResult = OK; } else { - // Use max stream duration to prevent us from waiting on a non-existent stream; - // when we cannot make out from the manifest what streams are included in a playlist - // we might assume extra streams. + // Use min stream duration, but ignore streams that never have any packet + // enqueued to prevent us from waiting on a non-existent stream; + // when we cannot make out from the manifest what streams are included in + // a playlist we might assume extra streams. + bufferedDurationUs = -1ll; for (size_t i = 0; i < mPacketSources.size(); ++i) { - if ((mStreamTypeMask & mPacketSources.keyAt(i)) == 0) { + if ((mStreamTypeMask & mPacketSources.keyAt(i)) == 0 + || mPacketSources[i]->getLatestEnqueuedMeta() == NULL) { continue; } @@ -647,46 +798,36 @@ void PlaylistFetcher::onMonitorQueue() { mPacketSources.valueAt(i)->getBufferedDurationUs(&finalResult); ALOGV("buffered %" PRId64 " for stream %d", bufferedStreamDurationUs, mPacketSources.keyAt(i)); - if (bufferedStreamDurationUs > bufferedDurationUs) { + if (bufferedDurationUs == -1ll + || bufferedStreamDurationUs < bufferedDurationUs) { bufferedDurationUs = bufferedStreamDurationUs; } } - } - downloadMore = (bufferedDurationUs < durationToBufferUs); - - // signal start if buffered up at least the target size - if (!mPrepared && bufferedDurationUs > targetDurationUs && downloadMore) { - mPrepared = true; - - ALOGV("prepared, buffered=%" PRId64 " > %" PRId64 "", - bufferedDurationUs, targetDurationUs); - sp<AMessage> msg = mNotify->dup(); - msg->setInt32("what", kWhatTemporarilyDoneFetching); - msg->post(); + if (bufferedDurationUs == -1ll) { + bufferedDurationUs = 0ll; + } } - if (finalResult == OK && downloadMore) { + if (finalResult == OK && bufferedDurationUs < kMinBufferedDurationUs) { ALOGV("monitoring, buffered=%" PRId64 " < %" PRId64 "", - bufferedDurationUs, durationToBufferUs); + bufferedDurationUs, kMinBufferedDurationUs); // delay the next download slightly; hopefully this gives other concurrent fetchers // a better chance to run. // onDownloadNext(); - sp<AMessage> msg = new AMessage(kWhatDownloadNext, id()); + sp<AMessage> msg = new AMessage(kWhatDownloadNext, this); msg->setInt32("generation", mMonitorQueueGeneration); msg->post(1000l); } else { - // Nothing to do yet, try again in a second. - - sp<AMessage> msg = mNotify->dup(); - msg->setInt32("what", kWhatTemporarilyDoneFetching); - msg->post(); - - int64_t delayUs = mPrepared ? kMaxMonitorDelayUs : targetDurationUs / 2; + // We'd like to maintain buffering above durationToBufferUs, so try + // again when buffer just about to go below durationToBufferUs + // (or after targetDurationUs / 2, whichever is smaller). + int64_t delayUs = bufferedDurationUs - kMinBufferedDurationUs + 1000000ll; + if (delayUs > targetDurationUs / 2) { + delayUs = targetDurationUs / 2; + } ALOGV("pausing for %" PRId64 ", buffered=%" PRId64 " > %" PRId64 "", - delayUs, bufferedDurationUs, durationToBufferUs); - // :TRICKY: need to enforce minimum delay because the delay to - // refresh the playlist will become 0 - postMonitorQueue(delayUs, mPrepared ? targetDurationUs * 2 : 0); + delayUs, bufferedDurationUs, kMinBufferedDurationUs); + postMonitorQueue(delayUs); } } @@ -715,6 +856,13 @@ status_t PlaylistFetcher::refreshPlaylist() { if (mPlaylist->isComplete() || mPlaylist->isEvent()) { updateDuration(); } + // Notify LiveSession to use target-duration based buffering level + // for up/down switch. Default LiveSession::kUpSwitchMark may not + // be reachable for live streams, as our max buffering amount is + // limited to 3 segments. + if (!mPlaylist->isComplete()) { + updateTargetDuration(); + } } mLastPlaylistFetchTimeUs = ALooper::GetNowUs(); @@ -727,10 +875,69 @@ bool PlaylistFetcher::bufferStartsWithTsSyncByte(const sp<ABuffer>& buffer) { return buffer->size() > 0 && buffer->data()[0] == 0x47; } -void PlaylistFetcher::onDownloadNext() { +bool PlaylistFetcher::shouldPauseDownload() { + if (mStreamTypeMask == LiveSession::STREAMTYPE_SUBTITLES) { + // doesn't apply to subtitles + return false; + } + + // Calculate threshold to abort current download + int32_t targetDurationSecs; + CHECK(mPlaylist->meta()->findInt32("target-duration", &targetDurationSecs)); + int64_t targetDurationUs = targetDurationSecs * 1000000ll; + int64_t thresholdUs = -1; + { + AutoMutex _l(mThresholdLock); + thresholdUs = (mThresholdRatio < 0.0f) ? + -1ll : mThresholdRatio * targetDurationUs; + } + + if (thresholdUs < 0) { + // never abort + return false; + } else if (thresholdUs == 0) { + // immediately abort + return true; + } + + // now we have a positive thresholdUs, abort if remaining + // portion to download is over that threshold. + if (mSegmentFirstPTS < 0) { + // this means we haven't even find the first access unit, + // abort now as we must be very far away from the end. + return true; + } + int64_t lastEnqueueUs = mSegmentFirstPTS; + for (size_t i = 0; i < mPacketSources.size(); ++i) { + if ((mStreamTypeMask & mPacketSources.keyAt(i)) == 0) { + continue; + } + sp<AMessage> meta = mPacketSources[i]->getLatestEnqueuedMeta(); + int32_t type; + if (meta == NULL || meta->findInt32("discontinuity", &type)) { + continue; + } + int64_t tmpUs; + CHECK(meta->findInt64("timeUs", &tmpUs)); + if (tmpUs > lastEnqueueUs) { + lastEnqueueUs = tmpUs; + } + } + lastEnqueueUs -= mSegmentFirstPTS; + if (targetDurationUs - lastEnqueueUs > thresholdUs) { + return true; + } + return false; +} + +bool PlaylistFetcher::initDownloadState( + AString &uri, + sp<AMessage> &itemMeta, + int32_t &firstSeqNumberInPlaylist, + int32_t &lastSeqNumberInPlaylist) { status_t err = refreshPlaylist(); - int32_t firstSeqNumberInPlaylist = 0; - int32_t lastSeqNumberInPlaylist = 0; + firstSeqNumberInPlaylist = 0; + lastSeqNumberInPlaylist = 0; bool discontinuity = false; if (mPlaylist != NULL) { @@ -746,6 +953,8 @@ void PlaylistFetcher::onDownloadNext() { } } + mSegmentFirstPTS = -1ll; + if (mPlaylist != NULL && mSeqNumber < 0) { CHECK_GE(mStartTimeUs, 0ll); @@ -773,7 +982,7 @@ void PlaylistFetcher::onDownloadNext() { // timestamps coming from the media container) is used to determine the position // inside a segments. mSeqNumber = getSeqNumberForTime(mSegmentStartTimeUs); - if (mAdaptive) { + if (mSeekMode == LiveSession::kSeekModeNextSegment) { // avoid double fetch/decode mSeqNumber += 1; } @@ -823,12 +1032,12 @@ void PlaylistFetcher::onDownloadNext() { mSeqNumber, firstSeqNumberInPlaylist, lastSeqNumberInPlaylist, delayUs, mNumRetries); postMonitorQueue(delayUs); - return; + return false; } if (err != OK) { notifyError(err); - return; + return false; } // we've missed the boat, let's start 3 segments prior to the latest sequence @@ -843,12 +1052,8 @@ void PlaylistFetcher::onDownloadNext() { // but since the segments we are supposed to fetch have already rolled off // the playlist, i.e. we have already missed the boat, we inevitably have to // skip. - for (size_t i = 0; i < mPacketSources.size(); i++) { - sp<ABuffer> formatChange = mSession->createFormatChangeBuffer(); - mPacketSources.valueAt(i)->queueAccessUnit(formatChange); - } - stopAsync(/* clear = */ false); - return; + notifyStopReached(); + return false; } mSeqNumber = lastSeqNumberInPlaylist - 3; if (mSeqNumber < firstSeqNumberInPlaylist) { @@ -858,45 +1063,49 @@ void PlaylistFetcher::onDownloadNext() { // fall through } else { - ALOGE("Cannot find sequence number %d in playlist " - "(contains %d - %d)", - mSeqNumber, firstSeqNumberInPlaylist, - firstSeqNumberInPlaylist + (int32_t)mPlaylist->size() - 1); + if (mPlaylist != NULL) { + ALOGE("Cannot find sequence number %d in playlist " + "(contains %d - %d)", + mSeqNumber, firstSeqNumberInPlaylist, + firstSeqNumberInPlaylist + (int32_t)mPlaylist->size() - 1); - notifyError(ERROR_END_OF_STREAM); - return; + notifyError(ERROR_END_OF_STREAM); + } else { + // It's possible that we were never able to download the playlist. + // In this case we should notify error, instead of EOS, as EOS during + // prepare means we succeeded in downloading everything. + ALOGE("Failed to download playlist!"); + notifyError(ERROR_IO); + } + + return false; } } mNumRetries = 0; - AString uri; - sp<AMessage> itemMeta; CHECK(mPlaylist->itemAt( mSeqNumber - firstSeqNumberInPlaylist, &uri, &itemMeta)); + CHECK(itemMeta->findInt32("discontinuity-sequence", &mDiscontinuitySeq)); + int32_t val; if (itemMeta->findInt32("discontinuity", &val) && val != 0) { - mDiscontinuitySeq++; + discontinuity = true; + } else if (mLastDiscontinuitySeq >= 0 + && mDiscontinuitySeq != mLastDiscontinuitySeq) { + // Seek jumped to a new discontinuity sequence. We need to signal + // a format change to decoder. Decoder needs to shutdown and be + // created again if seamless format change is unsupported. + ALOGV("saw discontinuity: mStartup %d, mLastDiscontinuitySeq %d, " + "mDiscontinuitySeq %d, mStartTimeUs %lld", + mStartup, mLastDiscontinuitySeq, mDiscontinuitySeq, (long long)mStartTimeUs); discontinuity = true; } + mLastDiscontinuitySeq = -1; - int64_t range_offset, range_length; - if (!itemMeta->findInt64("range-offset", &range_offset) - || !itemMeta->findInt64("range-length", &range_length)) { - range_offset = 0; - range_length = -1; - } - - ALOGV("fetching segment %d from (%d .. %d)", - mSeqNumber, firstSeqNumberInPlaylist, lastSeqNumberInPlaylist); - - ALOGV("fetching '%s'", uri.c_str()); - - sp<DataSource> source; - sp<ABuffer> buffer, tsBuffer; // decrypt a junk buffer to prefetch key; since a session uses only one http connection, // this avoids interleaved connections to the key and segment file. { @@ -906,16 +1115,120 @@ void PlaylistFetcher::onDownloadNext() { true /* first */); if (err != OK) { notifyError(err); + return false; + } + } + + if ((mStartup && !mTimeChangeSignaled) || discontinuity) { + // We need to signal a time discontinuity to ATSParser on the + // first segment after start, or on a discontinuity segment. + // Setting mNextPTSTimeUs informs extractAndQueueAccessUnitsXX() + // to send the time discontinuity. + if (mPlaylist->isComplete() || mPlaylist->isEvent()) { + // If this was a live event this made no sense since + // we don't have access to all the segment before the current + // one. + mNextPTSTimeUs = getSegmentStartTimeUs(mSeqNumber); + } + + // Setting mTimeChangeSignaled to true, so that if start time + // searching goes into 2nd segment (without a discontinuity), + // we don't reset time again. It causes corruption when pending + // data in ATSParser is cleared. + mTimeChangeSignaled = true; + } + + if (discontinuity) { + ALOGI("queueing discontinuity (explicit=%d)", discontinuity); + + // Signal a format discontinuity to ATSParser to clear partial data + // from previous streams. Not doing this causes bitstream corruption. + if (mTSParser != NULL) { + mTSParser->signalDiscontinuity( + ATSParser::DISCONTINUITY_FORMATCHANGE, NULL /* extra */); + } + + queueDiscontinuity( + ATSParser::DISCONTINUITY_FORMATCHANGE, + NULL /* extra */); + + if (mStartup && mStartTimeUsRelative && mFirstPTSValid) { + // This means we guessed mStartTimeUs to be in the previous + // segment (likely very close to the end), but either video or + // audio has not found start by the end of that segment. + // + // If this new segment is not a discontinuity, keep searching. + // + // If this new segment even got a discontinuity marker, just + // set mStartTimeUs=0, and take all samples from now on. + mStartTimeUs = 0; + mFirstPTSValid = false; + } + } + + ALOGV("fetching segment %d from (%d .. %d)", + mSeqNumber, firstSeqNumberInPlaylist, lastSeqNumberInPlaylist); + return true; +} + +void PlaylistFetcher::onDownloadNext() { + AString uri; + sp<AMessage> itemMeta; + sp<ABuffer> buffer; + sp<ABuffer> tsBuffer; + int32_t firstSeqNumberInPlaylist = 0; + int32_t lastSeqNumberInPlaylist = 0; + bool connectHTTP = true; + + if (mDownloadState->hasSavedState()) { + mDownloadState->restoreState( + uri, + itemMeta, + buffer, + tsBuffer, + firstSeqNumberInPlaylist, + lastSeqNumberInPlaylist); + connectHTTP = false; + ALOGV("resuming: '%s'", uri.c_str()); + } else { + if (!initDownloadState( + uri, + itemMeta, + firstSeqNumberInPlaylist, + lastSeqNumberInPlaylist)) { return; } + ALOGV("fetching: '%s'", uri.c_str()); + } + + int64_t range_offset, range_length; + if (!itemMeta->findInt64("range-offset", &range_offset) + || !itemMeta->findInt64("range-length", &range_length)) { + range_offset = 0; + range_length = -1; } // block-wise download - bool startup = mStartup; + bool shouldPause = false; ssize_t bytesRead; do { + sp<DataSource> source = mHTTPDataSource; + + int64_t startUs = ALooper::GetNowUs(); bytesRead = mSession->fetchFile( - uri.c_str(), &buffer, range_offset, range_length, kDownloadBlockSize, &source); + uri.c_str(), &buffer, range_offset, range_length, kDownloadBlockSize, + &source, NULL, connectHTTP); + + // add sample for bandwidth estimation (excluding subtitles) + if (bytesRead > 0 + && (mStreamTypeMask + & (LiveSession::STREAMTYPE_AUDIO + | LiveSession::STREAMTYPE_VIDEO))) { + int64_t delayUs = ALooper::GetNowUs() - startUs; + mSession->addBandwidthMeasurement(bytesRead, delayUs); + } + + connectHTTP = false; if (bytesRead < 0) { status_t err = bytesRead; @@ -941,28 +1254,7 @@ void PlaylistFetcher::onDownloadNext() { return; } - if (startup || discontinuity) { - // Signal discontinuity. - - if (mPlaylist->isComplete() || mPlaylist->isEvent()) { - // If this was a live event this made no sense since - // we don't have access to all the segment before the current - // one. - mNextPTSTimeUs = getSegmentStartTimeUs(mSeqNumber); - } - - if (discontinuity) { - ALOGI("queueing discontinuity (explicit=%d)", discontinuity); - - queueDiscontinuity( - ATSParser::DISCONTINUITY_FORMATCHANGE, - NULL /* extra */); - - discontinuity = false; - } - - startup = false; - } + bool startUp = mStartup; // save current start up state err = OK; if (bufferStartsWithTsSyncByte(buffer)) { @@ -976,7 +1268,6 @@ void PlaylistFetcher::onDownloadNext() { tsBuffer->setRange(tsOff, tsSize); } tsBuffer->setRange(tsBuffer->offset(), tsBuffer->size() + bytesRead); - err = extractAndQueueAccessUnitsFromTs(tsBuffer); } @@ -991,13 +1282,35 @@ void PlaylistFetcher::onDownloadNext() { return; } else if (err == ERROR_OUT_OF_RANGE) { // reached stopping point - stopAsync(/* clear = */ false); + notifyStopReached(); return; } else if (err != OK) { notifyError(err); return; } - + // If we're switching, post start notification + // this should only be posted when the last chunk is full processed by TSParser + if (mSeekMode != LiveSession::kSeekModeExactPosition && startUp != mStartup) { + CHECK(mStartTimeUsNotify != NULL); + mStartTimeUsNotify->post(); + mStartTimeUsNotify.clear(); + shouldPause = true; + } + if (shouldPause || shouldPauseDownload()) { + // save state and return if this is not the last chunk, + // leaving the fetcher in paused state. + if (bytesRead != 0) { + mDownloadState->saveState( + uri, + itemMeta, + buffer, + tsBuffer, + firstSeqNumberInPlaylist, + lastSeqNumberInPlaylist); + return; + } + shouldPause = true; + } } while (bytesRead != 0); if (bufferStartsWithTsSyncByte(buffer)) { @@ -1034,7 +1347,6 @@ void PlaylistFetcher::onDownloadNext() { return; } - err = OK; if (tsBuffer != NULL) { AString method; CHECK(buffer->meta()->findString("cipher-method", &method)); @@ -1048,30 +1360,40 @@ void PlaylistFetcher::onDownloadNext() { } // bulk extract non-ts files + bool startUp = mStartup; if (tsBuffer == NULL) { - err = extractAndQueueAccessUnits(buffer, itemMeta); + status_t err = extractAndQueueAccessUnits(buffer, itemMeta); if (err == -EAGAIN) { // starting sequence number too low/high postMonitorQueue(); return; } else if (err == ERROR_OUT_OF_RANGE) { // reached stopping point - stopAsync(/* clear = */false); + notifyStopReached(); + return; + } else if (err != OK) { + notifyError(err); return; } } - if (err != OK) { - notifyError(err); - return; - } - ++mSeqNumber; - postMonitorQueue(); + // if adapting, pause after found the next starting point + if (mSeekMode != LiveSession::kSeekModeExactPosition && startUp != mStartup) { + CHECK(mStartTimeUsNotify != NULL); + mStartTimeUsNotify->post(); + mStartTimeUsNotify.clear(); + shouldPause = true; + } + + if (!shouldPause) { + postMonitorQueue(); + } } -int32_t PlaylistFetcher::getSeqNumberWithAnchorTime(int64_t anchorTimeUs) const { +int32_t PlaylistFetcher::getSeqNumberWithAnchorTime( + int64_t anchorTimeUs, int64_t targetDiffUs) const { int32_t firstSeqNumberInPlaylist, lastSeqNumberInPlaylist; if (mPlaylist->meta() == NULL || !mPlaylist->meta()->findInt32("media-sequence", &firstSeqNumberInPlaylist)) { @@ -1080,7 +1402,8 @@ int32_t PlaylistFetcher::getSeqNumberWithAnchorTime(int64_t anchorTimeUs) const lastSeqNumberInPlaylist = firstSeqNumberInPlaylist + mPlaylist->size() - 1; int32_t index = mSeqNumber - firstSeqNumberInPlaylist - 1; - while (index >= 0 && anchorTimeUs > mStartTimeUs) { + // adjust anchorTimeUs to within targetDiffUs from mStartTimeUs + while (index >= 0 && anchorTimeUs - mStartTimeUs > targetDiffUs) { sp<AMessage> itemMeta; CHECK(mPlaylist->itemAt(index, NULL /* uri */, &itemMeta)); @@ -1182,6 +1505,7 @@ const sp<ABuffer> &PlaylistFetcher::setAccessUnitProperties( accessUnit->meta()->setInt32("discontinuitySeq", mDiscontinuitySeq); accessUnit->meta()->setInt64("segmentStartTimeUs", getSegmentStartTimeUs(mSeqNumber)); + accessUnit->meta()->setInt64("segmentDurationUs", getSegmentDurationUs(mSeqNumber)); return accessUnit; } @@ -1200,9 +1524,7 @@ status_t PlaylistFetcher::extractAndQueueAccessUnitsFromTs(const sp<ABuffer> &bu mTSParser->signalDiscontinuity( ATSParser::DISCONTINUITY_TIME, extra); - mAbsoluteTimeAnchorUs = mNextPTSTimeUs; mNextPTSTimeUs = -1ll; - mFirstPTSValid = false; } size_t offset = 0; @@ -1222,30 +1544,15 @@ status_t PlaylistFetcher::extractAndQueueAccessUnitsFromTs(const sp<ABuffer> &bu for (size_t i = mPacketSources.size(); i-- > 0;) { sp<AnotherPacketSource> packetSource = mPacketSources.valueAt(i); - const char *key; - ATSParser::SourceType type; const LiveSession::StreamType stream = mPacketSources.keyAt(i); - switch (stream) { - case LiveSession::STREAMTYPE_VIDEO: - type = ATSParser::VIDEO; - key = "timeUsVideo"; - break; - - case LiveSession::STREAMTYPE_AUDIO: - type = ATSParser::AUDIO; - key = "timeUsAudio"; - break; - - case LiveSession::STREAMTYPE_SUBTITLES: - { - ALOGE("MPEG2 Transport streams do not contain subtitles."); - return ERROR_MALFORMED; - break; - } - - default: - TRESPASS(); + if (stream == LiveSession::STREAMTYPE_SUBTITLES) { + ALOGE("MPEG2 Transport streams do not contain subtitles."); + return ERROR_MALFORMED; } + const char *key = LiveSession::getKeyForStream(stream); + ATSParser::SourceType type = + (stream == LiveSession::STREAMTYPE_AUDIO) ? + ATSParser::AUDIO : ATSParser::VIDEO; sp<AnotherPacketSource> source = static_cast<AnotherPacketSource *>( @@ -1255,14 +1562,72 @@ status_t PlaylistFetcher::extractAndQueueAccessUnitsFromTs(const sp<ABuffer> &bu continue; } - int64_t timeUs; + const char *mime; + sp<MetaData> format = source->getFormat(); + bool isAvc = format != NULL && format->findCString(kKeyMIMEType, &mime) + && !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC); + sp<ABuffer> accessUnit; status_t finalResult; while (source->hasBufferAvailable(&finalResult) && source->dequeueAccessUnit(&accessUnit) == OK) { + int64_t timeUs; CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + bool seeking = mSeekMode == LiveSession::kSeekModeExactPosition; + if (mSegmentFirstPTS < 0ll) { + mSegmentFirstPTS = timeUs; + if (!seeking) { + int32_t firstSeqNumberInPlaylist; + if (mPlaylist->meta() == NULL || !mPlaylist->meta()->findInt32( + "media-sequence", &firstSeqNumberInPlaylist)) { + firstSeqNumberInPlaylist = 0; + } + + int32_t targetDurationSecs; + CHECK(mPlaylist->meta()->findInt32("target-duration", &targetDurationSecs)); + int64_t targetDurationUs = targetDurationSecs * 1000000ll; + // mStartup + // mStartup is true until we have queued a packet for all the streams + // we are fetching. We queue packets whose timestamps are greater than + // mStartTimeUs. + // mSegmentStartTimeUs >= 0 + // mSegmentStartTimeUs is non-negative when adapting or switching tracks + // mSeqNumber > firstSeqNumberInPlaylist + // don't decrement mSeqNumber if it already points to the 1st segment + // timeUs - mStartTimeUs > targetDurationUs: + // This and the 2 above conditions should only happen when adapting in a live + // stream; the old fetcher has already fetched to mStartTimeUs; the new fetcher + // would start fetching after timeUs, which should be greater than mStartTimeUs; + // the old fetcher would then continue fetching data until timeUs. We don't want + // timeUs to be too far ahead of mStartTimeUs because we want the old fetcher to + // stop as early as possible. The definition of being "too far ahead" is + // arbitrary; here we use targetDurationUs as threshold. + int64_t targetDiffUs =(mSeekMode == LiveSession::kSeekModeNextSample + ? 0 : targetDurationUs); + if (mStartup && mSegmentStartTimeUs >= 0 + && mSeqNumber > firstSeqNumberInPlaylist + && timeUs - mStartTimeUs > targetDiffUs) { + // we just guessed a starting timestamp that is too high when adapting in a + // live stream; re-adjust based on the actual timestamp extracted from the + // media segment; if we didn't move backward after the re-adjustment + // (newSeqNumber), start at least 1 segment prior. + int32_t newSeqNumber = getSeqNumberWithAnchorTime( + timeUs, targetDiffUs); + if (newSeqNumber >= mSeqNumber) { + --mSeqNumber; + } else { + mSeqNumber = newSeqNumber; + } + mStartTimeUsNotify = mNotify->dup(); + mStartTimeUsNotify->setInt32("what", kWhatStartedAt); + mStartTimeUsNotify->setString("uri", mURI); + mIDRFound = false; + return -EAGAIN; + } + } + } if (mStartup) { if (!mFirstPTSValid) { mFirstTimeUs = timeUs; @@ -1275,73 +1640,28 @@ status_t PlaylistFetcher::extractAndQueueAccessUnitsFromTs(const sp<ABuffer> &bu } } - if (timeUs < mStartTimeUs) { - // buffer up to the closest preceding IDR frame - ALOGV("timeUs %" PRId64 " us < mStartTimeUs %" PRId64 " us", - timeUs, mStartTimeUs); - const char *mime; - sp<MetaData> format = source->getFormat(); - bool isAvc = false; - if (format != NULL && format->findCString(kKeyMIMEType, &mime) - && !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC)) { - isAvc = true; - } - if (isAvc && IsIDR(accessUnit)) { - mVideoBuffer->clear(); - } + bool startTimeReached = + seeking ? (timeUs >= mStartTimeUs) : true; + + if (!startTimeReached || (isAvc && !mIDRFound)) { + // buffer up to the closest preceding IDR frame in the next segement, + // or the closest succeeding IDR frame after the exact position if (isAvc) { - mVideoBuffer->queueAccessUnit(accessUnit); + if (IsIDR(accessUnit) && (seeking || startTimeReached)) { + mVideoBuffer->clear(); + mIDRFound = true; + } + if (mIDRFound && seeking && !startTimeReached) { + mVideoBuffer->queueAccessUnit(accessUnit); + } } - - continue; - } - } - - CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); - if (mStartTimeUsNotify != NULL && timeUs > mStartTimeUs) { - int32_t firstSeqNumberInPlaylist; - if (mPlaylist->meta() == NULL || !mPlaylist->meta()->findInt32( - "media-sequence", &firstSeqNumberInPlaylist)) { - firstSeqNumberInPlaylist = 0; - } - - int32_t targetDurationSecs; - CHECK(mPlaylist->meta()->findInt32("target-duration", &targetDurationSecs)); - int64_t targetDurationUs = targetDurationSecs * 1000000ll; - // mStartup - // mStartup is true until we have queued a packet for all the streams - // we are fetching. We queue packets whose timestamps are greater than - // mStartTimeUs. - // mSegmentStartTimeUs >= 0 - // mSegmentStartTimeUs is non-negative when adapting or switching tracks - // mSeqNumber > firstSeqNumberInPlaylist - // don't decrement mSeqNumber if it already points to the 1st segment - // timeUs - mStartTimeUs > targetDurationUs: - // This and the 2 above conditions should only happen when adapting in a live - // stream; the old fetcher has already fetched to mStartTimeUs; the new fetcher - // would start fetching after timeUs, which should be greater than mStartTimeUs; - // the old fetcher would then continue fetching data until timeUs. We don't want - // timeUs to be too far ahead of mStartTimeUs because we want the old fetcher to - // stop as early as possible. The definition of being "too far ahead" is - // arbitrary; here we use targetDurationUs as threshold. - if (mStartup && mSegmentStartTimeUs >= 0 - && mSeqNumber > firstSeqNumberInPlaylist - && timeUs - mStartTimeUs > targetDurationUs) { - // we just guessed a starting timestamp that is too high when adapting in a - // live stream; re-adjust based on the actual timestamp extracted from the - // media segment; if we didn't move backward after the re-adjustment - // (newSeqNumber), start at least 1 segment prior. - int32_t newSeqNumber = getSeqNumberWithAnchorTime(timeUs); - if (newSeqNumber >= mSeqNumber) { - --mSeqNumber; - } else { - mSeqNumber = newSeqNumber; + if (!startTimeReached || (isAvc && !mIDRFound)) { + continue; } - mStartTimeUsNotify = mNotify->dup(); - mStartTimeUsNotify->setInt32("what", kWhatStartedAt); - return -EAGAIN; } + } + if (mStartTimeUsNotify != NULL) { int32_t seq; if (!mStartTimeUsNotify->findInt32("discontinuitySeq", &seq)) { mStartTimeUsNotify->setInt32("discontinuitySeq", mDiscontinuitySeq); @@ -1357,8 +1677,6 @@ status_t PlaylistFetcher::extractAndQueueAccessUnitsFromTs(const sp<ABuffer> &bu if (streamMask == mStreamTypeMask) { mStartup = false; - mStartTimeUsNotify->post(); - mStartTimeUsNotify.clear(); } } } @@ -1372,7 +1690,6 @@ status_t PlaylistFetcher::extractAndQueueAccessUnitsFromTs(const sp<ABuffer> &bu || !mStopParams->findInt64(key, &stopTimeUs) || (discontinuitySeq == mDiscontinuitySeq && timeUs >= stopTimeUs)) { - packetSource->queueAccessUnit(mSession->createFormatChangeBuffer()); mStreamTypeMask &= ~stream; mPacketSources.removeItemsAt(i); break; @@ -1467,8 +1784,6 @@ status_t PlaylistFetcher::extractAndQueueAccessUnits( } if (mNextPTSTimeUs >= 0ll) { - mFirstPTSValid = false; - mAbsoluteTimeAnchorUs = mNextPTSTimeUs; mNextPTSTimeUs = -1ll; } @@ -1569,7 +1884,7 @@ status_t PlaylistFetcher::extractAndQueueAccessUnits( CHECK(packetSource->getFormat()->findInt32(kKeySampleRate, &sampleRate)); int64_t timeUs = (PTS * 100ll) / 9ll; - if (!mFirstPTSValid) { + if (mStartup && !mFirstPTSValid) { mFirstPTSValid = true; mFirstTimeUs = timeUs; } @@ -1621,10 +1936,13 @@ status_t PlaylistFetcher::extractAndQueueAccessUnits( CHECK(mPlaylist->meta()->findInt32("target-duration", &targetDurationSecs)); int64_t targetDurationUs = targetDurationSecs * 1000000ll; + int64_t targetDiffUs =(mSeekMode == LiveSession::kSeekModeNextSample + ? 0 : targetDurationUs); // Duplicated logic from how we handle .ts playlists. if (mStartup && mSegmentStartTimeUs >= 0 - && timeUs - mStartTimeUs > targetDurationUs) { - int32_t newSeqNumber = getSeqNumberWithAnchorTime(timeUs); + && timeUs - mStartTimeUs > targetDiffUs) { + int32_t newSeqNumber = getSeqNumberWithAnchorTime( + timeUs, targetDiffUs); if (newSeqNumber >= mSeqNumber) { --mSeqNumber; } else { @@ -1636,8 +1954,6 @@ status_t PlaylistFetcher::extractAndQueueAccessUnits( mStartTimeUsNotify->setInt64("timeUsAudio", timeUs); mStartTimeUsNotify->setInt32("discontinuitySeq", mDiscontinuitySeq); mStartTimeUsNotify->setInt32("streamMask", LiveSession::STREAMTYPE_AUDIO); - mStartTimeUsNotify->post(); - mStartTimeUsNotify.clear(); mStartup = false; } } @@ -1650,7 +1966,6 @@ status_t PlaylistFetcher::extractAndQueueAccessUnits( || discontinuitySeq > mDiscontinuitySeq || !mStopParams->findInt64("timeUsAudio", &stopTimeUs) || (discontinuitySeq == mDiscontinuitySeq && unitTimeUs >= stopTimeUs)) { - packetSource->queueAccessUnit(mSession->createFormatChangeBuffer()); mStreamTypeMask = 0; mPacketSources.clear(); return ERROR_OUT_OF_RANGE; @@ -1687,33 +2002,15 @@ void PlaylistFetcher::updateDuration() { msg->post(); } -int64_t PlaylistFetcher::resumeThreshold(const sp<AMessage> &msg) { - int64_t durationUs; - if (msg->findInt64("durationUs", &durationUs) && durationUs > 0) { - return kNumSkipFrames * durationUs; - } - - sp<RefBase> obj; - msg->findObject("format", &obj); - MetaData *format = static_cast<MetaData *>(obj.get()); - - const char *mime; - CHECK(format->findCString(kKeyMIMEType, &mime)); - bool audio = !strncasecmp(mime, "audio/", 6); - if (audio) { - // Assumes 1000 samples per frame. - int32_t sampleRate; - CHECK(format->findInt32(kKeySampleRate, &sampleRate)); - return kNumSkipFrames /* frames */ * 1000 /* samples */ - * (1000000 / sampleRate) /* sample duration (us) */; - } else { - int32_t frameRate; - if (format->findInt32(kKeyFrameRate, &frameRate) && frameRate > 0) { - return kNumSkipFrames * (1000000 / frameRate); - } - } +void PlaylistFetcher::updateTargetDuration() { + int32_t targetDurationSecs; + CHECK(mPlaylist->meta()->findInt32("target-duration", &targetDurationSecs)); + int64_t targetDurationUs = targetDurationSecs * 1000000ll; - return 500000ll; + sp<AMessage> msg = mNotify->dup(); + msg->setInt32("what", kWhatTargetDurationUpdate); + msg->setInt64("targetDurationUs", targetDurationUs); + msg->post(); } } // namespace android diff --git a/media/libstagefright/httplive/PlaylistFetcher.h b/media/libstagefright/httplive/PlaylistFetcher.h index 4e15f85..dab56df 100644 --- a/media/libstagefright/httplive/PlaylistFetcher.h +++ b/media/libstagefright/httplive/PlaylistFetcher.h @@ -36,6 +36,7 @@ class String8; struct PlaylistFetcher : public AHandler { static const int64_t kMinBufferedDurationUs; static const int32_t kDownloadBlockSize; + static const int64_t kFetcherResumeThreshold; enum { kWhatStarted, @@ -43,10 +44,11 @@ struct PlaylistFetcher : public AHandler { kWhatStopped, kWhatError, kWhatDurationUpdate, - kWhatTemporarilyDoneFetching, + kWhatTargetDurationUpdate, kWhatPrepared, kWhatPreparationFailed, kWhatStartedAt, + kWhatStopReached, }; PlaylistFetcher( @@ -64,10 +66,10 @@ struct PlaylistFetcher : public AHandler { int64_t startTimeUs = -1ll, // starting timestamps int64_t segmentStartTimeUs = -1ll, // starting position within playlist // startTimeUs!=segmentStartTimeUs only when playlist is live - int32_t startDiscontinuitySeq = 0, - bool adaptive = false); + int32_t startDiscontinuitySeq = -1, + LiveSession::SeekMode seekMode = LiveSession::kSeekModeExactPosition); - void pauseAsync(); + void pauseAsync(float thresholdRatio); void stopAsync(bool clear = true); @@ -95,6 +97,8 @@ private: kWhatDownloadNext = 'dlnx', }; + struct DownloadState; + static const int64_t kMaxMonitorDelayUs; static const int32_t kNumSkipFrames; @@ -105,6 +109,7 @@ private: sp<AMessage> mNotify; sp<AMessage> mStartTimeUsNotify; + sp<HTTPBase> mHTTPDataSource; sp<LiveSession> mSession; AString mURI; @@ -116,7 +121,7 @@ private: // adapting or switching tracks. int64_t mSegmentStartTimeUs; - ssize_t mDiscontinuitySeq; + int32_t mDiscontinuitySeq; bool mStartTimeUsRelative; sp<AMessage> mStopParams; // message containing the latest timestamps we should fetch. @@ -130,13 +135,16 @@ private: int32_t mSeqNumber; int32_t mNumRetries; bool mStartup; - bool mAdaptive; - bool mPrepared; + bool mIDRFound; + int32_t mSeekMode; + bool mTimeChangeSignaled; int64_t mNextPTSTimeUs; int32_t mMonitorQueueGeneration; const int32_t mSubtitleGeneration; + int32_t mLastDiscontinuitySeq; + enum RefreshState { INITIAL_MINIMUM_RELOAD_DELAY, FIRST_UNCHANGED_RELOAD_ATTEMPT, @@ -150,9 +158,8 @@ private: sp<ATSParser> mTSParser; bool mFirstPTSValid; - uint64_t mFirstPTS; int64_t mFirstTimeUs; - int64_t mAbsoluteTimeAnchorUs; + int64_t mSegmentFirstPTS; sp<AnotherPacketSource> mVideoBuffer; // Stores the initialization vector to decrypt the next block of cipher text, which can @@ -160,6 +167,11 @@ private: // the last block of cipher text (cipher-block chaining). unsigned char mAESInitVec[16]; + Mutex mThresholdLock; + float mThresholdRatio; + + sp<DownloadState> mDownloadState; + // Set first to true if decrypting the first segment of a playlist segment. When // first is true, reset the initialization vector based on the available // information in the manifest; otherwise, use the initialization vector as @@ -175,6 +187,8 @@ private: void postMonitorQueue(int64_t delayUs = 0, int64_t minDelayUs = 0); void cancelMonitorQueue(); + void setStoppingThreshold(float thresholdRatio); + bool shouldPauseDownload(); int64_t delayUsToRefreshPlaylist() const; status_t refreshPlaylist(); @@ -182,12 +196,19 @@ private: // Returns the media time in us of the segment specified by seqNumber. // This is computed by summing the durations of all segments before it. int64_t getSegmentStartTimeUs(int32_t seqNumber) const; + // Returns the duration time in us of the segment specified. + int64_t getSegmentDurationUs(int32_t seqNumber) const; status_t onStart(const sp<AMessage> &msg); void onPause(); void onStop(const sp<AMessage> &msg); void onMonitorQueue(); void onDownloadNext(); + bool initDownloadState( + AString &uri, + sp<AMessage> &itemMeta, + int32_t &firstSeqNumberInPlaylist, + int32_t &lastSeqNumberInPlaylist); // Resume a fetcher to continue until the stopping point stored in msg. status_t onResumeUntil(const sp<AMessage> &msg); @@ -201,20 +222,19 @@ private: status_t extractAndQueueAccessUnits( const sp<ABuffer> &buffer, const sp<AMessage> &itemMeta); + void notifyStopReached(); void notifyError(status_t err); void queueDiscontinuity( ATSParser::DiscontinuityType type, const sp<AMessage> &extra); - int32_t getSeqNumberWithAnchorTime(int64_t anchorTimeUs) const; + int32_t getSeqNumberWithAnchorTime( + int64_t anchorTimeUs, int64_t targetDurationUs) const; int32_t getSeqNumberForDiscontinuity(size_t discontinuitySeq) const; int32_t getSeqNumberForTime(int64_t timeUs) const; void updateDuration(); - - // Before resuming a fetcher in onResume, check the remaining duration is longer than that - // returned by resumeThreshold. - int64_t resumeThreshold(const sp<AMessage> &msg); + void updateTargetDuration(); DISALLOW_EVIL_CONSTRUCTORS(PlaylistFetcher); }; diff --git a/media/libstagefright/include/MPEG4Extractor.h b/media/libstagefright/include/MPEG4Extractor.h index 1fe6fcf..8c16251 100644 --- a/media/libstagefright/include/MPEG4Extractor.h +++ b/media/libstagefright/include/MPEG4Extractor.h @@ -83,6 +83,8 @@ private: Vector<SidxEntry> mSidxEntries; off64_t mMoofOffset; + bool mMoofFound; + bool mMdatFound; Vector<PsshInfo> mPssh; diff --git a/media/libstagefright/include/avc_utils.h b/media/libstagefright/include/avc_utils.h index c270bc1..dafa07e 100644 --- a/media/libstagefright/include/avc_utils.h +++ b/media/libstagefright/include/avc_utils.h @@ -36,6 +36,11 @@ enum { kAVCProfileCAVLC444Intra = 0x2c }; +struct NALPosition { + size_t nalOffset; + size_t nalSize; +}; + // Optionally returns sample aspect ratio as well. void FindAVCDimensions( const sp<ABuffer> &seqParamSet, diff --git a/media/libstagefright/mpeg2ts/ATSParser.cpp b/media/libstagefright/mpeg2ts/ATSParser.cpp index 482ccff..01c3dd6 100644 --- a/media/libstagefright/mpeg2ts/ATSParser.cpp +++ b/media/libstagefright/mpeg2ts/ATSParser.cpp @@ -35,6 +35,7 @@ #include <media/stagefright/Utils.h> #include <media/IStreamSource.h> #include <utils/KeyedVector.h> +#include <utils/Vector.h> #include <inttypes.h> @@ -86,14 +87,22 @@ struct ATSParser::Program : public RefBase { } private: + struct StreamInfo { + unsigned mType; + unsigned mPID; + }; + ATSParser *mParser; unsigned mProgramNumber; unsigned mProgramMapPID; KeyedVector<unsigned, sp<Stream> > mStreams; bool mFirstPTSValid; uint64_t mFirstPTS; + int64_t mLastRecoveredPTS; status_t parseProgramMap(ABitReader *br); + int64_t recoverPTS(uint64_t PTS_33bit); + bool switchPIDs(const Vector<StreamInfo> &infos); DISALLOW_EVIL_CONSTRUCTORS(Program); }; @@ -157,10 +166,12 @@ struct ATSParser::PSISection : public RefBase { PSISection(); status_t append(const void *data, size_t size); + void setSkipBytes(uint8_t skip); void clear(); bool isComplete() const; bool isEmpty() const; + bool isCRCOkay() const; const uint8_t *data() const; size_t size() const; @@ -170,6 +181,8 @@ protected: private: sp<ABuffer> mBuffer; + uint8_t mSkipBytes; + static uint32_t CRC_TABLE[]; DISALLOW_EVIL_CONSTRUCTORS(PSISection); }; @@ -182,7 +195,8 @@ ATSParser::Program::Program( mProgramNumber(programNumber), mProgramMapPID(programMapPID), mFirstPTSValid(false), - mFirstPTS(0) { + mFirstPTS(0), + mLastRecoveredPTS(-1ll) { ALOGV("new program number %u", programNumber); } @@ -237,10 +251,71 @@ void ATSParser::Program::signalEOS(status_t finalResult) { } } -struct StreamInfo { - unsigned mType; - unsigned mPID; -}; +bool ATSParser::Program::switchPIDs(const Vector<StreamInfo> &infos) { + bool success = false; + + if (mStreams.size() == infos.size()) { + // build type->PIDs map for old and new mapping + size_t i; + KeyedVector<int32_t, Vector<int32_t> > oldType2PIDs, newType2PIDs; + for (i = 0; i < mStreams.size(); ++i) { + ssize_t index = oldType2PIDs.indexOfKey(mStreams[i]->type()); + if (index < 0) { + oldType2PIDs.add(mStreams[i]->type(), Vector<int32_t>()); + } + oldType2PIDs.editValueFor(mStreams[i]->type()).push_back(mStreams[i]->pid()); + } + for (i = 0; i < infos.size(); ++i) { + ssize_t index = newType2PIDs.indexOfKey(infos[i].mType); + if (index < 0) { + newType2PIDs.add(infos[i].mType, Vector<int32_t>()); + } + newType2PIDs.editValueFor(infos[i].mType).push_back(infos[i].mPID); + } + + // we can recover if the number of streams for each type hasn't changed + if (oldType2PIDs.size() == newType2PIDs.size()) { + success = true; + for (i = 0; i < oldType2PIDs.size(); ++i) { + // KeyedVector is sorted, we just compare key and size of each index + if (oldType2PIDs.keyAt(i) != newType2PIDs.keyAt(i) + || oldType2PIDs[i].size() != newType2PIDs[i].size()) { + success = false; + break; + } + } + } + + if (success) { + // save current streams to temp + KeyedVector<int32_t, sp<Stream> > temp; + for (i = 0; i < mStreams.size(); ++i) { + temp.add(mStreams.keyAt(i), mStreams.editValueAt(i)); + } + + mStreams.clear(); + for (i = 0; i < temp.size(); ++i) { + // The two checks below shouldn't happen, + // we already checked above the stream count matches + ssize_t index = newType2PIDs.indexOfKey(temp[i]->type()); + CHECK(index >= 0); + Vector<int32_t> &newPIDs = newType2PIDs.editValueAt(index); + CHECK(newPIDs.size() > 0); + + // get the next PID for temp[i]->type() in the new PID map + Vector<int32_t>::iterator it = newPIDs.begin(); + + // change the PID of the stream, and add it back + temp.editValueAt(i)->setPID(*it); + mStreams.add(temp[i]->pid(), temp.editValueAt(i)); + + // removed the used PID + newPIDs.erase(it); + } + } + } + return success; +} status_t ATSParser::Program::parseProgramMap(ABitReader *br) { unsigned table_id = br->getBits(8); @@ -369,39 +444,8 @@ status_t ATSParser::Program::parseProgramMap(ABitReader *br) { } #endif - // The only case we can recover from is if we have two streams - // and they switched PIDs. - - bool success = false; - - if (mStreams.size() == 2 && infos.size() == 2) { - const StreamInfo &info1 = infos.itemAt(0); - const StreamInfo &info2 = infos.itemAt(1); - - sp<Stream> s1 = mStreams.editValueAt(0); - sp<Stream> s2 = mStreams.editValueAt(1); - - bool caseA = - info1.mPID == s1->pid() && info1.mType == s2->type() - && info2.mPID == s2->pid() && info2.mType == s1->type(); - - bool caseB = - info1.mPID == s2->pid() && info1.mType == s1->type() - && info2.mPID == s1->pid() && info2.mType == s2->type(); - - if (caseA || caseB) { - unsigned pid1 = s1->pid(); - unsigned pid2 = s2->pid(); - s1->setPID(pid2); - s2->setPID(pid1); - - mStreams.clear(); - mStreams.add(s1->pid(), s1); - mStreams.add(s2->pid(), s2); - - success = true; - } - } + // we can recover if number of streams for each type remain the same + bool success = switchPIDs(infos); if (!success) { ALOGI("Stream PIDs changed and we cannot recover."); @@ -425,6 +469,32 @@ status_t ATSParser::Program::parseProgramMap(ABitReader *br) { return OK; } +int64_t ATSParser::Program::recoverPTS(uint64_t PTS_33bit) { + // We only have the lower 33-bit of the PTS. It could overflow within a + // reasonable amount of time. To handle the wrap-around, use fancy math + // to get an extended PTS that is within [-0xffffffff, 0xffffffff] + // of the latest recovered PTS. + if (mLastRecoveredPTS < 0ll) { + // Use the original 33bit number for 1st frame, the reason is that + // if 1st frame wraps to negative that's far away from 0, we could + // never start. Only start wrapping around from 2nd frame. + mLastRecoveredPTS = static_cast<int64_t>(PTS_33bit); + } else { + mLastRecoveredPTS = static_cast<int64_t>( + ((mLastRecoveredPTS - PTS_33bit + 0x100000000ll) + & 0xfffffffe00000000ull) | PTS_33bit); + // We start from 0, but recovered PTS could be slightly below 0. + // Clamp it to 0 as rest of the pipeline doesn't take negative pts. + // (eg. video is read first and starts at 0, but audio starts at 0xfffffff0) + if (mLastRecoveredPTS < 0ll) { + ALOGI("Clamping negative recovered PTS (%" PRId64 ") to 0", mLastRecoveredPTS); + mLastRecoveredPTS = 0ll; + } + } + + return mLastRecoveredPTS; +} + sp<MediaSource> ATSParser::Program::getSource(SourceType type) { size_t index = (type == AUDIO) ? 0 : 0; @@ -455,6 +525,8 @@ bool ATSParser::Program::hasSource(SourceType type) const { } int64_t ATSParser::Program::convertPTSToTimestamp(uint64_t PTS) { + PTS = recoverPTS(PTS); + if (!(mParser->mFlags & TS_TIMESTAMPS_ARE_ABSOLUTE)) { if (!mFirstPTSValid) { mFirstPTSValid = true; @@ -1098,10 +1170,12 @@ status_t ATSParser::parsePID( if (payload_unit_start_indicator) { if (!section->isEmpty()) { - return ERROR_UNSUPPORTED; + ALOGW("parsePID encounters payload_unit_start_indicator when section is not empty"); + section->clear(); } unsigned skip = br->getBits(8); + section->setSkipBytes(skip + 1); // skip filler bytes + pointer field itself br->skipBits(skip * 8); } @@ -1116,6 +1190,9 @@ status_t ATSParser::parsePID( return OK; } + if (!section->isCRCOkay()) { + return BAD_VALUE; + } ABitReader sectionBits(section->data(), section->size()); if (PID == 0) { @@ -1338,7 +1415,79 @@ void ATSParser::updatePCR( //////////////////////////////////////////////////////////////////////////////// -ATSParser::PSISection::PSISection() { + +// CRC32 used for PSI section. The table was generated by following command: +// $ python pycrc.py --model crc-32-mpeg --algorithm table-driven --generate c +// Visit http://www.tty1.net/pycrc/index_en.html for more details. +uint32_t ATSParser::PSISection::CRC_TABLE[] = { + 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, + 0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005, + 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, + 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, + 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9, + 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, + 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, + 0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd, + 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, + 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, + 0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, + 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, + 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, + 0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95, + 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, + 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, + 0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae, + 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, + 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, + 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca, + 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, + 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, + 0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066, + 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, + 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, + 0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692, + 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, + 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, + 0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e, + 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, + 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, + 0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a, + 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, + 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, + 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f, + 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, + 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, + 0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b, + 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, + 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, + 0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, + 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, + 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, + 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3, + 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, + 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, + 0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f, + 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, + 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, + 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c, + 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, + 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, + 0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30, + 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, + 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, + 0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654, + 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, + 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, + 0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18, + 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, + 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, + 0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c, + 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, + 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 + }; + +ATSParser::PSISection::PSISection() : + mSkipBytes(0) { } ATSParser::PSISection::~PSISection() { @@ -1369,10 +1518,15 @@ status_t ATSParser::PSISection::append(const void *data, size_t size) { return OK; } +void ATSParser::PSISection::setSkipBytes(uint8_t skip) { + mSkipBytes = skip; +} + void ATSParser::PSISection::clear() { if (mBuffer != NULL) { mBuffer->setRange(0, 0); } + mSkipBytes = 0; } bool ATSParser::PSISection::isComplete() const { @@ -1396,4 +1550,30 @@ size_t ATSParser::PSISection::size() const { return mBuffer == NULL ? 0 : mBuffer->size(); } +bool ATSParser::PSISection::isCRCOkay() const { + if (!isComplete()) { + return false; + } + uint8_t* data = mBuffer->data(); + + // Return true if section_syntax_indicator says no section follows the field section_length. + if ((data[1] & 0x80) == 0) { + return true; + } + + unsigned sectionLength = U16_AT(data + 1) & 0xfff; + ALOGV("sectionLength %u, skip %u", sectionLength, mSkipBytes); + + // Skip the preceding field present when payload start indicator is on. + sectionLength -= mSkipBytes; + + uint32_t crc = 0xffffffff; + for(unsigned i = 0; i < sectionLength + 4 /* crc */; i++) { + uint8_t b = data[i]; + int index = ((crc >> 24) ^ (b & 0xff)) & 0xff; + crc = CRC_TABLE[index] ^ (crc << 8); + } + ALOGV("crc: %08x\n", crc); + return (crc == 0); +} } // namespace android diff --git a/media/libstagefright/mpeg2ts/ATSParser.h b/media/libstagefright/mpeg2ts/ATSParser.h index 75d76dc..5c50747 100644 --- a/media/libstagefright/mpeg2ts/ATSParser.h +++ b/media/libstagefright/mpeg2ts/ATSParser.h @@ -46,6 +46,9 @@ struct ATSParser : public RefBase { DISCONTINUITY_AUDIO_FORMAT | DISCONTINUITY_VIDEO_FORMAT | DISCONTINUITY_TIME, + DISCONTINUITY_FORMAT_ONLY = + DISCONTINUITY_AUDIO_FORMAT + | DISCONTINUITY_VIDEO_FORMAT, }; enum Flags { diff --git a/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp b/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp index f266fe7..c2f1527 100644 --- a/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp +++ b/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp @@ -19,6 +19,8 @@ #include "AnotherPacketSource.h" +#include "include/avc_utils.h" + #include <media/stagefright/foundation/ABuffer.h> #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/foundation/AMessage.h> @@ -38,6 +40,7 @@ const int64_t kNearEOSMarkUs = 2000000ll; // 2 secs AnotherPacketSource::AnotherPacketSource(const sp<MetaData> &meta) : mIsAudio(false), mIsVideo(false), + mEnabled(true), mFormat(NULL), mLastQueuedTimeUs(0), mEOSResult(OK), @@ -48,7 +51,10 @@ AnotherPacketSource::AnotherPacketSource(const sp<MetaData> &meta) } void AnotherPacketSource::setFormat(const sp<MetaData> &meta) { - CHECK(mFormat == NULL); + if (mFormat != NULL) { + // Only allowed to be set once. Requires explicit clear to reset. + return; + } mIsAudio = false; mIsVideo = false; @@ -91,13 +97,12 @@ sp<MetaData> AnotherPacketSource::getFormat() { while (it != mBuffers.end()) { sp<ABuffer> buffer = *it; int32_t discontinuity; - if (buffer->meta()->findInt32("discontinuity", &discontinuity)) { - break; - } - - sp<RefBase> object; - if (buffer->meta()->findObject("format", &object)) { - return mFormat = static_cast<MetaData*>(object.get()); + if (!buffer->meta()->findInt32("discontinuity", &discontinuity)) { + sp<RefBase> object; + if (buffer->meta()->findObject("format", &object)) { + setFormat(static_cast<MetaData*>(object.get())); + return mFormat; + } } ++it; @@ -131,7 +136,7 @@ status_t AnotherPacketSource::dequeueAccessUnit(sp<ABuffer> *buffer) { sp<RefBase> object; if ((*buffer)->meta()->findObject("format", &object)) { - mFormat = static_cast<MetaData*>(object.get()); + setFormat(static_cast<MetaData*>(object.get())); } return OK; @@ -153,7 +158,6 @@ status_t AnotherPacketSource::read( const sp<ABuffer> buffer = *mBuffers.begin(); mBuffers.erase(mBuffers.begin()); - mLatestDequeuedMeta = buffer->meta()->dup(); int32_t discontinuity; if (buffer->meta()->findInt32("discontinuity", &discontinuity)) { @@ -164,9 +168,11 @@ status_t AnotherPacketSource::read( return INFO_DISCONTINUITY; } + mLatestDequeuedMeta = buffer->meta()->dup(); + sp<RefBase> object; if (buffer->meta()->findObject("format", &object)) { - mFormat = static_cast<MetaData*>(object.get()); + setFormat(static_cast<MetaData*>(object.get())); } int64_t timeUs; @@ -203,20 +209,26 @@ void AnotherPacketSource::queueAccessUnit(const sp<ABuffer> &buffer) { return; } - int64_t lastQueuedTimeUs; - CHECK(buffer->meta()->findInt64("timeUs", &lastQueuedTimeUs)); - mLastQueuedTimeUs = lastQueuedTimeUs; - ALOGV("queueAccessUnit timeUs=%" PRIi64 " us (%.2f secs)", mLastQueuedTimeUs, mLastQueuedTimeUs / 1E6); - Mutex::Autolock autoLock(mLock); mBuffers.push_back(buffer); mCondition.signal(); int32_t discontinuity; if (buffer->meta()->findInt32("discontinuity", &discontinuity)) { + // discontinuity handling needs to be consistent with queueDiscontinuity() ++mQueuedDiscontinuityCount; + mLastQueuedTimeUs = 0ll; + mEOSResult = OK; + mLatestEnqueuedMeta = NULL; + return; } + int64_t lastQueuedTimeUs; + CHECK(buffer->meta()->findInt64("timeUs", &lastQueuedTimeUs)); + mLastQueuedTimeUs = lastQueuedTimeUs; + ALOGV("queueAccessUnit timeUs=%" PRIi64 " us (%.2f secs)", + mLastQueuedTimeUs, mLastQueuedTimeUs / 1E6); + if (mLatestEnqueuedMeta == NULL) { mLatestEnqueuedMeta = buffer->meta()->dup(); } else { @@ -296,6 +308,10 @@ void AnotherPacketSource::signalEOS(status_t result) { bool AnotherPacketSource::hasBufferAvailable(status_t *finalResult) { Mutex::Autolock autoLock(mLock); + *finalResult = OK; + if (!mEnabled) { + return false; + } if (!mBuffers.empty()) { return true; } @@ -304,6 +320,24 @@ bool AnotherPacketSource::hasBufferAvailable(status_t *finalResult) { return false; } +bool AnotherPacketSource::hasDataBufferAvailable(status_t *finalResult) { + Mutex::Autolock autoLock(mLock); + *finalResult = OK; + if (!mEnabled) { + return false; + } + List<sp<ABuffer> >::iterator it; + for (it = mBuffers.begin(); it != mBuffers.end(); it++) { + int32_t discontinuity; + if (!(*it)->meta()->findInt32("discontinuity", &discontinuity)) { + return true; + } + } + + *finalResult = mEOSResult; + return false; +} + int64_t AnotherPacketSource::getBufferedDurationUs(status_t *finalResult) { Mutex::Autolock autoLock(mLock); return getBufferedDurationUs_l(finalResult); @@ -422,4 +456,140 @@ sp<AMessage> AnotherPacketSource::getLatestDequeuedMeta() { return mLatestDequeuedMeta; } +void AnotherPacketSource::enable(bool enable) { + Mutex::Autolock autoLock(mLock); + mEnabled = enable; +} + +sp<AMessage> AnotherPacketSource::getMetaAfterLastDequeued(int64_t delayUs) { + Mutex::Autolock autoLock(mLock); + int64_t firstUs = -1; + int64_t lastUs = -1; + int64_t durationUs = 0; + + List<sp<ABuffer> >::iterator it; + for (it = mBuffers.begin(); it != mBuffers.end(); ++it) { + const sp<ABuffer> &buffer = *it; + int32_t discontinuity; + if (buffer->meta()->findInt32("discontinuity", &discontinuity)) { + durationUs += lastUs - firstUs; + firstUs = -1; + lastUs = -1; + continue; + } + int64_t timeUs; + if (buffer->meta()->findInt64("timeUs", &timeUs)) { + if (firstUs < 0) { + firstUs = timeUs; + } + if (lastUs < 0 || timeUs > lastUs) { + lastUs = timeUs; + } + if (durationUs + (lastUs - firstUs) >= delayUs) { + return buffer->meta(); + } + } + } + return mLatestEnqueuedMeta; +} + +void AnotherPacketSource::trimBuffersAfterTimeUs( + size_t discontinuitySeq, int64_t timeUs) { + ALOGV("trimBuffersAfterTimeUs: discontinuitySeq %zu, timeUs %lld", + discontinuitySeq, (long long)timeUs); + + Mutex::Autolock autoLock(mLock); + if (mBuffers.empty()) { + return; + } + + List<sp<ABuffer> >::iterator it; + sp<AMessage> newLatestEnqueuedMeta = NULL; + int64_t newLastQueuedTimeUs = 0; + size_t newDiscontinuityCount = 0; + for (it = mBuffers.begin(); it != mBuffers.end(); ++it) { + const sp<ABuffer> &buffer = *it; + int32_t discontinuity; + if (buffer->meta()->findInt32("discontinuity", &discontinuity)) { + newDiscontinuityCount++; + continue; + } + size_t curDiscontinuitySeq; + int64_t curTimeUs; + CHECK(buffer->meta()->findInt32( + "discontinuitySeq", (int32_t*)&curDiscontinuitySeq)); + CHECK(buffer->meta()->findInt64("timeUs", &curTimeUs)); + if ((curDiscontinuitySeq > discontinuitySeq + || (curDiscontinuitySeq == discontinuitySeq + && curTimeUs >= timeUs))) { + ALOGI("trimming from %lld (inclusive) to end", + (long long)curTimeUs); + break; + } + newLatestEnqueuedMeta = buffer->meta(); + newLastQueuedTimeUs = curTimeUs; + } + mBuffers.erase(it, mBuffers.end()); + mLatestEnqueuedMeta = newLatestEnqueuedMeta; + mLastQueuedTimeUs = newLastQueuedTimeUs; + mQueuedDiscontinuityCount = newDiscontinuityCount; +} + +sp<AMessage> AnotherPacketSource::trimBuffersBeforeTimeUs( + size_t discontinuitySeq, int64_t timeUs) { + ALOGV("trimBuffersBeforeTimeUs: discontinuitySeq %zu, timeUs %lld", + discontinuitySeq, (long long)timeUs); + sp<AMessage> meta; + Mutex::Autolock autoLock(mLock); + if (mBuffers.empty()) { + return NULL; + } + + sp<MetaData> format; + bool isAvc = false; + + List<sp<ABuffer> >::iterator it; + size_t discontinuityCount = 0; + for (it = mBuffers.begin(); it != mBuffers.end(); ++it) { + const sp<ABuffer> &buffer = *it; + int32_t discontinuity; + if (buffer->meta()->findInt32("discontinuity", &discontinuity)) { + format = NULL; + isAvc = false; + discontinuityCount++; + continue; + } + if (format == NULL) { + sp<RefBase> object; + if (buffer->meta()->findObject("format", &object)) { + const char* mime; + format = static_cast<MetaData*>(object.get()); + isAvc = format != NULL + && format->findCString(kKeyMIMEType, &mime) + && !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC); + } + } + if (isAvc && !IsIDR(buffer)) { + continue; + } + size_t curDiscontinuitySeq; + int64_t curTimeUs; + CHECK(buffer->meta()->findInt32( + "discontinuitySeq", (int32_t*)&curDiscontinuitySeq)); + CHECK(buffer->meta()->findInt64("timeUs", &curTimeUs)); + if ((curDiscontinuitySeq > discontinuitySeq + || (curDiscontinuitySeq == discontinuitySeq + && curTimeUs > timeUs))) { + ALOGI("trimming from beginning to %lld (not inclusive)", + (long long)curTimeUs); + meta = buffer->meta(); + break; + } + } + mBuffers.erase(mBuffers.begin(), it); + mQueuedDiscontinuityCount -= discontinuityCount; + mLatestDequeuedMeta = NULL; + return meta; +} + } // namespace android diff --git a/media/libstagefright/mpeg2ts/AnotherPacketSource.h b/media/libstagefright/mpeg2ts/AnotherPacketSource.h index 809a858..e126006 100644 --- a/media/libstagefright/mpeg2ts/AnotherPacketSource.h +++ b/media/libstagefright/mpeg2ts/AnotherPacketSource.h @@ -43,8 +43,12 @@ struct AnotherPacketSource : public MediaSource { void clear(); + // Returns true if we have any packets including discontinuities bool hasBufferAvailable(status_t *finalResult); + // Returns true if we have packets that's not discontinuities + bool hasDataBufferAvailable(status_t *finalResult); + // Returns the difference between the last and the first queued // presentation timestamps since the last discontinuity (if any). int64_t getBufferedDurationUs(status_t *finalResult); @@ -66,8 +70,14 @@ struct AnotherPacketSource : public MediaSource { bool isFinished(int64_t duration) const; + void enable(bool enable); + sp<AMessage> getLatestEnqueuedMeta(); sp<AMessage> getLatestDequeuedMeta(); + sp<AMessage> getMetaAfterLastDequeued(int64_t delayUs); + + void trimBuffersAfterTimeUs(size_t discontinuitySeq, int64_t timeUs); + sp<AMessage> trimBuffersBeforeTimeUs(size_t discontinuitySeq, int64_t timeUs); protected: virtual ~AnotherPacketSource(); @@ -78,6 +88,7 @@ private: bool mIsAudio; bool mIsVideo; + bool mEnabled; sp<MetaData> mFormat; int64_t mLastQueuedTimeUs; List<sp<ABuffer> > mBuffers; diff --git a/media/libstagefright/mpeg2ts/ESQueue.cpp b/media/libstagefright/mpeg2ts/ESQueue.cpp index a605595..88da275 100644 --- a/media/libstagefright/mpeg2ts/ESQueue.cpp +++ b/media/libstagefright/mpeg2ts/ESQueue.cpp @@ -617,8 +617,6 @@ sp<ABuffer> ElementaryStreamQueue::dequeueAccessUnitAAC() { // having to interpolate. // The final AAC frame may well extend into the next RangeInfo but // that's ok. - // TODO: the logic commented above is skipped because codec cannot take - // arbitrary sized input buffers; size_t offset = 0; while (offset < info.mLength) { if (offset + 7 > mBuffer->size()) { @@ -683,12 +681,9 @@ sp<ABuffer> ElementaryStreamQueue::dequeueAccessUnitAAC() { size_t headerSize __unused = protection_absent ? 7 : 9; offset += aac_frame_length; - // TODO: move back to concatenation when codec can support arbitrary input buffers. - // For now only queue a single buffer - break; } - int64_t timeUs = fetchTimestampAAC(offset); + int64_t timeUs = fetchTimestamp(offset); sp<ABuffer> accessUnit = new ABuffer(offset); memcpy(accessUnit->data(), mBuffer->data(), offset); @@ -735,50 +730,6 @@ int64_t ElementaryStreamQueue::fetchTimestamp(size_t size) { return timeUs; } -// TODO: avoid interpolating timestamps once codec supports arbitrary sized input buffers -int64_t ElementaryStreamQueue::fetchTimestampAAC(size_t size) { - int64_t timeUs = -1; - bool first = true; - - size_t samplesize = size; - while (size > 0) { - CHECK(!mRangeInfos.empty()); - - RangeInfo *info = &*mRangeInfos.begin(); - - if (first) { - timeUs = info->mTimestampUs; - first = false; - } - - if (info->mLength > size) { - int32_t sampleRate; - CHECK(mFormat->findInt32(kKeySampleRate, &sampleRate)); - info->mLength -= size; - size_t numSamples = 1024 * size / samplesize; - info->mTimestampUs += numSamples * 1000000ll / sampleRate; - size = 0; - } else { - size -= info->mLength; - - mRangeInfos.erase(mRangeInfos.begin()); - info = NULL; - } - - } - - if (timeUs == 0ll) { - ALOGV("Returning 0 timestamp"); - } - - return timeUs; -} - -struct NALPosition { - size_t nalOffset; - size_t nalSize; -}; - sp<ABuffer> ElementaryStreamQueue::dequeueAccessUnitH264() { const uint8_t *data = mBuffer->data(); @@ -786,6 +737,7 @@ sp<ABuffer> ElementaryStreamQueue::dequeueAccessUnitH264() { Vector<NALPosition> nals; size_t totalSize = 0; + size_t seiCount = 0; status_t err; const uint8_t *nalStart; @@ -815,6 +767,9 @@ sp<ABuffer> ElementaryStreamQueue::dequeueAccessUnitH264() { // next frame. flush = true; + } else if (nalType == 6 && nalSize > 0) { + // found non-zero sized SEI + ++seiCount; } if (flush) { @@ -823,21 +778,29 @@ sp<ABuffer> ElementaryStreamQueue::dequeueAccessUnitH264() { size_t auSize = 4 * nals.size() + totalSize; sp<ABuffer> accessUnit = new ABuffer(auSize); + sp<ABuffer> sei; + + if (seiCount > 0) { + sei = new ABuffer(seiCount * sizeof(NALPosition)); + accessUnit->meta()->setBuffer("sei", sei); + } #if !LOG_NDEBUG AString out; #endif size_t dstOffset = 0; + size_t seiIndex = 0; for (size_t i = 0; i < nals.size(); ++i) { const NALPosition &pos = nals.itemAt(i); unsigned nalType = mBuffer->data()[pos.nalOffset] & 0x1f; - if (nalType == 6) { - sp<ABuffer> sei = new ABuffer(pos.nalSize); - memcpy(sei->data(), mBuffer->data() + pos.nalOffset, pos.nalSize); - accessUnit->meta()->setBuffer("sei", sei); + if (nalType == 6 && pos.nalSize > 0) { + CHECK_LT(seiIndex, sei->size() / sizeof(NALPosition)); + NALPosition &seiPos = ((NALPosition *)sei->data())[seiIndex++]; + seiPos.nalOffset = dstOffset + 4; + seiPos.nalSize = pos.nalSize; } #if !LOG_NDEBUG diff --git a/media/libstagefright/mpeg2ts/ESQueue.h b/media/libstagefright/mpeg2ts/ESQueue.h index eb4b1c9..45b4624 100644 --- a/media/libstagefright/mpeg2ts/ESQueue.h +++ b/media/libstagefright/mpeg2ts/ESQueue.h @@ -77,7 +77,6 @@ private: // consume a logical (compressed) access unit of size "size", // returns its timestamp in us (or -1 if no time information). int64_t fetchTimestamp(size_t size); - int64_t fetchTimestampAAC(size_t size); DISALLOW_EVIL_CONSTRUCTORS(ElementaryStreamQueue); }; diff --git a/media/libstagefright/omx/Android.mk b/media/libstagefright/omx/Android.mk index aaa8334..07ea605 100644 --- a/media/libstagefright/omx/Android.mk +++ b/media/libstagefright/omx/Android.mk @@ -1,11 +1,8 @@ LOCAL_PATH:= $(call my-dir) include $(CLEAR_VARS) -ifeq ($(TARGET_DEVICE), manta) - LOCAL_CFLAGS += -DSURFACE_IS_BGR32 -endif - LOCAL_SRC_FILES:= \ + FrameDropper.cpp \ GraphicBufferSource.cpp \ OMX.cpp \ OMXMaster.cpp \ diff --git a/media/libstagefright/omx/FrameDropper.cpp b/media/libstagefright/omx/FrameDropper.cpp new file mode 100644 index 0000000..9fba0b7 --- /dev/null +++ b/media/libstagefright/omx/FrameDropper.cpp @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "FrameDropper" +#include <utils/Log.h> + +#include "FrameDropper.h" + +#include <media/stagefright/foundation/ADebug.h> + +namespace android { + +static const int64_t kMaxJitterUs = 2000; + +FrameDropper::FrameDropper() + : mDesiredMinTimeUs(-1), + mMinIntervalUs(0) { +} + +FrameDropper::~FrameDropper() { +} + +status_t FrameDropper::setMaxFrameRate(float maxFrameRate) { + if (maxFrameRate <= 0) { + ALOGE("framerate should be positive but got %f.", maxFrameRate); + return BAD_VALUE; + } + mMinIntervalUs = (int64_t) (1000000.0f / maxFrameRate); + return OK; +} + +bool FrameDropper::shouldDrop(int64_t timeUs) { + if (mMinIntervalUs <= 0) { + return false; + } + + if (mDesiredMinTimeUs < 0) { + mDesiredMinTimeUs = timeUs + mMinIntervalUs; + ALOGV("first frame %lld, next desired frame %lld", timeUs, mDesiredMinTimeUs); + return false; + } + + if (timeUs < (mDesiredMinTimeUs - kMaxJitterUs)) { + ALOGV("drop frame %lld, desired frame %lld, diff %lld", + timeUs, mDesiredMinTimeUs, mDesiredMinTimeUs - timeUs); + return true; + } + + int64_t n = (timeUs - mDesiredMinTimeUs + kMaxJitterUs) / mMinIntervalUs; + mDesiredMinTimeUs += (n + 1) * mMinIntervalUs; + ALOGV("keep frame %lld, next desired frame %lld, diff %lld", + timeUs, mDesiredMinTimeUs, mDesiredMinTimeUs - timeUs); + return false; +} + +} // namespace android diff --git a/media/libstagefright/omx/FrameDropper.h b/media/libstagefright/omx/FrameDropper.h new file mode 100644 index 0000000..c5a6d4b --- /dev/null +++ b/media/libstagefright/omx/FrameDropper.h @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FRAME_DROPPER_H_ + +#define FRAME_DROPPER_H_ + +#include <utils/Errors.h> +#include <utils/RefBase.h> + +#include <media/stagefright/foundation/ABase.h> + +namespace android { + +struct FrameDropper : public RefBase { + // No frames will be dropped until a valid max frame rate is set. + FrameDropper(); + + // maxFrameRate required to be positive. + status_t setMaxFrameRate(float maxFrameRate); + + // Returns false if max frame rate has not been set via setMaxFrameRate. + bool shouldDrop(int64_t timeUs); + +protected: + virtual ~FrameDropper(); + +private: + int64_t mDesiredMinTimeUs; + int64_t mMinIntervalUs; + + DISALLOW_EVIL_CONSTRUCTORS(FrameDropper); +}; + +} // namespace android + +#endif // FRAME_DROPPER_H_ diff --git a/media/libstagefright/omx/GraphicBufferSource.cpp b/media/libstagefright/omx/GraphicBufferSource.cpp index 44c7edc..477cfc6 100644 --- a/media/libstagefright/omx/GraphicBufferSource.cpp +++ b/media/libstagefright/omx/GraphicBufferSource.cpp @@ -28,8 +28,10 @@ #include <media/hardware/MetadataBufferType.h> #include <ui/GraphicBuffer.h> +#include <gui/BufferItem.h> #include <inttypes.h> +#include "FrameDropper.h" namespace android { @@ -53,9 +55,9 @@ GraphicBufferSource::GraphicBufferSource(OMXNodeInstance* nodeInstance, mRepeatAfterUs(-1ll), mRepeatLastFrameGeneration(0), mRepeatLastFrameTimestamp(-1ll), - mLatestSubmittedBufferId(-1), - mLatestSubmittedBufferFrameNum(0), - mLatestSubmittedBufferUseCount(0), + mLatestBufferId(-1), + mLatestBufferFrameNum(0), + mLatestBufferUseCount(0), mRepeatBufferDeferred(false), mTimePerCaptureUs(-1ll), mTimePerFrameUs(-1ll), @@ -152,9 +154,9 @@ void GraphicBufferSource::omxExecuting() { mLooper->registerHandler(mReflector); mLooper->start(); - if (mLatestSubmittedBufferId >= 0) { + if (mLatestBufferId >= 0) { sp<AMessage> msg = - new AMessage(kWhatRepeatLastFrame, mReflector->id()); + new AMessage(kWhatRepeatLastFrame, mReflector); msg->setInt32("generation", ++mRepeatLastFrameGeneration); msg->post(mRepeatAfterUs); @@ -287,8 +289,8 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) { ALOGV("cbi %d matches bq slot %d, handle=%p", cbi, id, mBufferSlot[id]->handle); - if (id == mLatestSubmittedBufferId) { - CHECK_GT(mLatestSubmittedBufferUseCount--, 0); + if (id == mLatestBufferId) { + CHECK_GT(mLatestBufferUseCount--, 0); } else { mConsumer->releaseBuffer(id, codecBuffer.mFrameNumber, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE); @@ -313,11 +315,11 @@ void GraphicBufferSource::codecBufferEmptied(OMX_BUFFERHEADERTYPE* header) { ALOGV("buffer freed, EOS pending"); submitEndOfInputStream_l(); } else if (mRepeatBufferDeferred) { - bool success = repeatLatestSubmittedBuffer_l(); + bool success = repeatLatestBuffer_l(); if (success) { - ALOGV("deferred repeatLatestSubmittedBuffer_l SUCCESS"); + ALOGV("deferred repeatLatestBuffer_l SUCCESS"); } else { - ALOGV("deferred repeatLatestSubmittedBuffer_l FAILURE"); + ALOGV("deferred repeatLatestBuffer_l FAILURE"); } mRepeatBufferDeferred = false; } @@ -359,7 +361,7 @@ void GraphicBufferSource::suspend(bool suspend) { mSuspended = true; while (mNumFramesAvailable > 0) { - BufferQueue::BufferItem item; + BufferItem item; status_t err = mConsumer->acquireBuffer(&item, 0); if (err == BufferQueue::NO_BUFFER_AVAILABLE) { @@ -382,12 +384,12 @@ void GraphicBufferSource::suspend(bool suspend) { mSuspended = false; if (mExecuting && mNumFramesAvailable == 0 && mRepeatBufferDeferred) { - if (repeatLatestSubmittedBuffer_l()) { - ALOGV("suspend/deferred repeatLatestSubmittedBuffer_l SUCCESS"); + if (repeatLatestBuffer_l()) { + ALOGV("suspend/deferred repeatLatestBuffer_l SUCCESS"); mRepeatBufferDeferred = false; } else { - ALOGV("suspend/deferred repeatLatestSubmittedBuffer_l FAILURE"); + ALOGV("suspend/deferred repeatLatestBuffer_l FAILURE"); } } } @@ -409,7 +411,7 @@ bool GraphicBufferSource::fillCodecBuffer_l() { ALOGV("fillCodecBuffer_l: acquiring buffer, avail=%zu", mNumFramesAvailable); - BufferQueue::BufferItem item; + BufferItem item; status_t err = mConsumer->acquireBuffer(&item, 0); if (err == BufferQueue::NO_BUFFER_AVAILABLE) { // shouldn't happen @@ -441,12 +443,22 @@ bool GraphicBufferSource::fillCodecBuffer_l() { // only submit sample if start time is unspecified, or sample // is queued after the specified start time + bool dropped = false; if (mSkipFramesBeforeNs < 0ll || item.mTimestamp >= mSkipFramesBeforeNs) { // if start time is set, offset time stamp by start time if (mSkipFramesBeforeNs > 0) { item.mTimestamp -= mSkipFramesBeforeNs; } - err = submitBuffer_l(item, cbi); + + int64_t timeUs = item.mTimestamp / 1000; + if (mFrameDropper != NULL && mFrameDropper->shouldDrop(timeUs)) { + ALOGV("skipping frame (%lld) to meet max framerate", static_cast<long long>(timeUs)); + // set err to OK so that the skipped frame can still be saved as the lastest frame + err = OK; + dropped = true; + } else { + err = submitBuffer_l(item, cbi); + } } if (err != OK) { @@ -455,46 +467,46 @@ bool GraphicBufferSource::fillCodecBuffer_l() { EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE); } else { ALOGV("buffer submitted (bq %d, cbi %d)", item.mBuf, cbi); - setLatestSubmittedBuffer_l(item); + setLatestBuffer_l(item, dropped); } return true; } -bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() { +bool GraphicBufferSource::repeatLatestBuffer_l() { CHECK(mExecuting && mNumFramesAvailable == 0); - if (mLatestSubmittedBufferId < 0 || mSuspended) { + if (mLatestBufferId < 0 || mSuspended) { return false; } - if (mBufferSlot[mLatestSubmittedBufferId] == NULL) { + if (mBufferSlot[mLatestBufferId] == NULL) { // This can happen if the remote side disconnects, causing // onBuffersReleased() to NULL out our copy of the slots. The // buffer is gone, so we have nothing to show. // // To be on the safe side we try to release the buffer. - ALOGD("repeatLatestSubmittedBuffer_l: slot was NULL"); + ALOGD("repeatLatestBuffer_l: slot was NULL"); mConsumer->releaseBuffer( - mLatestSubmittedBufferId, - mLatestSubmittedBufferFrameNum, + mLatestBufferId, + mLatestBufferFrameNum, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE); - mLatestSubmittedBufferId = -1; - mLatestSubmittedBufferFrameNum = 0; + mLatestBufferId = -1; + mLatestBufferFrameNum = 0; return false; } int cbi = findAvailableCodecBuffer_l(); if (cbi < 0) { // No buffers available, bail. - ALOGV("repeatLatestSubmittedBuffer_l: no codec buffers."); + ALOGV("repeatLatestBuffer_l: no codec buffers."); return false; } - BufferQueue::BufferItem item; - item.mBuf = mLatestSubmittedBufferId; - item.mFrameNumber = mLatestSubmittedBufferFrameNum; + BufferItem item; + item.mBuf = mLatestBufferId; + item.mFrameNumber = mLatestBufferFrameNum; item.mTimestamp = mRepeatLastFrameTimestamp; status_t err = submitBuffer_l(item, cbi); @@ -503,7 +515,7 @@ bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() { return false; } - ++mLatestSubmittedBufferUseCount; + ++mLatestBufferUseCount; /* repeat last frame up to kRepeatLastFrameCount times. * in case of static scene, a single repeat might not get rid of encoder @@ -513,7 +525,7 @@ bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() { mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000; if (mReflector != NULL) { - sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id()); + sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector); msg->setInt32("generation", ++mRepeatLastFrameGeneration); msg->post(mRepeatAfterUs); } @@ -522,31 +534,31 @@ bool GraphicBufferSource::repeatLatestSubmittedBuffer_l() { return true; } -void GraphicBufferSource::setLatestSubmittedBuffer_l( - const BufferQueue::BufferItem &item) { - ALOGV("setLatestSubmittedBuffer_l"); +void GraphicBufferSource::setLatestBuffer_l( + const BufferItem &item, bool dropped) { + ALOGV("setLatestBuffer_l"); - if (mLatestSubmittedBufferId >= 0) { - if (mLatestSubmittedBufferUseCount == 0) { + if (mLatestBufferId >= 0) { + if (mLatestBufferUseCount == 0) { mConsumer->releaseBuffer( - mLatestSubmittedBufferId, - mLatestSubmittedBufferFrameNum, + mLatestBufferId, + mLatestBufferFrameNum, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, Fence::NO_FENCE); } } - mLatestSubmittedBufferId = item.mBuf; - mLatestSubmittedBufferFrameNum = item.mFrameNumber; + mLatestBufferId = item.mBuf; + mLatestBufferFrameNum = item.mFrameNumber; mRepeatLastFrameTimestamp = item.mTimestamp + mRepeatAfterUs * 1000; - mLatestSubmittedBufferUseCount = 1; + mLatestBufferUseCount = dropped ? 0 : 1; mRepeatBufferDeferred = false; mRepeatLastFrameCount = kRepeatLastFrameCount; if (mReflector != NULL) { - sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector->id()); + sp<AMessage> msg = new AMessage(kWhatRepeatLastFrame, mReflector); msg->setInt32("generation", ++mRepeatLastFrameGeneration); msg->post(mRepeatAfterUs); } @@ -579,7 +591,7 @@ status_t GraphicBufferSource::signalEndOfInputStream() { return OK; } -int64_t GraphicBufferSource::getTimestamp(const BufferQueue::BufferItem &item) { +int64_t GraphicBufferSource::getTimestamp(const BufferItem &item) { int64_t timeUs = item.mTimestamp / 1000; if (mTimePerCaptureUs > 0ll) { @@ -640,7 +652,7 @@ int64_t GraphicBufferSource::getTimestamp(const BufferQueue::BufferItem &item) { } status_t GraphicBufferSource::submitBuffer_l( - const BufferQueue::BufferItem &item, int cbi) { + const BufferItem &item, int cbi) { ALOGV("submitBuffer_l cbi=%d", cbi); int64_t timeUs = getTimestamp(item); @@ -766,7 +778,7 @@ void GraphicBufferSource::onFrameAvailable(const BufferItem& /*item*/) { ALOGV("onFrameAvailable: suspended, ignoring frame"); } - BufferQueue::BufferItem item; + BufferItem item; status_t err = mConsumer->acquireBuffer(&item, 0); if (err == OK) { // If this is the first time we're seeing this buffer, add it to our @@ -841,6 +853,23 @@ status_t GraphicBufferSource::setMaxTimestampGapUs(int64_t maxGapUs) { return OK; } +status_t GraphicBufferSource::setMaxFps(float maxFps) { + Mutex::Autolock autoLock(mMutex); + + if (mExecuting) { + return INVALID_OPERATION; + } + + mFrameDropper = new FrameDropper(); + status_t err = mFrameDropper->setMaxFrameRate(maxFps); + if (err != OK) { + mFrameDropper.clear(); + return err; + } + + return OK; +} + void GraphicBufferSource::setSkipFramesBeforeUs(int64_t skipFramesBeforeUs) { Mutex::Autolock autoLock(mMutex); @@ -879,12 +908,12 @@ void GraphicBufferSource::onMessageReceived(const sp<AMessage> &msg) { break; } - bool success = repeatLatestSubmittedBuffer_l(); + bool success = repeatLatestBuffer_l(); if (success) { - ALOGV("repeatLatestSubmittedBuffer_l SUCCESS"); + ALOGV("repeatLatestBuffer_l SUCCESS"); } else { - ALOGV("repeatLatestSubmittedBuffer_l FAILURE"); + ALOGV("repeatLatestBuffer_l FAILURE"); mRepeatBufferDeferred = true; } break; diff --git a/media/libstagefright/omx/GraphicBufferSource.h b/media/libstagefright/omx/GraphicBufferSource.h index c8e3775..1067472 100644 --- a/media/libstagefright/omx/GraphicBufferSource.h +++ b/media/libstagefright/omx/GraphicBufferSource.h @@ -30,6 +30,8 @@ namespace android { +class FrameDropper; + /* * This class is used to feed OMX codecs from a Surface via BufferQueue. * @@ -119,6 +121,9 @@ public: // of suspension on input. status_t setMaxTimestampGapUs(int64_t maxGapUs); + // When set, the max frame rate fed to the encoder will be capped at maxFps. + status_t setMaxFps(float maxFps); + // Sets the time lapse (or slow motion) parameters. // data[0] is the time (us) between two frames for playback // data[1] is the time (us) between two frames for capture @@ -187,15 +192,15 @@ private: // Marks the mCodecBuffers entry as in-use, copies the GraphicBuffer // reference into the codec buffer, and submits the data to the codec. - status_t submitBuffer_l(const BufferQueue::BufferItem &item, int cbi); + status_t submitBuffer_l(const BufferItem &item, int cbi); // Submits an empty buffer, with the EOS flag set. Returns without // doing anything if we don't have a codec buffer available. void submitEndOfInputStream_l(); - void setLatestSubmittedBuffer_l(const BufferQueue::BufferItem &item); - bool repeatLatestSubmittedBuffer_l(); - int64_t getTimestamp(const BufferQueue::BufferItem &item); + void setLatestBuffer_l(const BufferItem &item, bool dropped); + bool repeatLatestBuffer_l(); + int64_t getTimestamp(const BufferItem &item); // Lock, covers all member variables. mutable Mutex mMutex; @@ -250,6 +255,8 @@ private: int64_t mPrevModifiedTimeUs; int64_t mSkipFramesBeforeNs; + sp<FrameDropper> mFrameDropper; + sp<ALooper> mLooper; sp<AHandlerReflector<GraphicBufferSource> > mReflector; @@ -258,11 +265,11 @@ private: int64_t mRepeatLastFrameTimestamp; int32_t mRepeatLastFrameCount; - int mLatestSubmittedBufferId; - uint64_t mLatestSubmittedBufferFrameNum; - int32_t mLatestSubmittedBufferUseCount; + int mLatestBufferId; + uint64_t mLatestBufferFrameNum; + int32_t mLatestBufferUseCount; - // The previously submitted buffer should've been repeated but + // The previous buffer should've been repeated but // no codec buffer was available at the time. bool mRepeatBufferDeferred; diff --git a/media/libstagefright/omx/OMXNodeInstance.cpp b/media/libstagefright/omx/OMXNodeInstance.cpp index ab7419f..4779d6a 100644 --- a/media/libstagefright/omx/OMXNodeInstance.cpp +++ b/media/libstagefright/omx/OMXNodeInstance.cpp @@ -1075,6 +1075,7 @@ inline static const char *asString(IOMX::InternalOptionType i, const char *def = case IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY: return "REPEAT_PREVIOUS_FRAME_DELAY"; case IOMX::INTERNAL_OPTION_MAX_TIMESTAMP_GAP: return "MAX_TIMESTAMP_GAP"; + case IOMX::INTERNAL_OPTION_MAX_FPS: return "MAX_FPS"; case IOMX::INTERNAL_OPTION_START_TIME: return "START_TIME"; case IOMX::INTERNAL_OPTION_TIME_LAPSE: return "TIME_LAPSE"; default: return def; @@ -1092,6 +1093,7 @@ status_t OMXNodeInstance::setInternalOption( case IOMX::INTERNAL_OPTION_SUSPEND: case IOMX::INTERNAL_OPTION_REPEAT_PREVIOUS_FRAME_DELAY: case IOMX::INTERNAL_OPTION_MAX_TIMESTAMP_GAP: + case IOMX::INTERNAL_OPTION_MAX_FPS: case IOMX::INTERNAL_OPTION_START_TIME: case IOMX::INTERNAL_OPTION_TIME_LAPSE: { @@ -1129,6 +1131,14 @@ status_t OMXNodeInstance::setInternalOption( int64_t maxGapUs = *(int64_t *)data; CLOG_CONFIG(setInternalOption, "gapUs=%lld", (long long)maxGapUs); return bufferSource->setMaxTimestampGapUs(maxGapUs); + } else if (type == IOMX::INTERNAL_OPTION_MAX_FPS) { + if (size != sizeof(float)) { + return INVALID_OPERATION; + } + + float maxFps = *(float *)data; + CLOG_CONFIG(setInternalOption, "maxFps=%f", maxFps); + return bufferSource->setMaxFps(maxFps); } else if (type == IOMX::INTERNAL_OPTION_START_TIME) { if (size != sizeof(int64_t)) { return INVALID_OPERATION; diff --git a/media/libstagefright/omx/SimpleSoftOMXComponent.cpp b/media/libstagefright/omx/SimpleSoftOMXComponent.cpp index 7f99dcd..801a1bd 100644 --- a/media/libstagefright/omx/SimpleSoftOMXComponent.cpp +++ b/media/libstagefright/omx/SimpleSoftOMXComponent.cpp @@ -58,7 +58,7 @@ OMX_ERRORTYPE SimpleSoftOMXComponent::sendCommand( OMX_COMMANDTYPE cmd, OMX_U32 param, OMX_PTR data) { CHECK(data == NULL); - sp<AMessage> msg = new AMessage(kWhatSendCommand, mHandler->id()); + sp<AMessage> msg = new AMessage(kWhatSendCommand, mHandler); msg->setInt32("cmd", cmd); msg->setInt32("param", param); msg->post(); @@ -307,7 +307,7 @@ OMX_ERRORTYPE SimpleSoftOMXComponent::freeBuffer( OMX_ERRORTYPE SimpleSoftOMXComponent::emptyThisBuffer( OMX_BUFFERHEADERTYPE *buffer) { - sp<AMessage> msg = new AMessage(kWhatEmptyThisBuffer, mHandler->id()); + sp<AMessage> msg = new AMessage(kWhatEmptyThisBuffer, mHandler); msg->setPointer("header", buffer); msg->post(); @@ -316,7 +316,7 @@ OMX_ERRORTYPE SimpleSoftOMXComponent::emptyThisBuffer( OMX_ERRORTYPE SimpleSoftOMXComponent::fillThisBuffer( OMX_BUFFERHEADERTYPE *buffer) { - sp<AMessage> msg = new AMessage(kWhatFillThisBuffer, mHandler->id()); + sp<AMessage> msg = new AMessage(kWhatFillThisBuffer, mHandler); msg->setPointer("header", buffer); msg->post(); diff --git a/media/libstagefright/omx/tests/Android.mk b/media/libstagefright/omx/tests/Android.mk index 447b29e..9be637a 100644 --- a/media/libstagefright/omx/tests/Android.mk +++ b/media/libstagefright/omx/tests/Android.mk @@ -20,3 +20,21 @@ LOCAL_MODULE_TAGS := tests LOCAL_32_BIT_ONLY := true include $(BUILD_EXECUTABLE) + +include $(CLEAR_VARS) + +LOCAL_MODULE := FrameDropper_test + +LOCAL_MODULE_TAGS := tests + +LOCAL_SRC_FILES := \ + FrameDropper_test.cpp \ + +LOCAL_SHARED_LIBRARIES := \ + libstagefright_omx \ + libutils \ + +LOCAL_C_INCLUDES := \ + frameworks/av/media/libstagefright/omx \ + +include $(BUILD_NATIVE_TEST) diff --git a/media/libstagefright/omx/tests/FrameDropper_test.cpp b/media/libstagefright/omx/tests/FrameDropper_test.cpp new file mode 100644 index 0000000..4ac72c4 --- /dev/null +++ b/media/libstagefright/omx/tests/FrameDropper_test.cpp @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "FrameDropper_test" +#include <utils/Log.h> + +#include <gtest/gtest.h> + +#include "FrameDropper.h" +#include <media/stagefright/foundation/ADebug.h> + +namespace android { + +struct TestFrame { + int64_t timeUs; + bool shouldDrop; +}; + +static const TestFrame testFrames20Fps[] = { + {1000000, false}, {1050000, false}, {1100000, false}, {1150000, false}, + {1200000, false}, {1250000, false}, {1300000, false}, {1350000, false}, + {1400000, false}, {1450000, false}, {1500000, false}, {1550000, false}, + {1600000, false}, {1650000, false}, {1700000, false}, {1750000, false}, + {1800000, false}, {1850000, false}, {1900000, false}, {1950000, false}, +}; + +static const TestFrame testFrames30Fps[] = { + {1000000, false}, {1033333, false}, {1066667, false}, {1100000, false}, + {1133333, false}, {1166667, false}, {1200000, false}, {1233333, false}, + {1266667, false}, {1300000, false}, {1333333, false}, {1366667, false}, + {1400000, false}, {1433333, false}, {1466667, false}, {1500000, false}, + {1533333, false}, {1566667, false}, {1600000, false}, {1633333, false}, +}; + +static const TestFrame testFrames40Fps[] = { + {1000000, false}, {1025000, true}, {1050000, false}, {1075000, false}, + {1100000, false}, {1125000, true}, {1150000, false}, {1175000, false}, + {1200000, false}, {1225000, true}, {1250000, false}, {1275000, false}, + {1300000, false}, {1325000, true}, {1350000, false}, {1375000, false}, + {1400000, false}, {1425000, true}, {1450000, false}, {1475000, false}, +}; + +static const TestFrame testFrames60Fps[] = { + {1000000, false}, {1016667, true}, {1033333, false}, {1050000, true}, + {1066667, false}, {1083333, true}, {1100000, false}, {1116667, true}, + {1133333, false}, {1150000, true}, {1166667, false}, {1183333, true}, + {1200000, false}, {1216667, true}, {1233333, false}, {1250000, true}, + {1266667, false}, {1283333, true}, {1300000, false}, {1316667, true}, +}; + +static const TestFrame testFramesVariableFps[] = { + // 40fps + {1000000, false}, {1025000, true}, {1050000, false}, {1075000, false}, + {1100000, false}, {1125000, true}, {1150000, false}, {1175000, false}, + {1200000, false}, {1225000, true}, {1250000, false}, {1275000, false}, + {1300000, false}, {1325000, true}, {1350000, false}, {1375000, false}, + {1400000, false}, {1425000, true}, {1450000, false}, {1475000, false}, + // a timestamp jump plus switch to 20fps + {2000000, false}, {2050000, false}, {2100000, false}, {2150000, false}, + {2200000, false}, {2250000, false}, {2300000, false}, {2350000, false}, + {2400000, false}, {2450000, false}, {2500000, false}, {2550000, false}, + {2600000, false}, {2650000, false}, {2700000, false}, {2750000, false}, + {2800000, false}, {2850000, false}, {2900000, false}, {2950000, false}, + // 60fps + {2966667, false}, {2983333, true}, {3000000, false}, {3016667, true}, + {3033333, false}, {3050000, true}, {3066667, false}, {3083333, true}, + {3100000, false}, {3116667, true}, {3133333, false}, {3150000, true}, + {3166667, false}, {3183333, true}, {3200000, false}, {3216667, true}, + {3233333, false}, {3250000, true}, {3266667, false}, {3283333, true}, +}; + +static const int kMaxTestJitterUs = 2000; +// return one of 1000, 0, -1000 as jitter. +static int GetJitter(size_t i) { + return (1 - (i % 3)) * (kMaxTestJitterUs / 2); +} + +class FrameDropperTest : public ::testing::Test { +public: + FrameDropperTest() : mFrameDropper(new FrameDropper()) { + EXPECT_EQ(OK, mFrameDropper->setMaxFrameRate(30.0)); + } + +protected: + void RunTest(const TestFrame* frames, size_t size) { + for (size_t i = 0; i < size; ++i) { + int jitter = GetJitter(i); + int64_t testTimeUs = frames[i].timeUs + jitter; + printf("time %lld, testTime %lld, jitter %d\n", frames[i].timeUs, testTimeUs, jitter); + EXPECT_EQ(frames[i].shouldDrop, mFrameDropper->shouldDrop(testTimeUs)); + } + } + + sp<FrameDropper> mFrameDropper; +}; + +TEST_F(FrameDropperTest, TestInvalidMaxFrameRate) { + EXPECT_NE(OK, mFrameDropper->setMaxFrameRate(-1.0)); + EXPECT_NE(OK, mFrameDropper->setMaxFrameRate(0)); +} + +TEST_F(FrameDropperTest, Test20Fps) { + RunTest(testFrames20Fps, ARRAY_SIZE(testFrames20Fps)); +} + +TEST_F(FrameDropperTest, Test30Fps) { + RunTest(testFrames30Fps, ARRAY_SIZE(testFrames30Fps)); +} + +TEST_F(FrameDropperTest, Test40Fps) { + RunTest(testFrames40Fps, ARRAY_SIZE(testFrames40Fps)); +} + +TEST_F(FrameDropperTest, Test60Fps) { + RunTest(testFrames60Fps, ARRAY_SIZE(testFrames60Fps)); +} + +TEST_F(FrameDropperTest, TestVariableFps) { + RunTest(testFramesVariableFps, ARRAY_SIZE(testFramesVariableFps)); +} + +} // namespace android diff --git a/media/libstagefright/rtsp/ARTPConnection.cpp b/media/libstagefright/rtsp/ARTPConnection.cpp index a6bd824..a86ab74 100644 --- a/media/libstagefright/rtsp/ARTPConnection.cpp +++ b/media/libstagefright/rtsp/ARTPConnection.cpp @@ -82,7 +82,7 @@ void ARTPConnection::addStream( size_t index, const sp<AMessage> ¬ify, bool injected) { - sp<AMessage> msg = new AMessage(kWhatAddStream, id()); + sp<AMessage> msg = new AMessage(kWhatAddStream, this); msg->setInt32("rtp-socket", rtpSocket); msg->setInt32("rtcp-socket", rtcpSocket); msg->setObject("session-desc", sessionDesc); @@ -93,7 +93,7 @@ void ARTPConnection::addStream( } void ARTPConnection::removeStream(int rtpSocket, int rtcpSocket) { - sp<AMessage> msg = new AMessage(kWhatRemoveStream, id()); + sp<AMessage> msg = new AMessage(kWhatRemoveStream, this); msg->setInt32("rtp-socket", rtpSocket); msg->setInt32("rtcp-socket", rtcpSocket); msg->post(); @@ -233,7 +233,7 @@ void ARTPConnection::postPollEvent() { return; } - sp<AMessage> msg = new AMessage(kWhatPollStreams, id()); + sp<AMessage> msg = new AMessage(kWhatPollStreams, this); msg->post(); mPollEventPending = true; @@ -639,7 +639,7 @@ sp<ARTPSource> ARTPConnection::findSource(StreamInfo *info, uint32_t srcId) { } void ARTPConnection::injectPacket(int index, const sp<ABuffer> &buffer) { - sp<AMessage> msg = new AMessage(kWhatInjectPacket, id()); + sp<AMessage> msg = new AMessage(kWhatInjectPacket, this); msg->setInt32("index", index); msg->setBuffer("buffer", buffer); msg->post(); diff --git a/media/libstagefright/rtsp/ARTPSession.cpp b/media/libstagefright/rtsp/ARTPSession.cpp index ba4e33c..e5acb06 100644 --- a/media/libstagefright/rtsp/ARTPSession.cpp +++ b/media/libstagefright/rtsp/ARTPSession.cpp @@ -82,7 +82,7 @@ status_t ARTPSession::setup(const sp<ASessionDescription> &desc) { info->mRTPSocket = rtpSocket; info->mRTCPSocket = rtcpSocket; - sp<AMessage> notify = new AMessage(kWhatAccessUnitComplete, id()); + sp<AMessage> notify = new AMessage(kWhatAccessUnitComplete, this); notify->setSize("track-index", mTracks.size() - 1); mRTPConn->addStream( diff --git a/media/libstagefright/rtsp/ARTPWriter.cpp b/media/libstagefright/rtsp/ARTPWriter.cpp index e1607bf..56c4aa6 100644 --- a/media/libstagefright/rtsp/ARTPWriter.cpp +++ b/media/libstagefright/rtsp/ARTPWriter.cpp @@ -146,7 +146,7 @@ status_t ARTPWriter::start(MetaData * /* params */) { TRESPASS(); } - (new AMessage(kWhatStart, mReflector->id()))->post(); + (new AMessage(kWhatStart, mReflector))->post(); while (!(mFlags & kFlagStarted)) { mCondition.wait(mLock); @@ -161,7 +161,7 @@ status_t ARTPWriter::stop() { return OK; } - (new AMessage(kWhatStop, mReflector->id()))->post(); + (new AMessage(kWhatStop, mReflector))->post(); while (mFlags & kFlagStarted) { mCondition.wait(mLock); @@ -213,8 +213,8 @@ void ARTPWriter::onMessageReceived(const sp<AMessage> &msg) { mCondition.signal(); } - (new AMessage(kWhatRead, mReflector->id()))->post(); - (new AMessage(kWhatSendSR, mReflector->id()))->post(); + (new AMessage(kWhatRead, mReflector))->post(); + (new AMessage(kWhatSendSR, mReflector))->post(); break; } diff --git a/media/libstagefright/rtsp/ARTSPConnection.cpp b/media/libstagefright/rtsp/ARTSPConnection.cpp index 60b3aaf..855ffdc 100644 --- a/media/libstagefright/rtsp/ARTSPConnection.cpp +++ b/media/libstagefright/rtsp/ARTSPConnection.cpp @@ -68,28 +68,28 @@ ARTSPConnection::~ARTSPConnection() { } void ARTSPConnection::connect(const char *url, const sp<AMessage> &reply) { - sp<AMessage> msg = new AMessage(kWhatConnect, id()); + sp<AMessage> msg = new AMessage(kWhatConnect, this); msg->setString("url", url); msg->setMessage("reply", reply); msg->post(); } void ARTSPConnection::disconnect(const sp<AMessage> &reply) { - sp<AMessage> msg = new AMessage(kWhatDisconnect, id()); + sp<AMessage> msg = new AMessage(kWhatDisconnect, this); msg->setMessage("reply", reply); msg->post(); } void ARTSPConnection::sendRequest( const char *request, const sp<AMessage> &reply) { - sp<AMessage> msg = new AMessage(kWhatSendRequest, id()); + sp<AMessage> msg = new AMessage(kWhatSendRequest, this); msg->setString("request", request); msg->setMessage("reply", reply); msg->post(); } void ARTSPConnection::observeBinaryData(const sp<AMessage> &reply) { - sp<AMessage> msg = new AMessage(kWhatObserveBinaryData, id()); + sp<AMessage> msg = new AMessage(kWhatObserveBinaryData, this); msg->setMessage("reply", reply); msg->post(); } @@ -286,7 +286,7 @@ void ARTSPConnection::onConnect(const sp<AMessage> &msg) { if (err < 0) { if (errno == EINPROGRESS) { - sp<AMessage> msg = new AMessage(kWhatCompleteConnection, id()); + sp<AMessage> msg = new AMessage(kWhatCompleteConnection, this); msg->setMessage("reply", reply); msg->setInt32("connection-id", mConnectionID); msg->post(); @@ -523,7 +523,7 @@ void ARTSPConnection::postReceiveReponseEvent() { return; } - sp<AMessage> msg = new AMessage(kWhatReceiveResponse, id()); + sp<AMessage> msg = new AMessage(kWhatReceiveResponse, this); msg->post(); mReceiveResponseEventPending = true; @@ -746,7 +746,7 @@ bool ARTSPConnection::receiveRTSPReponse() { AString request; CHECK(reply->findString("original-request", &request)); - sp<AMessage> msg = new AMessage(kWhatSendRequest, id()); + sp<AMessage> msg = new AMessage(kWhatSendRequest, this); msg->setMessage("reply", reply); msg->setString("request", request.c_str(), request.size()); diff --git a/media/libstagefright/rtsp/MyHandler.h b/media/libstagefright/rtsp/MyHandler.h index 3bf489b..0642343 100644 --- a/media/libstagefright/rtsp/MyHandler.h +++ b/media/libstagefright/rtsp/MyHandler.h @@ -169,10 +169,10 @@ struct MyHandler : public AHandler { looper()->registerHandler(mConn); (1 ? mNetLooper : looper())->registerHandler(mRTPConn); - sp<AMessage> notify = new AMessage('biny', id()); + sp<AMessage> notify = new AMessage('biny', this); mConn->observeBinaryData(notify); - sp<AMessage> reply = new AMessage('conn', id()); + sp<AMessage> reply = new AMessage('conn', this); mConn->connect(mOriginalSessionURL.c_str(), reply); } @@ -180,10 +180,10 @@ struct MyHandler : public AHandler { looper()->registerHandler(mConn); (1 ? mNetLooper : looper())->registerHandler(mRTPConn); - sp<AMessage> notify = new AMessage('biny', id()); + sp<AMessage> notify = new AMessage('biny', this); mConn->observeBinaryData(notify); - sp<AMessage> reply = new AMessage('sdpl', id()); + sp<AMessage> reply = new AMessage('sdpl', this); reply->setObject("description", desc); mConn->connect(mOriginalSessionURL.c_str(), reply); } @@ -210,11 +210,11 @@ struct MyHandler : public AHandler { } void disconnect() { - (new AMessage('abor', id()))->post(); + (new AMessage('abor', this))->post(); } void seek(int64_t timeUs) { - sp<AMessage> msg = new AMessage('seek', id()); + sp<AMessage> msg = new AMessage('seek', this); msg->setInt64("time", timeUs); mPauseGeneration++; msg->post(); @@ -225,14 +225,14 @@ struct MyHandler : public AHandler { } void pause() { - sp<AMessage> msg = new AMessage('paus', id()); + sp<AMessage> msg = new AMessage('paus', this); mPauseGeneration++; msg->setInt32("pausecheck", mPauseGeneration); msg->post(kPauseDelayUs); } void resume() { - sp<AMessage> msg = new AMessage('resu', id()); + sp<AMessage> msg = new AMessage('resu', this); mPauseGeneration++; msg->post(); } @@ -454,10 +454,10 @@ struct MyHandler : public AHandler { request.append("Accept: application/sdp\r\n"); request.append("\r\n"); - sp<AMessage> reply = new AMessage('desc', id()); + sp<AMessage> reply = new AMessage('desc', this); mConn->sendRequest(request.c_str(), reply); } else { - (new AMessage('disc', id()))->post(); + (new AMessage('disc', this))->post(); } break; } @@ -468,10 +468,10 @@ struct MyHandler : public AHandler { int32_t reconnect; if (msg->findInt32("reconnect", &reconnect) && reconnect) { - sp<AMessage> reply = new AMessage('conn', id()); + sp<AMessage> reply = new AMessage('conn', this); mConn->connect(mOriginalSessionURL.c_str(), reply); } else { - (new AMessage('quit', id()))->post(); + (new AMessage('quit', this))->post(); } break; } @@ -514,7 +514,7 @@ struct MyHandler : public AHandler { ALOGI("rewritten session url: '%s'", mSessionURL.c_str()); } - sp<AMessage> reply = new AMessage('conn', id()); + sp<AMessage> reply = new AMessage('conn', this); mConn->connect(mOriginalSessionURL.c_str(), reply); break; } @@ -586,7 +586,7 @@ struct MyHandler : public AHandler { } if (result != OK) { - sp<AMessage> reply = new AMessage('disc', id()); + sp<AMessage> reply = new AMessage('disc', this); mConn->disconnect(reply); } break; @@ -631,7 +631,7 @@ struct MyHandler : public AHandler { } if (result != OK) { - sp<AMessage> reply = new AMessage('disc', id()); + sp<AMessage> reply = new AMessage('disc', this); mConn->disconnect(reply); } break; @@ -703,7 +703,7 @@ struct MyHandler : public AHandler { mSessionID.erase(i, mSessionID.size() - i); } - sp<AMessage> notify = new AMessage('accu', id()); + sp<AMessage> notify = new AMessage('accu', this); notify->setSize("track-index", trackIndex); i = response->mHeaders.indexOfKey("transport"); @@ -769,10 +769,10 @@ struct MyHandler : public AHandler { request.append("\r\n"); - sp<AMessage> reply = new AMessage('play', id()); + sp<AMessage> reply = new AMessage('play', this); mConn->sendRequest(request.c_str(), reply); } else { - sp<AMessage> reply = new AMessage('disc', id()); + sp<AMessage> reply = new AMessage('disc', this); mConn->disconnect(reply); } break; @@ -797,7 +797,7 @@ struct MyHandler : public AHandler { } else { parsePlayResponse(response); - sp<AMessage> timeout = new AMessage('tiou', id()); + sp<AMessage> timeout = new AMessage('tiou', this); mCheckTimeoutGeneration++; timeout->setInt32("tioucheck", mCheckTimeoutGeneration); timeout->post(kStartupTimeoutUs); @@ -805,7 +805,7 @@ struct MyHandler : public AHandler { } if (result != OK) { - sp<AMessage> reply = new AMessage('disc', id()); + sp<AMessage> reply = new AMessage('disc', this); mConn->disconnect(reply); } @@ -831,7 +831,7 @@ struct MyHandler : public AHandler { request.append("\r\n"); request.append("\r\n"); - sp<AMessage> reply = new AMessage('opts', id()); + sp<AMessage> reply = new AMessage('opts', this); reply->setInt32("generation", mKeepAliveGeneration); mConn->sendRequest(request.c_str(), reply); break; @@ -894,7 +894,7 @@ struct MyHandler : public AHandler { mPausing = false; mSeekable = true; - sp<AMessage> reply = new AMessage('tear', id()); + sp<AMessage> reply = new AMessage('tear', this); int32_t reconnect; if (msg->findInt32("reconnect", &reconnect) && reconnect) { @@ -926,7 +926,7 @@ struct MyHandler : public AHandler { ALOGI("TEARDOWN completed with result %d (%s)", result, strerror(-result)); - sp<AMessage> reply = new AMessage('disc', id()); + sp<AMessage> reply = new AMessage('disc', this); int32_t reconnect; if (msg->findInt32("reconnect", &reconnect) && reconnect) { @@ -958,7 +958,7 @@ struct MyHandler : public AHandler { if (mNumAccessUnitsReceived == 0) { #if 1 ALOGI("stream ended? aborting."); - (new AMessage('abor', id()))->post(); + (new AMessage('abor', this))->post(); break; #else ALOGI("haven't seen an AU in a looong time."); @@ -1077,7 +1077,7 @@ struct MyHandler : public AHandler { request.append("\r\n"); - sp<AMessage> reply = new AMessage('pau2', id()); + sp<AMessage> reply = new AMessage('pau2', this); mConn->sendRequest(request.c_str(), reply); break; } @@ -1114,7 +1114,7 @@ struct MyHandler : public AHandler { request.append("\r\n"); - sp<AMessage> reply = new AMessage('res2', id()); + sp<AMessage> reply = new AMessage('res2', this); mConn->sendRequest(request.c_str(), reply); break; } @@ -1143,7 +1143,7 @@ struct MyHandler : public AHandler { // Post new timeout in order to make sure to use // fake timestamps if no new Sender Reports arrive - sp<AMessage> timeout = new AMessage('tiou', id()); + sp<AMessage> timeout = new AMessage('tiou', this); mCheckTimeoutGeneration++; timeout->setInt32("tioucheck", mCheckTimeoutGeneration); timeout->post(kStartupTimeoutUs); @@ -1152,7 +1152,7 @@ struct MyHandler : public AHandler { if (result != OK) { ALOGE("resume failed, aborting."); - (new AMessage('abor', id()))->post(); + (new AMessage('abor', this))->post(); } mPausing = false; @@ -1180,7 +1180,7 @@ struct MyHandler : public AHandler { mCheckPending = true; ++mCheckGeneration; - sp<AMessage> reply = new AMessage('see1', id()); + sp<AMessage> reply = new AMessage('see1', this); reply->setInt64("time", timeUs); if (mPausing) { @@ -1221,7 +1221,7 @@ struct MyHandler : public AHandler { // Start new timeoutgeneration to avoid getting timeout // before PLAY response arrive - sp<AMessage> timeout = new AMessage('tiou', id()); + sp<AMessage> timeout = new AMessage('tiou', this); mCheckTimeoutGeneration++; timeout->setInt32("tioucheck", mCheckTimeoutGeneration); timeout->post(kStartupTimeoutUs); @@ -1243,7 +1243,7 @@ struct MyHandler : public AHandler { request.append("\r\n"); - sp<AMessage> reply = new AMessage('see2', id()); + sp<AMessage> reply = new AMessage('see2', this); mConn->sendRequest(request.c_str(), reply); break; } @@ -1277,7 +1277,7 @@ struct MyHandler : public AHandler { // Post new timeout in order to make sure to use // fake timestamps if no new Sender Reports arrive - sp<AMessage> timeout = new AMessage('tiou', id()); + sp<AMessage> timeout = new AMessage('tiou', this); mCheckTimeoutGeneration++; timeout->setInt32("tioucheck", mCheckTimeoutGeneration); timeout->post(kStartupTimeoutUs); @@ -1293,7 +1293,7 @@ struct MyHandler : public AHandler { if (result != OK) { ALOGE("seek failed, aborting."); - (new AMessage('abor', id()))->post(); + (new AMessage('abor', this))->post(); } mPausing = false; @@ -1343,12 +1343,12 @@ struct MyHandler : public AHandler { mTryTCPInterleaving = true; - sp<AMessage> msg = new AMessage('abor', id()); + sp<AMessage> msg = new AMessage('abor', this); msg->setInt32("reconnect", true); msg->post(); } else { ALOGW("Never received any data, disconnecting."); - (new AMessage('abor', id()))->post(); + (new AMessage('abor', this))->post(); } } else { if (!mAllTracksHaveTime) { @@ -1369,7 +1369,7 @@ struct MyHandler : public AHandler { } void postKeepAlive() { - sp<AMessage> msg = new AMessage('aliv', id()); + sp<AMessage> msg = new AMessage('aliv', this); msg->setInt32("generation", mKeepAliveGeneration); msg->post((mKeepAliveTimeoutUs * 9) / 10); } @@ -1380,7 +1380,7 @@ struct MyHandler : public AHandler { } mCheckPending = true; - sp<AMessage> check = new AMessage('chek', id()); + sp<AMessage> check = new AMessage('chek', this); check->setInt32("generation", mCheckGeneration); check->post(kAccessUnitTimeoutUs); } @@ -1566,7 +1566,7 @@ private: if (source->initCheck() != OK) { ALOGW("Unsupported format. Ignoring track #%d.", index); - sp<AMessage> reply = new AMessage('setu', id()); + sp<AMessage> reply = new AMessage('setu', this); reply->setSize("index", index); reply->setInt32("result", ERROR_UNSUPPORTED); reply->post(); @@ -1652,7 +1652,7 @@ private: request.append("\r\n"); - sp<AMessage> reply = new AMessage('setu', id()); + sp<AMessage> reply = new AMessage('setu', this); reply->setSize("index", index); reply->setSize("track-index", mTracks.size() - 1); mConn->sendRequest(request.c_str(), reply); diff --git a/media/libstagefright/rtsp/MyTransmitter.h b/media/libstagefright/rtsp/MyTransmitter.h index 009a3b1..369f276 100644 --- a/media/libstagefright/rtsp/MyTransmitter.h +++ b/media/libstagefright/rtsp/MyTransmitter.h @@ -100,7 +100,7 @@ struct MyTransmitter : public AHandler { mLooper->registerHandler(this); mLooper->registerHandler(mConn); - sp<AMessage> reply = new AMessage('conn', id()); + sp<AMessage> reply = new AMessage('conn', this); mConn->connect(mServerURL.c_str(), reply); #ifdef ANDROID @@ -229,7 +229,7 @@ struct MyTransmitter : public AHandler { request.append("\r\n"); request.append(sdp); - sp<AMessage> reply = new AMessage('anno', id()); + sp<AMessage> reply = new AMessage('anno', this); mConn->sendRequest(request.c_str(), reply); } @@ -350,7 +350,7 @@ struct MyTransmitter : public AHandler { << result << " (" << strerror(-result) << ")"; if (result != OK) { - (new AMessage('quit', id()))->post(); + (new AMessage('quit', this))->post(); break; } @@ -381,7 +381,7 @@ struct MyTransmitter : public AHandler { if (response->mStatusCode == 401) { if (mAuthType != NONE) { LOG(INFO) << "FAILED to authenticate"; - (new AMessage('quit', id()))->post(); + (new AMessage('quit', this))->post(); break; } @@ -391,14 +391,14 @@ struct MyTransmitter : public AHandler { } if (result != OK || response->mStatusCode != 200) { - (new AMessage('quit', id()))->post(); + (new AMessage('quit', this))->post(); break; } unsigned rtpPort; ARTPConnection::MakePortPair(&mRTPSocket, &mRTCPSocket, &rtpPort); - // (new AMessage('poll', id()))->post(); + // (new AMessage('poll', this))->post(); AString request; request.append("SETUP "); @@ -414,7 +414,7 @@ struct MyTransmitter : public AHandler { request.append(";mode=record\r\n"); request.append("\r\n"); - sp<AMessage> reply = new AMessage('setu', id()); + sp<AMessage> reply = new AMessage('setu', this); mConn->sendRequest(request.c_str(), reply); break; } @@ -468,7 +468,7 @@ struct MyTransmitter : public AHandler { } if (result != OK || response->mStatusCode != 200) { - (new AMessage('quit', id()))->post(); + (new AMessage('quit', this))->post(); break; } @@ -535,7 +535,7 @@ struct MyTransmitter : public AHandler { request.append("\r\n"); request.append("\r\n"); - sp<AMessage> reply = new AMessage('reco', id()); + sp<AMessage> reply = new AMessage('reco', this); mConn->sendRequest(request.c_str(), reply); break; } @@ -558,13 +558,13 @@ struct MyTransmitter : public AHandler { } if (result != OK) { - (new AMessage('quit', id()))->post(); + (new AMessage('quit', this))->post(); break; } - (new AMessage('more', id()))->post(); - (new AMessage('sr ', id()))->post(); - (new AMessage('aliv', id()))->post(30000000ll); + (new AMessage('more', this))->post(); + (new AMessage('sr ', this))->post(); + (new AMessage('aliv', this))->post(30000000ll); break; } @@ -586,7 +586,7 @@ struct MyTransmitter : public AHandler { request.append("\r\n"); request.append("\r\n"); - sp<AMessage> reply = new AMessage('opts', id()); + sp<AMessage> reply = new AMessage('opts', this); mConn->sendRequest(request.c_str(), reply); break; } @@ -603,7 +603,7 @@ struct MyTransmitter : public AHandler { break; } - (new AMessage('aliv', id()))->post(30000000ll); + (new AMessage('aliv', this))->post(30000000ll); break; } @@ -702,7 +702,7 @@ struct MyTransmitter : public AHandler { request.append("\r\n"); request.append("\r\n"); - sp<AMessage> reply = new AMessage('paus', id()); + sp<AMessage> reply = new AMessage('paus', this); mConn->sendRequest(request.c_str(), reply); } break; @@ -753,7 +753,7 @@ struct MyTransmitter : public AHandler { request.append("\r\n"); request.append("\r\n"); - sp<AMessage> reply = new AMessage('tear', id()); + sp<AMessage> reply = new AMessage('tear', this); mConn->sendRequest(request.c_str(), reply); break; } @@ -775,7 +775,7 @@ struct MyTransmitter : public AHandler { CHECK(response != NULL); } - (new AMessage('quit', id()))->post(); + (new AMessage('quit', this))->post(); break; } @@ -784,14 +784,14 @@ struct MyTransmitter : public AHandler { LOG(INFO) << "disconnect completed"; mConnected = false; - (new AMessage('quit', id()))->post(); + (new AMessage('quit', this))->post(); break; } case 'quit': { if (mConnected) { - mConn->disconnect(new AMessage('disc', id())); + mConn->disconnect(new AMessage('disc', this)); break; } diff --git a/media/libstagefright/rtsp/SDPLoader.cpp b/media/libstagefright/rtsp/SDPLoader.cpp index a24eb69..0f46c83 100644 --- a/media/libstagefright/rtsp/SDPLoader.cpp +++ b/media/libstagefright/rtsp/SDPLoader.cpp @@ -51,7 +51,7 @@ SDPLoader::SDPLoader( void SDPLoader::load(const char *url, const KeyedVector<String8, String8> *headers) { mNetLooper->registerHandler(this); - sp<AMessage> msg = new AMessage(kWhatLoad, id()); + sp<AMessage> msg = new AMessage(kWhatLoad, this); msg->setString("url", url); if (headers != NULL) { diff --git a/media/libstagefright/rtsp/UDPPusher.cpp b/media/libstagefright/rtsp/UDPPusher.cpp index 47ea6f1..5c685a1 100644 --- a/media/libstagefright/rtsp/UDPPusher.cpp +++ b/media/libstagefright/rtsp/UDPPusher.cpp @@ -65,7 +65,7 @@ void UDPPusher::start() { mFirstTimeMs = fromlel(timeMs); mFirstTimeUs = ALooper::GetNowUs(); - (new AMessage(kWhatPush, id()))->post(); + (new AMessage(kWhatPush, this))->post(); } bool UDPPusher::onPush() { @@ -103,7 +103,7 @@ bool UDPPusher::onPush() { timeMs -= mFirstTimeMs; int64_t whenUs = mFirstTimeUs + timeMs * 1000ll; int64_t nowUs = ALooper::GetNowUs(); - (new AMessage(kWhatPush, id()))->post(whenUs - nowUs); + (new AMessage(kWhatPush, this))->post(whenUs - nowUs); return true; } diff --git a/media/libstagefright/timedtext/TimedTextPlayer.cpp b/media/libstagefright/timedtext/TimedTextPlayer.cpp index a070487..aecf666 100644 --- a/media/libstagefright/timedtext/TimedTextPlayer.cpp +++ b/media/libstagefright/timedtext/TimedTextPlayer.cpp @@ -56,25 +56,25 @@ TimedTextPlayer::~TimedTextPlayer() { } void TimedTextPlayer::start() { - (new AMessage(kWhatStart, id()))->post(); + (new AMessage(kWhatStart, this))->post(); } void TimedTextPlayer::pause() { - (new AMessage(kWhatPause, id()))->post(); + (new AMessage(kWhatPause, this))->post(); } void TimedTextPlayer::resume() { - (new AMessage(kWhatResume, id()))->post(); + (new AMessage(kWhatResume, this))->post(); } void TimedTextPlayer::seekToAsync(int64_t timeUs) { - sp<AMessage> msg = new AMessage(kWhatSeek, id()); + sp<AMessage> msg = new AMessage(kWhatSeek, this); msg->setInt64("seekTimeUs", timeUs); msg->post(); } void TimedTextPlayer::setDataSource(sp<TimedTextSource> source) { - sp<AMessage> msg = new AMessage(kWhatSetSource, id()); + sp<AMessage> msg = new AMessage(kWhatSetSource, this); msg->setObject("source", source); msg->post(); } @@ -231,7 +231,7 @@ void TimedTextPlayer::doRead(MediaSource::ReadOptions* options) { status_t err = mSource->read(&startTimeUs, &endTimeUs, &(parcelEvent->parcel), options); if (err == WOULD_BLOCK) { - sp<AMessage> msg = new AMessage(kWhatRetryRead, id()); + sp<AMessage> msg = new AMessage(kWhatRetryRead, this); if (options != NULL) { int64_t seekTimeUs = kInvalidTimeUs; MediaSource::ReadOptions::SeekMode seekMode = @@ -259,7 +259,7 @@ void TimedTextPlayer::doRead(MediaSource::ReadOptions* options) { void TimedTextPlayer::postTextEvent(const sp<ParcelEvent>& parcel, int64_t timeUs) { int64_t delayUs = delayUsFromCurrentTime(timeUs); - sp<AMessage> msg = new AMessage(kWhatSendSubtitle, id()); + sp<AMessage> msg = new AMessage(kWhatSendSubtitle, this); msg->setInt32("generation", mSendSubtitleGeneration); if (parcel != NULL) { msg->setObject("subtitle", parcel); diff --git a/media/libstagefright/webm/WebmWriter.cpp b/media/libstagefright/webm/WebmWriter.cpp index 069961b..737f144 100644 --- a/media/libstagefright/webm/WebmWriter.cpp +++ b/media/libstagefright/webm/WebmWriter.cpp @@ -80,38 +80,6 @@ WebmWriter::WebmWriter(int fd) mCuePoints); } -WebmWriter::WebmWriter(const char *filename) - : mInitCheck(NO_INIT), - mTimeCodeScale(1000000), - mStartTimestampUs(0), - mStartTimeOffsetMs(0), - mSegmentOffset(0), - mSegmentDataStart(0), - mInfoOffset(0), - mInfoSize(0), - mTracksOffset(0), - mCuesOffset(0), - mPaused(false), - mStarted(false), - mIsFileSizeLimitExplicitlyRequested(false), - mIsRealTimeRecording(false), - mStreamableFile(true), - mEstimatedCuesSize(0) { - mFd = open(filename, O_CREAT | O_LARGEFILE | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR); - if (mFd >= 0) { - ALOGV("fd %d; flags: %o", mFd, fcntl(mFd, F_GETFL, 0)); - mInitCheck = OK; - } - mStreams[kAudioIndex] = WebmStream(kAudioType, "Audio", &WebmWriter::audioTrack); - mStreams[kVideoIndex] = WebmStream(kVideoType, "Video", &WebmWriter::videoTrack); - mSinkThread = new WebmFrameSinkThread( - mFd, - mSegmentDataStart, - mStreams[kVideoIndex].mSink, - mStreams[kAudioIndex].mSink, - mCuePoints); -} - // static sp<WebmElement> WebmWriter::videoTrack(const sp<MetaData>& md) { int32_t width, height; diff --git a/media/libstagefright/webm/WebmWriter.h b/media/libstagefright/webm/WebmWriter.h index 36b6965..4ad770e 100644 --- a/media/libstagefright/webm/WebmWriter.h +++ b/media/libstagefright/webm/WebmWriter.h @@ -37,7 +37,6 @@ namespace android { class WebmWriter : public MediaWriter { public: WebmWriter(int fd); - WebmWriter(const char *filename); ~WebmWriter() { reset(); } diff --git a/media/libstagefright/wifi-display/MediaSender.cpp b/media/libstagefright/wifi-display/MediaSender.cpp index b1cdec0..6f0087f 100644 --- a/media/libstagefright/wifi-display/MediaSender.cpp +++ b/media/libstagefright/wifi-display/MediaSender.cpp @@ -121,7 +121,7 @@ status_t MediaSender::initAsync( } if (err == OK) { - sp<AMessage> notify = new AMessage(kWhatSenderNotify, id()); + sp<AMessage> notify = new AMessage(kWhatSenderNotify, this); notify->setInt32("generation", mGeneration); mTSSender = new RTPSender(mNetSession, notify); looper()->registerHandler(mTSSender); @@ -170,7 +170,7 @@ status_t MediaSender::initAsync( return INVALID_OPERATION; } - sp<AMessage> notify = new AMessage(kWhatSenderNotify, id()); + sp<AMessage> notify = new AMessage(kWhatSenderNotify, this); notify->setInt32("generation", mGeneration); notify->setSize("trackIndex", trackIndex); diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.cpp b/media/libstagefright/wifi-display/rtp/RTPSender.cpp index e88a3bd..4e72533 100644 --- a/media/libstagefright/wifi-display/rtp/RTPSender.cpp +++ b/media/libstagefright/wifi-display/rtp/RTPSender.cpp @@ -95,11 +95,11 @@ status_t RTPSender::initAsync( return INVALID_OPERATION; } - sp<AMessage> rtpNotify = new AMessage(kWhatRTPNotify, id()); + sp<AMessage> rtpNotify = new AMessage(kWhatRTPNotify, this); sp<AMessage> rtcpNotify; if (remoteRTCPPort >= 0) { - rtcpNotify = new AMessage(kWhatRTCPNotify, id()); + rtcpNotify = new AMessage(kWhatRTCPNotify, this); } CHECK_EQ(mRTPSessionID, 0); diff --git a/media/libstagefright/wifi-display/source/Converter.cpp b/media/libstagefright/wifi-display/source/Converter.cpp index 2834a66..8368945 100644 --- a/media/libstagefright/wifi-display/source/Converter.cpp +++ b/media/libstagefright/wifi-display/source/Converter.cpp @@ -93,7 +93,7 @@ Converter::~Converter() { void Converter::shutdownAsync() { ALOGV("shutdown"); - (new AMessage(kWhatShutdown, id()))->post(); + (new AMessage(kWhatShutdown, this))->post(); } status_t Converter::init() { @@ -482,11 +482,11 @@ void Converter::scheduleDoMoreWork() { #if 1 if (mEncoderActivityNotify == NULL) { - mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, id()); + mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, this); } mEncoder->requestActivityNotification(mEncoderActivityNotify->dup()); #else - sp<AMessage> notify = new AMessage(kWhatEncoderActivity, id()); + sp<AMessage> notify = new AMessage(kWhatEncoderActivity, this); notify->setInt64("whenUs", ALooper::GetNowUs()); mEncoder->requestActivityNotification(notify); #endif @@ -731,8 +731,7 @@ status_t Converter::doMoreWork() { // MediaSender will post the following message when HDCP // is done, to release the output buffer back to encoder. - sp<AMessage> notify(new AMessage( - kWhatReleaseOutputBuffer, id())); + sp<AMessage> notify(new AMessage(kWhatReleaseOutputBuffer, this)); notify->setInt32("bufferIndex", bufferIndex); buffer = new ABuffer( @@ -787,18 +786,18 @@ status_t Converter::doMoreWork() { } void Converter::requestIDRFrame() { - (new AMessage(kWhatRequestIDRFrame, id()))->post(); + (new AMessage(kWhatRequestIDRFrame, this))->post(); } void Converter::dropAFrame() { // Unsupported in surface input mode. CHECK(!(mFlags & FLAG_USE_SURFACE_INPUT)); - (new AMessage(kWhatDropAFrame, id()))->post(); + (new AMessage(kWhatDropAFrame, this))->post(); } void Converter::suspendEncoding(bool suspend) { - sp<AMessage> msg = new AMessage(kWhatSuspendEncoding, id()); + sp<AMessage> msg = new AMessage(kWhatSuspendEncoding, this); msg->setInt32("suspend", suspend); msg->post(); } diff --git a/media/libstagefright/wifi-display/source/MediaPuller.cpp b/media/libstagefright/wifi-display/source/MediaPuller.cpp index 86b918f..ce07a4e 100644 --- a/media/libstagefright/wifi-display/source/MediaPuller.cpp +++ b/media/libstagefright/wifi-display/source/MediaPuller.cpp @@ -63,21 +63,21 @@ status_t MediaPuller::postSynchronouslyAndReturnError( } status_t MediaPuller::start() { - return postSynchronouslyAndReturnError(new AMessage(kWhatStart, id())); + return postSynchronouslyAndReturnError(new AMessage(kWhatStart, this)); } void MediaPuller::stopAsync(const sp<AMessage> ¬ify) { - sp<AMessage> msg = new AMessage(kWhatStop, id()); + sp<AMessage> msg = new AMessage(kWhatStop, this); msg->setMessage("notify", notify); msg->post(); } void MediaPuller::pause() { - (new AMessage(kWhatPause, id()))->post(); + (new AMessage(kWhatPause, this))->post(); } void MediaPuller::resume() { - (new AMessage(kWhatResume, id()))->post(); + (new AMessage(kWhatResume, this))->post(); } void MediaPuller::onMessageReceived(const sp<AMessage> &msg) { @@ -105,7 +105,7 @@ void MediaPuller::onMessageReceived(const sp<AMessage> &msg) { sp<AMessage> response = new AMessage; response->setInt32("err", err); - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); response->postReply(replyID); break; @@ -215,7 +215,7 @@ void MediaPuller::onMessageReceived(const sp<AMessage> &msg) { } void MediaPuller::schedulePull() { - sp<AMessage> msg = new AMessage(kWhatPull, id()); + sp<AMessage> msg = new AMessage(kWhatPull, this); msg->setInt32("generation", mPullGeneration); msg->post(); } diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.cpp b/media/libstagefright/wifi-display/source/PlaybackSession.cpp index 2cb4786..6080943 100644 --- a/media/libstagefright/wifi-display/source/PlaybackSession.cpp +++ b/media/libstagefright/wifi-display/source/PlaybackSession.cpp @@ -214,7 +214,7 @@ void WifiDisplaySource::PlaybackSession::Track::stopAsync() { mConverter->shutdownAsync(); } - sp<AMessage> msg = new AMessage(kWhatMediaPullerStopped, id()); + sp<AMessage> msg = new AMessage(kWhatMediaPullerStopped, this); if (mStarted && mMediaPuller != NULL) { if (mRepeaterSource != NULL) { @@ -382,7 +382,7 @@ status_t WifiDisplaySource::PlaybackSession::init( size_t videoResolutionIndex, VideoFormats::ProfileType videoProfileType, VideoFormats::LevelType videoLevelType) { - sp<AMessage> notify = new AMessage(kWhatMediaSenderNotify, id()); + sp<AMessage> notify = new AMessage(kWhatMediaSenderNotify, this); mMediaSender = new MediaSender(mNetSession, notify); looper()->registerHandler(mMediaSender); @@ -440,7 +440,7 @@ void WifiDisplaySource::PlaybackSession::updateLiveness() { status_t WifiDisplaySource::PlaybackSession::play() { updateLiveness(); - (new AMessage(kWhatResume, id()))->post(); + (new AMessage(kWhatResume, this))->post(); return OK; } @@ -460,7 +460,7 @@ status_t WifiDisplaySource::PlaybackSession::onMediaSenderInitialized() { status_t WifiDisplaySource::PlaybackSession::pause() { updateLiveness(); - (new AMessage(kWhatPause, id()))->post(); + (new AMessage(kWhatPause, this))->post(); return OK; } @@ -786,7 +786,7 @@ status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer( size_t trackIndex = mTracks.size(); - sp<AMessage> notify = new AMessage(kWhatTrackNotify, id()); + sp<AMessage> notify = new AMessage(kWhatTrackNotify, this); notify->setSize("trackIndex", trackIndex); sp<Track> track = new Track(notify, format); @@ -833,7 +833,7 @@ void WifiDisplaySource::PlaybackSession::schedulePullExtractor() { int64_t whenUs = sampleTimeUs - mFirstSampleTimeUs + mFirstSampleTimeRealUs; - sp<AMessage> msg = new AMessage(kWhatPullExtractorSample, id()); + sp<AMessage> msg = new AMessage(kWhatPullExtractorSample, this); msg->setInt32("generation", mPullExtractorGeneration); msg->post(whenUs - nowUs); @@ -857,7 +857,7 @@ void WifiDisplaySource::PlaybackSession::onPullExtractor() { size_t trackIndex; CHECK_EQ((status_t)OK, mExtractor->getSampleTrackIndex(&trackIndex)); - sp<AMessage> msg = new AMessage(kWhatConverterNotify, id()); + sp<AMessage> msg = new AMessage(kWhatConverterNotify, this); msg->setSize( "trackIndex", mExtractorTrackToInternalTrack.valueFor(trackIndex)); @@ -955,7 +955,7 @@ status_t WifiDisplaySource::PlaybackSession::addSource( ? MEDIA_MIMETYPE_AUDIO_RAW : MEDIA_MIMETYPE_AUDIO_AAC); } - notify = new AMessage(kWhatConverterNotify, id()); + notify = new AMessage(kWhatConverterNotify, this); notify->setSize("trackIndex", trackIndex); sp<Converter> converter = new Converter(notify, codecLooper, format); @@ -970,7 +970,7 @@ status_t WifiDisplaySource::PlaybackSession::addSource( return err; } - notify = new AMessage(Converter::kWhatMediaPullerNotify, converter->id()); + notify = new AMessage(Converter::kWhatMediaPullerNotify, converter); notify->setSize("trackIndex", trackIndex); sp<MediaPuller> puller = new MediaPuller(source, notify); @@ -980,7 +980,7 @@ status_t WifiDisplaySource::PlaybackSession::addSource( *numInputBuffers = converter->getInputBufferCount(); } - notify = new AMessage(kWhatTrackNotify, id()); + notify = new AMessage(kWhatTrackNotify, this); notify->setSize("trackIndex", trackIndex); sp<Track> track = new Track( diff --git a/media/libstagefright/wifi-display/source/RepeaterSource.cpp b/media/libstagefright/wifi-display/source/RepeaterSource.cpp index 59d7e6e..af6b663 100644 --- a/media/libstagefright/wifi-display/source/RepeaterSource.cpp +++ b/media/libstagefright/wifi-display/source/RepeaterSource.cpp @@ -173,7 +173,7 @@ status_t RepeaterSource::read( } void RepeaterSource::postRead() { - (new AMessage(kWhatRead, mReflector->id()))->post(); + (new AMessage(kWhatRead, mReflector))->post(); } void RepeaterSource::onMessageReceived(const sp<AMessage> &msg) { diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp index 7eb8b73..14d0951 100644 --- a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp @@ -57,7 +57,7 @@ WifiDisplaySource::WifiDisplaySource( mNetSession(netSession), mClient(client), mSessionID(0), - mStopReplyID(0), + mStopReplyID(NULL), mChosenRTPPort(-1), mUsingPCMAudio(false), mClientSessionID(0), @@ -106,7 +106,7 @@ static status_t PostAndAwaitResponse( status_t WifiDisplaySource::start(const char *iface) { CHECK_EQ(mState, INITIALIZED); - sp<AMessage> msg = new AMessage(kWhatStart, id()); + sp<AMessage> msg = new AMessage(kWhatStart, this); msg->setString("iface", iface); sp<AMessage> response; @@ -114,21 +114,21 @@ status_t WifiDisplaySource::start(const char *iface) { } status_t WifiDisplaySource::stop() { - sp<AMessage> msg = new AMessage(kWhatStop, id()); + sp<AMessage> msg = new AMessage(kWhatStop, this); sp<AMessage> response; return PostAndAwaitResponse(msg, &response); } status_t WifiDisplaySource::pause() { - sp<AMessage> msg = new AMessage(kWhatPause, id()); + sp<AMessage> msg = new AMessage(kWhatPause, this); sp<AMessage> response; return PostAndAwaitResponse(msg, &response); } status_t WifiDisplaySource::resume() { - sp<AMessage> msg = new AMessage(kWhatResume, id()); + sp<AMessage> msg = new AMessage(kWhatResume, this); sp<AMessage> response; return PostAndAwaitResponse(msg, &response); @@ -138,7 +138,7 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { switch (msg->what()) { case kWhatStart: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); AString iface; @@ -167,7 +167,7 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { if (err == OK) { if (inet_aton(iface.c_str(), &mInterfaceAddr) != 0) { - sp<AMessage> notify = new AMessage(kWhatRTSPNotify, id()); + sp<AMessage> notify = new AMessage(kWhatRTSPNotify, this); err = mNetSession->createRTSPServer( mInterfaceAddr, port, notify, &mSessionID); @@ -310,7 +310,7 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { if (err == OK) { mState = AWAITING_CLIENT_TEARDOWN; - (new AMessage(kWhatTeardownTriggerTimedOut, id()))->post( + (new AMessage(kWhatTeardownTriggerTimedOut, this))->post( kTeardownTriggerTimeouSecs * 1000000ll); break; @@ -325,7 +325,7 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { case kWhatPause: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); status_t err = OK; @@ -345,7 +345,7 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { case kWhatResume: { - uint32_t replyID; + sp<AReplyToken> replyID; CHECK(msg->senderAwaitsResponse(&replyID)); status_t err = OK; @@ -492,7 +492,7 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { if (mState == AWAITING_CLIENT_TEARDOWN) { ALOGI("TEARDOWN trigger timed out, forcing disconnection."); - CHECK_NE(mStopReplyID, 0); + CHECK(mStopReplyID != NULL); finishStop(); break; } @@ -529,7 +529,7 @@ void WifiDisplaySource::onMessageReceived(const sp<AMessage> &msg) { // HDCPObserver::notify is completely handled before // we clear the HDCP instance and unload the shared // library :( - (new AMessage(kWhatFinishStop2, id()))->post(300000ll); + (new AMessage(kWhatFinishStop2, this))->post(300000ll); break; } @@ -1027,7 +1027,7 @@ void WifiDisplaySource::scheduleReaper() { } mReaperPending = true; - (new AMessage(kWhatReapDeadClients, id()))->post(kReaperIntervalUs); + (new AMessage(kWhatReapDeadClients, this))->post(kReaperIntervalUs); } void WifiDisplaySource::scheduleKeepAlive(int32_t sessionID) { @@ -1035,7 +1035,7 @@ void WifiDisplaySource::scheduleKeepAlive(int32_t sessionID) { // expire, make sure the timeout is greater than 5 secs to begin with. CHECK_GT(kPlaybackSessionTimeoutUs, 5000000ll); - sp<AMessage> msg = new AMessage(kWhatKeepAlive, id()); + sp<AMessage> msg = new AMessage(kWhatKeepAlive, this); msg->setInt32("sessionID", sessionID); msg->post(kPlaybackSessionTimeoutUs - 5000000ll); } @@ -1239,7 +1239,7 @@ status_t WifiDisplaySource::onSetupRequest( int32_t playbackSessionID = makeUniquePlaybackSessionID(); - sp<AMessage> notify = new AMessage(kWhatPlaybackSessionNotify, id()); + sp<AMessage> notify = new AMessage(kWhatPlaybackSessionNotify, this); notify->setInt32("playbackSessionID", playbackSessionID); notify->setInt32("sessionID", sessionID); @@ -1470,7 +1470,7 @@ status_t WifiDisplaySource::onTeardownRequest( mNetSession->sendRequest(sessionID, response.c_str()); if (mState == AWAITING_CLIENT_TEARDOWN) { - CHECK_NE(mStopReplyID, 0); + CHECK(mStopReplyID != NULL); finishStop(); } else { mClient->onDisplayError(IRemoteDisplayClient::kDisplayErrorUnknown); @@ -1707,7 +1707,7 @@ status_t WifiDisplaySource::makeHDCP() { return ERROR_UNSUPPORTED; } - sp<AMessage> notify = new AMessage(kWhatHDCPNotify, id()); + sp<AMessage> notify = new AMessage(kWhatHDCPNotify, this); mHDCPObserver = new HDCPObserver(notify); status_t err = mHDCP->setObserver(mHDCPObserver); diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.h b/media/libstagefright/wifi-display/source/WifiDisplaySource.h index 750265f..0f779e4 100644 --- a/media/libstagefright/wifi-display/source/WifiDisplaySource.h +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.h @@ -27,6 +27,7 @@ namespace android { +struct AReplyToken; struct IHDCP; struct IRemoteDisplayClient; struct ParsedMessage; @@ -121,7 +122,7 @@ private: struct in_addr mInterfaceAddr; int32_t mSessionID; - uint32_t mStopReplyID; + sp<AReplyToken> mStopReplyID; AString mWfdClientRtpPorts; int32_t mChosenRTPPort; // extracted from "wfd_client_rtp_ports" diff --git a/media/libstagefright/yuv/YUVImage.cpp b/media/libstagefright/yuv/YUVImage.cpp index bb3e2fd..c098135 100644 --- a/media/libstagefright/yuv/YUVImage.cpp +++ b/media/libstagefright/yuv/YUVImage.cpp @@ -374,13 +374,13 @@ uint8_t clamp(uint8_t v, uint8_t minValue, uint8_t maxValue) { void YUVImage::yuv2rgb(uint8_t yValue, uint8_t uValue, uint8_t vValue, uint8_t *r, uint8_t *g, uint8_t *b) const { - *r = yValue + (1.370705 * (vValue-128)); - *g = yValue - (0.698001 * (vValue-128)) - (0.337633 * (uValue-128)); - *b = yValue + (1.732446 * (uValue-128)); + int rTmp = yValue + (1.370705 * (vValue-128)); + int gTmp = yValue - (0.698001 * (vValue-128)) - (0.337633 * (uValue-128)); + int bTmp = yValue + (1.732446 * (uValue-128)); - *r = clamp(*r, 0, 255); - *g = clamp(*g, 0, 255); - *b = clamp(*b, 0, 255); + *r = clamp(rTmp, 0, 255); + *g = clamp(gTmp, 0, 255); + *b = clamp(bTmp, 0, 255); } bool YUVImage::writeToPPM(const char *filename) const { diff --git a/media/mediaserver/Android.mk b/media/mediaserver/Android.mk index 3a280f0..baf65f6 100644 --- a/media/mediaserver/Android.mk +++ b/media/mediaserver/Android.mk @@ -11,7 +11,7 @@ endif include $(CLEAR_VARS) LOCAL_SRC_FILES:= \ - main_mediaserver.cpp + main_mediaserver.cpp LOCAL_SHARED_LIBRARIES := \ libaudioflinger \ @@ -19,6 +19,7 @@ LOCAL_SHARED_LIBRARIES := \ libcamera_metadata\ libcameraservice \ libmedialogservice \ + libresourcemanagerservice \ libcutils \ libnbaio \ libmedia \ @@ -26,7 +27,8 @@ LOCAL_SHARED_LIBRARIES := \ libutils \ liblog \ libbinder \ - libsoundtriggerservice + libsoundtriggerservice \ + libradioservice LOCAL_STATIC_LIBRARIES := \ libregistermsext @@ -37,8 +39,10 @@ LOCAL_C_INCLUDES := \ frameworks/av/services/audioflinger \ frameworks/av/services/audiopolicy \ frameworks/av/services/camera/libcameraservice \ + frameworks/av/services/mediaresourcemanager \ $(call include-path-for, audio-utils) \ - frameworks/av/services/soundtrigger + frameworks/av/services/soundtrigger \ + frameworks/av/services/radio LOCAL_MODULE:= mediaserver LOCAL_32_BIT_ONLY := true diff --git a/media/mediaserver/main_mediaserver.cpp b/media/mediaserver/main_mediaserver.cpp index af1c9e6..99572f8 100644 --- a/media/mediaserver/main_mediaserver.cpp +++ b/media/mediaserver/main_mediaserver.cpp @@ -33,8 +33,9 @@ #include "CameraService.h" #include "MediaLogService.h" #include "MediaPlayerService.h" -#include "AudioPolicyService.h" +#include "service/AudioPolicyService.h" #include "SoundTriggerHwService.h" +#include "RadioService.h" using namespace android; @@ -130,6 +131,7 @@ int main(int argc __unused, char** argv) CameraService::instantiate(); AudioPolicyService::instantiate(); SoundTriggerHwService::instantiate(); + RadioService::instantiate(); registerExtensions(); ProcessState::self()->startThreadPool(); IPCThreadState::self()->joinThreadPool(); diff --git a/media/ndk/NdkMediaCodec.cpp b/media/ndk/NdkMediaCodec.cpp index ed00b72..80c1c2f 100644 --- a/media/ndk/NdkMediaCodec.cpp +++ b/media/ndk/NdkMediaCodec.cpp @@ -116,7 +116,7 @@ void CodecHandler::onMessageReceived(const sp<AMessage> &msg) { case kWhatStopActivityNotifications: { - uint32_t replyID; + sp<AReplyToken> replyID; msg->senderAwaitsResponse(&replyID); mCodec->mGeneration++; @@ -136,7 +136,7 @@ void CodecHandler::onMessageReceived(const sp<AMessage> &msg) { static void requestActivityNotification(AMediaCodec *codec) { - (new AMessage(kWhatRequestActivityNotifications, codec->mHandler->id()))->post(); + (new AMessage(kWhatRequestActivityNotifications, codec->mHandler))->post(); } extern "C" { @@ -219,7 +219,7 @@ media_status_t AMediaCodec_start(AMediaCodec *mData) { if (ret != OK) { return translate_error(ret); } - mData->mActivityNotification = new AMessage(kWhatActivityNotify, mData->mHandler->id()); + mData->mActivityNotification = new AMessage(kWhatActivityNotify, mData->mHandler); mData->mActivityNotification->setInt32("generation", mData->mGeneration); requestActivityNotification(mData); return AMEDIA_OK; @@ -229,7 +229,7 @@ EXPORT media_status_t AMediaCodec_stop(AMediaCodec *mData) { media_status_t ret = translate_error(mData->mCodec->stop()); - sp<AMessage> msg = new AMessage(kWhatStopActivityNotifications, mData->mHandler->id()); + sp<AMessage> msg = new AMessage(kWhatStopActivityNotifications, mData->mHandler); sp<AMessage> response; msg->postAndAwaitResponse(&response); mData->mActivityNotification.clear(); @@ -352,7 +352,8 @@ media_status_t AMediaCodec_releaseOutputBufferAtTime( } //EXPORT -media_status_t AMediaCodec_setNotificationCallback(AMediaCodec *mData, OnCodecEvent callback, void *userdata) { +media_status_t AMediaCodec_setNotificationCallback(AMediaCodec *mData, OnCodecEvent callback, + void *userdata) { mData->mCallback = callback; mData->mCallbackUserData = userdata; return AMEDIA_OK; diff --git a/media/ndk/NdkMediaDrm.cpp b/media/ndk/NdkMediaDrm.cpp index 7a1048c..83a5ba1 100644 --- a/media/ndk/NdkMediaDrm.cpp +++ b/media/ndk/NdkMediaDrm.cpp @@ -312,8 +312,10 @@ media_status_t AMediaDrm_getKeyRequest(AMediaDrm *mObj, const AMediaDrmScope *sc String8(optionalParameters[i].mValue)); } String8 defaultUrl; + DrmPlugin::KeyRequestType keyRequestType; status_t status = mObj->mDrm->getKeyRequest(*iter, mdInit, String8(mimeType), - mdKeyType, mdOptionalParameters, mObj->mKeyRequest, defaultUrl); + mdKeyType, mdOptionalParameters, mObj->mKeyRequest, defaultUrl, + &keyRequestType); if (status != OK) { return translateStatus(status); } else { @@ -725,4 +727,3 @@ media_status_t AMediaDrm_verify(AMediaDrm *mObj, const AMediaDrmSessionId *sessi } } // extern "C" - diff --git a/media/ndk/NdkMediaExtractor.cpp b/media/ndk/NdkMediaExtractor.cpp index db57d0b..0ecd64f 100644 --- a/media/ndk/NdkMediaExtractor.cpp +++ b/media/ndk/NdkMediaExtractor.cpp @@ -70,7 +70,8 @@ media_status_t AMediaExtractor_delete(AMediaExtractor *mData) { } EXPORT -media_status_t AMediaExtractor_setDataSourceFd(AMediaExtractor *mData, int fd, off64_t offset, off64_t length) { +media_status_t AMediaExtractor_setDataSourceFd(AMediaExtractor *mData, int fd, off64_t offset, + off64_t length) { ALOGV("setDataSource(%d, %lld, %lld)", fd, offset, length); return translate_error(mData->mImpl->setDataSource(fd, offset, length)); } |