diff options
98 files changed, 2535 insertions, 824 deletions
diff --git a/include/media/AudioTrack.h b/include/media/AudioTrack.h index a3cc396..72e51f9 100644 --- a/include/media/AudioTrack.h +++ b/include/media/AudioTrack.h @@ -430,7 +430,7 @@ public: * - NO_ERROR: successful operation * - BAD_VALUE: position is NULL */ - status_t getPosition(uint32_t *position) const; + status_t getPosition(uint32_t *position); /* For static buffer mode only, this returns the current playback position in frames * relative to start of buffer. It is analogous to the position units used by @@ -581,6 +581,7 @@ public: * if you need a high resolution mapping between frame position and presentation time, * consider implementing that at application level, based on the low resolution timestamps. * Returns NO_ERROR if timestamp is valid. + * The timestamp parameter is undefined on return, if status is not NO_ERROR. */ status_t getTimestamp(AudioTimestamp& timestamp); @@ -639,7 +640,7 @@ protected: // caller must hold lock on mLock for all _l methods - status_t createTrack_l(size_t epoch); + status_t createTrack_l(); // can only be called when mState != STATE_ACTIVE void flush_l(); @@ -659,6 +660,9 @@ protected: bool isDirect_l() const { return (mFlags & AUDIO_OUTPUT_FLAG_DIRECT) != 0; } + // increment mPosition by the delta of mServer, and return new value of mPosition + uint32_t updateAndGetPosition_l(); + // Next 4 fields may be changed if IAudioTrack is re-created, but always != 0 sp<IAudioTrack> mAudioTrack; sp<IMemory> mCblkMemory; @@ -731,6 +735,18 @@ protected: bool mMarkerReached; uint32_t mNewPosition; // in frames uint32_t mUpdatePeriod; // in frames, zero means no EVENT_NEW_POS + uint32_t mServer; // in frames, last known mProxy->getPosition() + // which is count of frames consumed by server, + // reset by new IAudioTrack, + // whether it is reset by stop() is TBD + uint32_t mPosition; // in frames, like mServer except continues + // monotonically after new IAudioTrack, + // and could be easily widened to uint64_t + uint32_t mReleased; // in frames, count of frames released to server + // but not necessarily consumed by server, + // reset by stop() but continues monotonically + // after new IAudioTrack to restore mPosition, + // and could be easily widened to uint64_t audio_output_flags_t mFlags; // const after set(), except for bits AUDIO_OUTPUT_FLAG_FAST and AUDIO_OUTPUT_FLAG_OFFLOAD. diff --git a/media/libmedia/CharacterEncodingDetector.h b/include/media/CharacterEncodingDetector.h index 7b5ed86..deaa377 100644 --- a/media/libmedia/CharacterEncodingDetector.h +++ b/include/media/CharacterEncodingDetector.h @@ -43,7 +43,7 @@ class CharacterEncodingDetector { const UCharsetMatch *getPreferred( const char *input, size_t len, const UCharsetMatch** ucma, size_t matches, - bool *goodmatch); + bool *goodmatch, int *highestmatch); bool isFrequent(const uint16_t *values, uint32_t c); diff --git a/include/media/IAudioTrack.h b/include/media/IAudioTrack.h index 5c8a484..619ac78 100644 --- a/include/media/IAudioTrack.h +++ b/include/media/IAudioTrack.h @@ -88,7 +88,7 @@ public: /* Send parameters to the audio hardware */ virtual status_t setParameters(const String8& keyValuePairs) = 0; - /* Return NO_ERROR if timestamp is valid */ + /* Return NO_ERROR if timestamp is valid. timestamp is undefined otherwise. */ virtual status_t getTimestamp(AudioTimestamp& timestamp) = 0; /* Signal the playback thread for a change in control block */ diff --git a/include/media/MediaProfiles.h b/include/media/MediaProfiles.h index 253c557..f061d22 100644 --- a/include/media/MediaProfiles.h +++ b/include/media/MediaProfiles.h @@ -54,7 +54,8 @@ enum camcorder_quality { CAMCORDER_QUALITY_HIGH_SPEED_480P = 2002, CAMCORDER_QUALITY_HIGH_SPEED_720P = 2003, CAMCORDER_QUALITY_HIGH_SPEED_1080P = 2004, - CAMCORDER_QUALITY_HIGH_SPEED_LIST_END = 2004, + CAMCORDER_QUALITY_HIGH_SPEED_2160P = 2005, + CAMCORDER_QUALITY_HIGH_SPEED_LIST_END = 2005, }; /** diff --git a/media/libmedia/StringArray.h b/include/media/StringArray.h index ae47085..ae47085 100644 --- a/media/libmedia/StringArray.h +++ b/include/media/StringArray.h diff --git a/include/media/mediaplayer.h b/include/media/mediaplayer.h index 2442219..9cc208e 100644 --- a/include/media/mediaplayer.h +++ b/include/media/mediaplayer.h @@ -278,6 +278,7 @@ private: bool mPrepareSync; status_t mPrepareStatus; audio_stream_type_t mStreamType; + Parcel* mAudioAttributesParcel; bool mLoop; float mLeftVolume; float mRightVolume; diff --git a/include/media/mediascanner.h b/include/media/mediascanner.h index 5213bdc..d555279 100644 --- a/include/media/mediascanner.h +++ b/include/media/mediascanner.h @@ -122,7 +122,6 @@ public: protected: // default encoding from MediaScanner::mLocale String8 mLocale; - CharacterEncodingDetector *mEncodingDetector; }; }; // namespace android diff --git a/include/media/nbaio/NBAIO.h b/include/media/nbaio/NBAIO.h index be0c15b..d422576 100644 --- a/include/media/nbaio/NBAIO.h +++ b/include/media/nbaio/NBAIO.h @@ -227,7 +227,7 @@ public: // Returns NO_ERROR if a timestamp is available. The timestamp includes the total number // of frames presented to an external observer, together with the value of CLOCK_MONOTONIC - // as of this presentation count. + // as of this presentation count. The timestamp parameter is undefined if error is returned. virtual status_t getTimestamp(AudioTimestamp& timestamp) { return INVALID_OPERATION; } protected: diff --git a/include/media/stagefright/MediaErrors.h b/include/media/stagefright/MediaErrors.h index 7540e07..2e663ec 100644 --- a/include/media/stagefright/MediaErrors.h +++ b/include/media/stagefright/MediaErrors.h @@ -58,20 +58,22 @@ enum { // drm/drm_framework_common.h DRM_ERROR_BASE = -2000, - ERROR_DRM_UNKNOWN = DRM_ERROR_BASE, - ERROR_DRM_NO_LICENSE = DRM_ERROR_BASE - 1, - ERROR_DRM_LICENSE_EXPIRED = DRM_ERROR_BASE - 2, - ERROR_DRM_SESSION_NOT_OPENED = DRM_ERROR_BASE - 3, - ERROR_DRM_DECRYPT_UNIT_NOT_INITIALIZED = DRM_ERROR_BASE - 4, - ERROR_DRM_DECRYPT = DRM_ERROR_BASE - 5, - ERROR_DRM_CANNOT_HANDLE = DRM_ERROR_BASE - 6, - ERROR_DRM_TAMPER_DETECTED = DRM_ERROR_BASE - 7, - ERROR_DRM_NOT_PROVISIONED = DRM_ERROR_BASE - 8, - ERROR_DRM_DEVICE_REVOKED = DRM_ERROR_BASE - 9, - ERROR_DRM_RESOURCE_BUSY = DRM_ERROR_BASE - 10, - - ERROR_DRM_VENDOR_MAX = DRM_ERROR_BASE - 500, - ERROR_DRM_VENDOR_MIN = DRM_ERROR_BASE - 999, + ERROR_DRM_UNKNOWN = DRM_ERROR_BASE, + ERROR_DRM_NO_LICENSE = DRM_ERROR_BASE - 1, + ERROR_DRM_LICENSE_EXPIRED = DRM_ERROR_BASE - 2, + ERROR_DRM_SESSION_NOT_OPENED = DRM_ERROR_BASE - 3, + ERROR_DRM_DECRYPT_UNIT_NOT_INITIALIZED = DRM_ERROR_BASE - 4, + ERROR_DRM_DECRYPT = DRM_ERROR_BASE - 5, + ERROR_DRM_CANNOT_HANDLE = DRM_ERROR_BASE - 6, + ERROR_DRM_TAMPER_DETECTED = DRM_ERROR_BASE - 7, + ERROR_DRM_NOT_PROVISIONED = DRM_ERROR_BASE - 8, + ERROR_DRM_DEVICE_REVOKED = DRM_ERROR_BASE - 9, + ERROR_DRM_RESOURCE_BUSY = DRM_ERROR_BASE - 10, + ERROR_DRM_INSUFFICIENT_OUTPUT_PROTECTION = DRM_ERROR_BASE - 11, + ERROR_DRM_LAST_USED_ERRORCODE = DRM_ERROR_BASE - 11, + + ERROR_DRM_VENDOR_MAX = DRM_ERROR_BASE - 500, + ERROR_DRM_VENDOR_MIN = DRM_ERROR_BASE - 999, // Heartbeat Error Codes HEARTBEAT_ERROR_BASE = -3000, @@ -100,7 +102,7 @@ enum ActionCode { // returns true if err is a recognized DRM error code static inline bool isCryptoError(status_t err) { - return (ERROR_DRM_RESOURCE_BUSY <= err && err <= ERROR_DRM_UNKNOWN) + return (ERROR_DRM_LAST_USED_ERRORCODE <= err && err <= ERROR_DRM_UNKNOWN) || (ERROR_DRM_VENDOR_MIN <= err && err <= ERROR_DRM_VENDOR_MAX); } diff --git a/include/media/stagefright/foundation/ALooperRoster.h b/include/media/stagefright/foundation/ALooperRoster.h index 940fc55..4d76b64 100644 --- a/include/media/stagefright/foundation/ALooperRoster.h +++ b/include/media/stagefright/foundation/ALooperRoster.h @@ -56,8 +56,6 @@ private: KeyedVector<uint32_t, sp<AMessage> > mReplies; - status_t postMessage_l(const sp<AMessage> &msg, int64_t delayUs); - DISALLOW_EVIL_CONSTRUCTORS(ALooperRoster); }; diff --git a/include/media/stagefright/foundation/AMessage.h b/include/media/stagefright/foundation/AMessage.h index 5846d6b..a9e235b 100644 --- a/include/media/stagefright/foundation/AMessage.h +++ b/include/media/stagefright/foundation/AMessage.h @@ -137,7 +137,9 @@ private: Rect rectValue; } u; const char *mName; + size_t mNameLength; Type mType; + void setName(const char *name, size_t len); }; enum { @@ -147,12 +149,14 @@ private: size_t mNumItems; Item *allocateItem(const char *name); - void freeItem(Item *item); + void freeItemValue(Item *item); const Item *findItem(const char *name, Type type) const; void setObjectInternal( const char *name, const sp<RefBase> &obj, Type type); + size_t findItemIndex(const char *name, size_t len) const; + DISALLOW_EVIL_CONSTRUCTORS(AMessage); }; diff --git a/media/libmedia/Android.mk b/media/libmedia/Android.mk index 37bc418..e012116 100644 --- a/media/libmedia/Android.mk +++ b/media/libmedia/Android.mk @@ -76,9 +76,10 @@ LOCAL_MODULE:= libmedia LOCAL_C_INCLUDES := \ $(TOP)/frameworks/native/include/media/openmax \ + $(TOP)/frameworks/av/include/media/ \ $(TOP)/frameworks/av/media/libstagefright \ - external/icu/icu4c/source/common \ - external/icu/icu4c/source/i18n \ + $(TOP)/external/icu/icu4c/source/common \ + $(TOP)/external/icu/icu4c/source/i18n \ $(call include-path-for, audio-effects) \ $(call include-path-for, audio-utils) diff --git a/media/libmedia/AudioSystem.cpp b/media/libmedia/AudioSystem.cpp index 3486d21..1742fbe 100644 --- a/media/libmedia/AudioSystem.cpp +++ b/media/libmedia/AudioSystem.cpp @@ -582,9 +582,13 @@ const sp<IAudioPolicyService>& AudioSystem::get_audio_policy_service() } binder->linkToDeath(gAudioPolicyServiceClient); gAudioPolicyService = interface_cast<IAudioPolicyService>(binder); - gAudioPolicyService->registerClient(gAudioPolicyServiceClient); gLock.unlock(); + // Registering the client takes the AudioPolicyService lock. + // Don't hold the AudioSystem lock at the same time. + gAudioPolicyService->registerClient(gAudioPolicyServiceClient); } else { + // There exists a benign race condition where gAudioPolicyService + // is set, but gAudioPolicyServiceClient is not yet registered. gLock.unlock(); } return gAudioPolicyService; diff --git a/media/libmedia/AudioTrack.cpp b/media/libmedia/AudioTrack.cpp index d87e6f5..ff7da83 100644 --- a/media/libmedia/AudioTrack.cpp +++ b/media/libmedia/AudioTrack.cpp @@ -398,7 +398,7 @@ status_t AudioTrack::set( } // create the IAudioTrack - status = createTrack_l(0 /*epoch*/); + status = createTrack_l(); if (status != NO_ERROR) { if (mAudioTrackThread != 0) { @@ -417,6 +417,9 @@ status_t AudioTrack::set( mMarkerReached = false; mNewPosition = 0; mUpdatePeriod = 0; + mServer = 0; + mPosition = 0; + mReleased = 0; AudioSystem::acquireAudioSessionId(mSessionId, mClientPid); mSequence = 1; mObservedSequence = mSequence; @@ -443,14 +446,16 @@ status_t AudioTrack::start() } else { mState = STATE_ACTIVE; } + (void) updateAndGetPosition_l(); if (previousState == STATE_STOPPED || previousState == STATE_FLUSHED) { // reset current position as seen by client to 0 - mProxy->setEpoch(mProxy->getEpoch() - mProxy->getPosition()); + mPosition = 0; + mReleased = 0; // force refresh of remaining frames by processAudioBuffer() as last // write before stop could be partial. mRefreshRemaining = true; } - mNewPosition = mProxy->getPosition() + mUpdatePeriod; + mNewPosition = mPosition + mUpdatePeriod; int32_t flags = android_atomic_and(~CBLK_DISABLED, &mCblk->mFlags); sp<AudioTrackThread> t = mAudioTrackThread; @@ -709,7 +714,7 @@ void AudioTrack::setLoop_l(uint32_t loopStart, uint32_t loopEnd, int loopCount) { // FIXME If setting a loop also sets position to start of loop, then // this is correct. Otherwise it should be removed. - mNewPosition = mProxy->getPosition() + mUpdatePeriod; + mNewPosition = updateAndGetPosition_l() + mUpdatePeriod; mLoopPeriod = loopCount != 0 ? loopEnd - loopStart : 0; mStaticProxy->setLoop(loopStart, loopEnd, loopCount); } @@ -751,7 +756,7 @@ status_t AudioTrack::setPositionUpdatePeriod(uint32_t updatePeriod) } AutoMutex lock(mLock); - mNewPosition = mProxy->getPosition() + updatePeriod; + mNewPosition = updateAndGetPosition_l() + updatePeriod; mUpdatePeriod = updatePeriod; return NO_ERROR; @@ -791,7 +796,7 @@ status_t AudioTrack::setPosition(uint32_t position) if (mState == STATE_ACTIVE) { return INVALID_OPERATION; } - mNewPosition = mProxy->getPosition() + mUpdatePeriod; + mNewPosition = updateAndGetPosition_l() + mUpdatePeriod; mLoopPeriod = 0; // FIXME Check whether loops and setting position are incompatible in old code. // If we use setLoop for both purposes we lose the capability to set the position while looping. @@ -800,7 +805,7 @@ status_t AudioTrack::setPosition(uint32_t position) return NO_ERROR; } -status_t AudioTrack::getPosition(uint32_t *position) const +status_t AudioTrack::getPosition(uint32_t *position) { if (position == NULL) { return BAD_VALUE; @@ -823,8 +828,8 @@ status_t AudioTrack::getPosition(uint32_t *position) const *position = dspFrames; } else { // IAudioTrack::stop() isn't synchronous; we don't know when presentation completes - *position = (mState == STATE_STOPPED || mState == STATE_FLUSHED) ? 0 : - mProxy->getPosition(); + *position = (mState == STATE_STOPPED || mState == STATE_FLUSHED) ? + 0 : updateAndGetPosition_l(); } return NO_ERROR; } @@ -881,7 +886,7 @@ status_t AudioTrack::attachAuxEffect(int effectId) // ------------------------------------------------------------------------- // must be called with mLock held -status_t AudioTrack::createTrack_l(size_t epoch) +status_t AudioTrack::createTrack_l() { status_t status; const sp<IAudioFlinger>& audioFlinger = AudioSystem::get_audio_flinger(); @@ -1184,7 +1189,6 @@ status_t AudioTrack::createTrack_l(size_t epoch) mProxy->setVolumeLR(GAIN_MINIFLOAT_PACKED_UNITY); mProxy->setSendLevel(mSendLevel); mProxy->setSampleRate(mSampleRate); - mProxy->setEpoch(epoch); mProxy->setMinimum(mNotificationFramesAct); mDeathNotifier = new DeathNotifier(this); @@ -1319,6 +1323,7 @@ void AudioTrack::releaseBuffer(Buffer* audioBuffer) buffer.mRaw = audioBuffer->raw; AutoMutex lock(mLock); + mReleased += stepCount; mInUnderrun = false; mProxy->releaseBuffer(&buffer); @@ -1531,7 +1536,7 @@ nsecs_t AudioTrack::processAudioBuffer() } // Get current position of server - size_t position = mProxy->getPosition(); + size_t position = updateAndGetPosition_l(); // Manage marker callback bool markerReached = false; @@ -1796,14 +1801,18 @@ status_t AudioTrack::restoreTrack_l(const char *from) return DEAD_OBJECT; } - // if the new IAudioTrack is created, createTrack_l() will modify the + // save the old static buffer position + size_t bufferPosition = mStaticProxy != NULL ? mStaticProxy->getBufferPosition() : 0; + + // If a new IAudioTrack is successfully created, createTrack_l() will modify the // following member variables: mAudioTrack, mCblkMemory and mCblk. - // It will also delete the strong references on previous IAudioTrack and IMemory + // It will also delete the strong references on previous IAudioTrack and IMemory. + // If a new IAudioTrack cannot be created, the previous (dead) instance will be left intact. + result = createTrack_l(); // take the frames that will be lost by track recreation into account in saved position - size_t position = mProxy->getPosition() + mProxy->getFramesFilled(); - size_t bufferPosition = mStaticProxy != NULL ? mStaticProxy->getBufferPosition() : 0; - result = createTrack_l(position /*epoch*/); + (void) updateAndGetPosition_l(); + mPosition = mReleased; if (result == NO_ERROR) { // continue playback from last known position, but @@ -1838,6 +1847,27 @@ status_t AudioTrack::restoreTrack_l(const char *from) return result; } +uint32_t AudioTrack::updateAndGetPosition_l() +{ + // This is the sole place to read server consumed frames + uint32_t newServer = mProxy->getPosition(); + int32_t delta = newServer - mServer; + mServer = newServer; + // TODO There is controversy about whether there can be "negative jitter" in server position. + // This should be investigated further, and if possible, it should be addressed. + // A more definite failure mode is infrequent polling by client. + // One could call (void)getPosition_l() in releaseBuffer(), + // so mReleased and mPosition are always lock-step as best possible. + // That should ensure delta never goes negative for infrequent polling + // unless the server has more than 2^31 frames in its buffer, + // in which case the use of uint32_t for these counters has bigger issues. + if (delta < 0) { + ALOGE("detected illegal retrograde motion by the server: mServer advanced by %d", delta); + delta = 0; + } + return mPosition += (uint32_t) delta; +} + status_t AudioTrack::setParameters(const String8& keyValuePairs) { AutoMutex lock(mLock); @@ -1854,9 +1884,34 @@ status_t AudioTrack::getTimestamp(AudioTimestamp& timestamp) if (mState != STATE_ACTIVE && mState != STATE_PAUSED) { return INVALID_OPERATION; } + // The presented frame count must always lag behind the consumed frame count. + // To avoid a race, read the presented frames first. This ensures that presented <= consumed. status_t status = mAudioTrack->getTimestamp(timestamp); if (status == NO_ERROR) { - timestamp.mPosition += mProxy->getEpoch(); + // Update the mapping between local consumed (mPosition) and server consumed (mServer) + (void) updateAndGetPosition_l(); + // Server consumed (mServer) and presented both use the same server time base, + // and server consumed is always >= presented. + // The delta between these represents the number of frames in the buffer pipeline. + // If this delta between these is greater than the client position, it means that + // actually presented is still stuck at the starting line (figuratively speaking), + // waiting for the first frame to go by. So we can't report a valid timestamp yet. + if ((uint32_t) (mServer - timestamp.mPosition) > mPosition) { + return INVALID_OPERATION; + } + // Convert timestamp position from server time base to client time base. + // TODO The following code should work OK now because timestamp.mPosition is 32-bit. + // But if we change it to 64-bit then this could fail. + // If (mPosition - mServer) can be negative then should use: + // (int32_t)(mPosition - mServer) + timestamp.mPosition += mPosition - mServer; + // Immediately after a call to getPosition_l(), mPosition and + // mServer both represent the same frame position. mPosition is + // in client's point of view, and mServer is in server's point of + // view. So the difference between them is the "fudge factor" + // between client and server views due to stop() and/or new + // IAudioTrack. And timestamp.mPosition is initially in server's + // point of view, so we need to apply the same fudge factor to it. } return status; } diff --git a/media/libmedia/CharacterEncodingDetector.cpp b/media/libmedia/CharacterEncodingDetector.cpp index 7d1ddfd..41994dc 100644 --- a/media/libmedia/CharacterEncodingDetector.cpp +++ b/media/libmedia/CharacterEncodingDetector.cpp @@ -18,7 +18,7 @@ #define LOG_TAG "CharacterEncodingDector" #include <utils/Log.h> -#include "CharacterEncodingDetector.h" +#include <CharacterEncodingDetector.h> #include "CharacterEncodingDetectorTables.h" #include "utils/Vector.h" @@ -118,10 +118,12 @@ void CharacterEncodingDetector::detectAndConvert() { int32_t matches; const UCharsetMatch** ucma = ucsdet_detectAll(csd, &matches, &status); bool goodmatch = true; + int highest = 0; const UCharsetMatch* bestCombinedMatch = getPreferred(buf, strlen(buf), - ucma, matches, &goodmatch); + ucma, matches, &goodmatch, &highest); - if (!goodmatch && strlen(buf) < 20) { + ALOGV("goodmatch: %s, highest: %d", goodmatch ? "true" : "false", highest); + if (!goodmatch && (highest < 15 || strlen(buf) < 20)) { ALOGV("not a good match, trying with more data"); // This string might be too short for ICU to do anything useful with. // (real world example: "Björk" in ISO-8859-1 might be detected as GB18030, because @@ -146,9 +148,10 @@ void CharacterEncodingDetector::detectAndConvert() { ucsdet_setText(csd, buf, strlen(buf), &status); ucma = ucsdet_detectAll(csd, &matches, &status); bestCombinedMatch = getPreferred(buf, strlen(buf), - ucma, matches, &goodmatch); - if (!goodmatch) { + ucma, matches, &goodmatch, &highest); + if (!goodmatch && highest <= 15) { ALOGV("still not a good match after adding printable tags"); + bestCombinedMatch = NULL; } } else { ALOGV("no printable tags to add"); @@ -157,6 +160,8 @@ void CharacterEncodingDetector::detectAndConvert() { if (bestCombinedMatch != NULL) { combinedenc = ucsdet_getName(bestCombinedMatch, &status); + } else { + combinedenc = "ISO-8859-1"; } } @@ -199,10 +204,17 @@ void CharacterEncodingDetector::detectAndConvert() { if (strcmp(enc,"UTF-8") != 0) { // only convert if the source encoding isn't already UTF-8 ALOGV("@@@ using converter %s for %s", enc, mNames.getEntry(i)); + status = U_ZERO_ERROR; UConverter *conv = ucnv_open(enc, &status); if (U_FAILURE(status)) { - ALOGE("could not create UConverter for %s", enc); - continue; + ALOGW("could not create UConverter for %s (%d), falling back to ISO-8859-1", + enc, status); + status = U_ZERO_ERROR; + conv = ucnv_open("ISO-8859-1", &status); + if (U_FAILURE(status)) { + ALOGW("could not create UConverter for ISO-8859-1 either"); + continue; + } } // convert from native encoding to UTF-8 @@ -224,7 +236,16 @@ void CharacterEncodingDetector::detectAndConvert() { } else { // zero terminate *target = 0; - mValues.setEntry(i, buffer); + // strip trailing spaces + while (--target > buffer && *target == ' ') { + *target = 0; + } + // skip leading spaces + char *start = buffer; + while (*start == ' ') { + start++; + } + mValues.setEntry(i, start); } delete[] buffer; @@ -261,7 +282,7 @@ void CharacterEncodingDetector::detectAndConvert() { const UCharsetMatch *CharacterEncodingDetector::getPreferred( const char *input, size_t len, const UCharsetMatch** ucma, size_t nummatches, - bool *goodmatch) { + bool *goodmatch, int *highestmatch) { *goodmatch = false; Vector<const UCharsetMatch*> matches; @@ -316,11 +337,17 @@ const UCharsetMatch *CharacterEncodingDetector::getPreferred( } ALOGV("%zu: %s %d", i, encname, confidence); + status = U_ZERO_ERROR; UConverter *conv = ucnv_open(encname, &status); + int demerit = 0; + if (U_FAILURE(status)) { + ALOGV("failed to open %s: %d", encname, status); + confidence = 0; + demerit += 1000; + } const char *source = input; const char *sourceLimit = input + len; status = U_ZERO_ERROR; - int demerit = 0; int frequentchars = 0; int totalchars = 0; while (true) { @@ -337,7 +364,8 @@ const UCharsetMatch *CharacterEncodingDetector::getPreferred( if (c < 0x20 || (c >= 0x7f && c <= 0x009f)) { ALOGV("control character %x", c); demerit += 100; - } else if ((c >= 0xa0 && c <= 0xbe) // symbols, superscripts + } else if ((c == 0xa0) // no-break space + || (c >= 0xa2 && c <= 0xbe) // symbols, superscripts || (c == 0xd7) || (c == 0xf7) // multiplication and division signs || (c >= 0x2000 && c <= 0x209f)) { // punctuation, superscripts ALOGV("unlikely character %x", c); @@ -408,10 +436,14 @@ const UCharsetMatch *CharacterEncodingDetector::getPreferred( } else { ALOGV("runner up: '%s' w/ %d confidence", ucsdet_getName(matches[runnerupidx], &status), runnerup); + if (runnerup < 0) { + runnerup = 0; + } if ((highest - runnerup) > 15) { *goodmatch = true; } } + *highestmatch = highest; return matches[highestidx]; } diff --git a/media/libmedia/MediaProfiles.cpp b/media/libmedia/MediaProfiles.cpp index d2e181b..e2e6042 100644 --- a/media/libmedia/MediaProfiles.cpp +++ b/media/libmedia/MediaProfiles.cpp @@ -87,6 +87,7 @@ const MediaProfiles::NameToTagMap MediaProfiles::sCamcorderQualityNameMap[] = { {"highspeed480p", CAMCORDER_QUALITY_HIGH_SPEED_480P}, {"highspeed720p", CAMCORDER_QUALITY_HIGH_SPEED_720P}, {"highspeed1080p", CAMCORDER_QUALITY_HIGH_SPEED_1080P}, + {"highspeed2160p", CAMCORDER_QUALITY_HIGH_SPEED_2160P}, }; #if LOG_NDEBUG diff --git a/media/libmedia/MediaScannerClient.cpp b/media/libmedia/MediaScannerClient.cpp index 1661f04..9f803cb 100644 --- a/media/libmedia/MediaScannerClient.cpp +++ b/media/libmedia/MediaScannerClient.cpp @@ -25,14 +25,10 @@ namespace android { -MediaScannerClient::MediaScannerClient() - : mEncodingDetector(NULL) -{ +MediaScannerClient::MediaScannerClient() { } -MediaScannerClient::~MediaScannerClient() -{ - delete mEncodingDetector; +MediaScannerClient::~MediaScannerClient() { } void MediaScannerClient::setLocale(const char* locale) @@ -40,31 +36,16 @@ void MediaScannerClient::setLocale(const char* locale) mLocale = locale; // not currently used } -void MediaScannerClient::beginFile() -{ - delete mEncodingDetector; - mEncodingDetector = new CharacterEncodingDetector(); +void MediaScannerClient::beginFile() { } status_t MediaScannerClient::addStringTag(const char* name, const char* value) { - mEncodingDetector->addTag(name, value); + handleStringTag(name, value); return OK; } -void MediaScannerClient::endFile() -{ - mEncodingDetector->detectAndConvert(); - - int size = mEncodingDetector->size(); - if (size) { - for (int i = 0; i < size; i++) { - const char *name; - const char *value; - mEncodingDetector->getTag(i, &name, &value); - handleStringTag(name, value); - } - } +void MediaScannerClient::endFile() { } } // namespace android diff --git a/media/libmedia/mediaplayer.cpp b/media/libmedia/mediaplayer.cpp index 6cd377a..9611ac7 100644 --- a/media/libmedia/mediaplayer.cpp +++ b/media/libmedia/mediaplayer.cpp @@ -50,6 +50,7 @@ MediaPlayer::MediaPlayer() mListener = NULL; mCookie = NULL; mStreamType = AUDIO_STREAM_MUSIC; + mAudioAttributesParcel = NULL; mCurrentPosition = -1; mSeekPosition = -1; mCurrentState = MEDIA_PLAYER_IDLE; @@ -68,6 +69,10 @@ MediaPlayer::MediaPlayer() MediaPlayer::~MediaPlayer() { ALOGV("destructor"); + if (mAudioAttributesParcel != NULL) { + delete mAudioAttributesParcel; + mAudioAttributesParcel = NULL; + } AudioSystem::releaseAudioSessionId(mAudioSessionId, -1); disconnect(); IPCThreadState::self()->flushCommands(); @@ -237,6 +242,9 @@ status_t MediaPlayer::prepareAsync_l() { if ( (mPlayer != 0) && ( mCurrentState & ( MEDIA_PLAYER_INITIALIZED | MEDIA_PLAYER_STOPPED) ) ) { mPlayer->setAudioStreamType(mStreamType); + if (mAudioAttributesParcel != NULL) { + mPlayer->setParameter(KEY_PARAMETER_AUDIO_ATTRIBUTES, *mAudioAttributesParcel); + } mCurrentState = MEDIA_PLAYER_PREPARING; return mPlayer->prepareAsync(); } @@ -662,8 +670,17 @@ status_t MediaPlayer::setParameter(int key, const Parcel& request) if (mPlayer != NULL) { return mPlayer->setParameter(key, request); } - ALOGV("setParameter: no active player"); - return INVALID_OPERATION; + switch (key) { + case KEY_PARAMETER_AUDIO_ATTRIBUTES: + // no player, save the marshalled audio attributes + if (mAudioAttributesParcel != NULL) { delete mAudioAttributesParcel; }; + mAudioAttributesParcel = new Parcel(); + mAudioAttributesParcel->appendFrom(&request, 0, request.dataSize()); + return OK; + default: + ALOGV("setParameter: no active player"); + return INVALID_OPERATION; + } } status_t MediaPlayer::getParameter(int key, Parcel *reply) diff --git a/media/libmediaplayerservice/MediaPlayerFactory.cpp b/media/libmediaplayerservice/MediaPlayerFactory.cpp index dacb144..3e0fc0d 100644 --- a/media/libmediaplayerservice/MediaPlayerFactory.cpp +++ b/media/libmediaplayerservice/MediaPlayerFactory.cpp @@ -60,7 +60,7 @@ status_t MediaPlayerFactory::registerFactory_l(IFactory* factory, return OK; } -player_type MediaPlayerFactory::getDefaultPlayerType() { +static player_type getDefaultPlayerType() { char value[PROPERTY_VALUE_MAX]; if (property_get("media.stagefright.use-awesome", value, NULL) && (!strcmp("1", value) || !strcasecmp("true", value))) { @@ -181,16 +181,19 @@ class StagefrightPlayerFactory : int64_t offset, int64_t /*length*/, float /*curScore*/) { - char buf[20]; - lseek(fd, offset, SEEK_SET); - read(fd, buf, sizeof(buf)); - lseek(fd, offset, SEEK_SET); - - uint32_t ident = *((uint32_t*)buf); - - // Ogg vorbis? - if (ident == 0x5367674f) // 'OggS' - return 1.0; + if (getDefaultPlayerType() + == STAGEFRIGHT_PLAYER) { + char buf[20]; + lseek(fd, offset, SEEK_SET); + read(fd, buf, sizeof(buf)); + lseek(fd, offset, SEEK_SET); + + uint32_t ident = *((uint32_t*)buf); + + // Ogg vorbis? + if (ident == 0x5367674f) // 'OggS' + return 1.0; + } return 0.0; } diff --git a/media/libmediaplayerservice/MediaPlayerFactory.h b/media/libmediaplayerservice/MediaPlayerFactory.h index 5ddde19..55ff918 100644 --- a/media/libmediaplayerservice/MediaPlayerFactory.h +++ b/media/libmediaplayerservice/MediaPlayerFactory.h @@ -71,7 +71,6 @@ class MediaPlayerFactory { static status_t registerFactory_l(IFactory* factory, player_type type); - static player_type getDefaultPlayerType(); static Mutex sLock; static tFactoryMap sFactoryMap; diff --git a/media/libmediaplayerservice/MediaPlayerService.cpp b/media/libmediaplayerservice/MediaPlayerService.cpp index b5bd988..c8cb7ed 100644 --- a/media/libmediaplayerservice/MediaPlayerService.cpp +++ b/media/libmediaplayerservice/MediaPlayerService.cpp @@ -204,6 +204,8 @@ const int32_t kAudioAttributesMarshallTagFlattenTags = 1; // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ // | content_type | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ +// | source | +// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ // | flags | // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ // | kAudioAttributesMarshallTagFlattenTags | // ignore tags if not found @@ -219,6 +221,7 @@ void unmarshallAudioAttributes(const Parcel& parcel, audio_attributes_t *attribu { attributes->usage = (audio_usage_t) parcel.readInt32(); attributes->content_type = (audio_content_type_t) parcel.readInt32(); + attributes->source = (audio_source_t) parcel.readInt32(); attributes->flags = (audio_flags_mask_t) parcel.readInt32(); const bool hasFlattenedTag = (parcel.readInt32() == kAudioAttributesMarshallTagFlattenTags); if (hasFlattenedTag) { diff --git a/media/libmediaplayerservice/StagefrightRecorder.cpp b/media/libmediaplayerservice/StagefrightRecorder.cpp index e2bcb1e..b904aa8 100644 --- a/media/libmediaplayerservice/StagefrightRecorder.cpp +++ b/media/libmediaplayerservice/StagefrightRecorder.cpp @@ -183,11 +183,7 @@ status_t StagefrightRecorder::setVideoEncoder(video_encoder ve) { return BAD_VALUE; } - if (ve == VIDEO_ENCODER_DEFAULT) { - mVideoEncoder = VIDEO_ENCODER_H263; - } else { - mVideoEncoder = ve; - } + mVideoEncoder = ve; return OK; } @@ -1033,6 +1029,7 @@ status_t StagefrightRecorder::setupRTPRecording() { if (mAudioSource != AUDIO_SOURCE_CNT) { source = createAudioSource(); } else { + setDefaultVideoEncoderIfNecessary(); sp<MediaSource> mediaSource; status_t err = setupMediaSource(&mediaSource); @@ -1074,6 +1071,7 @@ status_t StagefrightRecorder::setupMPEG2TSRecording() { if (mVideoSource < VIDEO_SOURCE_LIST_END) { if (mVideoEncoder != VIDEO_ENCODER_H264) { + ALOGE("MPEG2TS recording only supports H.264 encoding!"); return ERROR_UNSUPPORTED; } @@ -1108,6 +1106,12 @@ status_t StagefrightRecorder::setupMPEG2TSRecording() { void StagefrightRecorder::clipVideoFrameRate() { ALOGV("clipVideoFrameRate: encoder %d", mVideoEncoder); + if (mFrameRate == -1) { + mFrameRate = mEncoderProfiles->getCamcorderProfileParamByName( + "vid.fps", mCameraId, CAMCORDER_QUALITY_LOW); + ALOGW("Using default video fps %d", mFrameRate); + } + int minFrameRate = mEncoderProfiles->getVideoEncoderParamByName( "enc.vid.fps.min", mVideoEncoder); int maxFrameRate = mEncoderProfiles->getVideoEncoderParamByName( @@ -1243,6 +1247,27 @@ void StagefrightRecorder::setDefaultProfileIfNecessary() { } } +void StagefrightRecorder::setDefaultVideoEncoderIfNecessary() { + if (mVideoEncoder == VIDEO_ENCODER_DEFAULT) { + if (mOutputFormat == OUTPUT_FORMAT_WEBM) { + // default to VP8 for WEBM recording + mVideoEncoder = VIDEO_ENCODER_VP8; + } else { + // pick the default encoder for CAMCORDER_QUALITY_LOW + int videoCodec = mEncoderProfiles->getCamcorderProfileParamByName( + "vid.codec", mCameraId, CAMCORDER_QUALITY_LOW); + + if (videoCodec > VIDEO_ENCODER_DEFAULT && + videoCodec < VIDEO_ENCODER_LIST_END) { + mVideoEncoder = (video_encoder)videoCodec; + } else { + // default to H.264 if camcorder profile not available + mVideoEncoder = VIDEO_ENCODER_H264; + } + } + } +} + status_t StagefrightRecorder::checkAudioEncoderCapabilities() { clipAudioBitRate(); clipAudioSampleRate(); @@ -1562,6 +1587,7 @@ status_t StagefrightRecorder::setupMPEG4orWEBMRecording() { } if (mVideoSource < VIDEO_SOURCE_LIST_END) { + setDefaultVideoEncoderIfNecessary(); sp<MediaSource> mediaSource; err = setupMediaSource(&mediaSource); @@ -1721,7 +1747,7 @@ status_t StagefrightRecorder::reset() { // Default parameters mOutputFormat = OUTPUT_FORMAT_THREE_GPP; mAudioEncoder = AUDIO_ENCODER_AMR_NB; - mVideoEncoder = VIDEO_ENCODER_H263; + mVideoEncoder = VIDEO_ENCODER_DEFAULT; mVideoWidth = 176; mVideoHeight = 144; mFrameRate = -1; diff --git a/media/libmediaplayerservice/StagefrightRecorder.h b/media/libmediaplayerservice/StagefrightRecorder.h index 9062f30..54c38d3 100644 --- a/media/libmediaplayerservice/StagefrightRecorder.h +++ b/media/libmediaplayerservice/StagefrightRecorder.h @@ -178,6 +178,7 @@ private: void clipAudioSampleRate(); void clipNumberOfAudioChannels(); void setDefaultProfileIfNecessary(); + void setDefaultVideoEncoderIfNecessary(); StagefrightRecorder(const StagefrightRecorder &); diff --git a/media/libmediaplayerservice/nuplayer/GenericSource.cpp b/media/libmediaplayerservice/nuplayer/GenericSource.cpp index f257ef3..511871d 100644 --- a/media/libmediaplayerservice/nuplayer/GenericSource.cpp +++ b/media/libmediaplayerservice/nuplayer/GenericSource.cpp @@ -32,6 +32,7 @@ #include <media/stagefright/MediaExtractor.h> #include <media/stagefright/MediaSource.h> #include <media/stagefright/MetaData.h> +#include <media/stagefright/Utils.h> #include "../../libstagefright/include/DRMExtractor.h" #include "../../libstagefright/include/NuCachedSource2.h" #include "../../libstagefright/include/WVMExtractor.h" @@ -53,7 +54,8 @@ NuPlayer::GenericSource::GenericSource( mDrmManagerClient(NULL), mMetaDataSize(-1ll), mBitrate(-1ll), - mPollBufferingGeneration(0) { + mPollBufferingGeneration(0), + mPendingReadBufferTypes(0) { resetDataSource(); DataSource::RegisterDefaultSniffers(); } @@ -168,6 +170,8 @@ status_t NuPlayer::GenericSource::initFromDataSource() { if (mAudioTrack.mSource == NULL) { mAudioTrack.mIndex = i; mAudioTrack.mSource = track; + mAudioTrack.mPackets = + new AnotherPacketSource(mAudioTrack.mSource->getFormat()); if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_VORBIS)) { mAudioIsVorbis = true; @@ -179,6 +183,8 @@ status_t NuPlayer::GenericSource::initFromDataSource() { if (mVideoTrack.mSource == NULL) { mVideoTrack.mIndex = i; mVideoTrack.mSource = track; + mVideoTrack.mPackets = + new AnotherPacketSource(mVideoTrack.mSource->getFormat()); // check if the source requires secure buffers int32_t secure; @@ -318,7 +324,14 @@ void NuPlayer::GenericSource::onPrepareAsync() { } if (mVideoTrack.mSource != NULL) { - notifyVideoSizeChanged(getFormat(false /* audio */)); + sp<MetaData> meta = doGetFormatMeta(false /* audio */); + sp<AMessage> msg = new AMessage; + err = convertMetaDataToMessage(meta, &msg); + if(err != OK) { + notifyPreparedAndCleanup(err); + return; + } + notifyVideoSizeChanged(msg); } notifyFlagsChanged( @@ -419,18 +432,14 @@ void NuPlayer::GenericSource::start() { if (mAudioTrack.mSource != NULL) { CHECK_EQ(mAudioTrack.mSource->start(), (status_t)OK); - mAudioTrack.mPackets = - new AnotherPacketSource(mAudioTrack.mSource->getFormat()); - readBuffer(MEDIA_TRACK_TYPE_AUDIO); + postReadBuffer(MEDIA_TRACK_TYPE_AUDIO); } if (mVideoTrack.mSource != NULL) { CHECK_EQ(mVideoTrack.mSource->start(), (status_t)OK); - mVideoTrack.mPackets = - new AnotherPacketSource(mVideoTrack.mSource->getFormat()); - readBuffer(MEDIA_TRACK_TYPE_VIDEO); + postReadBuffer(MEDIA_TRACK_TYPE_VIDEO); } setDrmPlaybackStatusIfNeeded(Playback::START, getLastReadPosition() / 1000); @@ -459,6 +468,8 @@ void NuPlayer::GenericSource::setDrmPlaybackStatusIfNeeded(int playbackStatus, i if (mDecryptHandle != NULL) { mDrmManagerClient->setPlaybackStatus(mDecryptHandle, playbackStatus, position); } + mSubtitleTrack.mPackets = new AnotherPacketSource(NULL); + mTimedTextTrack.mPackets = new AnotherPacketSource(NULL); } status_t NuPlayer::GenericSource::feedMoreTSData() { @@ -615,6 +626,37 @@ void NuPlayer::GenericSource::onMessageReceived(const sp<AMessage> &msg) { } break; } + + case kWhatGetFormat: + { + onGetFormatMeta(msg); + break; + } + + case kWhatGetSelectedTrack: + { + onGetSelectedTrack(msg); + break; + } + + case kWhatSelectTrack: + { + onSelectTrack(msg); + break; + } + + case kWhatSeek: + { + onSeek(msg); + break; + } + + case kWhatReadBuffer: + { + onReadBuffer(msg); + break; + } + default: Source::onMessageReceived(msg); break; @@ -690,6 +732,34 @@ void NuPlayer::GenericSource::sendTextData( } sp<MetaData> NuPlayer::GenericSource::getFormatMeta(bool audio) { + sp<AMessage> msg = new AMessage(kWhatGetFormat, id()); + msg->setInt32("audio", audio); + + sp<AMessage> response; + void *format; + status_t err = msg->postAndAwaitResponse(&response); + if (err == OK && response != NULL) { + CHECK(response->findPointer("format", &format)); + return (MetaData *)format; + } else { + return NULL; + } +} + +void NuPlayer::GenericSource::onGetFormatMeta(sp<AMessage> msg) const { + int32_t audio; + CHECK(msg->findInt32("audio", &audio)); + + sp<AMessage> response = new AMessage; + sp<MetaData> format = doGetFormatMeta(audio); + response->setPointer("format", format.get()); + + uint32_t replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + response->postReply(replyID); +} + +sp<MetaData> NuPlayer::GenericSource::doGetFormatMeta(bool audio) const { sp<MediaSource> source = audio ? mAudioTrack.mSource : mVideoTrack.mSource; if (source == NULL) { @@ -709,7 +779,7 @@ status_t NuPlayer::GenericSource::dequeueAccessUnit( if (mIsWidevine && !audio) { // try to read a buffer as we may not have been able to the last time - readBuffer(MEDIA_TRACK_TYPE_VIDEO, -1ll); + postReadBuffer(MEDIA_TRACK_TYPE_VIDEO); } status_t finalResult; @@ -720,18 +790,7 @@ status_t NuPlayer::GenericSource::dequeueAccessUnit( status_t result = track->mPackets->dequeueAccessUnit(accessUnit); if (!track->mPackets->hasBufferAvailable(&finalResult)) { - readBuffer(audio? MEDIA_TRACK_TYPE_AUDIO : MEDIA_TRACK_TYPE_VIDEO, -1ll); - } - - if (mSubtitleTrack.mSource == NULL && mTimedTextTrack.mSource == NULL) { - return result; - } - - if (mSubtitleTrack.mSource != NULL) { - CHECK(mSubtitleTrack.mPackets != NULL); - } - if (mTimedTextTrack.mSource != NULL) { - CHECK(mTimedTextTrack.mPackets != NULL); + postReadBuffer(audio? MEDIA_TRACK_TYPE_AUDIO : MEDIA_TRACK_TYPE_VIDEO); } if (result != OK) { @@ -825,6 +884,35 @@ sp<AMessage> NuPlayer::GenericSource::getTrackInfo(size_t trackIndex) const { } ssize_t NuPlayer::GenericSource::getSelectedTrack(media_track_type type) const { + sp<AMessage> msg = new AMessage(kWhatGetSelectedTrack, id()); + msg->setInt32("type", type); + + sp<AMessage> response; + int32_t index; + status_t err = msg->postAndAwaitResponse(&response); + if (err == OK && response != NULL) { + CHECK(response->findInt32("index", &index)); + return index; + } else { + return -1; + } +} + +void NuPlayer::GenericSource::onGetSelectedTrack(sp<AMessage> msg) const { + int32_t tmpType; + CHECK(msg->findInt32("type", &tmpType)); + media_track_type type = (media_track_type)tmpType; + + sp<AMessage> response = new AMessage; + ssize_t index = doGetSelectedTrack(type); + response->setInt32("index", index); + + uint32_t replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + response->postReply(replyID); +} + +ssize_t NuPlayer::GenericSource::doGetSelectedTrack(media_track_type type) const { const Track *track = NULL; switch (type) { case MEDIA_TRACK_TYPE_VIDEO: @@ -852,6 +940,34 @@ ssize_t NuPlayer::GenericSource::getSelectedTrack(media_track_type type) const { status_t NuPlayer::GenericSource::selectTrack(size_t trackIndex, bool select) { ALOGV("%s track: %zu", select ? "select" : "deselect", trackIndex); + sp<AMessage> msg = new AMessage(kWhatSelectTrack, id()); + msg->setInt32("trackIndex", trackIndex); + msg->setInt32("select", trackIndex); + + sp<AMessage> response; + status_t err = msg->postAndAwaitResponse(&response); + if (err == OK && response != NULL) { + CHECK(response->findInt32("err", &err)); + } + + return err; +} + +void NuPlayer::GenericSource::onSelectTrack(sp<AMessage> msg) { + int32_t trackIndex, select; + CHECK(msg->findInt32("trackIndex", &trackIndex)); + CHECK(msg->findInt32("select", &select)); + + sp<AMessage> response = new AMessage; + status_t err = doSelectTrack(trackIndex, select); + response->setInt32("err", err); + + uint32_t replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + response->postReply(replyID); +} + +status_t NuPlayer::GenericSource::doSelectTrack(size_t trackIndex, bool select) { if (trackIndex >= mSources.size()) { return BAD_INDEX; } @@ -922,6 +1038,32 @@ status_t NuPlayer::GenericSource::selectTrack(size_t trackIndex, bool select) { } status_t NuPlayer::GenericSource::seekTo(int64_t seekTimeUs) { + sp<AMessage> msg = new AMessage(kWhatSeek, id()); + msg->setInt64("seekTimeUs", seekTimeUs); + + sp<AMessage> response; + status_t err = msg->postAndAwaitResponse(&response); + if (err == OK && response != NULL) { + CHECK(response->findInt32("err", &err)); + } + + return err; +} + +void NuPlayer::GenericSource::onSeek(sp<AMessage> msg) { + int64_t seekTimeUs; + CHECK(msg->findInt64("seekTimeUs", &seekTimeUs)); + + sp<AMessage> response = new AMessage; + status_t err = doSeek(seekTimeUs); + response->setInt32("err", err); + + uint32_t replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + response->postReply(replyID); +} + +status_t NuPlayer::GenericSource::doSeek(int64_t seekTimeUs) { if (mVideoTrack.mSource != NULL) { int64_t actualTimeUs; readBuffer(MEDIA_TRACK_TYPE_VIDEO, seekTimeUs, &actualTimeUs); @@ -1006,15 +1148,42 @@ sp<ABuffer> NuPlayer::GenericSource::mediaBufferToABuffer( return ab; } +void NuPlayer::GenericSource::postReadBuffer(media_track_type trackType) { + Mutex::Autolock _l(mReadBufferLock); + + if ((mPendingReadBufferTypes & (1 << trackType)) == 0) { + mPendingReadBufferTypes |= (1 << trackType); + sp<AMessage> msg = new AMessage(kWhatReadBuffer, id()); + msg->setInt32("trackType", trackType); + msg->post(); + } +} + +void NuPlayer::GenericSource::onReadBuffer(sp<AMessage> msg) { + int32_t tmpType; + CHECK(msg->findInt32("trackType", &tmpType)); + media_track_type trackType = (media_track_type)tmpType; + { + // only protect the variable change, as readBuffer may + // take considerable time. This may result in one extra + // read being processed, but that is benign. + Mutex::Autolock _l(mReadBufferLock); + mPendingReadBufferTypes &= ~(1 << trackType); + } + readBuffer(trackType); +} + void NuPlayer::GenericSource::readBuffer( media_track_type trackType, int64_t seekTimeUs, int64_t *actualTimeUs, bool formatChange) { Track *track; + size_t maxBuffers = 1; switch (trackType) { case MEDIA_TRACK_TYPE_VIDEO: track = &mVideoTrack; break; case MEDIA_TRACK_TYPE_AUDIO: track = &mAudioTrack; + maxBuffers = 64; break; case MEDIA_TRACK_TYPE_SUBTITLE: track = &mSubtitleTrack; @@ -1047,7 +1216,7 @@ void NuPlayer::GenericSource::readBuffer( options.setNonBlocking(); } - for (;;) { + for (size_t numBuffers = 0; numBuffers < maxBuffers; ) { MediaBuffer *mbuf; status_t err = track->mSource->read(&mbuf, &options); @@ -1078,7 +1247,7 @@ void NuPlayer::GenericSource::readBuffer( sp<ABuffer> buffer = mediaBufferToABuffer(mbuf, trackType, actualTimeUs); track->mPackets->queueAccessUnit(buffer); - break; + ++numBuffers; } else if (err == WOULD_BLOCK) { break; } else if (err == INFO_FORMAT_CHANGED) { diff --git a/media/libmediaplayerservice/nuplayer/GenericSource.h b/media/libmediaplayerservice/nuplayer/GenericSource.h index 1f13120..c70c48e 100644 --- a/media/libmediaplayerservice/nuplayer/GenericSource.h +++ b/media/libmediaplayerservice/nuplayer/GenericSource.h @@ -84,6 +84,11 @@ private: kWhatSendTimedTextData, kWhatChangeAVSource, kWhatPollBuffering, + kWhatGetFormat, + kWhatGetSelectedTrack, + kWhatSelectTrack, + kWhatSeek, + kWhatReadBuffer, }; Vector<sp<MediaSource> > mSources; @@ -126,6 +131,8 @@ private: off64_t mMetaDataSize; int64_t mBitrate; int32_t mPollBufferingGeneration; + uint32_t mPendingReadBufferTypes; + mutable Mutex mReadBufferLock; sp<ALooper> mLooper; @@ -140,6 +147,18 @@ private: void notifyPreparedAndCleanup(status_t err); + void onGetFormatMeta(sp<AMessage> msg) const; + sp<MetaData> doGetFormatMeta(bool audio) const; + + void onGetSelectedTrack(sp<AMessage> msg) const; + ssize_t doGetSelectedTrack(media_track_type type) const; + + void onSelectTrack(sp<AMessage> msg); + status_t doSelectTrack(size_t trackIndex, bool select); + + void onSeek(sp<AMessage> msg); + status_t doSeek(int64_t seekTimeUs); + void onPrepareAsync(); void fetchTextData( @@ -155,6 +174,8 @@ private: media_track_type trackType, int64_t *actualTimeUs = NULL); + void postReadBuffer(media_track_type trackType); + void onReadBuffer(sp<AMessage> msg); void readBuffer( media_track_type trackType, int64_t seekTimeUs = -1ll, int64_t *actualTimeUs = NULL, bool formatChange = false); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp index 76d25de..9020a8d 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp @@ -50,6 +50,10 @@ namespace android { +// TODO optimize buffer size for power consumption +// The offload read buffer size is 32 KB but 24 KB uses less power. +const size_t NuPlayer::kAggregateBufferSizeBytes = 24 * 1024; + struct NuPlayer::Action : public RefBase { Action() {} @@ -145,6 +149,7 @@ private: NuPlayer::NuPlayer() : mUIDValid(false), mSourceFlags(0), + mCurrentPositionUs(0), mVideoIsAVC(false), mOffloadAudio(false), mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER), @@ -540,6 +545,14 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { static_cast<NativeWindowWrapper *>(obj.get()))); if (obj != NULL) { + if (mStarted && mVideoDecoder != NULL) { + // Issue a seek to refresh the video screen only if started otherwise + // the extractor may not yet be started and will assert. + // If the video decoder is not set (perhaps audio only in this case) + // do not perform a seek as it is not needed. + mDeferredActions.push_back(new SeekAction(mCurrentPositionUs)); + } + // If there is a new surface texture, instantiate decoders // again if possible. mDeferredActions.push_back( @@ -721,7 +734,7 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { if (err == -EWOULDBLOCK) { if (mSource->feedMoreTSData() == OK) { - msg->post(10000ll); + msg->post(10 * 1000ll); } } } else if (what == Decoder::kWhatEOS) { @@ -860,6 +873,7 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) { } else if (what == Renderer::kWhatPosition) { int64_t positionUs; CHECK(msg->findInt64("positionUs", &positionUs)); + mCurrentPositionUs = positionUs; CHECK(msg->findInt64("videoLateByUs", &mVideoLateByUs)); @@ -984,6 +998,9 @@ void NuPlayer::finishFlushIfPossible() { ALOGV("both audio and video are flushed now."); + mPendingAudioAccessUnit.clear(); + mAggregateBuffer.clear(); + if (mTimeDiscontinuityPending) { mRenderer->signalTimeDiscontinuity(); mTimeDiscontinuityPending = false; @@ -1232,7 +1249,8 @@ status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) { CHECK(msg->findMessage("reply", &reply)); if ((audio && mFlushingAudio != NONE) - || (!audio && mFlushingVideo != NONE)) { + || (!audio && mFlushingVideo != NONE) + || mSource == NULL) { reply->setInt32("err", INFO_DISCONTINUITY); reply->post(); return OK; @@ -1240,14 +1258,37 @@ status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) { sp<ABuffer> accessUnit; + // Aggregate smaller buffers into a larger buffer. + // The goal is to reduce power consumption. + // Unfortunately this does not work with the software AAC decoder. + bool doBufferAggregation = (audio && mOffloadAudio);; + bool needMoreData = false; + bool dropAccessUnit; do { - status_t err = mSource->dequeueAccessUnit(audio, &accessUnit); + status_t err; + // Did we save an accessUnit earlier because of a discontinuity? + if (audio && (mPendingAudioAccessUnit != NULL)) { + accessUnit = mPendingAudioAccessUnit; + mPendingAudioAccessUnit.clear(); + err = mPendingAudioErr; + ALOGV("feedDecoderInputData() use mPendingAudioAccessUnit"); + } else { + err = mSource->dequeueAccessUnit(audio, &accessUnit); + } if (err == -EWOULDBLOCK) { return err; } else if (err != OK) { if (err == INFO_DISCONTINUITY) { + if (mAggregateBuffer != NULL) { + // We already have some data so save this for later. + mPendingAudioErr = err; + mPendingAudioAccessUnit = accessUnit; + accessUnit.clear(); + ALOGD("feedDecoderInputData() save discontinuity for later"); + break; + } int32_t type; CHECK(accessUnit->meta()->findInt32("discontinuity", &type)); @@ -1352,7 +1393,51 @@ status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) { dropAccessUnit = true; ++mNumFramesDropped; } - } while (dropAccessUnit); + + size_t smallSize = accessUnit->size(); + needMoreData = false; + if (doBufferAggregation && (mAggregateBuffer == NULL) + // Don't bother if only room for a few small buffers. + && (smallSize < (kAggregateBufferSizeBytes / 3))) { + // Create a larger buffer for combining smaller buffers from the extractor. + mAggregateBuffer = new ABuffer(kAggregateBufferSizeBytes); + mAggregateBuffer->setRange(0, 0); // start empty + } + + if (mAggregateBuffer != NULL) { + int64_t timeUs; + int64_t dummy; + bool smallTimestampValid = accessUnit->meta()->findInt64("timeUs", &timeUs); + bool bigTimestampValid = mAggregateBuffer->meta()->findInt64("timeUs", &dummy); + // Will the smaller buffer fit? + size_t bigSize = mAggregateBuffer->size(); + size_t roomLeft = mAggregateBuffer->capacity() - bigSize; + // Should we save this small buffer for the next big buffer? + // If the first small buffer did not have a timestamp then save + // any buffer that does have a timestamp until the next big buffer. + if ((smallSize > roomLeft) + || (!bigTimestampValid && (bigSize > 0) && smallTimestampValid)) { + mPendingAudioErr = err; + mPendingAudioAccessUnit = accessUnit; + accessUnit.clear(); + } else { + // Grab time from first small buffer if available. + if ((bigSize == 0) && smallTimestampValid) { + mAggregateBuffer->meta()->setInt64("timeUs", timeUs); + } + // Append small buffer to the bigger buffer. + memcpy(mAggregateBuffer->base() + bigSize, accessUnit->data(), smallSize); + bigSize += smallSize; + mAggregateBuffer->setRange(0, bigSize); + + // Keep looping until we run out of room in the mAggregateBuffer. + needMoreData = true; + + ALOGV("feedDecoderInputData() smallSize = %zu, bigSize = %zu, capacity = %zu", + smallSize, bigSize, mAggregateBuffer->capacity()); + } + } + } while (dropAccessUnit || needMoreData); // ALOGV("returned a valid buffer of %s data", audio ? "audio" : "video"); @@ -1368,7 +1453,15 @@ status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) { mCCDecoder->decode(accessUnit); } - reply->setBuffer("buffer", accessUnit); + if (mAggregateBuffer != NULL) { + ALOGV("feedDecoderInputData() reply with aggregated buffer, %zu", + mAggregateBuffer->size()); + reply->setBuffer("buffer", mAggregateBuffer); + mAggregateBuffer.clear(); + } else { + reply->setBuffer("buffer", accessUnit); + } + reply->post(); return OK; @@ -1536,6 +1629,10 @@ void NuPlayer::flushDecoder( ALOGE_IF(mFlushingVideo != NONE, "video flushDecoder() is called in state %d", mFlushingVideo); mFlushingVideo = newStatus; + + if (mCCDecoder != NULL) { + mCCDecoder->flush(); + } } } @@ -1661,6 +1758,14 @@ void NuPlayer::performSeek(int64_t seekTimeUs) { seekTimeUs, seekTimeUs / 1E6); + if (mSource == NULL) { + // This happens when reset occurs right before the loop mode + // asynchronously seeks to the start of the stream. + LOG_ALWAYS_FATAL_IF(mAudioDecoder != NULL || mVideoDecoder != NULL, + "mSource is NULL and decoders not NULL audio(%p) video(%p)", + mAudioDecoder.get(), mVideoDecoder.get()); + return; + } mSource->seekTo(seekTimeUs); ++mTimedTextGeneration; @@ -1723,6 +1828,9 @@ void NuPlayer::performReset() { ++mScanSourcesGeneration; mScanSourcesPending = false; + ++mAudioDecoderGeneration; + ++mVideoDecoderGeneration; + if (mRendererLooper != NULL) { if (mRenderer != NULL) { mRendererLooper->unregisterHandler(mRenderer->id()); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h index 511d752..2e951bd 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h @@ -67,6 +67,8 @@ struct NuPlayer : public AHandler { status_t getSelectedTrack(int32_t type, Parcel* reply) const; status_t selectTrack(size_t trackIndex, bool select); + static const size_t kAggregateBufferSizeBytes; + protected: virtual ~NuPlayer(); @@ -121,6 +123,7 @@ private: sp<Source> mSource; uint32_t mSourceFlags; sp<NativeWindowWrapper> mNativeWindow; + int64_t mCurrentPositionUs; sp<MediaPlayerBase::AudioSink> mAudioSink; sp<Decoder> mVideoDecoder; bool mVideoIsAVC; @@ -157,6 +160,12 @@ private: // notion of time has changed. bool mTimeDiscontinuityPending; + // Used by feedDecoderInputData to aggregate small buffers into + // one large buffer. + sp<ABuffer> mPendingAudioAccessUnit; + status_t mPendingAudioErr; + sp<ABuffer> mAggregateBuffer; + FlushStatus mFlushingAudio; FlushStatus mFlushingVideo; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp index 5aaf48c..87f85e7 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp @@ -122,14 +122,17 @@ void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) { mCodec->getName(&mComponentName); + status_t err; if (mNativeWindow != NULL) { // disconnect from surface as MediaCodec will reconnect - CHECK_EQ((int)NO_ERROR, - native_window_api_disconnect( - surface.get(), - NATIVE_WINDOW_API_MEDIA)); + err = native_window_api_disconnect( + surface.get(), NATIVE_WINDOW_API_MEDIA); + // We treat this as a warning, as this is a preparatory step. + // Codec will try to connect to the surface, which is where + // any error signaling will occur. + ALOGW_IF(err != OK, "failed to disconnect from surface: %d", err); } - status_t err = mCodec->configure( + err = mCodec->configure( format, surface, NULL /* crypto */, 0 /* flags */); if (err != OK) { ALOGE("Failed to configure %s decoder (err=%d)", mComponentName.c_str(), err); @@ -595,7 +598,18 @@ void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) { { if (!isStaleReply(msg)) { onInputBufferFilled(msg); + } else { + /* release any MediaBuffer passed in the stale buffer */ + sp<ABuffer> buffer; + MediaBuffer *mediaBuffer = NULL; + if (msg->findBuffer("buffer", &buffer) && + buffer->meta()->findPointer( + "mediaBuffer", (void **)&mediaBuffer) && + mediaBuffer != NULL) { + mediaBuffer->release(); + } } + break; } @@ -716,72 +730,28 @@ bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetF return seamless; } -struct NuPlayer::CCDecoder::CCData { +struct CCData { CCData(uint8_t type, uint8_t data1, uint8_t data2) : mType(type), mData1(data1), mData2(data2) { } + bool getChannel(size_t *channel) const { + if (mData1 >= 0x10 && mData1 <= 0x1f) { + *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0); + return true; + } + return false; + } uint8_t mType; uint8_t mData1; uint8_t mData2; }; -NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> ¬ify) - : mNotify(notify), - mTrackCount(0), - mSelectedTrack(-1) { -} - -size_t NuPlayer::CCDecoder::getTrackCount() const { - return mTrackCount; -} - -sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const { - CHECK(index == 0); - - sp<AMessage> format = new AMessage(); - - format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE); - format->setString("language", "und"); - format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608); - format->setInt32("auto", 1); - format->setInt32("default", 1); - format->setInt32("forced", 0); - - return format; -} - -status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) { - CHECK(index < mTrackCount); - - if (select) { - if (mSelectedTrack == (ssize_t)index) { - ALOGE("track %zu already selected", index); - return BAD_VALUE; - } - ALOGV("selected track %zu", index); - mSelectedTrack = index; - } else { - if (mSelectedTrack != (ssize_t)index) { - ALOGE("track %zu is not selected", index); - return BAD_VALUE; - } - ALOGV("unselected track %zu", index); - mSelectedTrack = -1; - } - - return OK; -} - -bool NuPlayer::CCDecoder::isSelected() const { - return mSelectedTrack >= 0 && mSelectedTrack < (int32_t)mTrackCount; -} - -bool NuPlayer::CCDecoder::isNullPad(CCData *cc) const { +static bool isNullPad(CCData *cc) { return cc->mData1 < 0x10 && cc->mData2 < 0x10; } -void NuPlayer::CCDecoder::dumpBytePair(const sp<ABuffer> &ccBuf) const { +static void dumpBytePair(const sp<ABuffer> &ccBuf) { size_t offset = 0; AString out; @@ -843,6 +813,78 @@ void NuPlayer::CCDecoder::dumpBytePair(const sp<ABuffer> &ccBuf) const { ALOGI("%s", out.c_str()); } +NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> ¬ify) + : mNotify(notify), + mCurrentChannel(0), + mSelectedTrack(-1) { + for (size_t i = 0; i < sizeof(mTrackIndices)/sizeof(mTrackIndices[0]); ++i) { + mTrackIndices[i] = -1; + } +} + +size_t NuPlayer::CCDecoder::getTrackCount() const { + return mFoundChannels.size(); +} + +sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const { + if (!isTrackValid(index)) { + return NULL; + } + + sp<AMessage> format = new AMessage(); + + format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE); + format->setString("language", "und"); + format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608); + //CC1, field 0 channel 0 + bool isDefaultAuto = (mFoundChannels[index] == 0); + format->setInt32("auto", isDefaultAuto); + format->setInt32("default", isDefaultAuto); + format->setInt32("forced", 0); + + return format; +} + +status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) { + if (!isTrackValid(index)) { + return BAD_VALUE; + } + + if (select) { + if (mSelectedTrack == (ssize_t)index) { + ALOGE("track %zu already selected", index); + return BAD_VALUE; + } + ALOGV("selected track %zu", index); + mSelectedTrack = index; + } else { + if (mSelectedTrack != (ssize_t)index) { + ALOGE("track %zu is not selected", index); + return BAD_VALUE; + } + ALOGV("unselected track %zu", index); + mSelectedTrack = -1; + } + + return OK; +} + +bool NuPlayer::CCDecoder::isSelected() const { + return mSelectedTrack >= 0 && mSelectedTrack < (int32_t) getTrackCount(); +} + +bool NuPlayer::CCDecoder::isTrackValid(size_t index) const { + return index < getTrackCount(); +} + +int32_t NuPlayer::CCDecoder::getTrackIndex(size_t channel) const { + if (channel < sizeof(mTrackIndices)/sizeof(mTrackIndices[0])) { + return mTrackIndices[channel]; + } + return -1; +} + +// returns true if a new CC track is found bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) { int64_t timeUs; CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); @@ -852,7 +894,7 @@ bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) { return false; } - bool hasCC = false; + bool trackAdded = false; NALBitReader br(sei->data() + 1, sei->size() - 1); // sei_message() @@ -887,8 +929,6 @@ bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) { && itu_t_t35_provider_code == 0x0031 && user_identifier == 'GA94' && user_data_type_code == 0x3) { - hasCC = true; - // MPEG_cc_data() // ATSC A/53 Part 4: 6.2.3.1 br.skipBits(1); //process_em_data_flag @@ -918,6 +958,12 @@ bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) { && (cc_type == 0 || cc_type == 1)) { CCData cc(cc_type, cc_data_1, cc_data_2); if (!isNullPad(&cc)) { + size_t channel; + if (cc.getChannel(&channel) && getTrackIndex(channel) < 0) { + mTrackIndices[channel] = mFoundChannels.size(); + mFoundChannels.push_back(channel); + trackAdded = true; + } memcpy(ccBuf->data() + ccBuf->size(), (void *)&cc, sizeof(cc)); ccBuf->setRange(0, ccBuf->size() + sizeof(CCData)); @@ -940,13 +986,33 @@ bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) { br.skipBits(payload_size * 8); } - return hasCC; + return trackAdded; } -void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) { - if (extractFromSEI(accessUnit) && mTrackCount == 0) { - mTrackCount++; +sp<ABuffer> NuPlayer::CCDecoder::filterCCBuf( + const sp<ABuffer> &ccBuf, size_t index) { + sp<ABuffer> filteredCCBuf = new ABuffer(ccBuf->size()); + filteredCCBuf->setRange(0, 0); + + size_t cc_count = ccBuf->size() / sizeof(CCData); + const CCData* cc_data = (const CCData*)ccBuf->data(); + for (size_t i = 0; i < cc_count; ++i) { + size_t channel; + if (cc_data[i].getChannel(&channel)) { + mCurrentChannel = channel; + } + if (mCurrentChannel == mFoundChannels[index]) { + memcpy(filteredCCBuf->data() + filteredCCBuf->size(), + (void *)&cc_data[i], sizeof(CCData)); + filteredCCBuf->setRange(0, filteredCCBuf->size() + sizeof(CCData)); + } + } + + return filteredCCBuf; +} +void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) { + if (extractFromSEI(accessUnit)) { ALOGI("Found CEA-608 track"); sp<AMessage> msg = mNotify->dup(); msg->setInt32("what", kWhatTrackAdded); @@ -956,13 +1022,18 @@ void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) { } void NuPlayer::CCDecoder::display(int64_t timeUs) { + if (!isTrackValid(mSelectedTrack)) { + ALOGE("Could not find current track(index=%d)", mSelectedTrack); + return; + } + ssize_t index = mCCMap.indexOfKey(timeUs); if (index < 0) { ALOGV("cc for timestamp %" PRId64 " not found", timeUs); return; } - sp<ABuffer> &ccBuf = mCCMap.editValueAt(index); + sp<ABuffer> ccBuf = filterCCBuf(mCCMap.valueAt(index), mSelectedTrack); if (ccBuf->size() > 0) { #if 0 @@ -983,5 +1054,9 @@ void NuPlayer::CCDecoder::display(int64_t timeUs) { mCCMap.removeItemsAt(0, index + 1); } +void NuPlayer::CCDecoder::flush() { + mCCMap.clear(); +} + } // namespace android diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h index 67bddb8..cc1bdff 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h @@ -126,18 +126,20 @@ struct NuPlayer::CCDecoder : public RefBase { bool isSelected() const; void decode(const sp<ABuffer> &accessUnit); void display(int64_t timeUs); + void flush(); private: - struct CCData; - sp<AMessage> mNotify; KeyedVector<int64_t, sp<ABuffer> > mCCMap; - size_t mTrackCount; + size_t mCurrentChannel; int32_t mSelectedTrack; + int32_t mTrackIndices[4]; + Vector<size_t> mFoundChannels; - bool isNullPad(CCData *cc) const; - void dumpBytePair(const sp<ABuffer> &ccBuf) const; + bool isTrackValid(size_t index) const; + int32_t getTrackIndex(size_t channel) const; bool extractFromSEI(const sp<ABuffer> &accessUnit); + sp<ABuffer> filterCCBuf(const sp<ABuffer> &ccBuf, size_t index); DISALLOW_EVIL_CONSTRUCTORS(CCDecoder); }; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp index c9be0dd..f7aacdd 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.cpp @@ -30,8 +30,10 @@ namespace android { -static const int kMaxPendingBuffers = 10; -static const int kMaxCachedBytes = 200000; +static const size_t kMaxCachedBytes = 200000; +// The buffers will contain a bit less than kAggregateBufferSizeBytes. +// So we can start off with just enough buffers to keep the cache full. +static const size_t kMaxPendingBuffers = 1 + (kMaxCachedBytes / NuPlayer::kAggregateBufferSizeBytes); NuPlayer::DecoderPassThrough::DecoderPassThrough( const sp<AMessage> ¬ify) @@ -39,7 +41,8 @@ NuPlayer::DecoderPassThrough::DecoderPassThrough( mNotify(notify), mBufferGeneration(0), mReachedEOS(true), - mPendingBuffers(0), + mPendingBuffersToFill(0), + mPendingBuffersToDrain(0), mCachedBytes(0), mComponentName("pass through decoder") { mDecoderLooper = new ALooper; @@ -79,12 +82,13 @@ bool NuPlayer::DecoderPassThrough::supportsSeamlessFormatChange( void NuPlayer::DecoderPassThrough::onConfigure(const sp<AMessage> &format) { ALOGV("[%s] onConfigure", mComponentName.c_str()); - mPendingBuffers = 0; mCachedBytes = 0; + mPendingBuffersToFill = 0; + mPendingBuffersToDrain = 0; mReachedEOS = false; ++mBufferGeneration; - requestABuffer(); + requestMaxBuffers(); sp<AMessage> notify = mNotify->dup(); notify->setInt32("what", kWhatOutputFormatChanged); @@ -98,12 +102,15 @@ bool NuPlayer::DecoderPassThrough::isStaleReply(const sp<AMessage> &msg) { return generation != mBufferGeneration; } -void NuPlayer::DecoderPassThrough::requestABuffer() { - if (mCachedBytes >= kMaxCachedBytes || mReachedEOS) { - ALOGV("[%s] mReachedEOS=%d, max pending buffers(%d:%d)", - mComponentName.c_str(), (mReachedEOS ? 1 : 0), - mPendingBuffers, kMaxPendingBuffers); - return; +bool NuPlayer::DecoderPassThrough::requestABuffer() { + if (mCachedBytes >= kMaxCachedBytes) { + ALOGV("[%s] mCachedBytes = %zu", + mComponentName.c_str(), mCachedBytes); + return false; + } + if (mReachedEOS) { + ALOGV("[%s] reached EOS", mComponentName.c_str()); + return false; } sp<AMessage> reply = new AMessage(kWhatInputBufferFilled, id()); @@ -113,16 +120,16 @@ void NuPlayer::DecoderPassThrough::requestABuffer() { notify->setInt32("what", kWhatFillThisBuffer); notify->setMessage("reply", reply); notify->post(); - mPendingBuffers++; + mPendingBuffersToFill++; + ALOGV("requestABuffer: #ToFill = %zu, #ToDrain = %zu", mPendingBuffersToFill, + mPendingBuffersToDrain); - sp<AMessage> message = new AMessage(kWhatRequestABuffer, id()); - message->setInt32("generation", mBufferGeneration); - message->post(); - return; + return true; } void android::NuPlayer::DecoderPassThrough::onInputBufferFilled( const sp<AMessage> &msg) { + --mPendingBuffersToFill; if (mReachedEOS) { return; } @@ -150,14 +157,17 @@ void android::NuPlayer::DecoderPassThrough::onInputBufferFilled( notify->setBuffer("buffer", buffer); notify->setMessage("reply", reply); notify->post(); + ++mPendingBuffersToDrain; + ALOGV("onInputBufferFilled: #ToFill = %zu, #ToDrain = %zu, cachedBytes = %zu", + mPendingBuffersToFill, mPendingBuffersToDrain, mCachedBytes); } void NuPlayer::DecoderPassThrough::onBufferConsumed(int32_t size) { - mPendingBuffers--; + --mPendingBuffersToDrain; mCachedBytes -= size; - sp<AMessage> message = new AMessage(kWhatRequestABuffer, id()); - message->setInt32("generation", mBufferGeneration); - message->post(); + ALOGV("onBufferConsumed: #ToFill = %zu, #ToDrain = %zu, cachedBytes = %zu", + mPendingBuffersToFill, mPendingBuffersToDrain, mCachedBytes); + requestABuffer(); } void NuPlayer::DecoderPassThrough::onFlush() { @@ -166,11 +176,20 @@ void NuPlayer::DecoderPassThrough::onFlush() { sp<AMessage> notify = mNotify->dup(); notify->setInt32("what", kWhatFlushCompleted); notify->post(); - mPendingBuffers = 0; + mPendingBuffersToFill = 0; + mPendingBuffersToDrain = 0; mCachedBytes = 0; mReachedEOS = false; } +void NuPlayer::DecoderPassThrough::requestMaxBuffers() { + for (size_t i = 0; i < kMaxPendingBuffers; i++) { + if (!requestABuffer()) { + break; + } + } +} + void NuPlayer::DecoderPassThrough::onShutdown() { ++mBufferGeneration; @@ -228,7 +247,7 @@ void NuPlayer::DecoderPassThrough::onMessageReceived(const sp<AMessage> &msg) { case kWhatResume: { - requestABuffer(); + requestMaxBuffers(); break; } diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h index 8590856..fb20257 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoderPassThrough.h @@ -55,19 +55,26 @@ private: sp<AMessage> mNotify; sp<ALooper> mDecoderLooper; - void requestABuffer(); + /** Returns true if a buffer was requested. + * Returns false if at EOS or cache already full. + */ + bool requestABuffer(); bool isStaleReply(const sp<AMessage> &msg); void onConfigure(const sp<AMessage> &format); void onFlush(); void onInputBufferFilled(const sp<AMessage> &msg); void onBufferConsumed(int32_t size); + void requestMaxBuffers(); void onShutdown(); int32_t mBufferGeneration; - bool mReachedEOS; - int32_t mPendingBuffers; - int32_t mCachedBytes; + bool mReachedEOS; + // TODO mPendingBuffersToFill and mPendingBuffersToDrain are only for + // debugging. They can be removed when the power investigation is done. + size_t mPendingBuffersToFill; + size_t mPendingBuffersToDrain; + size_t mCachedBytes; AString mComponentName; DISALLOW_EVIL_CONSTRUCTORS(DecoderPassThrough); diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp index 2423f5f..7dd54c1 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp @@ -626,11 +626,14 @@ void NuPlayerDriver::notifyListener_l( switch (msg) { case MEDIA_PLAYBACK_COMPLETE: { - if (mLooping) { - mLock.unlock(); - mPlayer->seekToAsync(0); - mLock.lock(); - break; + if (mState != STATE_RESET_IN_PROGRESS) { + if (mLooping) { + mPlayer->seekToAsync(0); + break; + } + + mPlayer->pause(); + mState = STATE_PAUSED; } // fall through } diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp index 49941f8..067784b 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp @@ -58,7 +58,8 @@ NuPlayer::Renderer::Renderer( mVideoRenderingStartGeneration(0), mAudioRenderingStartGeneration(0), mLastPositionUpdateUs(-1ll), - mVideoLateByUs(0ll) { + mVideoLateByUs(0ll), + mVideoSampleReceived(false) { } NuPlayer::Renderer::~Renderer() { @@ -491,7 +492,9 @@ int64_t NuPlayer::Renderer::getAudioPendingPlayoutUs() { } void NuPlayer::Renderer::postDrainVideoQueue() { - if (mDrainVideoQueuePending || mSyncQueues || mPaused) { + if (mDrainVideoQueuePending + || mSyncQueues + || (mPaused && mVideoSampleReceived)) { return; } @@ -570,16 +573,22 @@ void NuPlayer::Renderer::onDrainVideoQueue() { realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs; } - mVideoLateByUs = ALooper::GetNowUs() - realTimeUs; - bool tooLate = (mVideoLateByUs > 40000); + bool tooLate = false; - if (tooLate) { - ALOGV("video late by %lld us (%.2f secs)", - mVideoLateByUs, mVideoLateByUs / 1E6); + if (!mPaused) { + mVideoLateByUs = ALooper::GetNowUs() - realTimeUs; + tooLate = (mVideoLateByUs > 40000); + + if (tooLate) { + ALOGV("video late by %lld us (%.2f secs)", + mVideoLateByUs, mVideoLateByUs / 1E6); + } else { + ALOGV("rendering video at media time %.2f secs", + (mFlags & FLAG_REAL_TIME ? realTimeUs : + (realTimeUs + mAnchorTimeMediaUs - mAnchorTimeRealUs)) / 1E6); + } } else { - ALOGV("rendering video at media time %.2f secs", - (mFlags & FLAG_REAL_TIME ? realTimeUs : - (realTimeUs + mAnchorTimeMediaUs - mAnchorTimeRealUs)) / 1E6); + mVideoLateByUs = 0ll; } entry->mNotifyConsumed->setInt32("render", !tooLate); @@ -587,12 +596,15 @@ void NuPlayer::Renderer::onDrainVideoQueue() { mVideoQueue.erase(mVideoQueue.begin()); entry = NULL; - if (!mVideoRenderingStarted) { - mVideoRenderingStarted = true; - notifyVideoRenderingStart(); - } + mVideoSampleReceived = true; - notifyIfMediaRenderingStarted(); + if (!mPaused) { + if (!mVideoRenderingStarted) { + mVideoRenderingStarted = true; + notifyVideoRenderingStart(); + } + notifyIfMediaRenderingStarted(); + } notifyPosition(); } @@ -791,6 +803,7 @@ void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) { prepareForMediaRenderingStart(); } + mVideoSampleReceived = false; notifyFlushComplete(audio); } @@ -880,8 +893,10 @@ void NuPlayer::Renderer::notifyPosition() { } void NuPlayer::Renderer::onPause() { - CHECK(!mPaused); - + if (mPaused) { + ALOGW("Renderer::onPause() called while already paused!"); + return; + } { Mutex::Autolock autoLock(mLock); ++mAudioQueueGeneration; diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h index 8da6458..5c7d2d7 100644 --- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h +++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h @@ -119,6 +119,7 @@ private: bool mSyncQueues; bool mPaused; + bool mVideoSampleReceived; bool mVideoRenderingStarted; int32_t mVideoRenderingStartGeneration; int32_t mAudioRenderingStartGeneration; diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp index 19a5908..9b03b71 100644 --- a/media/libstagefright/ACodec.cpp +++ b/media/libstagefright/ACodec.cpp @@ -2313,7 +2313,6 @@ static OMX_U32 setPFramesSpacing(int32_t iFramesInterval, int32_t frameRate) { return 0; } OMX_U32 ret = frameRate * iFramesInterval; - CHECK(ret > 1); return ret; } diff --git a/media/libstagefright/Android.mk b/media/libstagefright/Android.mk index be9af5e..193f8a7 100644 --- a/media/libstagefright/Android.mk +++ b/media/libstagefright/Android.mk @@ -62,6 +62,7 @@ LOCAL_SRC_FILES:= \ avc_utils.cpp \ LOCAL_C_INCLUDES:= \ + $(TOP)/frameworks/av/include/media/ \ $(TOP)/frameworks/av/include/media/stagefright/timedtext \ $(TOP)/frameworks/native/include/media/hardware \ $(TOP)/frameworks/native/include/media/openmax \ @@ -70,6 +71,8 @@ LOCAL_C_INCLUDES:= \ $(TOP)/external/openssl/include \ $(TOP)/external/libvpx/libwebm \ $(TOP)/system/netd/include \ + $(TOP)/external/icu/icu4c/source/common \ + $(TOP)/external/icu/icu4c/source/i18n \ LOCAL_SHARED_LIBRARIES := \ libbinder \ diff --git a/media/libstagefright/MPEG4Extractor.cpp b/media/libstagefright/MPEG4Extractor.cpp index 0064293..1729f93 100644 --- a/media/libstagefright/MPEG4Extractor.cpp +++ b/media/libstagefright/MPEG4Extractor.cpp @@ -2810,7 +2810,6 @@ status_t MPEG4Extractor::updateAudioTrackInfoFromESDS_MPEG4Audio( { if (objectType == AOT_SBR || objectType == AOT_PS) { - const int32_t extensionSamplingFrequency = br.getBits(4); objectType = br.getBits(5); if (objectType == AOT_ESCAPE) { @@ -2828,9 +2827,30 @@ status_t MPEG4Extractor::updateAudioTrackInfoFromESDS_MPEG4Audio( const int32_t coreCoderDelay = br.getBits(14); } - const int32_t extensionFlag = br.getBits(1); + int32_t extensionFlag = -1; + if (br.numBitsLeft() > 0) { + extensionFlag = br.getBits(1); + } else { + switch (objectType) { + // 14496-3 4.5.1.1 extensionFlag + case AOT_AAC_LC: + extensionFlag = 0; + break; + case AOT_ER_AAC_LC: + case AOT_ER_AAC_SCAL: + case AOT_ER_BSAC: + case AOT_ER_AAC_LD: + extensionFlag = 1; + break; + default: + TRESPASS(); + break; + } + ALOGW("csd missing extension flag; assuming %d for object type %u.", + extensionFlag, objectType); + } - if (numChannels == 0 ) { + if (numChannels == 0) { int32_t channelsEffectiveNum = 0; int32_t channelsNum = 0; const int32_t ElementInstanceTag = br.getBits(4); diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp index fc2dd30..0bfc6e4 100644 --- a/media/libstagefright/MediaCodec.cpp +++ b/media/libstagefright/MediaCodec.cpp @@ -270,7 +270,20 @@ status_t MediaCodec::configure( } sp<AMessage> response; - return PostAndAwaitResponse(msg, &response); + status_t err = PostAndAwaitResponse(msg, &response); + + if (err != OK && err != INVALID_OPERATION) { + // MediaCodec now set state to UNINITIALIZED upon any fatal error. + // To maintain backward-compatibility, do a reset() to put codec + // back into INITIALIZED state. + // But don't reset if the err is INVALID_OPERATION, which means + // the configure failure is due to wrong state. + + ALOGE("configure failed with err 0x%08x, resetting...", err); + reset(); + } + + return err; } status_t MediaCodec::createInputSurface( diff --git a/media/libstagefright/OMXCodec.cpp b/media/libstagefright/OMXCodec.cpp index 78758da..a8806c8 100644 --- a/media/libstagefright/OMXCodec.cpp +++ b/media/libstagefright/OMXCodec.cpp @@ -994,7 +994,6 @@ static OMX_U32 setPFramesSpacing(int32_t iFramesInterval, int32_t frameRate) { return 0; } OMX_U32 ret = frameRate * iFramesInterval - 1; - CHECK(ret > 1); return ret; } diff --git a/media/libstagefright/StagefrightMetadataRetriever.cpp b/media/libstagefright/StagefrightMetadataRetriever.cpp index 8cc41e7..101fc8a 100644 --- a/media/libstagefright/StagefrightMetadataRetriever.cpp +++ b/media/libstagefright/StagefrightMetadataRetriever.cpp @@ -32,6 +32,7 @@ #include <media/stagefright/MetaData.h> #include <media/stagefright/OMXCodec.h> #include <media/stagefright/MediaDefs.h> +#include <CharacterEncodingDetector.h> namespace android { @@ -450,32 +451,58 @@ void StagefrightMetadataRetriever::parseMetaData() { struct Map { int from; int to; + const char *name; }; static const Map kMap[] = { - { kKeyMIMEType, METADATA_KEY_MIMETYPE }, - { kKeyCDTrackNumber, METADATA_KEY_CD_TRACK_NUMBER }, - { kKeyDiscNumber, METADATA_KEY_DISC_NUMBER }, - { kKeyAlbum, METADATA_KEY_ALBUM }, - { kKeyArtist, METADATA_KEY_ARTIST }, - { kKeyAlbumArtist, METADATA_KEY_ALBUMARTIST }, - { kKeyAuthor, METADATA_KEY_AUTHOR }, - { kKeyComposer, METADATA_KEY_COMPOSER }, - { kKeyDate, METADATA_KEY_DATE }, - { kKeyGenre, METADATA_KEY_GENRE }, - { kKeyTitle, METADATA_KEY_TITLE }, - { kKeyYear, METADATA_KEY_YEAR }, - { kKeyWriter, METADATA_KEY_WRITER }, - { kKeyCompilation, METADATA_KEY_COMPILATION }, - { kKeyLocation, METADATA_KEY_LOCATION }, + { kKeyMIMEType, METADATA_KEY_MIMETYPE, NULL }, + { kKeyCDTrackNumber, METADATA_KEY_CD_TRACK_NUMBER, "tracknumber" }, + { kKeyDiscNumber, METADATA_KEY_DISC_NUMBER, "discnumber" }, + { kKeyAlbum, METADATA_KEY_ALBUM, "album" }, + { kKeyArtist, METADATA_KEY_ARTIST, "artist" }, + { kKeyAlbumArtist, METADATA_KEY_ALBUMARTIST, "albumartist" }, + { kKeyAuthor, METADATA_KEY_AUTHOR, NULL }, + { kKeyComposer, METADATA_KEY_COMPOSER, "composer" }, + { kKeyDate, METADATA_KEY_DATE, NULL }, + { kKeyGenre, METADATA_KEY_GENRE, "genre" }, + { kKeyTitle, METADATA_KEY_TITLE, "title" }, + { kKeyYear, METADATA_KEY_YEAR, "year" }, + { kKeyWriter, METADATA_KEY_WRITER, "writer" }, + { kKeyCompilation, METADATA_KEY_COMPILATION, "compilation" }, + { kKeyLocation, METADATA_KEY_LOCATION, NULL }, }; + static const size_t kNumMapEntries = sizeof(kMap) / sizeof(kMap[0]); + CharacterEncodingDetector *detector = new CharacterEncodingDetector(); + for (size_t i = 0; i < kNumMapEntries; ++i) { const char *value; if (meta->findCString(kMap[i].from, &value)) { - mMetaData.add(kMap[i].to, String8(value)); + if (kMap[i].name) { + // add to charset detector + detector->addTag(kMap[i].name, value); + } else { + // directly add to output list + mMetaData.add(kMap[i].to, String8(value)); + } + } + } + + detector->detectAndConvert(); + int size = detector->size(); + if (size) { + for (int i = 0; i < size; i++) { + const char *name; + const char *value; + detector->getTag(i, &name, &value); + for (size_t j = 0; j < kNumMapEntries; ++j) { + if (kMap[j].name && !strcmp(kMap[j].name, name)) { + mMetaData.add(kMap[j].to, String8(value)); + } + } } } + delete detector; const void *data; uint32_t type; diff --git a/media/libstagefright/TimedEventQueue.cpp b/media/libstagefright/TimedEventQueue.cpp index da50c56..1fdb244 100644 --- a/media/libstagefright/TimedEventQueue.cpp +++ b/media/libstagefright/TimedEventQueue.cpp @@ -338,7 +338,7 @@ void TimedEventQueue::acquireWakeLock_l() status_t status = mPowerManager->acquireWakeLock(POWERMANAGER_PARTIAL_WAKE_LOCK, binder, String16("TimedEventQueue"), - String16("media")); + String16("media")); // not oneway IPCThreadState::self()->restoreCallingIdentity(token); if (status == NO_ERROR) { mWakeLockToken = binder; @@ -363,7 +363,7 @@ void TimedEventQueue::releaseWakeLock_l(bool force) CHECK(mWakeLockToken != 0); if (mPowerManager != 0) { int64_t token = IPCThreadState::self()->clearCallingIdentity(); - mPowerManager->releaseWakeLock(mWakeLockToken, 0); + mPowerManager->releaseWakeLock(mWakeLockToken, 0); // not oneway IPCThreadState::self()->restoreCallingIdentity(token); } mWakeLockToken.clear(); diff --git a/media/libstagefright/codecs/aacdec/SoftAAC2.cpp b/media/libstagefright/codecs/aacdec/SoftAAC2.cpp index 8b4dd6f..4569c1c 100644 --- a/media/libstagefright/codecs/aacdec/SoftAAC2.cpp +++ b/media/libstagefright/codecs/aacdec/SoftAAC2.cpp @@ -14,8 +14,8 @@ * limitations under the License. */ -#define LOG_TAG "SoftAAC2" //#define LOG_NDEBUG 0 +#define LOG_TAG "SoftAAC2" #include <utils/Log.h> #include "SoftAAC2.h" @@ -68,7 +68,6 @@ SoftAAC2::SoftAAC2( mOutputBufferCount(0), mSignalledError(false), mLastInHeader(NULL), - mCurrentInputTime(0), mOutputPortSettingsChange(NONE) { initPorts(); CHECK_EQ(initDecoder(), (status_t)OK); @@ -610,9 +609,24 @@ void SoftAAC2::onQueueFilled(OMX_U32 /* portIndex */) { notify(OMX_EventError, OMX_ErrorStreamCorrupt, ERROR_MALFORMED, NULL); return; } + + // insert buffer size and time stamp + mBufferSizes.add(inBufferLength[0]); + if (mLastInHeader != inHeader) { + mBufferTimestamps.add(inHeader->nTimeStamp); + mLastInHeader = inHeader; + } else { + int64_t currentTime = mBufferTimestamps.top(); + currentTime += mStreamInfo->aacSamplesPerFrame * + 1000000ll / mStreamInfo->sampleRate; + mBufferTimestamps.add(currentTime); + } } else { inBuffer[0] = inHeader->pBuffer + inHeader->nOffset; inBufferLength[0] = inHeader->nFilledLen; + mLastInHeader = inHeader; + mBufferTimestamps.add(inHeader->nTimeStamp); + mBufferSizes.add(inHeader->nFilledLen); } // Fill and decode @@ -621,136 +635,136 @@ void SoftAAC2::onQueueFilled(OMX_U32 /* portIndex */) { INT prevSampleRate = mStreamInfo->sampleRate; INT prevNumChannels = mStreamInfo->numChannels; - if (inHeader != mLastInHeader) { - mLastInHeader = inHeader; - mCurrentInputTime = inHeader->nTimeStamp; - } else { - if (mStreamInfo->sampleRate) { - mCurrentInputTime += mStreamInfo->aacSamplesPerFrame * - 1000000ll / mStreamInfo->sampleRate; - } else { - ALOGW("no sample rate yet"); - } - } - mAnchorTimes.add(mCurrentInputTime); aacDecoder_Fill(mAACDecoder, inBuffer, inBufferLength, bytesValid); - // run DRC check - mDrcWrap.submitStreamData(mStreamInfo); - mDrcWrap.update(); + // run DRC check + mDrcWrap.submitStreamData(mStreamInfo); + mDrcWrap.update(); - AAC_DECODER_ERROR decoderErr = - aacDecoder_DecodeFrame(mAACDecoder, - tmpOutBuffer, - 2048 * MAX_CHANNEL_COUNT, - 0 /* flags */); + UINT inBufferUsedLength = inBufferLength[0] - bytesValid[0]; + inHeader->nFilledLen -= inBufferUsedLength; + inHeader->nOffset += inBufferUsedLength; - if (decoderErr != AAC_DEC_OK) { - ALOGW("aacDecoder_DecodeFrame decoderErr = 0x%4.4x", decoderErr); - } - - if (decoderErr == AAC_DEC_NOT_ENOUGH_BITS) { - ALOGE("AAC_DEC_NOT_ENOUGH_BITS should never happen"); - mSignalledError = true; - notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); - return; - } + AAC_DECODER_ERROR decoderErr; + do { + if (outputDelayRingBufferSamplesLeft() < + (mStreamInfo->frameSize * mStreamInfo->numChannels)) { + ALOGV("skipping decode: not enough space left in ringbuffer"); + break; + } - if (bytesValid[0] != 0) { - ALOGE("bytesValid[0] != 0 should never happen"); - mSignalledError = true; - notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); - return; - } + int numconsumed = mStreamInfo->numTotalBytes + mStreamInfo->numBadBytes; + decoderErr = aacDecoder_DecodeFrame(mAACDecoder, + tmpOutBuffer, + 2048 * MAX_CHANNEL_COUNT, + 0 /* flags */); - size_t numOutBytes = - mStreamInfo->frameSize * sizeof(int16_t) * mStreamInfo->numChannels; + numconsumed = (mStreamInfo->numTotalBytes + mStreamInfo->numBadBytes) - numconsumed; + if (numconsumed != 0) { + mDecodedSizes.add(numconsumed); + } - if (decoderErr == AAC_DEC_OK) { - if (!outputDelayRingBufferPutSamples(tmpOutBuffer, - mStreamInfo->frameSize * mStreamInfo->numChannels)) { - mSignalledError = true; - notify(OMX_EventError, OMX_ErrorUndefined, decoderErr, NULL); - return; + if (decoderErr == AAC_DEC_NOT_ENOUGH_BITS) { + break; } - UINT inBufferUsedLength = inBufferLength[0] - bytesValid[0]; - inHeader->nFilledLen -= inBufferUsedLength; - inHeader->nOffset += inBufferUsedLength; - } else { - ALOGW("AAC decoder returned error 0x%4.4x, substituting silence", decoderErr); - memset(tmpOutBuffer, 0, numOutBytes); // TODO: check for overflow + if (decoderErr != AAC_DEC_OK) { + ALOGW("aacDecoder_DecodeFrame decoderErr = 0x%4.4x", decoderErr); + } - if (!outputDelayRingBufferPutSamples(tmpOutBuffer, - mStreamInfo->frameSize * mStreamInfo->numChannels)) { + if (bytesValid[0] != 0) { + ALOGE("bytesValid[0] != 0 should never happen"); mSignalledError = true; - notify(OMX_EventError, OMX_ErrorUndefined, decoderErr, NULL); + notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); return; } - // Discard input buffer. - inHeader->nFilledLen = 0; - - aacDecoder_SetParam(mAACDecoder, AAC_TPDEC_CLEAR_BUFFER, 1); - - // fall through - } - - /* - * AAC+/eAAC+ streams can be signalled in two ways: either explicitly - * or implicitly, according to MPEG4 spec. AAC+/eAAC+ is a dual - * rate system and the sampling rate in the final output is actually - * doubled compared with the core AAC decoder sampling rate. - * - * Explicit signalling is done by explicitly defining SBR audio object - * type in the bitstream. Implicit signalling is done by embedding - * SBR content in AAC extension payload specific to SBR, and hence - * requires an AAC decoder to perform pre-checks on actual audio frames. - * - * Thus, we could not say for sure whether a stream is - * AAC+/eAAC+ until the first data frame is decoded. - */ - if (mInputBufferCount <= 2 || mOutputBufferCount > 1) { // TODO: <= 1 - if (mStreamInfo->sampleRate != prevSampleRate || - mStreamInfo->numChannels != prevNumChannels) { - ALOGI("Reconfiguring decoder: %d->%d Hz, %d->%d channels", - prevSampleRate, mStreamInfo->sampleRate, - prevNumChannels, mStreamInfo->numChannels); + size_t numOutBytes = + mStreamInfo->frameSize * sizeof(int16_t) * mStreamInfo->numChannels; - notify(OMX_EventPortSettingsChanged, 1, 0, NULL); - mOutputPortSettingsChange = AWAITING_DISABLED; + if (decoderErr == AAC_DEC_OK) { + if (!outputDelayRingBufferPutSamples(tmpOutBuffer, + mStreamInfo->frameSize * mStreamInfo->numChannels)) { + mSignalledError = true; + notify(OMX_EventError, OMX_ErrorUndefined, decoderErr, NULL); + return; + } + } else { + ALOGW("AAC decoder returned error 0x%4.4x, substituting silence", decoderErr); + + memset(tmpOutBuffer, 0, numOutBytes); // TODO: check for overflow - if (inHeader->nFilledLen == 0) { - inInfo->mOwnedByUs = false; - mInputBufferCount++; - inQueue.erase(inQueue.begin()); - mLastInHeader = NULL; - inInfo = NULL; - notifyEmptyBufferDone(inHeader); - inHeader = NULL; + if (!outputDelayRingBufferPutSamples(tmpOutBuffer, + mStreamInfo->frameSize * mStreamInfo->numChannels)) { + mSignalledError = true; + notify(OMX_EventError, OMX_ErrorUndefined, decoderErr, NULL); + return; } + + // Discard input buffer. + inHeader->nFilledLen = 0; + + aacDecoder_SetParam(mAACDecoder, AAC_TPDEC_CLEAR_BUFFER, 1); + + // fall through + } + + /* + * AAC+/eAAC+ streams can be signalled in two ways: either explicitly + * or implicitly, according to MPEG4 spec. AAC+/eAAC+ is a dual + * rate system and the sampling rate in the final output is actually + * doubled compared with the core AAC decoder sampling rate. + * + * Explicit signalling is done by explicitly defining SBR audio object + * type in the bitstream. Implicit signalling is done by embedding + * SBR content in AAC extension payload specific to SBR, and hence + * requires an AAC decoder to perform pre-checks on actual audio frames. + * + * Thus, we could not say for sure whether a stream is + * AAC+/eAAC+ until the first data frame is decoded. + */ + if (mInputBufferCount <= 2 || mOutputBufferCount > 1) { // TODO: <= 1 + if (mStreamInfo->sampleRate != prevSampleRate || + mStreamInfo->numChannels != prevNumChannels) { + ALOGI("Reconfiguring decoder: %d->%d Hz, %d->%d channels", + prevSampleRate, mStreamInfo->sampleRate, + prevNumChannels, mStreamInfo->numChannels); + + notify(OMX_EventPortSettingsChanged, 1, 0, NULL); + mOutputPortSettingsChange = AWAITING_DISABLED; + + if (inHeader->nFilledLen == 0) { + inInfo->mOwnedByUs = false; + mInputBufferCount++; + inQueue.erase(inQueue.begin()); + mLastInHeader = NULL; + inInfo = NULL; + notifyEmptyBufferDone(inHeader); + inHeader = NULL; + } + return; + } + } else if (!mStreamInfo->sampleRate || !mStreamInfo->numChannels) { + ALOGW("Invalid AAC stream"); + mSignalledError = true; + notify(OMX_EventError, OMX_ErrorUndefined, decoderErr, NULL); return; } - } else if (!mStreamInfo->sampleRate || !mStreamInfo->numChannels) { - ALOGW("Invalid AAC stream"); - mSignalledError = true; - notify(OMX_EventError, OMX_ErrorUndefined, decoderErr, NULL); - return; - } - if (inHeader->nFilledLen == 0) { - inInfo->mOwnedByUs = false; - mInputBufferCount++; - inQueue.erase(inQueue.begin()); - mLastInHeader = NULL; - inInfo = NULL; - notifyEmptyBufferDone(inHeader); - inHeader = NULL; - } else { - ALOGV("inHeader->nFilledLen = %d", inHeader->nFilledLen); - } + if (inHeader && inHeader->nFilledLen == 0) { + inInfo->mOwnedByUs = false; + mInputBufferCount++; + inQueue.erase(inQueue.begin()); + mLastInHeader = NULL; + inInfo = NULL; + notifyEmptyBufferDone(inHeader); + inHeader = NULL; + } else { + ALOGV("inHeader->nFilledLen = %d", inHeader ? inHeader->nFilledLen : 0); + } + } while (decoderErr == AAC_DEC_OK); } int32_t outputDelay = mStreamInfo->outputDelay * mStreamInfo->numChannels; @@ -809,8 +823,9 @@ void SoftAAC2::onQueueFilled(OMX_U32 /* portIndex */) { INT_PCM *outBuffer = reinterpret_cast<INT_PCM *>(outHeader->pBuffer + outHeader->nOffset); + int samplesize = mStreamInfo->numChannels * sizeof(int16_t); if (outHeader->nOffset - + mStreamInfo->frameSize * mStreamInfo->numChannels * sizeof(int16_t) + + mStreamInfo->frameSize * samplesize > outHeader->nAllocLen) { ALOGE("buffer overflow"); mSignalledError = true; @@ -818,17 +833,67 @@ void SoftAAC2::onQueueFilled(OMX_U32 /* portIndex */) { return; } - int32_t ns = outputDelayRingBufferGetSamples(outBuffer, - mStreamInfo->frameSize * mStreamInfo->numChannels); // TODO: check for overflow - if (ns != mStreamInfo->frameSize * mStreamInfo->numChannels) { - ALOGE("not a complete frame of samples available"); - mSignalledError = true; - notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); - return; + + int available = outputDelayRingBufferSamplesAvailable(); + int numSamples = outHeader->nAllocLen / sizeof(int16_t); + if (numSamples > available) { + numSamples = available; + } + int64_t currentTime = 0; + if (available) { + + int numFrames = numSamples / (mStreamInfo->frameSize * mStreamInfo->numChannels); + numSamples = numFrames * (mStreamInfo->frameSize * mStreamInfo->numChannels); + + ALOGV("%d samples available (%d), or %d frames", + numSamples, available, numFrames); + int64_t *nextTimeStamp = &mBufferTimestamps.editItemAt(0); + currentTime = *nextTimeStamp; + int32_t *currentBufLeft = &mBufferSizes.editItemAt(0); + for (int i = 0; i < numFrames; i++) { + int32_t decodedSize = mDecodedSizes.itemAt(0); + mDecodedSizes.removeAt(0); + ALOGV("decoded %d of %d", decodedSize, *currentBufLeft); + if (*currentBufLeft > decodedSize) { + // adjust/interpolate next time stamp + *currentBufLeft -= decodedSize; + *nextTimeStamp += mStreamInfo->aacSamplesPerFrame * + 1000000ll / mStreamInfo->sampleRate; + ALOGV("adjusted nextTimeStamp/size to %lld/%d", + *nextTimeStamp, *currentBufLeft); + } else { + // move to next timestamp in list + if (mBufferTimestamps.size() > 0) { + mBufferTimestamps.removeAt(0); + nextTimeStamp = &mBufferTimestamps.editItemAt(0); + mBufferSizes.removeAt(0); + currentBufLeft = &mBufferSizes.editItemAt(0); + ALOGV("moved to next time/size: %lld/%d", + *nextTimeStamp, *currentBufLeft); + } + // try to limit output buffer size to match input buffers + // (e.g when an input buffer contained 4 "sub" frames, output + // at most 4 decoded units in the corresponding output buffer) + // This is optional. Remove the next three lines to fill the output + // buffer with as many units as available. + numFrames = i + 1; + numSamples = numFrames * mStreamInfo->frameSize * mStreamInfo->numChannels; + break; + } + } + + ALOGV("getting %d from ringbuffer", numSamples); + int32_t ns = outputDelayRingBufferGetSamples(outBuffer, numSamples); + if (ns != numSamples) { + ALOGE("not a complete frame of samples available"); + mSignalledError = true; + notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); + return; + } } - outHeader->nFilledLen = mStreamInfo->frameSize * mStreamInfo->numChannels - * sizeof(int16_t); + outHeader->nFilledLen = numSamples * sizeof(int16_t); + if (mEndOfInput && !outQueue.empty() && outputDelayRingBufferSamplesAvailable() == 0) { outHeader->nFlags = OMX_BUFFERFLAG_EOS; mEndOfOutput = true; @@ -836,13 +901,13 @@ void SoftAAC2::onQueueFilled(OMX_U32 /* portIndex */) { outHeader->nFlags = 0; } - outHeader->nTimeStamp = mAnchorTimes.isEmpty() ? 0 : mAnchorTimes.itemAt(0); - mAnchorTimes.removeAt(0); + outHeader->nTimeStamp = currentTime; mOutputBufferCount++; outInfo->mOwnedByUs = false; outQueue.erase(outQueue.begin()); outInfo = NULL; + ALOGV("out timestamp %lld / %d", outHeader->nTimeStamp, outHeader->nFilledLen); notifyFillBufferDone(outHeader); outHeader = NULL; } @@ -877,8 +942,10 @@ void SoftAAC2::onQueueFilled(OMX_U32 /* portIndex */) { outHeader->nFilledLen = 0; outHeader->nFlags = OMX_BUFFERFLAG_EOS; - outHeader->nTimeStamp = mAnchorTimes.itemAt(0); - mAnchorTimes.removeAt(0); + outHeader->nTimeStamp = mBufferTimestamps.itemAt(0); + mBufferTimestamps.clear(); + mBufferSizes.clear(); + mDecodedSizes.clear(); mOutputBufferCount++; outInfo->mOwnedByUs = false; @@ -899,7 +966,9 @@ void SoftAAC2::onPortFlushCompleted(OMX_U32 portIndex) { // depend on fragments from the last one decoded. // drain all existing data drainDecoder(); - mAnchorTimes.clear(); + mBufferTimestamps.clear(); + mBufferSizes.clear(); + mDecodedSizes.clear(); mLastInHeader = NULL; } else { while (outputDelayRingBufferSamplesAvailable() > 0) { @@ -955,7 +1024,9 @@ void SoftAAC2::onReset() { mOutputDelayRingBufferReadPos = 0; mEndOfInput = false; mEndOfOutput = false; - mAnchorTimes.clear(); + mBufferTimestamps.clear(); + mBufferSizes.clear(); + mDecodedSizes.clear(); mLastInHeader = NULL; // To make the codec behave the same before and after a reset, we need to invalidate the diff --git a/media/libstagefright/codecs/aacdec/SoftAAC2.h b/media/libstagefright/codecs/aacdec/SoftAAC2.h index 865bd15..9fcb598 100644 --- a/media/libstagefright/codecs/aacdec/SoftAAC2.h +++ b/media/libstagefright/codecs/aacdec/SoftAAC2.h @@ -59,8 +59,9 @@ private: size_t mOutputBufferCount; bool mSignalledError; OMX_BUFFERHEADERTYPE *mLastInHeader; - int64_t mCurrentInputTime; - Vector<int64_t> mAnchorTimes; + Vector<int32_t> mBufferSizes; + Vector<int32_t> mDecodedSizes; + Vector<int64_t> mBufferTimestamps; CDrcPresModeWrapper mDrcWrap; diff --git a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp index 0d1ab71..5b2ab84 100644 --- a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp +++ b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp @@ -134,6 +134,12 @@ void SoftMPEG4::onQueueFilled(OMX_U32 /* portIndex */) { } uint8_t *bitstream = inHeader->pBuffer + inHeader->nOffset; + uint32_t *start_code = (uint32_t *)bitstream; + bool volHeader = *start_code == 0xB0010000; + if (volHeader) { + PVCleanUpVideoDecoder(mHandle); + mInitialized = false; + } if (!mInitialized) { uint8_t *vol_data[1]; @@ -141,7 +147,7 @@ void SoftMPEG4::onQueueFilled(OMX_U32 /* portIndex */) { vol_data[0] = NULL; - if (inHeader->nFlags & OMX_BUFFERFLAG_CODECCONFIG) { + if ((inHeader->nFlags & OMX_BUFFERFLAG_CODECCONFIG) || volHeader) { vol_data[0] = bitstream; vol_size = inHeader->nFilledLen; } @@ -169,21 +175,26 @@ void SoftMPEG4::onQueueFilled(OMX_U32 /* portIndex */) { PVSetPostProcType((VideoDecControls *) mHandle, 0); + bool hasFrameData = false; if (inHeader->nFlags & OMX_BUFFERFLAG_CODECCONFIG) { inInfo->mOwnedByUs = false; inQueue.erase(inQueue.begin()); inInfo = NULL; notifyEmptyBufferDone(inHeader); inHeader = NULL; + } else if (volHeader) { + hasFrameData = true; } mInitialized = true; - if (mode == MPEG4_MODE && portSettingsChanged()) { + if (mode == MPEG4_MODE && handlePortSettingsChange()) { return; } - continue; + if (!hasFrameData) { + continue; + } } if (!mFramesConfigured) { @@ -223,7 +234,9 @@ void SoftMPEG4::onQueueFilled(OMX_U32 /* portIndex */) { return; } - if (portSettingsChanged()) { + // H263 doesn't have VOL header, the frame size information is in short header, i.e. the + // decoder may detect size change after PVDecodeVideoFrame. + if (handlePortSettingsChange()) { return; } @@ -269,7 +282,7 @@ void SoftMPEG4::onQueueFilled(OMX_U32 /* portIndex */) { } } -bool SoftMPEG4::portSettingsChanged() { +bool SoftMPEG4::handlePortSettingsChange() { uint32_t disp_width, disp_height; PVGetVideoDimensions(mHandle, (int32 *)&disp_width, (int32 *)&disp_height); @@ -282,25 +295,20 @@ bool SoftMPEG4::portSettingsChanged() { ALOGV("disp_width = %d, disp_height = %d, buf_width = %d, buf_height = %d", disp_width, disp_height, buf_width, buf_height); - if (mCropWidth != disp_width - || mCropHeight != disp_height) { + bool cropChanged = false; + if (mCropWidth != disp_width || mCropHeight != disp_height) { mCropLeft = 0; mCropTop = 0; mCropWidth = disp_width; mCropHeight = disp_height; - - notify(OMX_EventPortSettingsChanged, - 1, - OMX_IndexConfigCommonOutputCrop, - NULL); + cropChanged = true; } - if (buf_width != mWidth || buf_height != mHeight) { - mWidth = buf_width; - mHeight = buf_height; - - updatePortDefinitions(); - + bool portWillReset = false; + const bool fakeStride = true; + SoftVideoDecoderOMXComponent::handlePortSettingsChange( + &portWillReset, buf_width, buf_height, cropChanged, fakeStride); + if (portWillReset) { if (mMode == MODE_H263) { PVCleanUpVideoDecoder(mHandle); @@ -318,13 +326,9 @@ bool SoftMPEG4::portSettingsChanged() { } mFramesConfigured = false; - - notify(OMX_EventPortSettingsChanged, 1, 0, NULL); - mOutputPortSettingsChange = AWAITING_DISABLED; - return true; } - return false; + return portWillReset; } void SoftMPEG4::onPortFlushCompleted(OMX_U32 portIndex) { diff --git a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.h b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.h index de14aaf..8a06a00 100644 --- a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.h +++ b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.h @@ -67,7 +67,7 @@ private: status_t initDecoder(); virtual void updatePortDefinitions(); - bool portSettingsChanged(); + bool handlePortSettingsChange(); DISALLOW_EVIL_CONSTRUCTORS(SoftMPEG4); }; diff --git a/media/libstagefright/codecs/m4v_h263/dec/src/vop.cpp b/media/libstagefright/codecs/m4v_h263/dec/src/vop.cpp index b3c350f..b03ec8c 100644 --- a/media/libstagefright/codecs/m4v_h263/dec/src/vop.cpp +++ b/media/libstagefright/codecs/m4v_h263/dec/src/vop.cpp @@ -1426,7 +1426,7 @@ PV_STATUS DecodeShortHeader(VideoDecData *video, Vop *currVop) video->nBitsForMBID = CalcNumBits((uint)video->nTotalMB - 1); /* otherwise calculate above */ } size = (int32)video->width * video->height; - if (video->currVop->predictionType == P_VOP && size > video->videoDecControls->size) + if (currVop->predictionType == P_VOP && size > video->videoDecControls->size) { status = PV_FAIL; goto return_point; diff --git a/media/libstagefright/codecs/on2/dec/SoftVPX.cpp b/media/libstagefright/codecs/on2/dec/SoftVPX.cpp index a4258dd..828577a 100644 --- a/media/libstagefright/codecs/on2/dec/SoftVPX.cpp +++ b/media/libstagefright/codecs/on2/dec/SoftVPX.cpp @@ -23,9 +23,6 @@ #include <media/stagefright/foundation/ADebug.h> #include <media/stagefright/MediaDefs.h> -#include "vpx/vpx_decoder.h" -#include "vpx/vpx_codec.h" -#include "vpx/vp8dx.h" namespace android { @@ -41,7 +38,8 @@ SoftVPX::SoftVPX( NULL /* profileLevels */, 0 /* numProfileLevels */, 320 /* width */, 240 /* height */, callbacks, appData, component), mMode(codingType == OMX_VIDEO_CodingVP8 ? MODE_VP8 : MODE_VP9), - mCtx(NULL) { + mCtx(NULL), + mImg(NULL) { initPorts(kNumBuffers, 768 * 1024 /* inputBufferSize */, kNumBuffers, codingType == OMX_VIDEO_CodingVP8 ? MEDIA_MIMETYPE_VIDEO_VP8 : MEDIA_MIMETYPE_VIDEO_VP9); @@ -118,49 +116,31 @@ void SoftVPX::onQueueFilled(OMX_U32 /* portIndex */) { } } - if (vpx_codec_decode( - (vpx_codec_ctx_t *)mCtx, - inHeader->pBuffer + inHeader->nOffset, - inHeader->nFilledLen, - NULL, - 0)) { - ALOGE("on2 decoder failed to decode frame."); + if (mImg == NULL) { + if (vpx_codec_decode( + (vpx_codec_ctx_t *)mCtx, + inHeader->pBuffer + inHeader->nOffset, + inHeader->nFilledLen, + NULL, + 0)) { + ALOGE("on2 decoder failed to decode frame."); - notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); - return; + notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL); + return; + } + vpx_codec_iter_t iter = NULL; + mImg = vpx_codec_get_frame((vpx_codec_ctx_t *)mCtx, &iter); } - vpx_codec_iter_t iter = NULL; - vpx_image_t *img = vpx_codec_get_frame((vpx_codec_ctx_t *)mCtx, &iter); - - if (img != NULL) { - CHECK_EQ(img->fmt, IMG_FMT_I420); - - uint32_t width = img->d_w; - uint32_t height = img->d_h; - - if (width != mWidth || height != mHeight) { - mWidth = width; - mHeight = height; - - if (!mIsAdaptive || width > mAdaptiveMaxWidth || height > mAdaptiveMaxHeight) { - if (mIsAdaptive) { - if (width > mAdaptiveMaxWidth) { - mAdaptiveMaxWidth = width; - } - if (height > mAdaptiveMaxHeight) { - mAdaptiveMaxHeight = height; - } - } - updatePortDefinitions(); - notify(OMX_EventPortSettingsChanged, kOutputPortIndex, 0, NULL); - mOutputPortSettingsChange = AWAITING_DISABLED; - return; - } else { - updatePortDefinitions(); - notify(OMX_EventPortSettingsChanged, kOutputPortIndex, - OMX_IndexConfigCommonOutputCrop, NULL); - } + if (mImg != NULL) { + CHECK_EQ(mImg->fmt, IMG_FMT_I420); + + uint32_t width = mImg->d_w; + uint32_t height = mImg->d_h; + bool portWillReset = false; + handlePortSettingsChange(&portWillReset, width, height); + if (portWillReset) { + return; } outHeader->nOffset = 0; @@ -168,37 +148,16 @@ void SoftVPX::onQueueFilled(OMX_U32 /* portIndex */) { outHeader->nFlags = EOSseen ? OMX_BUFFERFLAG_EOS : 0; outHeader->nTimeStamp = inHeader->nTimeStamp; - uint32_t buffer_stride = mIsAdaptive ? mAdaptiveMaxWidth : mWidth; - uint32_t buffer_height = mIsAdaptive ? mAdaptiveMaxHeight : mHeight; - - const uint8_t *srcLine = (const uint8_t *)img->planes[PLANE_Y]; uint8_t *dst = outHeader->pBuffer; - for (size_t i = 0; i < buffer_height; ++i) { - if (i < img->d_h) { - memcpy(dst, srcLine, img->d_w); - srcLine += img->stride[PLANE_Y]; - } - dst += buffer_stride; - } - - srcLine = (const uint8_t *)img->planes[PLANE_U]; - for (size_t i = 0; i < buffer_height / 2; ++i) { - if (i < img->d_h / 2) { - memcpy(dst, srcLine, img->d_w / 2); - srcLine += img->stride[PLANE_U]; - } - dst += buffer_stride / 2; - } - - srcLine = (const uint8_t *)img->planes[PLANE_V]; - for (size_t i = 0; i < buffer_height / 2; ++i) { - if (i < img->d_h / 2) { - memcpy(dst, srcLine, img->d_w / 2); - srcLine += img->stride[PLANE_V]; - } - dst += buffer_stride / 2; - } - + const uint8_t *srcY = (const uint8_t *)mImg->planes[PLANE_Y]; + const uint8_t *srcU = (const uint8_t *)mImg->planes[PLANE_U]; + const uint8_t *srcV = (const uint8_t *)mImg->planes[PLANE_V]; + size_t srcYStride = mImg->stride[PLANE_Y]; + size_t srcUStride = mImg->stride[PLANE_U]; + size_t srcVStride = mImg->stride[PLANE_V]; + copyYV12FrameToOutputBuffer(dst, srcY, srcU, srcV, srcYStride, srcUStride, srcVStride); + + mImg = NULL; outInfo->mOwnedByUs = false; outQueue.erase(outQueue.begin()); outInfo = NULL; diff --git a/media/libstagefright/codecs/on2/dec/SoftVPX.h b/media/libstagefright/codecs/on2/dec/SoftVPX.h index cd5eb28..8f68693 100644 --- a/media/libstagefright/codecs/on2/dec/SoftVPX.h +++ b/media/libstagefright/codecs/on2/dec/SoftVPX.h @@ -20,6 +20,10 @@ #include "SoftVideoDecoderOMXComponent.h" +#include "vpx/vpx_decoder.h" +#include "vpx/vpx_codec.h" +#include "vpx/vp8dx.h" + namespace android { struct SoftVPX : public SoftVideoDecoderOMXComponent { @@ -47,6 +51,8 @@ private: void *mCtx; + vpx_image_t *mImg; + status_t initDecoder(); DISALLOW_EVIL_CONSTRUCTORS(SoftVPX); diff --git a/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp b/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp index a7bde97..cf3c3e3 100644 --- a/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp +++ b/media/libstagefright/codecs/on2/h264dec/SoftAVC.cpp @@ -58,7 +58,6 @@ SoftAVC::SoftAVC( 320 /* width */, 240 /* height */, callbacks, appData, component), mHandle(NULL), mInputBufferCount(0), - mPictureSize(mWidth * mHeight * 3 / 2), mFirstPicture(NULL), mFirstPictureId(-1), mPicId(0), @@ -118,7 +117,7 @@ void SoftAVC::onQueueFilled(OMX_U32 /* portIndex */) { } H264SwDecRet ret = H264SWDEC_PIC_RDY; - bool portSettingsChanged = false; + bool portWillReset = false; while ((mEOSStatus != INPUT_DATA_AVAILABLE || !inQueue.empty()) && outQueue.size() == kNumOutputBuffers) { @@ -161,17 +160,13 @@ void SoftAVC::onQueueFilled(OMX_U32 /* portIndex */) { H264SwDecInfo decoderInfo; CHECK(H264SwDecGetInfo(mHandle, &decoderInfo) == H264SWDEC_OK); - if (handlePortSettingChangeEvent(&decoderInfo)) { - portSettingsChanged = true; - } - - if (decoderInfo.croppingFlag && - handleCropRectEvent(&decoderInfo.cropParams)) { - portSettingsChanged = true; - } + bool cropChanged = handleCropChange(decoderInfo); + handlePortSettingsChange( + &portWillReset, decoderInfo.picWidth, decoderInfo.picHeight, + cropChanged); } } else { - if (portSettingsChanged) { + if (portWillReset) { if (H264SwDecNextPicture(mHandle, &decodedPicture, 0) == H264SWDEC_PIC_RDY) { @@ -199,8 +194,7 @@ void SoftAVC::onQueueFilled(OMX_U32 /* portIndex */) { inInfo->mOwnedByUs = false; notifyEmptyBufferDone(inHeader); - if (portSettingsChanged) { - portSettingsChanged = false; + if (portWillReset) { return; } @@ -215,44 +209,33 @@ void SoftAVC::onQueueFilled(OMX_U32 /* portIndex */) { } } -bool SoftAVC::handlePortSettingChangeEvent(const H264SwDecInfo *info) { - if (mWidth != info->picWidth || mHeight != info->picHeight) { - mWidth = info->picWidth; - mHeight = info->picHeight; - mPictureSize = mWidth * mHeight * 3 / 2; - updatePortDefinitions(); - notify(OMX_EventPortSettingsChanged, 1, 0, NULL); - mOutputPortSettingsChange = AWAITING_DISABLED; - return true; +bool SoftAVC::handleCropChange(const H264SwDecInfo& decInfo) { + if (!decInfo.croppingFlag) { + return false; } - return false; -} - -bool SoftAVC::handleCropRectEvent(const CropParams *crop) { - if (mCropLeft != crop->cropLeftOffset || - mCropTop != crop->cropTopOffset || - mCropWidth != crop->cropOutWidth || - mCropHeight != crop->cropOutHeight) { - mCropLeft = crop->cropLeftOffset; - mCropTop = crop->cropTopOffset; - mCropWidth = crop->cropOutWidth; - mCropHeight = crop->cropOutHeight; - - notify(OMX_EventPortSettingsChanged, 1, - OMX_IndexConfigCommonOutputCrop, NULL); - - return true; + const CropParams& crop = decInfo.cropParams; + if (mCropLeft == crop.cropLeftOffset && + mCropTop == crop.cropTopOffset && + mCropWidth == crop.cropOutWidth && + mCropHeight == crop.cropOutHeight) { + return false; } - return false; + + mCropLeft = crop.cropLeftOffset; + mCropTop = crop.cropTopOffset; + mCropWidth = crop.cropOutWidth; + mCropHeight = crop.cropOutHeight; + return true; } void SoftAVC::saveFirstOutputBuffer(int32_t picId, uint8_t *data) { CHECK(mFirstPicture == NULL); mFirstPictureId = picId; - mFirstPicture = new uint8_t[mPictureSize]; - memcpy(mFirstPicture, data, mPictureSize); + uint32_t pictureSize = mWidth * mHeight * 3 / 2; + mFirstPicture = new uint8_t[pictureSize]; + memcpy(mFirstPicture, data, pictureSize); } void SoftAVC::drainOneOutputBuffer(int32_t picId, uint8_t* data) { @@ -263,9 +246,17 @@ void SoftAVC::drainOneOutputBuffer(int32_t picId, uint8_t* data) { OMX_BUFFERHEADERTYPE *header = mPicToHeaderMap.valueFor(picId); outHeader->nTimeStamp = header->nTimeStamp; outHeader->nFlags = header->nFlags; - outHeader->nFilledLen = mPictureSize; - memcpy(outHeader->pBuffer + outHeader->nOffset, - data, mPictureSize); + outHeader->nFilledLen = mWidth * mHeight * 3 / 2; + + uint8_t *dst = outHeader->pBuffer + outHeader->nOffset; + const uint8_t *srcY = data; + const uint8_t *srcU = srcY + mWidth * mHeight; + const uint8_t *srcV = srcU + mWidth * mHeight / 4; + size_t srcYStride = mWidth; + size_t srcUStride = mWidth / 2; + size_t srcVStride = srcUStride; + copyYV12FrameToOutputBuffer(dst, srcY, srcU, srcV, srcYStride, srcUStride, srcVStride); + mPicToHeaderMap.removeItem(picId); delete header; outInfo->mOwnedByUs = false; diff --git a/media/libstagefright/codecs/on2/h264dec/SoftAVC.h b/media/libstagefright/codecs/on2/h264dec/SoftAVC.h index ee69926..253a406 100644 --- a/media/libstagefright/codecs/on2/h264dec/SoftAVC.h +++ b/media/libstagefright/codecs/on2/h264dec/SoftAVC.h @@ -55,8 +55,6 @@ private: size_t mInputBufferCount; - uint32_t mPictureSize; - uint8_t *mFirstPicture; int32_t mFirstPictureId; @@ -75,8 +73,7 @@ private: void drainAllOutputBuffers(bool eos); void drainOneOutputBuffer(int32_t picId, uint8_t *data); void saveFirstOutputBuffer(int32_t pidId, uint8_t *data); - bool handleCropRectEvent(const CropParams* crop); - bool handlePortSettingChangeEvent(const H264SwDecInfo *info); + bool handleCropChange(const H264SwDecInfo& decInfo); DISALLOW_EVIL_CONSTRUCTORS(SoftAVC); }; diff --git a/media/libstagefright/colorconversion/SoftwareRenderer.cpp b/media/libstagefright/colorconversion/SoftwareRenderer.cpp index cc98da0..1899b40 100644 --- a/media/libstagefright/colorconversion/SoftwareRenderer.cpp +++ b/media/libstagefright/colorconversion/SoftwareRenderer.cpp @@ -65,8 +65,8 @@ void SoftwareRenderer::resetFormatIfChanged(const sp<AMessage> &format) { CHECK(format->findInt32("color-format", &colorFormatNew)); int32_t widthNew, heightNew; - CHECK(format->findInt32("width", &widthNew)); - CHECK(format->findInt32("height", &heightNew)); + CHECK(format->findInt32("stride", &widthNew)); + CHECK(format->findInt32("slice-height", &heightNew)); int32_t cropLeftNew, cropTopNew, cropRightNew, cropBottomNew; if (!format->findRect( diff --git a/media/libstagefright/foundation/ALooper.cpp b/media/libstagefright/foundation/ALooper.cpp index ebf9d8d..88b1c92 100644 --- a/media/libstagefright/foundation/ALooper.cpp +++ b/media/libstagefright/foundation/ALooper.cpp @@ -68,14 +68,14 @@ int64_t ALooper::GetNowUs() { ALooper::ALooper() : mRunningLocally(false) { + // clean up stale AHandlers. Doing it here instead of in the destructor avoids + // the side effect of objects being deleted from the unregister function recursively. + gLooperRoster.unregisterStaleHandlers(); } ALooper::~ALooper() { stop(); - - // Since this looper is "dead" (or as good as dead by now), - // have ALooperRoster unregister any handlers still registered for it. - gLooperRoster.unregisterStaleHandlers(); + // stale AHandlers are now cleaned up in the constructor of the next ALooper to come along } void ALooper::setName(const char *name) { diff --git a/media/libstagefright/foundation/ALooperRoster.cpp b/media/libstagefright/foundation/ALooperRoster.cpp index 0f44b52..e0dc768 100644 --- a/media/libstagefright/foundation/ALooperRoster.cpp +++ b/media/libstagefright/foundation/ALooperRoster.cpp @@ -99,35 +99,13 @@ void ALooperRoster::unregisterStaleHandlers() { status_t ALooperRoster::postMessage( const sp<AMessage> &msg, int64_t delayUs) { - Mutex::Autolock autoLock(mLock); - return postMessage_l(msg, delayUs); -} - -status_t ALooperRoster::postMessage_l( - const sp<AMessage> &msg, int64_t delayUs) { - ssize_t index = mHandlers.indexOfKey(msg->target()); - if (index < 0) { - ALOGW("failed to post message '%s'. Target handler not registered.", - msg->debugString().c_str()); - return -ENOENT; - } - - const HandlerInfo &info = mHandlers.valueAt(index); - - sp<ALooper> looper = info.mLooper.promote(); + sp<ALooper> looper = findLooper(msg->target()); if (looper == NULL) { - ALOGW("failed to post message. " - "Target handler %d still registered, but object gone.", - msg->target()); - - mHandlers.removeItemsAt(index); return -ENOENT; } - looper->post(msg, delayUs); - return OK; } @@ -181,18 +159,23 @@ sp<ALooper> ALooperRoster::findLooper(ALooper::handler_id handlerID) { status_t ALooperRoster::postAndAwaitResponse( const sp<AMessage> &msg, sp<AMessage> *response) { + sp<ALooper> looper = findLooper(msg->target()); + + if (looper == NULL) { + ALOGW("failed to post message. " + "Target handler %d still registered, but object gone.", + msg->target()); + response->clear(); + return -ENOENT; + } + Mutex::Autolock autoLock(mLock); uint32_t replyID = mNextReplyID++; msg->setInt32("replyID", replyID); - status_t err = postMessage_l(msg, 0 /* delayUs */); - - if (err != OK) { - response->clear(); - return err; - } + looper->post(msg, 0 /* delayUs */); ssize_t index; while ((index = mReplies.indexOfKey(replyID)) < 0) { diff --git a/media/libstagefright/foundation/AMessage.cpp b/media/libstagefright/foundation/AMessage.cpp index d268aa4..bc3e3fb 100644 --- a/media/libstagefright/foundation/AMessage.cpp +++ b/media/libstagefright/foundation/AMessage.cpp @@ -14,6 +14,11 @@ * limitations under the License. */ +#define LOG_TAG "AMessage" +//#define LOG_NDEBUG 0 +//#define DUMP_STATS +#include <cutils/log.h> + #include "AMessage.h" #include <ctype.h> @@ -60,12 +65,14 @@ ALooper::handler_id AMessage::target() const { void AMessage::clear() { for (size_t i = 0; i < mNumItems; ++i) { Item *item = &mItems[i]; - freeItem(item); + delete[] item->mName; + item->mName = NULL; + freeItemValue(item); } mNumItems = 0; } -void AMessage::freeItem(Item *item) { +void AMessage::freeItemValue(Item *item) { switch (item->mType) { case kTypeString: { @@ -88,25 +95,85 @@ void AMessage::freeItem(Item *item) { } } -AMessage::Item *AMessage::allocateItem(const char *name) { - name = AAtomizer::Atomize(name); +#ifdef DUMP_STATS +#include <utils/Mutex.h> + +Mutex gLock; +static int32_t gFindItemCalls = 1; +static int32_t gDupCalls = 1; +static int32_t gAverageNumItems = 0; +static int32_t gAverageNumChecks = 0; +static int32_t gAverageNumMemChecks = 0; +static int32_t gAverageDupItems = 0; +static int32_t gLastChecked = -1; + +static void reportStats() { + int32_t time = (ALooper::GetNowUs() / 1000); + if (time / 1000 != gLastChecked / 1000) { + gLastChecked = time; + ALOGI("called findItemIx %zu times (for len=%.1f i=%.1f/%.1f mem) dup %zu times (for len=%.1f)", + gFindItemCalls, + gAverageNumItems / (float)gFindItemCalls, + gAverageNumChecks / (float)gFindItemCalls, + gAverageNumMemChecks / (float)gFindItemCalls, + gDupCalls, + gAverageDupItems / (float)gDupCalls); + gFindItemCalls = gDupCalls = 1; + gAverageNumItems = gAverageNumChecks = gAverageNumMemChecks = gAverageDupItems = 0; + gLastChecked = time; + } +} +#endif +inline size_t AMessage::findItemIndex(const char *name, size_t len) const { +#ifdef DUMP_STATS + size_t memchecks = 0; +#endif size_t i = 0; - while (i < mNumItems && mItems[i].mName != name) { - ++i; + for (; i < mNumItems; i++) { + if (len != mItems[i].mNameLength) { + continue; + } +#ifdef DUMP_STATS + ++memchecks; +#endif + if (!memcmp(mItems[i].mName, name, len)) { + break; + } + } +#ifdef DUMP_STATS + { + Mutex::Autolock _l(gLock); + ++gFindItemCalls; + gAverageNumItems += mNumItems; + gAverageNumMemChecks += memchecks; + gAverageNumChecks += i; + reportStats(); } +#endif + return i; +} + +// assumes item's name was uninitialized or NULL +void AMessage::Item::setName(const char *name, size_t len) { + mNameLength = len; + mName = new char[len + 1]; + memcpy((void*)mName, name, len + 1); +} +AMessage::Item *AMessage::allocateItem(const char *name) { + size_t len = strlen(name); + size_t i = findItemIndex(name, len); Item *item; if (i < mNumItems) { item = &mItems[i]; - freeItem(item); + freeItemValue(item); } else { CHECK(mNumItems < kMaxNumItems); i = mNumItems++; item = &mItems[i]; - - item->mName = name; + item->setName(name, len); } return item; @@ -114,31 +181,18 @@ AMessage::Item *AMessage::allocateItem(const char *name) { const AMessage::Item *AMessage::findItem( const char *name, Type type) const { - name = AAtomizer::Atomize(name); - - for (size_t i = 0; i < mNumItems; ++i) { + size_t i = findItemIndex(name, strlen(name)); + if (i < mNumItems) { const Item *item = &mItems[i]; + return item->mType == type ? item : NULL; - if (item->mName == name) { - return item->mType == type ? item : NULL; - } } - return NULL; } bool AMessage::contains(const char *name) const { - name = AAtomizer::Atomize(name); - - for (size_t i = 0; i < mNumItems; ++i) { - const Item *item = &mItems[i]; - - if (item->mName == name) { - return true; - } - } - - return false; + size_t i = findItemIndex(name, strlen(name)); + return i < mNumItems; } #define BASIC_TYPE(NAME,FIELDNAME,TYPENAME) \ @@ -297,11 +351,20 @@ sp<AMessage> AMessage::dup() const { sp<AMessage> msg = new AMessage(mWhat, mTarget); msg->mNumItems = mNumItems; +#ifdef DUMP_STATS + { + Mutex::Autolock _l(gLock); + ++gDupCalls; + gAverageDupItems += mNumItems; + reportStats(); + } +#endif + for (size_t i = 0; i < mNumItems; ++i) { const Item *from = &mItems[i]; Item *to = &msg->mItems[i]; - to->mName = from->mName; + to->setName(from->mName, from->mNameLength); to->mType = from->mType; switch (from->mType) { @@ -472,11 +535,11 @@ sp<AMessage> AMessage::FromParcel(const Parcel &parcel) { sp<AMessage> msg = new AMessage(what); msg->mNumItems = static_cast<size_t>(parcel.readInt32()); - for (size_t i = 0; i < msg->mNumItems; ++i) { Item *item = &msg->mItems[i]; - item->mName = AAtomizer::Atomize(parcel.readCString()); + const char *name = parcel.readCString(); + item->setName(name, strlen(name)); item->mType = static_cast<Type>(parcel.readInt32()); switch (item->mType) { diff --git a/media/libstagefright/httplive/LiveSession.cpp b/media/libstagefright/httplive/LiveSession.cpp index 7b18348..3720085 100644 --- a/media/libstagefright/httplive/LiveSession.cpp +++ b/media/libstagefright/httplive/LiveSession.cpp @@ -81,6 +81,7 @@ LiveSession::LiveSession( mDiscontinuities.add(indexToType(i), new AnotherPacketSource(NULL /* meta */)); mPacketSources.add(indexToType(i), new AnotherPacketSource(NULL /* meta */)); mPacketSources2.add(indexToType(i), new AnotherPacketSource(NULL /* meta */)); + mBuffering[i] = false; } } @@ -133,8 +134,26 @@ status_t LiveSession::dequeueAccessUnit( sp<AnotherPacketSource> packetSource = mPacketSources.valueFor(stream); + ssize_t idx = typeToIndex(stream); if (!packetSource->hasBufferAvailable(&finalResult)) { - return finalResult == OK ? -EAGAIN : finalResult; + if (finalResult == OK) { + mBuffering[idx] = true; + return -EAGAIN; + } else { + return finalResult; + } + } + + if (mBuffering[idx]) { + if (mSwitchInProgress + || packetSource->isFinished(0) + || packetSource->getEstimatedDurationUs() > 10000000ll) { + mBuffering[idx] = false; + } + } + + if (mBuffering[idx]) { + return -EAGAIN; } // wait for counterpart @@ -498,7 +517,7 @@ void LiveSession::onMessageReceived(const sp<AMessage> &msg) { break; } - onCheckBandwidth(); + onCheckBandwidth(msg); break; } @@ -531,6 +550,19 @@ void LiveSession::onMessageReceived(const sp<AMessage> &msg) { onSwapped(msg); break; } + + case kWhatCheckSwitchDown: + { + onCheckSwitchDown(); + break; + } + + case kWhatSwitchDown: + { + onSwitchDown(); + break; + } + default: TRESPASS(); break; @@ -554,6 +586,21 @@ LiveSession::StreamType LiveSession::indexToType(int idx) { return (StreamType)(1 << idx); } +// static +ssize_t LiveSession::typeToIndex(int32_t type) { + switch (type) { + case STREAMTYPE_AUDIO: + return 0; + case STREAMTYPE_VIDEO: + return 1; + case STREAMTYPE_SUBTITLES: + return 2; + default: + return -1; + }; + return -1; +} + void LiveSession::onConnect(const sp<AMessage> &msg) { AString url; CHECK(msg->findString("url", &url)); @@ -643,6 +690,9 @@ void LiveSession::finishDisconnect() { // (finishDisconnect, onFinishDisconnect2) cancelBandwidthSwitch(); + // cancel switch down monitor + mSwitchDownMonitor.clear(); + for (size_t i = 0; i < mFetcherInfos.size(); ++i) { mFetcherInfos.valueAt(i).mFetcher->stopAsync(); } @@ -919,14 +969,22 @@ size_t LiveSession::getBandwidthIndex() { } } - // Consider only 80% of the available bandwidth usable. - bandwidthBps = (bandwidthBps * 8) / 10; - // Pick the highest bandwidth stream below or equal to estimated bandwidth. index = mBandwidthItems.size() - 1; - while (index > 0 && mBandwidthItems.itemAt(index).mBandwidth - > (size_t)bandwidthBps) { + while (index > 0) { + // consider only 80% of the available bandwidth, but if we are switching up, + // be even more conservative (70%) to avoid overestimating and immediately + // switching back. + size_t adjustedBandwidthBps = bandwidthBps; + if (index > mCurBandwidthIndex) { + adjustedBandwidthBps = adjustedBandwidthBps * 7 / 10; + } else { + adjustedBandwidthBps = adjustedBandwidthBps * 8 / 10; + } + if (mBandwidthItems.itemAt(index).mBandwidth <= adjustedBandwidthBps) { + break; + } --index; } } @@ -1228,12 +1286,6 @@ void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { CHECK(msg->findInt32("streamMask", (int32_t *)&streamMask)); CHECK(msg->findInt32("resumeMask", (int32_t *)&resumeMask)); - for (size_t i = 0; i < kMaxStreams; ++i) { - if (streamMask & indexToType(i)) { - CHECK(msg->findString(mStreams[i].uriKey().c_str(), &mStreams[i].mUri)); - } - } - int64_t timeUs; int32_t pickTrack; bool switching = false; @@ -1249,7 +1301,20 @@ void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { mRealTimeBaseUs = ALooper::GetNowUs() - timeUs; } + for (size_t i = 0; i < kMaxStreams; ++i) { + if (streamMask & indexToType(i)) { + if (switching) { + CHECK(msg->findString(mStreams[i].uriKey().c_str(), &mStreams[i].mNewUri)); + } else { + CHECK(msg->findString(mStreams[i].uriKey().c_str(), &mStreams[i].mUri)); + } + } + } + mNewStreamMask = streamMask | resumeMask; + if (switching) { + mSwapMask = mStreamMask & ~resumeMask; + } // Of all existing fetchers: // * Resume fetchers that are still needed and assign them original packet sources. @@ -1299,7 +1364,7 @@ void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { } AString uri; - uri = mStreams[i].mUri; + uri = switching ? mStreams[i].mNewUri : mStreams[i].mUri; sp<PlaylistFetcher> fetcher = addFetcher(uri.c_str()); CHECK(fetcher != NULL); @@ -1312,7 +1377,8 @@ void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { // TRICKY: looping from i as earlier streams are already removed from streamMask for (size_t j = i; j < kMaxStreams; ++j) { - if ((streamMask & indexToType(j)) && uri == mStreams[j].mUri) { + const AString &streamUri = switching ? mStreams[j].mNewUri : mStreams[j].mUri; + if ((streamMask & indexToType(j)) && uri == streamUri) { sources[j] = mPacketSources.valueFor(indexToType(j)); if (timeUs >= 0) { @@ -1394,13 +1460,13 @@ void LiveSession::onChangeConfiguration3(const sp<AMessage> &msg) { // All fetchers have now been started, the configuration change // has completed. + cancelCheckBandwidthEvent(); scheduleCheckBandwidthEvent(); ALOGV("XXX configuration change completed."); mReconfigurationInProgress = false; if (switching) { mSwitchInProgress = true; - mSwapMask = streamMask; } else { mStreamMask = mNewStreamMask; } @@ -1419,6 +1485,15 @@ void LiveSession::onSwapped(const sp<AMessage> &msg) { int32_t stream; CHECK(msg->findInt32("stream", &stream)); + + ssize_t idx = typeToIndex(stream); + CHECK(idx >= 0); + if ((mNewStreamMask & stream) && mStreams[idx].mNewUri.empty()) { + ALOGW("swapping stream type %d %s to empty stream", stream, mStreams[idx].mUri.c_str()); + } + mStreams[idx].mUri = mStreams[idx].mNewUri; + mStreams[idx].mNewUri.clear(); + mSwapMask &= ~stream; if (mSwapMask != 0) { return; @@ -1430,11 +1505,58 @@ void LiveSession::onSwapped(const sp<AMessage> &msg) { StreamType extraStream = (StreamType) (extraStreams & ~(extraStreams - 1)); swapPacketSource(extraStream); extraStreams &= ~extraStream; + + idx = typeToIndex(extraStream); + CHECK(idx >= 0); + if (mStreams[idx].mNewUri.empty()) { + ALOGW("swapping extra stream type %d %s to empty stream", + extraStream, mStreams[idx].mUri.c_str()); + } + mStreams[idx].mUri = mStreams[idx].mNewUri; + mStreams[idx].mNewUri.clear(); } tryToFinishBandwidthSwitch(); } +void LiveSession::onCheckSwitchDown() { + if (mSwitchDownMonitor == NULL) { + return; + } + + for (size_t i = 0; i < kMaxStreams; ++i) { + int32_t targetDuration; + sp<AnotherPacketSource> packetSource = mPacketSources.valueFor(indexToType(i)); + sp<AMessage> meta = packetSource->getLatestDequeuedMeta(); + + if (meta != NULL && meta->findInt32("targetDuration", &targetDuration) ) { + int64_t bufferedDurationUs = packetSource->getEstimatedDurationUs(); + int64_t targetDurationUs = targetDuration * 1000000ll; + + if (bufferedDurationUs < targetDurationUs / 3) { + (new AMessage(kWhatSwitchDown, id()))->post(); + break; + } + } + } + + mSwitchDownMonitor->post(1000000ll); +} + +void LiveSession::onSwitchDown() { + if (mReconfigurationInProgress || mSwitchInProgress || mCurBandwidthIndex == 0) { + return; + } + + ssize_t bandwidthIndex = getBandwidthIndex(); + if (bandwidthIndex < mCurBandwidthIndex) { + changeConfiguration(-1, bandwidthIndex, false); + return; + } + + changeConfiguration(-1, mCurBandwidthIndex - 1, false); +} + // Mark switch done when: // 1. all old buffers are swapped out void LiveSession::tryToFinishBandwidthSwitch() { @@ -1472,6 +1594,28 @@ void LiveSession::cancelBandwidthSwitch() { mSwitchGeneration++; mSwitchInProgress = false; mSwapMask = 0; + + for (size_t i = 0; i < mFetcherInfos.size(); ++i) { + FetcherInfo& info = mFetcherInfos.editValueAt(i); + if (info.mToBeRemoved) { + info.mToBeRemoved = false; + } + } + + for (size_t i = 0; i < kMaxStreams; ++i) { + if (!mStreams[i].mNewUri.empty()) { + ssize_t j = mFetcherInfos.indexOfKey(mStreams[i].mNewUri); + if (j < 0) { + mStreams[i].mNewUri.clear(); + continue; + } + + const FetcherInfo &info = mFetcherInfos.valueAt(j); + info.mFetcher->stopAsync(); + mFetcherInfos.removeItemsAt(j); + mStreams[i].mNewUri.clear(); + } + } } bool LiveSession::canSwitchBandwidthTo(size_t bandwidthIndex) { @@ -1492,20 +1636,16 @@ bool LiveSession::canSwitchBandwidthTo(size_t bandwidthIndex) { } } -void LiveSession::onCheckBandwidth() { +void LiveSession::onCheckBandwidth(const sp<AMessage> &msg) { size_t bandwidthIndex = getBandwidthIndex(); if (canSwitchBandwidthTo(bandwidthIndex)) { changeConfiguration(-1ll /* timeUs */, bandwidthIndex); } else { - scheduleCheckBandwidthEvent(); + // Come back and check again 10 seconds later in case there is nothing to do now. + // If we DO change configuration, once that completes it'll schedule a new + // check bandwidth event with an incremented mCheckBandwidthGeneration. + msg->post(10000000ll); } - - // Handling the kWhatCheckBandwidth even here does _not_ automatically - // schedule another one on return, only an explicit call to - // scheduleCheckBandwidthEvent will do that. - // This ensures that only one configuration change is ongoing at any - // one time, once that completes it'll schedule another check bandwidth - // event. } void LiveSession::postPrepared(status_t err) { @@ -1522,6 +1662,9 @@ void LiveSession::postPrepared(status_t err) { notify->post(); mInPreparationPhase = false; + + mSwitchDownMonitor = new AMessage(kWhatCheckSwitchDown, id()); + mSwitchDownMonitor->post(); } } // namespace android diff --git a/media/libstagefright/httplive/LiveSession.h b/media/libstagefright/httplive/LiveSession.h index 5423f0f..6be86cf 100644 --- a/media/libstagefright/httplive/LiveSession.h +++ b/media/libstagefright/httplive/LiveSession.h @@ -108,6 +108,8 @@ private: kWhatChangeConfiguration3 = 'chC3', kWhatFinishDisconnect2 = 'fin2', kWhatSwapped = 'swap', + kWhatCheckSwitchDown = 'ckSD', + kWhatSwitchDown = 'sDwn', }; struct BandwidthItem { @@ -124,7 +126,7 @@ private: struct StreamItem { const char *mType; - AString mUri; + AString mUri, mNewUri; size_t mCurDiscontinuitySeq; int64_t mLastDequeuedTimeUs; int64_t mLastSampleDurationUs; @@ -151,6 +153,7 @@ private: sp<IMediaHTTPService> mHTTPService; bool mInPreparationPhase; + bool mBuffering[kMaxStreams]; sp<HTTPBase> mHTTPDataSource; KeyedVector<String8, String8> mExtraHeaders; @@ -202,6 +205,7 @@ private: bool mFirstTimeUsValid; int64_t mFirstTimeUs; int64_t mLastSeekTimeUs; + sp<AMessage> mSwitchDownMonitor; KeyedVector<size_t, int64_t> mDiscontinuityAbsStartTimesUs; KeyedVector<size_t, int64_t> mDiscontinuityOffsetTimesUs; @@ -239,6 +243,7 @@ private: static int SortByBandwidth(const BandwidthItem *, const BandwidthItem *); static StreamType indexToType(int idx); + static ssize_t typeToIndex(int32_t type); void changeConfiguration( int64_t timeUs, size_t bandwidthIndex, bool pickTrack = false); @@ -246,6 +251,8 @@ private: void onChangeConfiguration2(const sp<AMessage> &msg); void onChangeConfiguration3(const sp<AMessage> &msg); void onSwapped(const sp<AMessage> &msg); + void onCheckSwitchDown(); + void onSwitchDown(); void tryToFinishBandwidthSwitch(); void scheduleCheckBandwidthEvent(); @@ -257,7 +264,7 @@ private: void cancelBandwidthSwitch(); bool canSwitchBandwidthTo(size_t bandwidthIndex); - void onCheckBandwidth(); + void onCheckBandwidth(const sp<AMessage> &msg); void finishDisconnect(); diff --git a/media/libstagefright/httplive/PlaylistFetcher.cpp b/media/libstagefright/httplive/PlaylistFetcher.cpp index 4d5d79e..1166762 100644 --- a/media/libstagefright/httplive/PlaylistFetcher.cpp +++ b/media/libstagefright/httplive/PlaylistFetcher.cpp @@ -490,11 +490,11 @@ status_t PlaylistFetcher::onStart(const sp<AMessage> &msg) { mStreamTypeMask = streamTypeMask; - mStartTimeUs = startTimeUs; mSegmentStartTimeUs = segmentStartTimeUs; mDiscontinuitySeq = startDiscontinuitySeq; - if (mStartTimeUs >= 0ll) { + if (startTimeUs >= 0) { + mStartTimeUs = startTimeUs; mSeqNumber = -1; mStartup = true; mPrepared = false; @@ -737,12 +737,6 @@ void PlaylistFetcher::onDownloadNext() { const int32_t lastSeqNumberInPlaylist = firstSeqNumberInPlaylist + (int32_t)mPlaylist->size() - 1; - if (mStartup && mSeqNumber >= 0 - && (mSeqNumber < firstSeqNumberInPlaylist || mSeqNumber > lastSeqNumberInPlaylist)) { - // in case we guessed wrong during reconfiguration, try fetching the latest content. - mSeqNumber = lastSeqNumberInPlaylist; - } - if (mDiscontinuitySeq < 0) { mDiscontinuitySeq = mPlaylist->getDiscontinuitySeq(); } @@ -951,7 +945,7 @@ void PlaylistFetcher::onDownloadNext() { } if (err == -EAGAIN) { - // starting sequence number too low + // starting sequence number too low/high mTSParser.clear(); postMonitorQueue(); return; @@ -1023,12 +1017,39 @@ void PlaylistFetcher::onDownloadNext() { return; } - mStartup = false; ++mSeqNumber; postMonitorQueue(); } +int32_t PlaylistFetcher::getSeqNumberWithAnchorTime(int64_t anchorTimeUs) const { + int32_t firstSeqNumberInPlaylist, lastSeqNumberInPlaylist; + if (mPlaylist->meta() == NULL + || !mPlaylist->meta()->findInt32("media-sequence", &firstSeqNumberInPlaylist)) { + firstSeqNumberInPlaylist = 0; + } + lastSeqNumberInPlaylist = firstSeqNumberInPlaylist + mPlaylist->size() - 1; + + int32_t index = mSeqNumber - firstSeqNumberInPlaylist - 1; + while (index >= 0 && anchorTimeUs > mStartTimeUs) { + sp<AMessage> itemMeta; + CHECK(mPlaylist->itemAt(index, NULL /* uri */, &itemMeta)); + + int64_t itemDurationUs; + CHECK(itemMeta->findInt64("durationUs", &itemDurationUs)); + + anchorTimeUs -= itemDurationUs; + --index; + } + + int32_t newSeqNumber = firstSeqNumberInPlaylist + index + 1; + if (newSeqNumber <= lastSeqNumberInPlaylist) { + return newSeqNumber; + } else { + return lastSeqNumberInPlaylist; + } +} + int32_t PlaylistFetcher::getSeqNumberForDiscontinuity(size_t discontinuitySeq) const { int32_t firstSeqNumberInPlaylist; if (mPlaylist->meta() == NULL @@ -1198,60 +1219,84 @@ status_t PlaylistFetcher::extractAndQueueAccessUnitsFromTs(const sp<ABuffer> &bu if (timeUs < 0) { timeUs = 0; } - } else if (mAdaptive && timeUs > mStartTimeUs) { - int32_t seq; - if (mStartTimeUsNotify != NULL - && !mStartTimeUsNotify->findInt32("discontinuitySeq", &seq)) { - mStartTimeUsNotify->setInt32("discontinuitySeq", mDiscontinuitySeq); + } + + if (timeUs < mStartTimeUs) { + // buffer up to the closest preceding IDR frame + ALOGV("timeUs %" PRId64 " us < mStartTimeUs %" PRId64 " us", + timeUs, mStartTimeUs); + const char *mime; + sp<MetaData> format = source->getFormat(); + bool isAvc = false; + if (format != NULL && format->findCString(kKeyMIMEType, &mime) + && !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC)) { + isAvc = true; + } + if (isAvc && IsIDR(accessUnit)) { + mVideoBuffer->clear(); } - int64_t startTimeUs; - if (mStartTimeUsNotify != NULL - && !mStartTimeUsNotify->findInt64(key, &startTimeUs)) { - mStartTimeUsNotify->setInt64(key, timeUs); - - uint32_t streamMask = 0; - mStartTimeUsNotify->findInt32("streamMask", (int32_t *) &streamMask); - streamMask |= mPacketSources.keyAt(i); - mStartTimeUsNotify->setInt32("streamMask", streamMask); - - if (streamMask == mStreamTypeMask) { - mStartTimeUsNotify->post(); - mStartTimeUsNotify.clear(); - } + if (isAvc) { + mVideoBuffer->queueAccessUnit(accessUnit); } + + continue; } + } - if (timeUs < mStartTimeUs) { - if (mAdaptive) { - int32_t targetDuration; - mPlaylist->meta()->findInt32("target-duration", &targetDuration); - int32_t incr = (mStartTimeUs - timeUs) / 1000000 / targetDuration; - if (incr == 0) { - // increment mSeqNumber by at least one - incr = 1; - } - mSeqNumber += incr; - err = -EAGAIN; - break; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + if (mStartTimeUsNotify != NULL && timeUs > mStartTimeUs) { + + int32_t targetDurationSecs; + CHECK(mPlaylist->meta()->findInt32("target-duration", &targetDurationSecs)); + int64_t targetDurationUs = targetDurationSecs * 1000000ll; + // mStartup + // mStartup is true until we have queued a packet for all the streams + // we are fetching. We queue packets whose timestamps are greater than + // mStartTimeUs. + // mSegmentStartTimeUs >= 0 + // mSegmentStartTimeUs is non-negative when adapting or switching tracks + // timeUs - mStartTimeUs > targetDurationUs: + // This and the 2 above conditions should only happen when adapting in a live + // stream; the old fetcher has already fetched to mStartTimeUs; the new fetcher + // would start fetching after timeUs, which should be greater than mStartTimeUs; + // the old fetcher would then continue fetching data until timeUs. We don't want + // timeUs to be too far ahead of mStartTimeUs because we want the old fetcher to + // stop as early as possible. The definition of being "too far ahead" is + // arbitrary; here we use targetDurationUs as threshold. + if (mStartup && mSegmentStartTimeUs >= 0 + && timeUs - mStartTimeUs > targetDurationUs) { + // we just guessed a starting timestamp that is too high when adapting in a + // live stream; re-adjust based on the actual timestamp extracted from the + // media segment; if we didn't move backward after the re-adjustment + // (newSeqNumber), start at least 1 segment prior. + int32_t newSeqNumber = getSeqNumberWithAnchorTime(timeUs); + if (newSeqNumber >= mSeqNumber) { + --mSeqNumber; } else { - // buffer up to the closest preceding IDR frame - ALOGV("timeUs %" PRId64 " us < mStartTimeUs %" PRId64 " us", - timeUs, mStartTimeUs); - const char *mime; - sp<MetaData> format = source->getFormat(); - bool isAvc = false; - if (format != NULL && format->findCString(kKeyMIMEType, &mime) - && !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC)) { - isAvc = true; - } - if (isAvc && IsIDR(accessUnit)) { - mVideoBuffer->clear(); - } - if (isAvc) { - mVideoBuffer->queueAccessUnit(accessUnit); - } - - continue; + mSeqNumber = newSeqNumber; + } + mStartTimeUsNotify = mNotify->dup(); + mStartTimeUsNotify->setInt32("what", kWhatStartedAt); + return -EAGAIN; + } + + int32_t seq; + if (!mStartTimeUsNotify->findInt32("discontinuitySeq", &seq)) { + mStartTimeUsNotify->setInt32("discontinuitySeq", mDiscontinuitySeq); + } + int64_t startTimeUs; + if (!mStartTimeUsNotify->findInt64(key, &startTimeUs)) { + mStartTimeUsNotify->setInt64(key, timeUs); + + uint32_t streamMask = 0; + mStartTimeUsNotify->findInt32("streamMask", (int32_t *) &streamMask); + streamMask |= mPacketSources.keyAt(i); + mStartTimeUsNotify->setInt32("streamMask", streamMask); + + if (streamMask == mStreamTypeMask) { + mStartup = false; + mStartTimeUsNotify->post(); + mStartTimeUsNotify.clear(); } } } diff --git a/media/libstagefright/httplive/PlaylistFetcher.h b/media/libstagefright/httplive/PlaylistFetcher.h index daefb26..4ba37fa 100644 --- a/media/libstagefright/httplive/PlaylistFetcher.h +++ b/media/libstagefright/httplive/PlaylistFetcher.h @@ -104,7 +104,12 @@ private: uint32_t mStreamTypeMask; int64_t mStartTimeUs; + + // Start time relative to the beginning of the first segment in the initial + // playlist. It's value is initialized to a non-negative value only when we are + // adapting or switching tracks. int64_t mSegmentStartTimeUs; + ssize_t mDiscontinuitySeq; bool mStartTimeUsRelative; sp<AMessage> mStopParams; // message containing the latest timestamps we should fetch. diff --git a/media/libstagefright/include/SoftVideoDecoderOMXComponent.h b/media/libstagefright/include/SoftVideoDecoderOMXComponent.h index ee553d9..8cb8ed7 100644 --- a/media/libstagefright/include/SoftVideoDecoderOMXComponent.h +++ b/media/libstagefright/include/SoftVideoDecoderOMXComponent.h @@ -63,7 +63,15 @@ protected: OMX_U32 numOutputBuffers, const char *mimeType); - virtual void updatePortDefinitions(); + virtual void updatePortDefinitions(bool updateCrop = true); + + void handlePortSettingsChange( + bool *portWillReset, uint32_t width, uint32_t height, + bool cropChanged = false, bool fakeStride = false); + + void copyYV12FrameToOutputBuffer( + uint8_t *dst, const uint8_t *srcY, const uint8_t *srcU, const uint8_t *srcV, + size_t srcYStride, size_t srcUStride, size_t srcVStride); enum { kInputPortIndex = 0, diff --git a/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp b/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp index 010063f..c74c3e7 100644 --- a/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp +++ b/media/libstagefright/mpeg2ts/AnotherPacketSource.cpp @@ -42,7 +42,8 @@ AnotherPacketSource::AnotherPacketSource(const sp<MetaData> &meta) mLastQueuedTimeUs(0), mEOSResult(OK), mLatestEnqueuedMeta(NULL), - mLatestDequeuedMeta(NULL) { + mLatestDequeuedMeta(NULL), + mQueuedDiscontinuityCount(0) { setFormat(meta); } @@ -122,6 +123,7 @@ status_t AnotherPacketSource::dequeueAccessUnit(sp<ABuffer> *buffer) { mFormat.clear(); } + --mQueuedDiscontinuityCount; return INFO_DISCONTINUITY; } @@ -210,6 +212,11 @@ void AnotherPacketSource::queueAccessUnit(const sp<ABuffer> &buffer) { mBuffers.push_back(buffer); mCondition.signal(); + int32_t discontinuity; + if (buffer->meta()->findInt32("discontinuity", &discontinuity)) { + ++mQueuedDiscontinuityCount; + } + if (mLatestEnqueuedMeta == NULL) { mLatestEnqueuedMeta = buffer->meta(); } else { @@ -226,6 +233,7 @@ void AnotherPacketSource::clear() { mBuffers.clear(); mEOSResult = OK; + mQueuedDiscontinuityCount = 0; mFormat = NULL; mLatestEnqueuedMeta = NULL; @@ -262,6 +270,7 @@ void AnotherPacketSource::queueDiscontinuity( mEOSResult = OK; mLastQueuedTimeUs = 0; mLatestEnqueuedMeta = NULL; + ++mQueuedDiscontinuityCount; sp<ABuffer> buffer = new ABuffer(0); buffer->meta()->setInt32("discontinuity", static_cast<int32_t>(type)); @@ -291,7 +300,10 @@ bool AnotherPacketSource::hasBufferAvailable(status_t *finalResult) { int64_t AnotherPacketSource::getBufferedDurationUs(status_t *finalResult) { Mutex::Autolock autoLock(mLock); + return getBufferedDurationUs_l(finalResult); +} +int64_t AnotherPacketSource::getBufferedDurationUs_l(status_t *finalResult) { *finalResult = mEOSResult; if (mBuffers.empty()) { @@ -300,6 +312,7 @@ int64_t AnotherPacketSource::getBufferedDurationUs(status_t *finalResult) { int64_t time1 = -1; int64_t time2 = -1; + int64_t durationUs = 0; List<sp<ABuffer> >::iterator it = mBuffers.begin(); while (it != mBuffers.end()) { @@ -307,20 +320,64 @@ int64_t AnotherPacketSource::getBufferedDurationUs(status_t *finalResult) { int64_t timeUs; if (buffer->meta()->findInt64("timeUs", &timeUs)) { - if (time1 < 0) { + if (time1 < 0 || timeUs < time1) { time1 = timeUs; } - time2 = timeUs; + if (time2 < 0 || timeUs > time2) { + time2 = timeUs; + } } else { // This is a discontinuity, reset everything. + durationUs += time2 - time1; time1 = time2 = -1; } ++it; } - return time2 - time1; + return durationUs + (time2 - time1); +} + +// A cheaper but less precise version of getBufferedDurationUs that we would like to use in +// LiveSession::dequeueAccessUnit to trigger downwards adaptation. +int64_t AnotherPacketSource::getEstimatedDurationUs() { + Mutex::Autolock autoLock(mLock); + if (mBuffers.empty()) { + return 0; + } + + if (mQueuedDiscontinuityCount > 0) { + status_t finalResult; + return getBufferedDurationUs_l(&finalResult); + } + + List<sp<ABuffer> >::iterator it = mBuffers.begin(); + sp<ABuffer> buffer = *it; + + int64_t startTimeUs; + buffer->meta()->findInt64("timeUs", &startTimeUs); + if (startTimeUs < 0) { + return 0; + } + + it = mBuffers.end(); + --it; + buffer = *it; + + int64_t endTimeUs; + buffer->meta()->findInt64("timeUs", &endTimeUs); + if (endTimeUs < 0) { + return 0; + } + + int64_t diffUs; + if (endTimeUs > startTimeUs) { + diffUs = endTimeUs - startTimeUs; + } else { + diffUs = startTimeUs - endTimeUs; + } + return diffUs; } status_t AnotherPacketSource::nextBufferTime(int64_t *timeUs) { diff --git a/media/libstagefright/mpeg2ts/AnotherPacketSource.h b/media/libstagefright/mpeg2ts/AnotherPacketSource.h index 0c717d7..809a858 100644 --- a/media/libstagefright/mpeg2ts/AnotherPacketSource.h +++ b/media/libstagefright/mpeg2ts/AnotherPacketSource.h @@ -49,6 +49,8 @@ struct AnotherPacketSource : public MediaSource { // presentation timestamps since the last discontinuity (if any). int64_t getBufferedDurationUs(status_t *finalResult); + int64_t getEstimatedDurationUs(); + status_t nextBufferTime(int64_t *timeUs); void queueAccessUnit(const sp<ABuffer> &buffer); @@ -83,7 +85,10 @@ private: sp<AMessage> mLatestEnqueuedMeta; sp<AMessage> mLatestDequeuedMeta; + size_t mQueuedDiscontinuityCount; + bool wasFormatChange(int32_t discontinuityType) const; + int64_t getBufferedDurationUs_l(status_t *finalResult); DISALLOW_EVIL_CONSTRUCTORS(AnotherPacketSource); }; diff --git a/media/libstagefright/omx/SoftVideoDecoderOMXComponent.cpp b/media/libstagefright/omx/SoftVideoDecoderOMXComponent.cpp index 69b572e..1cb1859 100644 --- a/media/libstagefright/omx/SoftVideoDecoderOMXComponent.cpp +++ b/media/libstagefright/omx/SoftVideoDecoderOMXComponent.cpp @@ -123,13 +123,15 @@ void SoftVideoDecoderOMXComponent::initPorts( updatePortDefinitions(); } -void SoftVideoDecoderOMXComponent::updatePortDefinitions() { +void SoftVideoDecoderOMXComponent::updatePortDefinitions(bool updateCrop) { OMX_PARAM_PORTDEFINITIONTYPE *def = &editPortInfo(kInputPortIndex)->mDef; def->format.video.nFrameWidth = mWidth; def->format.video.nFrameHeight = mHeight; def->format.video.nStride = def->format.video.nFrameWidth; def->format.video.nSliceHeight = def->format.video.nFrameHeight; + def->nBufferSize = def->format.video.nFrameWidth * def->format.video.nFrameHeight * 3 / 2; + def = &editPortInfo(kOutputPortIndex)->mDef; def->format.video.nFrameWidth = mIsAdaptive ? mAdaptiveMaxWidth : mWidth; def->format.video.nFrameHeight = mIsAdaptive ? mAdaptiveMaxHeight : mHeight; @@ -140,10 +142,87 @@ void SoftVideoDecoderOMXComponent::updatePortDefinitions() { (def->format.video.nFrameWidth * def->format.video.nFrameHeight * 3) / 2; - mCropLeft = 0; - mCropTop = 0; - mCropWidth = mWidth; - mCropHeight = mHeight; + if (updateCrop) { + mCropLeft = 0; + mCropTop = 0; + mCropWidth = mWidth; + mCropHeight = mHeight; + } +} + +void SoftVideoDecoderOMXComponent::handlePortSettingsChange( + bool *portWillReset, uint32_t width, uint32_t height, bool cropChanged, bool fakeStride) { + *portWillReset = false; + bool sizeChanged = (width != mWidth || height != mHeight); + + if (sizeChanged || cropChanged) { + mWidth = width; + mHeight = height; + + bool updateCrop = !cropChanged; + if ((sizeChanged && !mIsAdaptive) + || width > mAdaptiveMaxWidth + || height > mAdaptiveMaxHeight) { + if (mIsAdaptive) { + if (width > mAdaptiveMaxWidth) { + mAdaptiveMaxWidth = width; + } + if (height > mAdaptiveMaxHeight) { + mAdaptiveMaxHeight = height; + } + } + updatePortDefinitions(updateCrop); + notify(OMX_EventPortSettingsChanged, kOutputPortIndex, 0, NULL); + mOutputPortSettingsChange = AWAITING_DISABLED; + *portWillReset = true; + } else { + updatePortDefinitions(updateCrop); + + if (fakeStride) { + // MAJOR HACK that is not pretty, it's just to fool the renderer to read the correct + // data. + // Some software decoders (e.g. SoftMPEG4) fill decoded frame directly to output + // buffer without considering the output buffer stride and slice height. So this is + // used to signal how the buffer is arranged. The alternative is to re-arrange the + // output buffer in SoftMPEG4, but that results in memcopies. + OMX_PARAM_PORTDEFINITIONTYPE *def = &editPortInfo(kOutputPortIndex)->mDef; + def->format.video.nStride = mWidth; + def->format.video.nSliceHeight = mHeight; + } + + notify(OMX_EventPortSettingsChanged, kOutputPortIndex, + OMX_IndexConfigCommonOutputCrop, NULL); + } + } +} + +void SoftVideoDecoderOMXComponent::copyYV12FrameToOutputBuffer( + uint8_t *dst, const uint8_t *srcY, const uint8_t *srcU, const uint8_t *srcV, + size_t srcYStride, size_t srcUStride, size_t srcVStride) { + size_t dstYStride = mIsAdaptive ? mAdaptiveMaxWidth : mWidth; + size_t dstUVStride = dstYStride / 2; + size_t dstHeight = mIsAdaptive ? mAdaptiveMaxHeight : mHeight; + uint8_t *dstStart = dst; + + for (size_t i = 0; i < mHeight; ++i) { + memcpy(dst, srcY, mWidth); + srcY += srcYStride; + dst += dstYStride; + } + + dst = dstStart + dstYStride * dstHeight; + for (size_t i = 0; i < mHeight / 2; ++i) { + memcpy(dst, srcU, mWidth / 2); + srcU += srcUStride; + dst += dstUVStride; + } + + dst = dstStart + (5 * dstYStride * dstHeight) / 4; + for (size_t i = 0; i < mHeight / 2; ++i) { + memcpy(dst, srcV, mWidth / 2); + srcV += srcVStride; + dst += dstUVStride; + } } OMX_ERRORTYPE SoftVideoDecoderOMXComponent::internalGetParameter( diff --git a/services/audioflinger/AudioFlinger.cpp b/services/audioflinger/AudioFlinger.cpp index 1f77b2f..e200857 100644 --- a/services/audioflinger/AudioFlinger.cpp +++ b/services/audioflinger/AudioFlinger.cpp @@ -418,6 +418,13 @@ status_t AudioFlinger::dump(int fd, const Vector<String16>& args) mRecordThreads.valueAt(i)->dump(fd, args); } + // dump orphan effect chains + if (mOrphanEffectChains.size() != 0) { + write(fd, " Orphan Effect Chains\n", strlen(" Orphan Effect Chains\n")); + for (size_t i = 0; i < mOrphanEffectChains.size(); i++) { + mOrphanEffectChains.valueAt(i)->dump(fd, args); + } + } // dump all hardware devs for (size_t i = 0; i < mAudioHwDevs.size(); i++) { audio_hw_device_t *dev = mAudioHwDevs.valueAt(i)->hwDevice(); @@ -796,9 +803,14 @@ status_t AudioFlinger::setMicMute(bool state) } AutoMutex lock(mHardwareLock); - audio_hw_device_t *dev = mPrimaryHardwareDev->hwDevice(); mHardwareStatus = AUDIO_HW_SET_MIC_MUTE; - ret = dev->set_mic_mute(dev, state); + for (size_t i = 0; i < mAudioHwDevs.size(); i++) { + audio_hw_device_t *dev = mAudioHwDevs.valueAt(i)->hwDevice(); + status_t result = dev->set_mic_mute(dev, state); + if (result != NO_ERROR) { + ret = result; + } + } mHardwareStatus = AUDIO_HW_IDLE; return ret; } @@ -1416,7 +1428,7 @@ sp<IAudioRecord> AudioFlinger::openRecord( *sessionId = lSessionId; } } - ALOGV("openRecord() lSessionId: %d", lSessionId); + ALOGV("openRecord() lSessionId: %d input %d", lSessionId, input); // TODO: the uid should be passed in as a parameter to openRecord recordTrack = thread->createRecordTrack_l(client, sampleRate, format, channelMask, @@ -1941,9 +1953,8 @@ sp<AudioFlinger::RecordThread> AudioFlinger::openInput_l(audio_module_handle_t m TEE_SINK_NEW, // copy input using a new pipe TEE_SINK_OLD, // copy input using an existing pipe } kind; - NBAIO_Format format = Format_from_SR_C(inStream->common.get_sample_rate(&inStream->common), - audio_channel_count_from_in_mask( - inStream->common.get_channels(&inStream->common))); + NBAIO_Format format = Format_from_SR_C(halconfig.sample_rate, + audio_channel_count_from_in_mask(halconfig.channel_mask), halconfig.format); if (!mTeeSinkInputEnabled) { kind = TEE_SINK_NO; } else if (!Format_isValid(format)) { @@ -2023,6 +2034,16 @@ status_t AudioFlinger::closeInput_nonvirtual(audio_io_handle_t input) } ALOGV("closeInput() %d", input); + { + // If we still have effect chains, it means that a client still holds a handle + // on at least one effect. We must keep the chain alive in case a new record + // thread is opened for a new capture on the same session + Mutex::Autolock _sl(thread->mLock); + Vector< sp<EffectChain> > effectChains = thread->getEffectChains_l(); + for (size_t i = 0; i < effectChains.size(); i++) { + putOrphanEffectChain_l(effectChains[i]); + } + } audioConfigChanged(AudioSystem::INPUT_CLOSED, input, NULL); mRecordThreads.removeItem(input); } @@ -2452,6 +2473,13 @@ sp<IEffect> AudioFlinger::createEffect( lStatus = BAD_VALUE; goto Exit; } + } else { + // Check if one effect chain was awaiting for an effect to be created on this + // session and used it instead of creating a new one. + sp<EffectChain> chain = getOrphanEffectChain_l((audio_session_t)sessionId); + if (chain != 0) { + thread->addEffectChain_l(chain); + } } sp<Client> client = registerPid(pid); @@ -2624,6 +2652,49 @@ void AudioFlinger::onNonOffloadableGlobalEffectEnable() } +status_t AudioFlinger::putOrphanEffectChain_l(const sp<AudioFlinger::EffectChain>& chain) +{ + audio_session_t session = (audio_session_t)chain->sessionId(); + ssize_t index = mOrphanEffectChains.indexOfKey(session); + ALOGV("putOrphanEffectChain_l session %d index %d", session, index); + if (index >= 0) { + ALOGW("putOrphanEffectChain_l chain for session %d already present", session); + return ALREADY_EXISTS; + } + mOrphanEffectChains.add(session, chain); + return NO_ERROR; +} + +sp<AudioFlinger::EffectChain> AudioFlinger::getOrphanEffectChain_l(audio_session_t session) +{ + sp<EffectChain> chain; + ssize_t index = mOrphanEffectChains.indexOfKey(session); + ALOGV("getOrphanEffectChain_l session %d index %d", session, index); + if (index >= 0) { + chain = mOrphanEffectChains.valueAt(index); + mOrphanEffectChains.removeItemsAt(index); + } + return chain; +} + +bool AudioFlinger::updateOrphanEffectChains(const sp<AudioFlinger::EffectModule>& effect) +{ + Mutex::Autolock _l(mLock); + audio_session_t session = (audio_session_t)effect->sessionId(); + ssize_t index = mOrphanEffectChains.indexOfKey(session); + ALOGV("updateOrphanEffectChains session %d index %d", session, index); + if (index >= 0) { + sp<EffectChain> chain = mOrphanEffectChains.valueAt(index); + if (chain->removeEffect_l(effect) == 0) { + ALOGV("updateOrphanEffectChains removing effect chain at index %d", index); + mOrphanEffectChains.removeItemsAt(index); + } + return true; + } + return false; +} + + struct Entry { #define MAX_NAME 32 // %Y%m%d%H%M%S_%d.wav char mName[MAX_NAME]; @@ -2700,24 +2771,26 @@ void AudioFlinger::dumpTee(int fd, const sp<NBAIO_Source>& source, audio_io_hand // if 2 dumpsys are done within 1 second, and rotation didn't work, then discard 2nd int teeFd = open(teePath, O_WRONLY | O_CREAT | O_EXCL | O_NOFOLLOW, S_IRUSR | S_IWUSR); if (teeFd >= 0) { + // FIXME use libsndfile char wavHeader[44]; memcpy(wavHeader, "RIFF\0\0\0\0WAVEfmt \20\0\0\0\1\0\2\0\104\254\0\0\0\0\0\0\4\0\20\0data\0\0\0\0", sizeof(wavHeader)); NBAIO_Format format = teeSource->format(); unsigned channelCount = Format_channelCount(format); - ALOG_ASSERT(channelCount <= FCC_2); uint32_t sampleRate = Format_sampleRate(format); + size_t frameSize = Format_frameSize(format); wavHeader[22] = channelCount; // number of channels wavHeader[24] = sampleRate; // sample rate wavHeader[25] = sampleRate >> 8; - wavHeader[32] = channelCount * 2; // block alignment + wavHeader[32] = frameSize; // block alignment + wavHeader[33] = frameSize >> 8; write(teeFd, wavHeader, sizeof(wavHeader)); size_t total = 0; bool firstRead = true; +#define TEE_SINK_READ 1024 // frames per I/O operation + void *buffer = malloc(TEE_SINK_READ * frameSize); for (;;) { -#define TEE_SINK_READ 1024 - short buffer[TEE_SINK_READ * FCC_2]; size_t count = TEE_SINK_READ; ssize_t actual = teeSource->read(buffer, count, AudioBufferProvider::kInvalidPTS); @@ -2730,14 +2803,17 @@ void AudioFlinger::dumpTee(int fd, const sp<NBAIO_Source>& source, audio_io_hand break; } ALOG_ASSERT(actual <= (ssize_t)count); - write(teeFd, buffer, actual * channelCount * sizeof(short)); + write(teeFd, buffer, actual * frameSize); total += actual; } + free(buffer); lseek(teeFd, (off_t) 4, SEEK_SET); - uint32_t temp = 44 + total * channelCount * sizeof(short) - 8; + uint32_t temp = 44 + total * frameSize - 8; + // FIXME not big-endian safe write(teeFd, &temp, sizeof(temp)); lseek(teeFd, (off_t) 40, SEEK_SET); - temp = total * channelCount * sizeof(short); + temp = total * frameSize; + // FIXME not big-endian safe write(teeFd, &temp, sizeof(temp)); close(teeFd); if (fd >= 0) { diff --git a/services/audioflinger/AudioFlinger.h b/services/audioflinger/AudioFlinger.h index 753314f..1003017 100644 --- a/services/audioflinger/AudioFlinger.h +++ b/services/audioflinger/AudioFlinger.h @@ -569,6 +569,23 @@ private: bool isNonOffloadableGlobalEffectEnabled_l(); void onNonOffloadableGlobalEffectEnable(); + // Store an effect chain to mOrphanEffectChains keyed vector. + // Called when a thread exits and effects are still attached to it. + // If effects are later created on the same session, they will reuse the same + // effect chain and same instances in the effect library. + // return ALREADY_EXISTS if a chain with the same session already exists in + // mOrphanEffectChains. Note that this should never happen as there is only one + // chain for a given session and it is attached to only one thread at a time. + status_t putOrphanEffectChain_l(const sp<EffectChain>& chain); + // Get an effect chain for the specified session in mOrphanEffectChains and remove + // it if found. Returns 0 if not found (this is the most common case). + sp<EffectChain> getOrphanEffectChain_l(audio_session_t session); + // Called when the last effect handle on an effect instance is removed. If this + // effect belongs to an effect chain in mOrphanEffectChains, the chain is updated + // and removed from mOrphanEffectChains if it does not contain any effect. + // Return true if the effect was found in mOrphanEffectChains, false otherwise. + bool updateOrphanEffectChains(const sp<EffectModule>& effect); + class AudioHwDevice { public: enum Flags { @@ -713,6 +730,9 @@ private: Vector < sp<SyncEvent> > mPendingSyncEvents; // sync events awaiting for a session // to be created + // Effect chains without a valid thread + DefaultKeyedVector< audio_session_t , sp<EffectChain> > mOrphanEffectChains; + private: sp<Client> registerPid(pid_t pid); // always returns non-0 diff --git a/services/audioflinger/Effects.cpp b/services/audioflinger/Effects.cpp index 365f271..15f1f23 100644 --- a/services/audioflinger/Effects.cpp +++ b/services/audioflinger/Effects.cpp @@ -68,7 +68,8 @@ AudioFlinger::EffectModule::EffectModule(ThreadBase *thread, mStatus(NO_INIT), mState(IDLE), // mMaxDisableWaitCnt is set by configure() and not used before then // mDisableWaitCnt is set by process() and updateState() and not used before then - mSuspended(false) + mSuspended(false), + mAudioFlinger(thread->mAudioFlinger) { ALOGV("Constructor %p", this); int lStatus; @@ -197,9 +198,19 @@ size_t AudioFlinger::EffectModule::disconnect(EffectHandle *handle, bool unpinIf // destructor before we exit sp<EffectModule> keep(this); { - sp<ThreadBase> thread = mThread.promote(); - if (thread != 0) { - thread->disconnectEffect(keep, handle, unpinIfLast); + if (removeHandle(handle) == 0) { + if (!isPinned() || unpinIfLast) { + sp<ThreadBase> thread = mThread.promote(); + if (thread != 0) { + Mutex::Autolock _l(thread->mLock); + thread->removeEffect_l(this); + } + sp<AudioFlinger> af = mAudioFlinger.promote(); + if (af != 0) { + af->updateOrphanEffectChains(this); + } + AudioSystem::unregisterEffect(mId); + } } } return mHandles.size(); @@ -1911,4 +1922,13 @@ bool AudioFlinger::EffectChain::isNonOffloadableEnabled() return false; } +void AudioFlinger::EffectChain::setThread(const sp<ThreadBase>& thread) +{ + Mutex::Autolock _l(mLock); + mThread = thread; + for (size_t i = 0; i < mEffects.size(); i++) { + mEffects[i]->setThread(thread); + } +} + }; // namespace android diff --git a/services/audioflinger/Effects.h b/services/audioflinger/Effects.h index 4170fd4..eaf90e7 100644 --- a/services/audioflinger/Effects.h +++ b/services/audioflinger/Effects.h @@ -153,6 +153,7 @@ mutable Mutex mLock; // mutex for process, commands and handl uint32_t mDisableWaitCnt; // current process() calls count during disable period. bool mSuspended; // effect is suspended: temporarily disabled by framework bool mOffloaded; // effect is currently offloaded to the audio DSP + wp<AudioFlinger> mAudioFlinger; }; // The EffectHandle class implements the IEffect interface. It provides resources @@ -347,6 +348,8 @@ protected: void clearInputBuffer_l(sp<ThreadBase> thread); + void setThread(const sp<ThreadBase>& thread); + wp<ThreadBase> mThread; // parent mixer thread Mutex mLock; // mutex protecting effect list Vector< sp<EffectModule> > mEffects; // list of effect modules diff --git a/services/audioflinger/FastMixer.cpp b/services/audioflinger/FastMixer.cpp index 9e15293..2678cbf 100644 --- a/services/audioflinger/FastMixer.cpp +++ b/services/audioflinger/FastMixer.cpp @@ -420,7 +420,7 @@ void FastMixer::onWork() // if non-NULL, then duplicate write() to this non-blocking sink NBAIO_Sink* teeSink; if ((teeSink = current->mTeeSink) != NULL) { - (void) teeSink->write(mMixerBuffer, frameCount); + (void) teeSink->write(buffer, frameCount); } // FIXME write() is non-blocking and lock-free for a properly implemented NBAIO sink, // but this code should be modified to handle both non-blocking and blocking sinks diff --git a/services/audioflinger/PatchPanel.cpp b/services/audioflinger/PatchPanel.cpp index 2d0a25f..7544052 100644 --- a/services/audioflinger/PatchPanel.cpp +++ b/services/audioflinger/PatchPanel.cpp @@ -593,10 +593,10 @@ status_t AudioFlinger::PatchPanel::releaseAudioPatch(audio_patch_handle_t handle status = BAD_VALUE; break; } - status = thread->sendReleaseAudioPatchConfigEvent(mPatches[index]->mHalHandle); + status = thread->sendReleaseAudioPatchConfigEvent(removedPatch->mHalHandle); } else { audio_hw_device_t *hwDevice = audioHwDevice->hwDevice(); - status = hwDevice->release_audio_patch(hwDevice, mPatches[index]->mHalHandle); + status = hwDevice->release_audio_patch(hwDevice, removedPatch->mHalHandle); } } else { sp<ThreadBase> thread = audioflinger->checkRecordThread_l( @@ -632,7 +632,7 @@ status_t AudioFlinger::PatchPanel::releaseAudioPatch(audio_patch_handle_t handle } AudioHwDevice *audioHwDevice = audioflinger->mAudioHwDevs.valueAt(index); if (audioHwDevice->version() >= AUDIO_DEVICE_API_VERSION_3_0) { - status = thread->sendReleaseAudioPatchConfigEvent(mPatches[index]->mHalHandle); + status = thread->sendReleaseAudioPatchConfigEvent(removedPatch->mHalHandle); } else { AudioParameter param; param.addInt(String8(AUDIO_PARAMETER_STREAM_ROUTING), 0); diff --git a/services/audioflinger/Threads.cpp b/services/audioflinger/Threads.cpp index f721d5c..3d17c89 100644 --- a/services/audioflinger/Threads.cpp +++ b/services/audioflinger/Threads.cpp @@ -662,12 +662,14 @@ void AudioFlinger::ThreadBase::acquireWakeLock_l(int uid) binder, getWakeLockTag(), String16("media"), - uid); + uid, + true /* FIXME force oneway contrary to .aidl */); } else { status = mPowerManager->acquireWakeLock(POWERMANAGER_PARTIAL_WAKE_LOCK, binder, getWakeLockTag(), - String16("media")); + String16("media"), + true /* FIXME force oneway contrary to .aidl */); } if (status == NO_ERROR) { mWakeLockToken = binder; @@ -687,7 +689,8 @@ void AudioFlinger::ThreadBase::releaseWakeLock_l() if (mWakeLockToken != 0) { ALOGV("releaseWakeLock_l() %s", mName); if (mPowerManager != 0) { - mPowerManager->releaseWakeLock(mWakeLockToken, 0); + mPowerManager->releaseWakeLock(mWakeLockToken, 0, + true /* FIXME force oneway contrary to .aidl */); } mWakeLockToken.clear(); } @@ -723,7 +726,8 @@ void AudioFlinger::ThreadBase::updateWakeLockUids_l(const SortedVector<int> &uid if (mPowerManager != 0) { sp<IBinder> binder = new BBinder(); status_t status; - status = mPowerManager->updateWakeLockUids(mWakeLockToken, uids.size(), uids.array()); + status = mPowerManager->updateWakeLockUids(mWakeLockToken, uids.size(), uids.array(), + true /* FIXME force oneway contrary to .aidl */); ALOGV("acquireWakeLock_l() %s status %d", mName, status); } } @@ -1143,21 +1147,6 @@ void AudioFlinger::ThreadBase::setMode(audio_mode_t mode) } } -void AudioFlinger::ThreadBase::disconnectEffect(const sp<EffectModule>& effect, - EffectHandle *handle, - bool unpinIfLast) { - - Mutex::Autolock _l(mLock); - ALOGV("disconnectEffect() %p effect %p", this, effect.get()); - // delete the effect module if removing last handle on it - if (effect->removeHandle(handle) == 0) { - if (!effect->isPinned() || unpinIfLast) { - removeEffect_l(effect); - AudioSystem::unregisterEffect(effect->id()); - } - } -} - void AudioFlinger::ThreadBase::getAudioPortConfig(struct audio_port_config *config) { config->type = AUDIO_PORT_TYPE_MIX; @@ -2274,7 +2263,7 @@ status_t AudioFlinger::PlaybackThread::addEffectChain_l(const sp<EffectChain>& c } } } - + chain->setThread(this); chain->setInBuffer(buffer, ownsBuffer); chain->setOutBuffer(reinterpret_cast<int16_t*>(mEffectBufferEnabled ? mEffectBuffer : mSinkBuffer)); @@ -3662,6 +3651,10 @@ track_is_ready: ; // remove all the tracks that need to be... removeTracks_l(*tracksToRemove); + if (getEffectChain_l(AUDIO_SESSION_OUTPUT_MIX) != 0) { + mEffectBufferValid = true; + } + // sink or mix buffer must be cleared if all tracks are connected to an // effect chain as in this case the mixer will not write to the sink or mix buffer // and track effects will accumulate into it @@ -6180,10 +6173,11 @@ status_t AudioFlinger::RecordThread::addEffectChain_l(const sp<EffectChain>& cha { // only one chain per input thread if (mEffectChains.size() != 0) { + ALOGW("addEffectChain_l() already one chain %p on thread %p", chain.get(), this); return INVALID_OPERATION; } ALOGV("addEffectChain_l() %p on thread %p", chain.get(), this); - + chain->setThread(this); chain->setInBuffer(NULL); chain->setOutBuffer(NULL); diff --git a/services/audioflinger/Threads.h b/services/audioflinger/Threads.h index 648502b..fd025b5 100644 --- a/services/audioflinger/Threads.h +++ b/services/audioflinger/Threads.h @@ -283,9 +283,6 @@ public: effect_descriptor_t *desc, int *enabled, status_t *status /*non-NULL*/); - void disconnectEffect(const sp< EffectModule>& effect, - EffectHandle *handle, - bool unpinIfLast); // return values for hasAudioSession (bit field) enum effect_state { diff --git a/services/audioflinger/TrackBase.h b/services/audioflinger/TrackBase.h index 864daa5..98bf96e 100644 --- a/services/audioflinger/TrackBase.h +++ b/services/audioflinger/TrackBase.h @@ -166,6 +166,7 @@ protected: sp<NBAIO_Source> mTeeSource; bool mTerminated; track_type mType; // must be one of TYPE_DEFAULT, TYPE_OUTPUT, TYPE_PATCH ... + audio_io_handle_t mThreadIoHandle; // I/O handle of the thread the track is attached to }; // PatchProxyBufferProvider interface is implemented by PatchTrack and PatchRecord. diff --git a/services/audioflinger/Tracks.cpp b/services/audioflinger/Tracks.cpp index c5ab832..c0a75b9 100644 --- a/services/audioflinger/Tracks.cpp +++ b/services/audioflinger/Tracks.cpp @@ -96,7 +96,8 @@ AudioFlinger::ThreadBase::TrackBase::TrackBase( mServerProxy(NULL), mId(android_atomic_inc(&nextTrackId)), mTerminated(false), - mType(type) + mType(type), + mThreadIoHandle(thread->id()) { // if the caller is us, trust the specified uid if (IPCThreadState::self()->getCallingPid() != getpid_cached || clientUid == -1) { @@ -182,7 +183,7 @@ AudioFlinger::ThreadBase::TrackBase::TrackBase( #ifdef TEE_SINK if (mTeeSinkTrackEnabled) { - NBAIO_Format pipeFormat = Format_from_SR_C(mSampleRate, mChannelCount); + NBAIO_Format pipeFormat = Format_from_SR_C(mSampleRate, mChannelCount, mFormat); if (Format_isValid(pipeFormat)) { Pipe *pipe = new Pipe(mTeeSinkTrackFrames, pipeFormat); size_t numCounterOffers = 0; @@ -482,14 +483,15 @@ void AudioFlinger::PlaybackThread::Track::destroy() // this Track with its member mTrack. sp<Track> keep(this); { // scope for mLock + bool wasActive = false; sp<ThreadBase> thread = mThread.promote(); if (thread != 0) { Mutex::Autolock _l(thread->mLock); PlaybackThread *playbackThread = (PlaybackThread *)thread.get(); - bool wasActive = playbackThread->destroyTrack_l(this); - if (isExternalTrack() && !wasActive) { - AudioSystem::releaseOutput(thread->id()); - } + wasActive = playbackThread->destroyTrack_l(this); + } + if (isExternalTrack() && !wasActive) { + AudioSystem::releaseOutput(mThreadIoHandle); } } } @@ -2050,7 +2052,7 @@ void AudioFlinger::RecordThread::RecordTrack::stop() if (thread != 0) { RecordThread *recordThread = (RecordThread *)thread.get(); if (recordThread->stop(this) && isExternalTrack()) { - AudioSystem::stopInput(recordThread->id(), (audio_session_t)mSessionId); + AudioSystem::stopInput(mThreadIoHandle, (audio_session_t)mSessionId); } } } @@ -2060,14 +2062,14 @@ void AudioFlinger::RecordThread::RecordTrack::destroy() // see comments at AudioFlinger::PlaybackThread::Track::destroy() sp<RecordTrack> keep(this); { + if (isExternalTrack()) { + if (mState == ACTIVE || mState == RESUMING) { + AudioSystem::stopInput(mThreadIoHandle, (audio_session_t)mSessionId); + } + AudioSystem::releaseInput(mThreadIoHandle, (audio_session_t)mSessionId); + } sp<ThreadBase> thread = mThread.promote(); if (thread != 0) { - if (isExternalTrack()) { - if (mState == ACTIVE || mState == RESUMING) { - AudioSystem::stopInput(thread->id(), (audio_session_t)mSessionId); - } - AudioSystem::releaseInput(thread->id(), (audio_session_t)mSessionId); - } Mutex::Autolock _l(thread->mLock); RecordThread *recordThread = (RecordThread *) thread.get(); recordThread->destroyTrack_l(this); diff --git a/services/audiopolicy/AudioPolicyManager.cpp b/services/audiopolicy/AudioPolicyManager.cpp index a805923..d5f6c1e 100644 --- a/services/audiopolicy/AudioPolicyManager.cpp +++ b/services/audiopolicy/AudioPolicyManager.cpp @@ -64,6 +64,7 @@ struct StringToEnum { const StringToEnum sDeviceNameToEnumTable[] = { STRING_TO_ENUM(AUDIO_DEVICE_OUT_EARPIECE), STRING_TO_ENUM(AUDIO_DEVICE_OUT_SPEAKER), + STRING_TO_ENUM(AUDIO_DEVICE_OUT_SPEAKER_SAFE), STRING_TO_ENUM(AUDIO_DEVICE_OUT_WIRED_HEADSET), STRING_TO_ENUM(AUDIO_DEVICE_OUT_WIRED_HEADPHONE), STRING_TO_ENUM(AUDIO_DEVICE_OUT_BLUETOOTH_SCO), @@ -485,7 +486,9 @@ void AudioPolicyManager::updateCallRouting(audio_devices_t rxDevice, int delayMs // request to reuse existing output stream if one is already opened to reach the RX device SortedVector<audio_io_handle_t> outputs = getOutputsForDevice(rxDevice, mOutputs); - audio_io_handle_t output = selectOutput(outputs, AUDIO_OUTPUT_FLAG_NONE); + audio_io_handle_t output = selectOutput(outputs, + AUDIO_OUTPUT_FLAG_NONE, + AUDIO_FORMAT_INVALID); if (output != AUDIO_IO_HANDLE_NONE) { sp<AudioOutputDescriptor> outputDesc = mOutputs.valueFor(output); ALOG_ASSERT(!outputDesc->isDuplicated(), @@ -524,7 +527,9 @@ void AudioPolicyManager::updateCallRouting(audio_devices_t rxDevice, int delayMs SortedVector<audio_io_handle_t> outputs = getOutputsForDevice(AUDIO_DEVICE_OUT_TELEPHONY_TX, mOutputs); - audio_io_handle_t output = selectOutput(outputs, AUDIO_OUTPUT_FLAG_NONE); + audio_io_handle_t output = selectOutput(outputs, + AUDIO_OUTPUT_FLAG_NONE, + AUDIO_FORMAT_INVALID); // request to reuse existing output stream if one is already opened to reach the TX // path output device if (output != AUDIO_IO_HANDLE_NONE) { @@ -1016,7 +1021,9 @@ audio_io_handle_t AudioPolicyManager::getOutputForDevice( // routing change will happen when startOutput() will be called SortedVector<audio_io_handle_t> outputs = getOutputsForDevice(device, mOutputs); - output = selectOutput(outputs, flags); + // at this stage we should ignore the DIRECT flag as no direct output could be found earlier + flags = (audio_output_flags_t)(flags & ~AUDIO_OUTPUT_FLAG_DIRECT); + output = selectOutput(outputs, flags, format); } ALOGW_IF((output == 0), "getOutput() could not find output for stream %d, samplingRate %d," "format %d, channels %x, flags %x", stream, samplingRate, format, channelMask, flags); @@ -1027,7 +1034,8 @@ audio_io_handle_t AudioPolicyManager::getOutputForDevice( } audio_io_handle_t AudioPolicyManager::selectOutput(const SortedVector<audio_io_handle_t>& outputs, - audio_output_flags_t flags) + audio_output_flags_t flags, + audio_format_t format) { // select one output among several that provide a path to a particular device or set of // devices (the list was previously build by getOutputsForDevice()). @@ -1050,6 +1058,17 @@ audio_io_handle_t AudioPolicyManager::selectOutput(const SortedVector<audio_io_h for (size_t i = 0; i < outputs.size(); i++) { sp<AudioOutputDescriptor> outputDesc = mOutputs.valueFor(outputs[i]); if (!outputDesc->isDuplicated()) { + // if a valid format is specified, skip output if not compatible + if (format != AUDIO_FORMAT_INVALID) { + if (outputDesc->mFlags & AUDIO_OUTPUT_FLAG_DIRECT) { + if (format != outputDesc->mFormat) { + continue; + } + } else if (!audio_is_linear_pcm(format)) { + continue; + } + } + int commonFlags = popcount(outputDesc->mProfile->mFlags & flags); if (commonFlags > maxCommonFlags) { outputFlags = outputs[i]; @@ -2275,14 +2294,14 @@ status_t AudioPolicyManager::createAudioPatch(const struct audio_patch *patch, } sp<DeviceDescriptor> srcDeviceDesc = mAvailableInputDevices.getDeviceFromId(patch->sources[0].id); + if (srcDeviceDesc == 0) { + return BAD_VALUE; + } //update source and sink with our own data as the data passed in the patch may // be incomplete. struct audio_patch newPatch = *patch; srcDeviceDesc->toAudioPortConfig(&newPatch.sources[0], &patch->sources[0]); - if (srcDeviceDesc == 0) { - return BAD_VALUE; - } for (size_t i = 0; i < patch->num_sinks; i++) { if (patch->sinks[i].type != AUDIO_PORT_TYPE_DEVICE) { @@ -2307,7 +2326,9 @@ status_t AudioPolicyManager::createAudioPatch(const struct audio_patch *patch, mOutputs); // if the sink device is reachable via an opened output stream, request to go via // this output stream by adding a second source to the patch description - audio_io_handle_t output = selectOutput(outputs, AUDIO_OUTPUT_FLAG_NONE); + audio_io_handle_t output = selectOutput(outputs, + AUDIO_OUTPUT_FLAG_NONE, + AUDIO_FORMAT_INVALID); if (output != AUDIO_IO_HANDLE_NONE) { sp<AudioOutputDescriptor> outputDesc = mOutputs.valueFor(output); if (outputDesc->isDuplicated()) { @@ -3649,8 +3670,11 @@ void AudioPolicyManager::checkOutputForStrategy(routing_strategy strategy) void AudioPolicyManager::checkOutputForAllStrategies() { - checkOutputForStrategy(STRATEGY_ENFORCED_AUDIBLE); + if (mForceUse[AUDIO_POLICY_FORCE_FOR_SYSTEM] == AUDIO_POLICY_FORCE_SYSTEM_ENFORCED) + checkOutputForStrategy(STRATEGY_ENFORCED_AUDIBLE); checkOutputForStrategy(STRATEGY_PHONE); + if (mForceUse[AUDIO_POLICY_FORCE_FOR_SYSTEM] != AUDIO_POLICY_FORCE_SYSTEM_ENFORCED) + checkOutputForStrategy(STRATEGY_ENFORCED_AUDIBLE); checkOutputForStrategy(STRATEGY_SONIFICATION); checkOutputForStrategy(STRATEGY_SONIFICATION_RESPECTFUL); checkOutputForStrategy(STRATEGY_MEDIA); @@ -3731,23 +3755,28 @@ audio_devices_t AudioPolicyManager::getNewOutputDevice(audio_io_handle_t output, } // check the following by order of priority to request a routing change if necessary: - // 1: the strategy enforced audible is active on the output: + // 1: the strategy enforced audible is active and enforced on the output: // use device for strategy enforced audible // 2: we are in call or the strategy phone is active on the output: // use device for strategy phone - // 3: the strategy sonification is active on the output: + // 3: the strategy for enforced audible is active but not enforced on the output: + // use the device for strategy enforced audible + // 4: the strategy sonification is active on the output: // use device for strategy sonification - // 4: the strategy "respectful" sonification is active on the output: + // 5: the strategy "respectful" sonification is active on the output: // use device for strategy "respectful" sonification - // 5: the strategy media is active on the output: + // 6: the strategy media is active on the output: // use device for strategy media - // 6: the strategy DTMF is active on the output: + // 7: the strategy DTMF is active on the output: // use device for strategy DTMF - if (outputDesc->isStrategyActive(STRATEGY_ENFORCED_AUDIBLE)) { + if (outputDesc->isStrategyActive(STRATEGY_ENFORCED_AUDIBLE) && + mForceUse[AUDIO_POLICY_FORCE_FOR_SYSTEM] == AUDIO_POLICY_FORCE_SYSTEM_ENFORCED) { device = getDeviceForStrategy(STRATEGY_ENFORCED_AUDIBLE, fromCache); } else if (isInCall() || outputDesc->isStrategyActive(STRATEGY_PHONE)) { device = getDeviceForStrategy(STRATEGY_PHONE, fromCache); + } else if (outputDesc->isStrategyActive(STRATEGY_ENFORCED_AUDIBLE)) { + device = getDeviceForStrategy(STRATEGY_ENFORCED_AUDIBLE, fromCache); } else if (outputDesc->isStrategyActive(STRATEGY_SONIFICATION)) { device = getDeviceForStrategy(STRATEGY_SONIFICATION, fromCache); } else if (outputDesc->isStrategyActive(STRATEGY_SONIFICATION_RESPECTFUL)) { @@ -3804,6 +3833,14 @@ audio_devices_t AudioPolicyManager::getDevicesForStream(audio_stream_type_t stre break; } } + + /*Filter SPEAKER_SAFE out of results, as AudioService doesn't know about it + and doesn't really need to.*/ + if (devices & AUDIO_DEVICE_OUT_SPEAKER_SAFE) { + devices |= AUDIO_DEVICE_OUT_SPEAKER; + devices &= ~AUDIO_DEVICE_OUT_SPEAKER_SAFE; + } + return devices; } @@ -3906,12 +3943,20 @@ audio_devices_t AudioPolicyManager::getDeviceForStrategy(routing_strategy strate // the isStreamActive() method only informs about the activity of a stream, not // if it's for local playback. Note also that we use the same delay between both tests device = getDeviceForStrategy(STRATEGY_SONIFICATION, false /*fromCache*/); + //user "safe" speaker if available instead of normal speaker to avoid triggering + //other acoustic safety mechanisms for notification + if (device == AUDIO_DEVICE_OUT_SPEAKER && (availableOutputDeviceTypes & AUDIO_DEVICE_OUT_SPEAKER_SAFE)) + device = AUDIO_DEVICE_OUT_SPEAKER_SAFE; } else if (isStreamActive(AUDIO_STREAM_MUSIC, SONIFICATION_RESPECTFUL_AFTER_MUSIC_DELAY)) { // while media is playing (or has recently played), use the same device device = getDeviceForStrategy(STRATEGY_MEDIA, false /*fromCache*/); } else { // when media is not playing anymore, fall back on the sonification behavior device = getDeviceForStrategy(STRATEGY_SONIFICATION, false /*fromCache*/); + //user "safe" speaker if available instead of normal speaker to avoid triggering + //other acoustic safety mechanisms for notification + if (device == AUDIO_DEVICE_OUT_SPEAKER && (availableOutputDeviceTypes & AUDIO_DEVICE_OUT_SPEAKER_SAFE)) + device = AUDIO_DEVICE_OUT_SPEAKER_SAFE; } break; @@ -4634,13 +4679,24 @@ audio_devices_t AudioPolicyManager::getDeviceForVolume(audio_devices_t device) // - one A2DP device + another device: happens with duplicated output. In this case // retain the device on the A2DP output as the other must not correspond to an active // selection if not the speaker. + // - HDMI-CEC system audio mode only output: give priority to available item in order. if (device & AUDIO_DEVICE_OUT_SPEAKER) { device = AUDIO_DEVICE_OUT_SPEAKER; + } else if (device & AUDIO_DEVICE_OUT_HDMI_ARC) { + device = AUDIO_DEVICE_OUT_HDMI_ARC; + } else if (device & AUDIO_DEVICE_OUT_AUX_LINE) { + device = AUDIO_DEVICE_OUT_AUX_LINE; + } else if (device & AUDIO_DEVICE_OUT_SPDIF) { + device = AUDIO_DEVICE_OUT_SPDIF; } else { device = (audio_devices_t)(device & AUDIO_DEVICE_OUT_ALL_A2DP); } } + /*SPEAKER_SAFE is an alias of SPEAKER for purposes of volume control*/ + if (device == AUDIO_DEVICE_OUT_SPEAKER_SAFE) + device = AUDIO_DEVICE_OUT_SPEAKER; + ALOGW_IF(popcount(device) != 1, "getDeviceForVolume() invalid device combination: %08x", device); @@ -6032,14 +6088,26 @@ uint32_t AudioPolicyManager::AudioPort::pickSamplingRate() const return 0; } + // For direct outputs, pick minimum sampling rate: this helps ensuring that the + // channel count / sampling rate combination chosen will be supported by the connected + // sink + if ((mType == AUDIO_PORT_TYPE_MIX) && (mRole == AUDIO_PORT_ROLE_SOURCE) && + (mFlags & (AUDIO_OUTPUT_FLAG_DIRECT | AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD))) { + uint32_t samplingRate = UINT_MAX; + for (size_t i = 0; i < mSamplingRates.size(); i ++) { + if ((mSamplingRates[i] < samplingRate) && (mSamplingRates[i] > 0)) { + samplingRate = mSamplingRates[i]; + } + } + return (samplingRate == UINT_MAX) ? 0 : samplingRate; + } + uint32_t samplingRate = 0; uint32_t maxRate = MAX_MIXER_SAMPLING_RATE; // For mixed output and inputs, use max mixer sampling rates. Do not // limit sampling rate otherwise - if ((mType != AUDIO_PORT_TYPE_MIX) || - ((mRole == AUDIO_PORT_ROLE_SOURCE) && - (mFlags & (AUDIO_OUTPUT_FLAG_DIRECT | AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD)))) { + if (mType != AUDIO_PORT_TYPE_MIX) { maxRate = UINT_MAX; } for (size_t i = 0; i < mSamplingRates.size(); i ++) { @@ -6056,16 +6124,35 @@ audio_channel_mask_t AudioPolicyManager::AudioPort::pickChannelMask() const if (mChannelMasks.size() == 1 && mChannelMasks[0] == 0) { return AUDIO_CHANNEL_NONE; } - audio_channel_mask_t channelMask = AUDIO_CHANNEL_NONE; + + // For direct outputs, pick minimum channel count: this helps ensuring that the + // channel count / sampling rate combination chosen will be supported by the connected + // sink + if ((mType == AUDIO_PORT_TYPE_MIX) && (mRole == AUDIO_PORT_ROLE_SOURCE) && + (mFlags & (AUDIO_OUTPUT_FLAG_DIRECT | AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD))) { + uint32_t channelCount = UINT_MAX; + for (size_t i = 0; i < mChannelMasks.size(); i ++) { + uint32_t cnlCount; + if (mUseInChannelMask) { + cnlCount = audio_channel_count_from_in_mask(mChannelMasks[i]); + } else { + cnlCount = audio_channel_count_from_out_mask(mChannelMasks[i]); + } + if ((cnlCount < channelCount) && (cnlCount > 0)) { + channelMask = mChannelMasks[i]; + channelCount = cnlCount; + } + } + return channelMask; + } + uint32_t channelCount = 0; uint32_t maxCount = MAX_MIXER_CHANNEL_COUNT; // For mixed output and inputs, use max mixer channel count. Do not // limit channel count otherwise - if ((mType != AUDIO_PORT_TYPE_MIX) || - ((mRole == AUDIO_PORT_ROLE_SOURCE) && - (mFlags & (AUDIO_OUTPUT_FLAG_DIRECT | AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD)))) { + if (mType != AUDIO_PORT_TYPE_MIX) { maxCount = UINT_MAX; } for (size_t i = 0; i < mChannelMasks.size(); i ++) { @@ -6077,6 +6164,7 @@ audio_channel_mask_t AudioPolicyManager::AudioPort::pickChannelMask() const } if ((cnlCount > channelCount) && (cnlCount <= maxCount)) { channelMask = mChannelMasks[i]; + channelCount = cnlCount; } } return channelMask; diff --git a/services/audiopolicy/AudioPolicyManager.h b/services/audiopolicy/AudioPolicyManager.h index 57e015e..da0d95d 100644 --- a/services/audiopolicy/AudioPolicyManager.h +++ b/services/audiopolicy/AudioPolicyManager.h @@ -713,7 +713,8 @@ protected: uint32_t delayMs); audio_io_handle_t selectOutput(const SortedVector<audio_io_handle_t>& outputs, - audio_output_flags_t flags); + audio_output_flags_t flags, + audio_format_t format); // samplingRate parameter is an in/out and so may be modified sp<IOProfile> getInputProfile(audio_devices_t device, uint32_t& samplingRate, diff --git a/services/audiopolicy/AudioPolicyService.cpp b/services/audiopolicy/AudioPolicyService.cpp index 7f14960..50bb8c7 100644 --- a/services/audiopolicy/AudioPolicyService.cpp +++ b/services/audiopolicy/AudioPolicyService.cpp @@ -765,7 +765,16 @@ void AudioPolicyService::AudioCommandThread::insertCommand_l(sp<AudioCommand>& c sp<AudioCommand> command2 = mAudioCommands[i]; // commands are sorted by increasing time stamp: no need to scan the rest of mAudioCommands if (command2->mTime <= command->mTime) break; - if (command2->mCommand != command->mCommand) continue; + + // create audio patch or release audio patch commands are equivalent + // with regard to filtering + if ((command->mCommand == CREATE_AUDIO_PATCH) || + (command->mCommand == RELEASE_AUDIO_PATCH)) { + if ((command2->mCommand != CREATE_AUDIO_PATCH) && + (command2->mCommand != RELEASE_AUDIO_PATCH)) { + continue; + } + } else if (command2->mCommand != command->mCommand) continue; switch (command->mCommand) { case SET_PARAMETERS: { @@ -817,6 +826,31 @@ void AudioPolicyService::AudioCommandThread::insertCommand_l(sp<AudioCommand>& c // command status as the command is now delayed delayMs = 1; } break; + + case CREATE_AUDIO_PATCH: + case RELEASE_AUDIO_PATCH: { + audio_patch_handle_t handle; + if (command->mCommand == CREATE_AUDIO_PATCH) { + handle = ((CreateAudioPatchData *)command->mParam.get())->mHandle; + } else { + handle = ((ReleaseAudioPatchData *)command->mParam.get())->mHandle; + } + audio_patch_handle_t handle2; + if (command2->mCommand == CREATE_AUDIO_PATCH) { + handle2 = ((CreateAudioPatchData *)command2->mParam.get())->mHandle; + } else { + handle2 = ((ReleaseAudioPatchData *)command2->mParam.get())->mHandle; + } + if (handle != handle2) break; + ALOGV("Filtering out %s audio patch command for handle %d", + (command->mCommand == CREATE_AUDIO_PATCH) ? "create" : "release", handle); + removedCommands.add(command2); + command->mTime = command2->mTime; + // force delayMs to non 0 so that code below does not request to wait for + // command status as the command is now delayed + delayMs = 1; + } break; + case START_TONE: case STOP_TONE: default: diff --git a/services/camera/libcameraservice/Android.mk b/services/camera/libcameraservice/Android.mk index 9d6ab23..e184d97 100644 --- a/services/camera/libcameraservice/Android.mk +++ b/services/camera/libcameraservice/Android.mk @@ -34,6 +34,7 @@ LOCAL_SRC_FILES:= \ api1/client2/JpegProcessor.cpp \ api1/client2/CallbackProcessor.cpp \ api1/client2/ZslProcessor.cpp \ + api1/client2/ZslProcessorInterface.cpp \ api1/client2/BurstCapture.cpp \ api1/client2/JpegCompressor.cpp \ api1/client2/CaptureSequencer.cpp \ diff --git a/services/camera/libcameraservice/api1/Camera2Client.cpp b/services/camera/libcameraservice/api1/Camera2Client.cpp index bc40971..fe2f299 100644 --- a/services/camera/libcameraservice/api1/Camera2Client.cpp +++ b/services/camera/libcameraservice/api1/Camera2Client.cpp @@ -434,6 +434,9 @@ void Camera2Client::disconnect() { mCallbackProcessor->deleteStream(); mZslProcessor->deleteStream(); + // Remove all ZSL stream state before disconnect; needed to work around b/15408128. + mZslProcessor->disconnect(); + ALOGV("Camera %d: Disconnecting device", mCameraId); mDevice->disconnect(); @@ -918,6 +921,13 @@ void Camera2Client::stopPreviewL() { "stop preview: %s (%d)", __FUNCTION__, mCameraId, strerror(-res), res); } + { + // Ideally we should recover the override after recording stopped, but + // right now recording stream will live until here, so we are forced to + // recover here. TODO: find a better way to handle that (b/17495165) + SharedParameters::Lock l(mParameters); + l.mParameters.recoverOverriddenJpegSize(); + } // no break case Parameters::WAITING_FOR_PREVIEW_WINDOW: { SharedParameters::Lock l(mParameters); @@ -1072,22 +1082,57 @@ status_t Camera2Client::startRecordingL(Parameters ¶ms, bool restart) { // and we can't fail record start without stagefright asserting. params.previewCallbackFlags = 0; - res = updateProcessorStream< - StreamingProcessor, - &StreamingProcessor::updateRecordingStream>(mStreamingProcessor, - params); + bool recordingStreamNeedsUpdate; + res = mStreamingProcessor->recordingStreamNeedsUpdate(params, &recordingStreamNeedsUpdate); if (res != OK) { - ALOGE("%s: Camera %d: Unable to update recording stream: %s (%d)", - __FUNCTION__, mCameraId, strerror(-res), res); + ALOGE("%s: Camera %d: Can't query recording stream", + __FUNCTION__, mCameraId); return res; } + if (recordingStreamNeedsUpdate) { + // Need to stop stream here so updateProcessorStream won't trigger configureStream + // Right now camera device cannot handle configureStream failure gracefully + // when device is streaming + res = mStreamingProcessor->stopStream(); + if (res != OK) { + ALOGE("%s: Camera %d: Can't stop streaming to update record stream", + __FUNCTION__, mCameraId); + return res; + } + res = mDevice->waitUntilDrained(); + if (res != OK) { + ALOGE("%s: Camera %d: Waiting to stop streaming failed: %s (%d)", + __FUNCTION__, mCameraId, strerror(-res), res); + } + res = updateProcessorStream< + StreamingProcessor, + &StreamingProcessor::updateRecordingStream>(mStreamingProcessor, + params); + + if (res != OK) { + ALOGE("%s: Camera %d: Unable to update recording stream: %s (%d)", + __FUNCTION__, mCameraId, strerror(-res), res); + return res; + } + } + Vector<int32_t> outputStreams; outputStreams.push(getPreviewStreamId()); outputStreams.push(getRecordingStreamId()); res = mStreamingProcessor->startStream(StreamingProcessor::RECORD, outputStreams); + + // startStream might trigger a configureStream call and device might fail + // configureStream due to jpeg size > video size. Try again with jpeg size overridden + // to video size. + if (res == BAD_VALUE) { + overrideVideoSnapshotSize(params); + res = mStreamingProcessor->startStream(StreamingProcessor::RECORD, + outputStreams); + } + if (res != OK) { ALOGE("%s: Camera %d: Unable to start recording stream: %s (%d)", __FUNCTION__, mCameraId, strerror(-res), res); @@ -1271,6 +1316,9 @@ status_t Camera2Client::cancelAutoFocus() { return OK; } + if (l.mParameters.zslMode) { + mZslProcessor->clearZslQueue(); + } } syncWithDevice(); @@ -1320,6 +1368,12 @@ status_t Camera2Client::takePicture(int msgType) { int lastJpegStreamId = mJpegProcessor->getStreamId(); res = updateProcessorStream(mJpegProcessor, l.mParameters); + // If video snapshot fail to configureStream, try override video snapshot size to + // video size + if (res == BAD_VALUE && l.mParameters.state == Parameters::VIDEO_SNAPSHOT) { + overrideVideoSnapshotSize(l.mParameters); + res = updateProcessorStream(mJpegProcessor, l.mParameters); + } if (res != OK) { ALOGE("%s: Camera %d: Can't set up still image stream: %s (%d)", __FUNCTION__, mCameraId, strerror(-res), res); @@ -1359,8 +1413,14 @@ status_t Camera2Client::setParameters(const String8& params) { SharedParameters::Lock l(mParameters); + Parameters::focusMode_t focusModeBefore = l.mParameters.focusMode; res = l.mParameters.set(params); if (res != OK) return res; + Parameters::focusMode_t focusModeAfter = l.mParameters.focusMode; + + if (l.mParameters.zslMode && focusModeAfter != focusModeBefore) { + mZslProcessor->clearZslQueue(); + } res = updateRequests(l.mParameters); @@ -1894,6 +1954,18 @@ status_t Camera2Client::updateProcessorStream(sp<ProcessorT> processor, return res; } +status_t Camera2Client::overrideVideoSnapshotSize(Parameters ¶ms) { + ALOGV("%s: Camera %d: configure still size to video size before recording" + , __FUNCTION__, mCameraId); + params.overrideJpegSizeByVideoSize(); + status_t res = updateProcessorStream(mJpegProcessor, params); + if (res != OK) { + ALOGE("%s: Camera %d: Can't override video snapshot size to video size: %s (%d)", + __FUNCTION__, mCameraId, strerror(-res), res); + } + return res; +} + const char* Camera2Client::kAutofocusLabel = "autofocus"; const char* Camera2Client::kTakepictureLabel = "take_picture"; diff --git a/services/camera/libcameraservice/api1/Camera2Client.h b/services/camera/libcameraservice/api1/Camera2Client.h index f5c3a30..d68bb29 100644 --- a/services/camera/libcameraservice/api1/Camera2Client.h +++ b/services/camera/libcameraservice/api1/Camera2Client.h @@ -208,6 +208,9 @@ private: // Wait until the camera device has received the latest control settings status_t syncWithDevice(); + + // Video snapshot jpeg size overriding helper function + status_t overrideVideoSnapshotSize(Parameters ¶ms); }; }; // namespace android diff --git a/services/camera/libcameraservice/api1/CameraClient.cpp b/services/camera/libcameraservice/api1/CameraClient.cpp index 33bdaa3..1a4d9a6 100644 --- a/services/camera/libcameraservice/api1/CameraClient.cpp +++ b/services/camera/libcameraservice/api1/CameraClient.cpp @@ -122,6 +122,16 @@ status_t CameraClient::dump(int fd, const Vector<String16>& args) { mClientPid); len = (len > SIZE - 1) ? SIZE - 1 : len; write(fd, buffer, len); + + len = snprintf(buffer, SIZE, "Latest set parameters:\n"); + len = (len > SIZE - 1) ? SIZE - 1 : len; + write(fd, buffer, len); + + mLatestSetParameters.dump(fd, args); + + const char *enddump = "\n\n"; + write(fd, enddump, strlen(enddump)); + return mHardware->dump(fd, args); } @@ -550,6 +560,7 @@ status_t CameraClient::setParameters(const String8& params) { status_t result = checkPidAndHardware(); if (result != NO_ERROR) return result; + mLatestSetParameters = CameraParameters(params); CameraParameters p(params); return mHardware->setParameters(p); } diff --git a/services/camera/libcameraservice/api1/CameraClient.h b/services/camera/libcameraservice/api1/CameraClient.h index 6779f5e..63a9d0f 100644 --- a/services/camera/libcameraservice/api1/CameraClient.h +++ b/services/camera/libcameraservice/api1/CameraClient.h @@ -142,6 +142,9 @@ private: // of the original one), we allocate mPreviewBuffer and reuse it if possible. sp<MemoryHeapBase> mPreviewBuffer; + // Debugging information + CameraParameters mLatestSetParameters; + // We need to avoid the deadlock when the incoming command thread and // the CameraHardwareInterface callback thread both want to grab mLock. // An extra flag is used to tell the callback thread that it should stop diff --git a/services/camera/libcameraservice/api1/client2/CaptureSequencer.cpp b/services/camera/libcameraservice/api1/client2/CaptureSequencer.cpp index cb9aca6..9849f4d 100644 --- a/services/camera/libcameraservice/api1/client2/CaptureSequencer.cpp +++ b/services/camera/libcameraservice/api1/client2/CaptureSequencer.cpp @@ -445,11 +445,18 @@ CaptureSequencer::CaptureState CaptureSequencer::manageStandardPrecaptureWait( if (mNewAEState) { if (!mAeInPrecapture) { // Waiting to see PRECAPTURE state - if (mAETriggerId == mTriggerId && - mAEState == ANDROID_CONTROL_AE_STATE_PRECAPTURE) { - ALOGV("%s: Got precapture start", __FUNCTION__); - mAeInPrecapture = true; - mTimeoutCount = kMaxTimeoutsForPrecaptureEnd; + if (mAETriggerId == mTriggerId) { + if (mAEState == ANDROID_CONTROL_AE_STATE_PRECAPTURE) { + ALOGV("%s: Got precapture start", __FUNCTION__); + mAeInPrecapture = true; + mTimeoutCount = kMaxTimeoutsForPrecaptureEnd; + } else if (mAEState == ANDROID_CONTROL_AE_STATE_CONVERGED || + mAEState == ANDROID_CONTROL_AE_STATE_FLASH_REQUIRED) { + // It is legal to transit to CONVERGED or FLASH_REQUIRED + // directly after a trigger. + ALOGV("%s: AE is already in good state, start capture", __FUNCTION__); + return STANDARD_CAPTURE; + } } } else { // Waiting to see PRECAPTURE state end diff --git a/services/camera/libcameraservice/api1/client2/Parameters.cpp b/services/camera/libcameraservice/api1/client2/Parameters.cpp index e7f9a78..aa9d746 100644 --- a/services/camera/libcameraservice/api1/client2/Parameters.cpp +++ b/services/camera/libcameraservice/api1/client2/Parameters.cpp @@ -76,9 +76,29 @@ status_t Parameters::initialize(const CameraMetadata *info, int deviceVersion) { res = getFilteredSizes(MAX_VIDEO_SIZE, &availableVideoSizes); if (res != OK) return res; - // TODO: Pick more intelligently - previewWidth = availablePreviewSizes[0].width; - previewHeight = availablePreviewSizes[0].height; + // Select initial preview and video size that's under the initial bound and + // on the list of both preview and recording sizes + previewWidth = 0; + previewHeight = 0; + for (size_t i = 0 ; i < availablePreviewSizes.size(); i++) { + int newWidth = availablePreviewSizes[i].width; + int newHeight = availablePreviewSizes[i].height; + if (newWidth >= previewWidth && newHeight >= previewHeight && + newWidth <= MAX_INITIAL_PREVIEW_WIDTH && + newHeight <= MAX_INITIAL_PREVIEW_HEIGHT) { + for (size_t j = 0; j < availableVideoSizes.size(); j++) { + if (availableVideoSizes[j].width == newWidth && + availableVideoSizes[j].height == newHeight) { + previewWidth = newWidth; + previewHeight = newHeight; + } + } + } + } + if (previewWidth == 0) { + ALOGE("%s: No initial preview size can be found!", __FUNCTION__); + return BAD_VALUE; + } videoWidth = previewWidth; videoHeight = previewHeight; @@ -249,6 +269,9 @@ status_t Parameters::initialize(const CameraMetadata *info, int deviceVersion) { // TODO: Pick maximum pictureWidth = availableJpegSizes[0].width; pictureHeight = availableJpegSizes[0].height; + pictureWidthLastSet = pictureWidth; + pictureHeightLastSet = pictureHeight; + pictureSizeOverriden = false; params.setPictureSize(pictureWidth, pictureHeight); @@ -960,6 +983,13 @@ status_t Parameters::buildFastInfo() { bool fixedLens = minFocusDistance.count == 0 || minFocusDistance.data.f[0] == 0; + camera_metadata_ro_entry_t focusDistanceCalibration = + staticInfo(ANDROID_LENS_INFO_FOCUS_DISTANCE_CALIBRATION, 0, 0, + false); + bool canFocusInfinity = (focusDistanceCalibration.count && + focusDistanceCalibration.data.u8[0] != + ANDROID_LENS_INFO_FOCUS_DISTANCE_CALIBRATION_UNCALIBRATED); + camera_metadata_ro_entry_t availableFocalLengths = staticInfo(ANDROID_LENS_INFO_AVAILABLE_FOCAL_LENGTHS); if (!availableFocalLengths.count) return NO_INIT; @@ -1010,6 +1040,13 @@ status_t Parameters::buildFastInfo() { sceneModeOverrides.data.u8[i * kModesPerSceneMode + 2]; switch(afMode) { case ANDROID_CONTROL_AF_MODE_OFF: + if (!fixedLens && !canFocusInfinity) { + ALOGE("%s: Camera %d: Scene mode override lists asks for" + " fixed focus on a device with focuser but not" + " calibrated for infinity focus", __FUNCTION__, + cameraId); + return NO_INIT; + } modes.focusMode = fixedLens ? FOCUS_MODE_FIXED : FOCUS_MODE_INFINITY; break; @@ -1381,8 +1418,8 @@ status_t Parameters::set(const String8& paramString) { // PICTURE_SIZE newParams.getPictureSize(&validatedParams.pictureWidth, &validatedParams.pictureHeight); - if (validatedParams.pictureWidth == pictureWidth || - validatedParams.pictureHeight == pictureHeight) { + if (validatedParams.pictureWidth != pictureWidth || + validatedParams.pictureHeight != pictureHeight) { Vector<Size> availablePictureSizes = getAvailableJpegSizes(); for (i = 0; i < availablePictureSizes.size(); i++) { if ((availablePictureSizes[i].width == @@ -1798,6 +1835,7 @@ status_t Parameters::set(const String8& paramString) { /** Update internal parameters */ *this = validatedParams; + updateOverriddenJpegSize(); /** Update external parameters calculated from the internal ones */ @@ -2115,6 +2153,52 @@ status_t Parameters::updateRequestJpeg(CameraMetadata *request) const { return OK; } +status_t Parameters::overrideJpegSizeByVideoSize() { + if (pictureSizeOverriden) { + ALOGV("Picture size has been overridden. Skip overriding"); + return OK; + } + + pictureSizeOverriden = true; + pictureWidthLastSet = pictureWidth; + pictureHeightLastSet = pictureHeight; + pictureWidth = videoWidth; + pictureHeight = videoHeight; + // This change of picture size is invisible to app layer. + // Do not update app visible params + return OK; +} + +status_t Parameters::updateOverriddenJpegSize() { + if (!pictureSizeOverriden) { + ALOGV("Picture size has not been overridden. Skip checking"); + return OK; + } + + pictureWidthLastSet = pictureWidth; + pictureHeightLastSet = pictureHeight; + + if (pictureWidth <= videoWidth && pictureHeight <= videoHeight) { + // Picture size is now smaller than video size. No need to override anymore + return recoverOverriddenJpegSize(); + } + + pictureWidth = videoWidth; + pictureHeight = videoHeight; + + return OK; +} + +status_t Parameters::recoverOverriddenJpegSize() { + if (!pictureSizeOverriden) { + ALOGV("Picture size has not been overridden. Skip recovering"); + return OK; + } + pictureSizeOverriden = false; + pictureWidth = pictureWidthLastSet; + pictureHeight = pictureHeightLastSet; + return OK; +} const char* Parameters::getStateName(State state) { #define CASE_ENUM_TO_CHAR(x) case x: return(#x); break; diff --git a/services/camera/libcameraservice/api1/client2/Parameters.h b/services/camera/libcameraservice/api1/client2/Parameters.h index d9d33c4..815cc55 100644 --- a/services/camera/libcameraservice/api1/client2/Parameters.h +++ b/services/camera/libcameraservice/api1/client2/Parameters.h @@ -52,6 +52,9 @@ struct Parameters { int previewTransform; // set by CAMERA_CMD_SET_DISPLAY_ORIENTATION int pictureWidth, pictureHeight; + // Store the picture size before they are overriden by video snapshot + int pictureWidthLastSet, pictureHeightLastSet; + bool pictureSizeOverriden; int32_t jpegThumbSize[2]; uint8_t jpegQuality, jpegThumbQuality; @@ -176,8 +179,13 @@ struct Parameters { // Number of zoom steps to simulate static const unsigned int NUM_ZOOM_STEPS = 100; // Max preview size allowed + // This is set to a 1:1 value to allow for any aspect ratio that has + // a max long side of 1920 pixels static const unsigned int MAX_PREVIEW_WIDTH = 1920; - static const unsigned int MAX_PREVIEW_HEIGHT = 1080; + static const unsigned int MAX_PREVIEW_HEIGHT = 1920; + // Initial max preview/recording size bound + static const int MAX_INITIAL_PREVIEW_WIDTH = 1920; + static const int MAX_INITIAL_PREVIEW_HEIGHT = 1080; // Aspect ratio tolerance static const float ASPECT_RATIO_TOLERANCE = 0.001; @@ -253,6 +261,12 @@ struct Parameters { // Add/update JPEG entries in metadata status_t updateRequestJpeg(CameraMetadata *request) const; + /* Helper functions to override jpeg size for video snapshot */ + // Override jpeg size by video size. Called during startRecording. + status_t overrideJpegSizeByVideoSize(); + // Recover overridden jpeg size. Called during stopRecording. + status_t recoverOverriddenJpegSize(); + // Calculate the crop region rectangle based on current stream sizes struct CropRegion { float left; @@ -348,6 +362,12 @@ private: // Get max size (from the size array) that matches the given aspect ratio. Size getMaxSizeForRatio(float ratio, const int32_t* sizeArray, size_t count); + // Helper function for overriding jpeg size for video snapshot + // Check if overridden jpeg size needs to be updated after Parameters::set. + // The behavior of this function is tailored to the implementation of Parameters::set. + // Do not use this function for other purpose. + status_t updateOverriddenJpegSize(); + struct StreamConfiguration { int32_t format; int32_t width; diff --git a/services/camera/libcameraservice/api1/client2/StreamingProcessor.cpp b/services/camera/libcameraservice/api1/client2/StreamingProcessor.cpp index ab0af0d..9e7fff8 100644 --- a/services/camera/libcameraservice/api1/client2/StreamingProcessor.cpp +++ b/services/camera/libcameraservice/api1/client2/StreamingProcessor.cpp @@ -318,6 +318,44 @@ status_t StreamingProcessor::updateRecordingRequest(const Parameters ¶ms) { return OK; } +status_t StreamingProcessor::recordingStreamNeedsUpdate( + const Parameters ¶ms, bool *needsUpdate) { + status_t res; + + if (needsUpdate == 0) { + ALOGE("%s: Camera %d: invalid argument", __FUNCTION__, mId); + return INVALID_OPERATION; + } + + if (mRecordingStreamId == NO_STREAM) { + *needsUpdate = true; + return OK; + } + + sp<CameraDeviceBase> device = mDevice.promote(); + if (device == 0) { + ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); + return INVALID_OPERATION; + } + + uint32_t currentWidth, currentHeight; + res = device->getStreamInfo(mRecordingStreamId, + ¤tWidth, ¤tHeight, 0); + if (res != OK) { + ALOGE("%s: Camera %d: Error querying recording output stream info: " + "%s (%d)", __FUNCTION__, mId, + strerror(-res), res); + return res; + } + + if (mRecordingConsumer == 0 || currentWidth != (uint32_t)params.videoWidth || + currentHeight != (uint32_t)params.videoHeight) { + *needsUpdate = true; + } + *needsUpdate = false; + return res; +} + status_t StreamingProcessor::updateRecordingStream(const Parameters ¶ms) { ATRACE_CALL(); status_t res; diff --git a/services/camera/libcameraservice/api1/client2/StreamingProcessor.h b/services/camera/libcameraservice/api1/client2/StreamingProcessor.h index 833bb8f..8466af4 100644 --- a/services/camera/libcameraservice/api1/client2/StreamingProcessor.h +++ b/services/camera/libcameraservice/api1/client2/StreamingProcessor.h @@ -54,6 +54,9 @@ class StreamingProcessor: status_t setRecordingBufferCount(size_t count); status_t updateRecordingRequest(const Parameters ¶ms); + // If needsUpdate is set to true, a updateRecordingStream call with params will recreate + // recording stream + status_t recordingStreamNeedsUpdate(const Parameters ¶ms, bool *needsUpdate); status_t updateRecordingStream(const Parameters ¶ms); status_t deleteRecordingStream(); int getRecordingStreamId() const; diff --git a/services/camera/libcameraservice/api1/client2/ZslProcessor.cpp b/services/camera/libcameraservice/api1/client2/ZslProcessor.cpp index 8fb876e..bb72206 100644 --- a/services/camera/libcameraservice/api1/client2/ZslProcessor.cpp +++ b/services/camera/libcameraservice/api1/client2/ZslProcessor.cpp @@ -48,6 +48,7 @@ ZslProcessor::ZslProcessor( mDevice(client->getCameraDevice()), mSequencer(sequencer), mId(client->getCameraId()), + mDeleted(false), mZslBufferAvailable(false), mZslStreamId(NO_STREAM), mZslReprocessStreamId(NO_STREAM), @@ -62,7 +63,7 @@ ZslProcessor::ZslProcessor( ZslProcessor::~ZslProcessor() { ALOGV("%s: Exit", __FUNCTION__); - deleteStream(); + disconnect(); } void ZslProcessor::onFrameAvailable() { @@ -153,7 +154,7 @@ status_t ZslProcessor::updateStream(const Parameters ¶ms) { mId, strerror(-res), res); return res; } - if (currentWidth != (uint32_t)params.fastInfo.arrayWidth || + if (mDeleted || currentWidth != (uint32_t)params.fastInfo.arrayWidth || currentHeight != (uint32_t)params.fastInfo.arrayHeight) { res = device->deleteReprocessStream(mZslReprocessStreamId); if (res != OK) { @@ -175,6 +176,8 @@ status_t ZslProcessor::updateStream(const Parameters ¶ms) { } } + mDeleted = false; + if (mZslStreamId == NO_STREAM) { // Create stream for HAL production // TODO: Sort out better way to select resolution for ZSL @@ -209,6 +212,14 @@ status_t ZslProcessor::updateStream(const Parameters ¶ms) { status_t ZslProcessor::deleteStream() { ATRACE_CALL(); + Mutex::Autolock l(mInputMutex); + // WAR(b/15408128): do not delete stream unless client is being disconnected. + mDeleted = true; + return OK; +} + +status_t ZslProcessor::disconnect() { + ATRACE_CALL(); status_t res; Mutex::Autolock l(mInputMutex); diff --git a/services/camera/libcameraservice/api1/client2/ZslProcessor.h b/services/camera/libcameraservice/api1/client2/ZslProcessor.h index f4cf0c8..b6533cf 100644 --- a/services/camera/libcameraservice/api1/client2/ZslProcessor.h +++ b/services/camera/libcameraservice/api1/client2/ZslProcessor.h @@ -67,6 +67,7 @@ class ZslProcessor: status_t updateStream(const Parameters ¶ms); status_t deleteStream(); + status_t disconnect(); int getStreamId() const; status_t pushToReprocess(int32_t requestId); @@ -86,6 +87,8 @@ class ZslProcessor: wp<CaptureSequencer> mSequencer; int mId; + bool mDeleted; + mutable Mutex mInputMutex; bool mZslBufferAvailable; Condition mZslBufferAvailableSignal; diff --git a/services/camera/libcameraservice/api1/client2/ZslProcessor3.cpp b/services/camera/libcameraservice/api1/client2/ZslProcessor3.cpp index 2d31275..de31e23 100644 --- a/services/camera/libcameraservice/api1/client2/ZslProcessor3.cpp +++ b/services/camera/libcameraservice/api1/client2/ZslProcessor3.cpp @@ -44,6 +44,7 @@ ZslProcessor3::ZslProcessor3( sp<Camera2Client> client, wp<CaptureSequencer> sequencer): Thread(false), + mLatestClearedBufferTimestamp(0), mState(RUNNING), mClient(client), mSequencer(sequencer), @@ -107,7 +108,6 @@ void ZslProcessor3::onResultAvailable(const CaptureResult &result) { ALOGE("%s: metadata doesn't have timestamp, skip this result", __FUNCTION__); return; } - (void)timestamp; entry = result.mMetadata.find(ANDROID_REQUEST_FRAME_COUNT); if (entry.count == 0) { @@ -120,6 +120,9 @@ void ZslProcessor3::onResultAvailable(const CaptureResult &result) { if (mState != RUNNING) return; + // Corresponding buffer has been cleared. No need to push into mFrameList + if (timestamp <= mLatestClearedBufferTimestamp) return; + mFrameList.editItemAt(mFrameListHead) = result.mMetadata; mFrameListHead = (mFrameListHead + 1) % mFrameListDepth; } @@ -241,6 +244,46 @@ int ZslProcessor3::getStreamId() const { return mZslStreamId; } +status_t ZslProcessor3::updateRequestWithDefaultStillRequest(CameraMetadata &request) const { + sp<Camera2Client> client = mClient.promote(); + if (client == 0) { + ALOGE("%s: Camera %d: Client does not exist", __FUNCTION__, mId); + return INVALID_OPERATION; + } + sp<Camera3Device> device = + static_cast<Camera3Device*>(client->getCameraDevice().get()); + if (device == 0) { + ALOGE("%s: Camera %d: Device does not exist", __FUNCTION__, mId); + return INVALID_OPERATION; + } + + CameraMetadata stillTemplate; + device->createDefaultRequest(CAMERA3_TEMPLATE_STILL_CAPTURE, &stillTemplate); + + // Find some of the post-processing tags, and assign the value from template to the request. + // Only check the aberration mode and noise reduction mode for now, as they are very important + // for image quality. + uint32_t postProcessingTags[] = { + ANDROID_NOISE_REDUCTION_MODE, + ANDROID_COLOR_CORRECTION_ABERRATION_MODE, + ANDROID_COLOR_CORRECTION_MODE, + ANDROID_TONEMAP_MODE, + ANDROID_SHADING_MODE, + ANDROID_HOT_PIXEL_MODE, + ANDROID_EDGE_MODE + }; + + camera_metadata_entry_t entry; + for (size_t i = 0; i < sizeof(postProcessingTags) / sizeof(uint32_t); i++) { + entry = stillTemplate.find(postProcessingTags[i]); + if (entry.count > 0) { + request.update(postProcessingTags[i], entry.data.u8, 1); + } + } + + return OK; +} + status_t ZslProcessor3::pushToReprocess(int32_t requestId) { ALOGV("%s: Send in reprocess request with id %d", __FUNCTION__, requestId); @@ -366,6 +409,13 @@ status_t ZslProcessor3::pushToReprocess(int32_t requestId) { } } + // Update post-processing settings + res = updateRequestWithDefaultStillRequest(request); + if (res != OK) { + ALOGW("%s: Unable to update post-processing tags, the reprocessed image quality " + "may be compromised", __FUNCTION__); + } + mLatestCapturedRequest = request; res = client->getCameraDevice()->capture(request); if (res != OK ) { @@ -392,7 +442,7 @@ status_t ZslProcessor3::clearZslQueueLocked() { if (mZslStream != 0) { // clear result metadata list first. clearZslResultQueueLocked(); - return mZslStream->clearInputRingBuffer(); + return mZslStream->clearInputRingBuffer(&mLatestClearedBufferTimestamp); } return OK; } @@ -454,6 +504,23 @@ void ZslProcessor3::dumpZslQueue(int fd) const { } } +bool ZslProcessor3::isFixedFocusMode(uint8_t afMode) const { + switch (afMode) { + case ANDROID_CONTROL_AF_MODE_AUTO: + case ANDROID_CONTROL_AF_MODE_CONTINUOUS_VIDEO: + case ANDROID_CONTROL_AF_MODE_CONTINUOUS_PICTURE: + case ANDROID_CONTROL_AF_MODE_MACRO: + return false; + break; + case ANDROID_CONTROL_AF_MODE_OFF: + case ANDROID_CONTROL_AF_MODE_EDOF: + return true; + default: + ALOGE("%s: unknown focus mode %d", __FUNCTION__, afMode); + return false; + } +} + nsecs_t ZslProcessor3::getCandidateTimestampLocked(size_t* metadataIdx) const { /** * Find the smallest timestamp we know about so far @@ -499,8 +566,16 @@ nsecs_t ZslProcessor3::getCandidateTimestampLocked(size_t* metadataIdx) const { continue; } - // Check AF state if device has focuser - if (mHasFocuser) { + entry = frame.find(ANDROID_CONTROL_AF_MODE); + if (entry.count == 0) { + ALOGW("%s: ZSL queue frame has no AF mode field!", + __FUNCTION__); + continue; + } + uint8_t afMode = entry.data.u8[0]; + + // Check AF state if device has focuser and focus mode isn't fixed + if (mHasFocuser && !isFixedFocusMode(afMode)) { // Make sure the candidate frame has good focus. entry = frame.find(ANDROID_CONTROL_AF_STATE); if (entry.count == 0) { diff --git a/services/camera/libcameraservice/api1/client2/ZslProcessor3.h b/services/camera/libcameraservice/api1/client2/ZslProcessor3.h index daa352b..fc9f70c 100644 --- a/services/camera/libcameraservice/api1/client2/ZslProcessor3.h +++ b/services/camera/libcameraservice/api1/client2/ZslProcessor3.h @@ -82,6 +82,7 @@ class ZslProcessor3 : private: static const nsecs_t kWaitDuration = 10000000; // 10 ms + nsecs_t mLatestClearedBufferTimestamp; enum { RUNNING, @@ -132,6 +133,11 @@ class ZslProcessor3 : void dumpZslQueue(int id) const; nsecs_t getCandidateTimestampLocked(size_t* metadataIdx) const; + + bool isFixedFocusMode(uint8_t afMode) const; + + // Update the post-processing metadata with the default still capture request template + status_t updateRequestWithDefaultStillRequest(CameraMetadata &request) const; }; diff --git a/services/camera/libcameraservice/api1/client2/ZslProcessorInterface.cpp b/services/camera/libcameraservice/api1/client2/ZslProcessorInterface.cpp new file mode 100644 index 0000000..9efeaba --- /dev/null +++ b/services/camera/libcameraservice/api1/client2/ZslProcessorInterface.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ZslProcessorInterface.h" + +namespace android { +namespace camera2 { + +status_t ZslProcessorInterface::disconnect() { + return OK; +} + +}; //namespace camera2 +}; //namespace android + diff --git a/services/camera/libcameraservice/api1/client2/ZslProcessorInterface.h b/services/camera/libcameraservice/api1/client2/ZslProcessorInterface.h index 183c0c2..9e266e7 100644 --- a/services/camera/libcameraservice/api1/client2/ZslProcessorInterface.h +++ b/services/camera/libcameraservice/api1/client2/ZslProcessorInterface.h @@ -19,6 +19,8 @@ #include <utils/Errors.h> #include <utils/RefBase.h> +#include <utils/String16.h> +#include <utils/Vector.h> namespace android { namespace camera2 { @@ -37,6 +39,9 @@ public: // Delete the underlying CameraDevice streams virtual status_t deleteStream() = 0; + // Clear any additional state necessary before the CameraDevice is disconnected + virtual status_t disconnect(); + /** * Submits a ZSL capture request (id = requestId) * diff --git a/services/camera/libcameraservice/device3/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp index 6f78db5..6a7f9e7 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.cpp +++ b/services/camera/libcameraservice/device3/Camera3Device.cpp @@ -601,10 +601,18 @@ sp<Camera3Device::CaptureRequest> Camera3Device::setUpRequestLocked( if (mStatus == STATUS_UNCONFIGURED || mNeedConfig) { res = configureStreamsLocked(); + // Stream configuration failed due to unsupported configuration. + // Device back to unconfigured state. Client might try other configuraitons + if (res == BAD_VALUE && mStatus == STATUS_UNCONFIGURED) { + CLOGE("No streams configured"); + return NULL; + } + // Stream configuration failed for other reason. Fatal. if (res != OK) { SET_ERR_L("Can't set up streams: %s (%d)", strerror(-res), res); return NULL; } + // Stream configuration successfully configure to empty stream configuration. if (mStatus == STATUS_UNCONFIGURED) { CLOGE("No streams configured"); return NULL; @@ -1036,6 +1044,11 @@ status_t Camera3Device::createDefaultRequest(int templateId, return INVALID_OPERATION; } + if (!mRequestTemplateCache[templateId].isEmpty()) { + *request = mRequestTemplateCache[templateId]; + return OK; + } + const camera_metadata_t *rawRequest; ATRACE_BEGIN("camera3->construct_default_request_settings"); rawRequest = mHal3Device->ops->construct_default_request_settings( @@ -1047,6 +1060,7 @@ status_t Camera3Device::createDefaultRequest(int templateId, return DEAD_OBJECT; } *request = rawRequest; + mRequestTemplateCache[templateId] = rawRequest; return OK; } @@ -1078,6 +1092,10 @@ status_t Camera3Device::waitUntilDrainedLocked() { ALOGV("%s: Camera %d: Waiting until idle", __FUNCTION__, mId); status_t res = waitUntilStateThenRelock(/*active*/ false, kShutdownTimeout); + if (res != OK) { + SET_ERR_L("Error waiting for HAL to drain: %s (%d)", strerror(-res), + res); + } return res; } diff --git a/services/camera/libcameraservice/device3/Camera3Device.h b/services/camera/libcameraservice/device3/Camera3Device.h index b99ed7e..ec6bba1 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.h +++ b/services/camera/libcameraservice/device3/Camera3Device.h @@ -174,6 +174,8 @@ class Camera3Device : CameraMetadata mDeviceInfo; + CameraMetadata mRequestTemplateCache[CAMERA3_TEMPLATE_COUNT]; + uint32_t mDeviceVersion; enum Status { diff --git a/services/camera/libcameraservice/device3/Camera3Stream.cpp b/services/camera/libcameraservice/device3/Camera3Stream.cpp index 29ce38c..3c0e908 100644 --- a/services/camera/libcameraservice/device3/Camera3Stream.cpp +++ b/services/camera/libcameraservice/device3/Camera3Stream.cpp @@ -233,8 +233,7 @@ status_t Camera3Stream::cancelConfiguration() { camera3_stream::usage = oldUsage; camera3_stream::max_buffers = oldMaxBuffers; - mState = STATE_CONSTRUCTED; - + mState = (mState == STATE_IN_RECONFIG) ? STATE_CONFIGURED : STATE_CONSTRUCTED; return OK; } diff --git a/services/camera/libcameraservice/device3/Camera3ZslStream.cpp b/services/camera/libcameraservice/device3/Camera3ZslStream.cpp index 92bf81b..81330ea 100644 --- a/services/camera/libcameraservice/device3/Camera3ZslStream.cpp +++ b/services/camera/libcameraservice/device3/Camera3ZslStream.cpp @@ -315,20 +315,24 @@ status_t Camera3ZslStream::enqueueInputBufferByTimestamp( return OK; } -status_t Camera3ZslStream::clearInputRingBuffer() { +status_t Camera3ZslStream::clearInputRingBuffer(nsecs_t* latestTimestamp) { Mutex::Autolock l(mLock); - return clearInputRingBufferLocked(); + return clearInputRingBufferLocked(latestTimestamp); } -status_t Camera3ZslStream::clearInputRingBufferLocked() { +status_t Camera3ZslStream::clearInputRingBufferLocked(nsecs_t* latestTimestamp) { + + if (latestTimestamp) { + *latestTimestamp = mProducer->getLatestTimestamp(); + } mInputBufferQueue.clear(); return mProducer->clear(); } status_t Camera3ZslStream::disconnectLocked() { - clearInputRingBufferLocked(); + clearInputRingBufferLocked(NULL); return Camera3OutputStream::disconnectLocked(); } diff --git a/services/camera/libcameraservice/device3/Camera3ZslStream.h b/services/camera/libcameraservice/device3/Camera3ZslStream.h index d89c38d..5323a49 100644 --- a/services/camera/libcameraservice/device3/Camera3ZslStream.h +++ b/services/camera/libcameraservice/device3/Camera3ZslStream.h @@ -59,8 +59,10 @@ class Camera3ZslStream : /** * Clears the buffers that can be used by enqueueInputBufferByTimestamp + * latestTimestamp will be filled with the largest timestamp of buffers + * being cleared, 0 if there is no buffer being clear. */ - status_t clearInputRingBuffer(); + status_t clearInputRingBuffer(nsecs_t* latestTimestamp); protected: @@ -100,7 +102,7 @@ class Camera3ZslStream : // Disconnet the Camera3ZslStream specific bufferQueues. virtual status_t disconnectLocked(); - status_t clearInputRingBufferLocked(); + status_t clearInputRingBufferLocked(nsecs_t* latestTimestamp); }; // class Camera3ZslStream diff --git a/services/camera/libcameraservice/gui/RingBufferConsumer.cpp b/services/camera/libcameraservice/gui/RingBufferConsumer.cpp index e4ec5fd..f8562ec 100644 --- a/services/camera/libcameraservice/gui/RingBufferConsumer.cpp +++ b/services/camera/libcameraservice/gui/RingBufferConsumer.cpp @@ -41,7 +41,8 @@ RingBufferConsumer::RingBufferConsumer(const sp<IGraphicBufferConsumer>& consume uint32_t consumerUsage, int bufferCount) : ConsumerBase(consumer), - mBufferCount(bufferCount) + mBufferCount(bufferCount), + mLatestTimestamp(0) { mConsumer->setConsumerUsageBits(consumerUsage); mConsumer->setMaxAcquiredBufferCount(bufferCount); @@ -152,6 +153,14 @@ status_t RingBufferConsumer::clear() { return OK; } +nsecs_t RingBufferConsumer::getLatestTimestamp() { + Mutex::Autolock _l(mMutex); + if (mBufferItemList.size() == 0) { + return 0; + } + return mLatestTimestamp; +} + void RingBufferConsumer::pinBufferLocked(const BufferItem& item) { List<RingBufferItem>::iterator it, end; @@ -302,6 +311,13 @@ void RingBufferConsumer::onFrameAvailable() { item.mTimestamp, mBufferItemList.size(), mBufferCount); + if (item.mTimestamp < mLatestTimestamp) { + BI_LOGE("Timestamp decreases from %" PRId64 " to %" PRId64, + mLatestTimestamp, item.mTimestamp); + } + + mLatestTimestamp = item.mTimestamp; + item.mGraphicBuffer = mSlots[item.mBuf].mGraphicBuffer; } // end of mMutex lock diff --git a/services/camera/libcameraservice/gui/RingBufferConsumer.h b/services/camera/libcameraservice/gui/RingBufferConsumer.h index a03736d..da97a11 100644 --- a/services/camera/libcameraservice/gui/RingBufferConsumer.h +++ b/services/camera/libcameraservice/gui/RingBufferConsumer.h @@ -159,6 +159,9 @@ class RingBufferConsumer : public ConsumerBase, // Release all the non-pinned buffers in the ring buffer status_t clear(); + // Return 0 if RingBuffer is empty, otherwise return timestamp of latest buffer. + nsecs_t getLatestTimestamp(); + private: // Override ConsumerBase::onFrameAvailable @@ -180,6 +183,9 @@ class RingBufferConsumer : public ConsumerBase, // List of acquired buffers in our ring buffer List<RingBufferItem> mBufferItemList; const int mBufferCount; + + // Timestamp of latest buffer + nsecs_t mLatestTimestamp; }; } // namespace android |