diff options
Diffstat (limited to 'media/libstagefright')
| -rw-r--r-- | media/libstagefright/MPEG4Writer.cpp | 205 |
1 files changed, 181 insertions, 24 deletions
diff --git a/media/libstagefright/MPEG4Writer.cpp b/media/libstagefright/MPEG4Writer.cpp index 546df47..90b1aab 100644 --- a/media/libstagefright/MPEG4Writer.cpp +++ b/media/libstagefright/MPEG4Writer.cpp @@ -40,6 +40,7 @@ namespace android { static const int64_t kMax32BitFileSize = 0x007fffffffLL; static const uint8_t kNalUnitTypeSeqParamSet = 0x07; static const uint8_t kNalUnitTypePicParamSet = 0x08; +static const int64_t kVideoMediaTimeAdjustPeriodTimeUs = 10000000LL; // 10s class MPEG4Writer::Track { public: @@ -148,6 +149,28 @@ private: int64_t mPreviousTrackTimeUs; int64_t mTrackEveryTimeDurationUs; + // Has the media time adjustment for video started? + bool mIsMediaTimeAdjustmentOn; + // The time stamp when previous media time adjustment period starts + int64_t mPrevMediaTimeAdjustTimestampUs; + // Number of vidoe frames whose time stamp may be adjusted + int64_t mMediaTimeAdjustNumFrames; + // The sample number when previous meida time adjustmnet period starts + int64_t mPrevMediaTimeAdjustSample; + // The total accumulated drift time within a period of + // kVideoMediaTimeAdjustPeriodTimeUs. + int64_t mTotalDriftTimeToAdjustUs; + // The total accumalated drift time since the start of the recording + // excluding the current time adjustment period + int64_t mPrevTotalAccumDriftTimeUs; + + // Update the audio track's drift information. + void updateDriftTime(const sp<MetaData>& meta); + + // Adjust the time stamp of the video track according to + // the drift time information from the audio track. + void adjustMediaTime(int64_t *timestampUs); + static void *ThreadWrapper(void *me); status_t threadEntry(); @@ -319,7 +342,7 @@ int64_t MPEG4Writer::estimateMoovBoxSize(int32_t bitRate) { size = MAX_MOOV_BOX_SIZE; } - LOGI("limits: %lld/%lld bytes/us, bit rate: %d bps and the estimated" + LOGV("limits: %lld/%lld bytes/us, bit rate: %d bps and the estimated" " moov size %lld bytes", mMaxFileSizeLimitBytes, mMaxFileDurationLimitUs, bitRate, size); return factor * size; @@ -346,7 +369,7 @@ status_t MPEG4Writer::start(MetaData *param) { // If file size is set to be larger than the 32 bit file // size limit, treat it as an error. if (mMaxFileSizeLimitBytes > kMax32BitFileSize) { - LOGW("32-bi file size limit (%lld bytes) too big. " + LOGW("32-bit file size limit (%lld bytes) too big. " "It is changed to %lld bytes", mMaxFileSizeLimitBytes, kMax32BitFileSize); mMaxFileSizeLimitBytes = kMax32BitFileSize; @@ -1149,6 +1172,12 @@ status_t MPEG4Writer::Track::start(MetaData *params) { mNumStscTableEntries = 0; mNumSttsTableEntries = 0; mMdatSizeBytes = 0; + mIsMediaTimeAdjustmentOn = false; + mPrevMediaTimeAdjustTimestampUs = 0; + mMediaTimeAdjustNumFrames = 0; + mPrevMediaTimeAdjustSample = 0; + mTotalDriftTimeToAdjustUs = 0; + mPrevTotalAccumDriftTimeUs = 0; pthread_create(&mThread, &attr, ThreadWrapper, this); pthread_attr_destroy(&attr); @@ -1437,6 +1466,145 @@ status_t MPEG4Writer::Track::makeAVCCodecSpecificData( return OK; } +/* +* The video track's media time adjustment for real-time applications +* is described as follows: +* +* First, the media time adjustment is done for every period of +* kVideoMediaTimeAdjustPeriodTimeUs. kVideoMediaTimeAdjustPeriodTimeUs +* is currently a fixed value chosen heuristically. The value of +* kVideoMediaTimeAdjustPeriodTimeUs should not be very large or very small +* for two considerations: on one hand, a relatively large value +* helps reduce large fluctuation of drift time in the audio encoding +* path; while on the other hand, a relatively small value helps keep +* restoring synchronization in audio/video more frequently. Note for the +* very first period of kVideoMediaTimeAdjustPeriodTimeUs, there is +* no media time adjustment for the video track. +* +* Second, the total accumulated audio track time drift found +* in a period of kVideoMediaTimeAdjustPeriodTimeUs is distributed +* over a stream of incoming video frames. The number of video frames +* affected is determined based on the number of recorded video frames +* within the past kVideoMediaTimeAdjustPeriodTimeUs period. +* We choose to distribute the drift time over only a portion +* (rather than all) of the total number of recorded video frames +* in order to make sure that the video track media time adjustment is +* completed for the current period before the next video track media +* time adjustment period starts. Currently, the portion chosen is a +* half (0.5). +* +* Last, various additional checks are performed to ensure that +* the actual audio encoding path does not have too much drift. +* In particular, 1) we want to limit the average incremental time +* adjustment for each video frame to be less than a threshold +* for a single period of kVideoMediaTimeAdjustPeriodTimeUs. +* Currently, the threshold is set to 5 ms. If the average incremental +* media time adjustment for a video frame is larger than the +* threshold, the audio encoding path has too much time drift. +* 2) We also want to limit the total time drift in the audio +* encoding path to be less than a threshold for a period of +* kVideoMediaTimeAdjustPeriodTimeUs. Currently, the threshold +* is 0.5% of kVideoMediaTimeAdjustPeriodTimeUs. If the time drift of +* the audio encoding path is larger than the threshold, the audio +* encoding path has too much time drift. We treat the large time +* drift of the audio encoding path as errors, since there is no +* way to keep audio/video in synchronization for real-time +* applications if the time drift is too large unless we drop some +* video frames, which has its own problems that we don't want +* to get into for the time being. +*/ +void MPEG4Writer::Track::adjustMediaTime(int64_t *timestampUs) { + if (*timestampUs - mPrevMediaTimeAdjustTimestampUs >= + kVideoMediaTimeAdjustPeriodTimeUs) { + + LOGV("New media time adjustment period at %lld us", *timestampUs); + mIsMediaTimeAdjustmentOn = true; + mMediaTimeAdjustNumFrames = + (mNumSamples - mPrevMediaTimeAdjustSample) >> 1; + + mPrevMediaTimeAdjustTimestampUs = *timestampUs; + mPrevMediaTimeAdjustSample = mNumSamples; + int64_t totalAccumDriftTimeUs = mOwner->getDriftTimeUs(); + mTotalDriftTimeToAdjustUs = + totalAccumDriftTimeUs - mPrevTotalAccumDriftTimeUs; + + mPrevTotalAccumDriftTimeUs = totalAccumDriftTimeUs; + + // Check on incremental adjusted time per frame + int64_t adjustTimePerFrameUs = + mTotalDriftTimeToAdjustUs / mMediaTimeAdjustNumFrames; + + if (adjustTimePerFrameUs < 0) { + adjustTimePerFrameUs = -adjustTimePerFrameUs; + } + if (adjustTimePerFrameUs >= 5000) { + LOGE("Adjusted time per video frame is %lld us", + adjustTimePerFrameUs); + CHECK(!"Video frame time adjustment is too large!"); + } + + // Check on total accumulated time drift within a period of + // kVideoMediaTimeAdjustPeriodTimeUs. + int64_t driftPercentage = (mTotalDriftTimeToAdjustUs * 1000) + / kVideoMediaTimeAdjustPeriodTimeUs; + + if (driftPercentage < 0) { + driftPercentage = -driftPercentage; + } + if (driftPercentage > 5) { + LOGE("Audio track has time drift %lld us over %lld us", + mTotalDriftTimeToAdjustUs, + kVideoMediaTimeAdjustPeriodTimeUs); + + CHECK(!"The audio track media time drifts too much!"); + } + + } + + if (mIsMediaTimeAdjustmentOn) { + if (mNumSamples - mPrevMediaTimeAdjustSample <= + mMediaTimeAdjustNumFrames) { + + // Do media time incremental adjustment + int64_t incrementalAdjustTimeUs = + (mTotalDriftTimeToAdjustUs * + (mNumSamples - mPrevMediaTimeAdjustSample)) + / mMediaTimeAdjustNumFrames; + + *timestampUs += + (incrementalAdjustTimeUs + mPrevTotalAccumDriftTimeUs); + + LOGV("Incremental video frame media time adjustment: %lld us", + (incrementalAdjustTimeUs + mPrevTotalAccumDriftTimeUs)); + } else { + // Within the remaining adjustment period, + // no incremental adjustment is needed. + *timestampUs += + (mTotalDriftTimeToAdjustUs + mPrevTotalAccumDriftTimeUs); + + LOGV("Fixed video frame media time adjustment: %lld us", + (mTotalDriftTimeToAdjustUs + mPrevTotalAccumDriftTimeUs)); + } + } +} + +/* + * Updates the drift time from the audio track so that + * the video track can get the updated drift time information + * from the file writer. The fluctuation of the drift time of the audio + * encoding path is smoothed out with a simple filter by giving a larger + * weight to more recently drift time. The filter coefficients, 0.5 and 0.5, + * are heuristically determined. + */ +void MPEG4Writer::Track::updateDriftTime(const sp<MetaData>& meta) { + int64_t driftTimeUs = 0; + if (meta->findInt64(kKeyDriftTime, &driftTimeUs)) { + int64_t prevDriftTimeUs = mOwner->getDriftTimeUs(); + int64_t timeUs = (driftTimeUs + prevDriftTimeUs) >> 1; + mOwner->setDriftTimeUs(timeUs); + } +} + status_t MPEG4Writer::Track::threadEntry() { int32_t count = 0; const int64_t interleaveDurationUs = mOwner->interleaveDuration(); @@ -1587,24 +1755,16 @@ status_t MPEG4Writer::Track::threadEntry() { timestampUs -= previousPausedDurationUs; CHECK(timestampUs >= 0); - if (mIsRealTimeRecording && !mIsAudio) { - // The minor adjustment on the timestamp is heuristic/experimental - // We are adjusting the timestamp to reduce the fluctuation of the duration - // of neighboring samples. This in turn helps reduce the track header size, - // especially, the number of entries in the "stts" box. - if (mNumSamples > 1) { - int64_t currDriftTimeUs = mOwner->getDriftTimeUs(); - int64_t durationUs = timestampUs + currDriftTimeUs - lastTimestampUs; - int64_t diffUs = (durationUs > lastDurationUs) - ? durationUs - lastDurationUs - : lastDurationUs - durationUs; - if (diffUs <= 5000) { // XXX: Magic number 5ms - timestampUs = lastTimestampUs + lastDurationUs; - } else { - timestampUs += currDriftTimeUs; - } + + // Media time adjustment for real-time applications + if (mIsRealTimeRecording) { + if (mIsAudio) { + updateDriftTime(meta_data); + } else { + adjustMediaTime(×tampUs); } } + CHECK(timestampUs >= 0); if (mNumSamples > 1) { if (timestampUs <= lastTimestampUs) { @@ -1656,12 +1816,6 @@ status_t MPEG4Writer::Track::threadEntry() { lastDurationUs = timestampUs - lastTimestampUs; lastDurationTicks = currDurationTicks; lastTimestampUs = timestampUs; - if (mIsRealTimeRecording && mIsAudio) { - int64_t driftTimeUs = 0; - if (meta_data->findInt64(kKeyDriftTime, &driftTimeUs)) { - mOwner->setDriftTimeUs(driftTimeUs); - } - } if (isSync != 0) { addOneStssTableEntry(mNumSamples); @@ -1735,6 +1889,9 @@ status_t MPEG4Writer::Track::threadEntry() { mReachedEOS = true; LOGI("Received total/0-length (%d/%d) buffers and encoded %d frames. - %s", count, nZeroLengthFrames, mNumSamples, mIsAudio? "audio": "video"); + if (mIsAudio) { + LOGI("Audio track drift time: %lld us", mOwner->getDriftTimeUs()); + } if (err == ERROR_END_OF_STREAM) { return OK; |