summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJames Dong <jdong@google.com>2010-09-01 18:48:35 -0700
committerJames Dong <jdong@google.com>2010-09-01 20:45:39 -0700
commitd707fcb3e29707ca4a5935c294ef0b38eb5aba5f (patch)
tree8f223d9d5656ba34a0fdfe2e7ee4992f7a0bf3e3
parent9b93478fef2915a1d0cbb1fe17d2788ef8b5b230 (diff)
downloadframeworks_av-d707fcb3e29707ca4a5935c294ef0b38eb5aba5f.zip
frameworks_av-d707fcb3e29707ca4a5935c294ef0b38eb5aba5f.tar.gz
frameworks_av-d707fcb3e29707ca4a5935c294ef0b38eb5aba5f.tar.bz2
Calculate audio media drift time from AudioSource
The problem was that the time to receive an output buffer from an audio encoder is different because the encoder does not need to read from the source for all output buffers. This leads to large fluctuation in terms of wall clock duration between two neighboring audio sample outputs from the audio encoder. As a result, the media time for the video track after adjustment using the drifting changes wildly sometimes. This patch addresses this issue by only updating the media drift time when an audio source input buffer is read. the wall clock for the audio track is also calculated at the same time when the input audio buffer is read at AudioSource. bug - 2959800 Change-Id: I3174aa182f744784b540f0a7198524d4eee8bd7b
-rw-r--r--include/media/stagefright/AudioSource.h1
-rw-r--r--include/media/stagefright/MPEG4Writer.h2
-rw-r--r--include/media/stagefright/MetaData.h1
-rw-r--r--media/libstagefright/AudioSource.cpp15
-rw-r--r--media/libstagefright/MPEG4Writer.cpp29
-rw-r--r--media/libstagefright/codecs/aacenc/AACEncoder.cpp15
-rw-r--r--media/libstagefright/codecs/amrnb/enc/AMRNBEncoder.cpp16
-rw-r--r--media/libstagefright/codecs/amrwbenc/AMRWBEncoder.cpp16
8 files changed, 63 insertions, 32 deletions
diff --git a/include/media/stagefright/AudioSource.h b/include/media/stagefright/AudioSource.h
index 1af4254..a5cec78 100644
--- a/include/media/stagefright/AudioSource.h
+++ b/include/media/stagefright/AudioSource.h
@@ -72,6 +72,7 @@ private:
int64_t mPrevSampleTimeUs;
int64_t mTotalLostFrames;
int64_t mPrevLostBytes;
+ int64_t mInitialReadTimeUs;
MediaBufferGroup *mGroup;
diff --git a/include/media/stagefright/MPEG4Writer.h b/include/media/stagefright/MPEG4Writer.h
index 2412f6a..9716e98 100644
--- a/include/media/stagefright/MPEG4Writer.h
+++ b/include/media/stagefright/MPEG4Writer.h
@@ -132,7 +132,7 @@ private:
// Adjust other track media clock (presumably wall clock)
// based on audio track media clock with the drift time.
int64_t mDriftTimeUs;
- void addDriftTimeUs(int64_t driftTimeUs);
+ void setDriftTimeUs(int64_t driftTimeUs);
int64_t getDriftTimeUs();
void lock();
diff --git a/include/media/stagefright/MetaData.h b/include/media/stagefright/MetaData.h
index 43354c2..3b31e68 100644
--- a/include/media/stagefright/MetaData.h
+++ b/include/media/stagefright/MetaData.h
@@ -48,6 +48,7 @@ enum {
kKeyTime = 'time', // int64_t (usecs)
kKeyNTPTime = 'ntpT', // uint64_t (ntp-timestamp)
kKeyTargetTime = 'tarT', // int64_t (usecs)
+ kKeyDriftTime = 'dftT', // int64_t (usecs)
kKeyDuration = 'dura', // int64_t (usecs)
kKeyColorFormat = 'colf',
kKeyPlatformPrivate = 'priv', // pointer
diff --git a/media/libstagefright/AudioSource.cpp b/media/libstagefright/AudioSource.cpp
index bcae913..c2f79e8 100644
--- a/media/libstagefright/AudioSource.cpp
+++ b/media/libstagefright/AudioSource.cpp
@@ -84,6 +84,7 @@ status_t AudioSource::start(MetaData *params) {
mTrackMaxAmplitude = false;
mMaxAmplitude = 0;
+ mInitialReadTimeUs = 0;
mStartTimeUs = 0;
int64_t startTimeUs;
if (params && params->findInt64(kKeyTime, &startTimeUs)) {
@@ -210,6 +211,7 @@ status_t AudioSource::read(
return NO_INIT;
}
+ int64_t readTimeUs = systemTime() / 1000;
*out = NULL;
MediaBuffer *buffer;
@@ -223,9 +225,10 @@ status_t AudioSource::read(
if (numFramesRecorded == 0 && mPrevSampleTimeUs == 0) {
+ mInitialReadTimeUs = readTimeUs;
// Initial delay
if (mStartTimeUs > 0) {
- mStartTimeUs = systemTime() / 1000 - mStartTimeUs;
+ mStartTimeUs = readTimeUs - mStartTimeUs;
} else {
// Assume latency is constant.
mStartTimeUs += mRecord->latency() * 1000;
@@ -271,7 +274,10 @@ status_t AudioSource::read(
}
memset(buffer->data(), 0, numLostBytes);
buffer->set_range(0, numLostBytes);
- buffer->meta_data()->setInt64(kKeyTime, mPrevSampleTimeUs);
+ if (numFramesRecorded == 0) {
+ buffer->meta_data()->setInt64(kKeyTime, mStartTimeUs);
+ }
+ buffer->meta_data()->setInt64(kKeyDriftTime, readTimeUs - mInitialReadTimeUs);
mPrevSampleTimeUs = timestampUs;
*out = buffer;
return OK;
@@ -309,7 +315,10 @@ status_t AudioSource::read(
trackMaxAmplitude((int16_t *) buffer->data(), n >> 1);
}
- buffer->meta_data()->setInt64(kKeyTime, mPrevSampleTimeUs);
+ if (numFramesRecorded == 0) {
+ buffer->meta_data()->setInt64(kKeyTime, mStartTimeUs);
+ }
+ buffer->meta_data()->setInt64(kKeyDriftTime, readTimeUs - mInitialReadTimeUs);
CHECK(timestampUs > mPrevSampleTimeUs);
mPrevSampleTimeUs = timestampUs;
LOGV("initial delay: %lld, sample rate: %d, timestamp: %lld",
diff --git a/media/libstagefright/MPEG4Writer.cpp b/media/libstagefright/MPEG4Writer.cpp
index a15b84e..af2b4c4 100644
--- a/media/libstagefright/MPEG4Writer.cpp
+++ b/media/libstagefright/MPEG4Writer.cpp
@@ -1430,9 +1430,6 @@ status_t MPEG4Writer::Track::threadEntry() {
int64_t previousPausedDurationUs = 0;
int64_t timestampUs;
- int64_t wallClockTimeUs = 0;
- int64_t lastWallClockTimeUs = 0;
-
sp<MetaData> meta_data;
bool collectStats = collectStatisticalData();
@@ -1542,14 +1539,15 @@ status_t MPEG4Writer::Track::threadEntry() {
// of neighboring samples. This in turn helps reduce the track header size,
// especially, the number of entries in the "stts" box.
if (mNumSamples > 1) {
- int64_t durationUs = timestampUs + mOwner->getDriftTimeUs() - lastTimestampUs;
+ int64_t currDriftTimeUs = mOwner->getDriftTimeUs();
+ int64_t durationUs = timestampUs + currDriftTimeUs - lastTimestampUs;
int64_t diffUs = (durationUs > lastDurationUs)
? durationUs - lastDurationUs
: lastDurationUs - durationUs;
if (diffUs <= 5000) { // XXX: Magic number 5ms
timestampUs = lastTimestampUs + lastDurationUs;
} else {
- timestampUs += mOwner->getDriftTimeUs();
+ timestampUs += currDriftTimeUs;
}
}
}
@@ -1557,12 +1555,6 @@ status_t MPEG4Writer::Track::threadEntry() {
if (mNumSamples > 1) {
if (timestampUs <= lastTimestampUs) {
LOGW("Frame arrives too late!");
-#if 0
- // Drop the late frame.
- copy->release();
- copy = NULL;
- continue;
-#else
// Don't drop the late frame, since dropping a frame may cause
// problems later during playback
@@ -1573,7 +1565,6 @@ status_t MPEG4Writer::Track::threadEntry() {
} else {
timestampUs = lastTimestampUs + (1000000LL + (mTimeScale >> 1)) / mTimeScale;
}
-#endif
}
}
@@ -1613,12 +1604,10 @@ status_t MPEG4Writer::Track::threadEntry() {
lastDurationTicks = currDurationTicks;
lastTimestampUs = timestampUs;
if (mIsRealTimeRecording && mIsAudio) {
- wallClockTimeUs = systemTime() / 1000;
- int64_t wallClockDurationUs = wallClockTimeUs - lastWallClockTimeUs;
- if (mNumSamples > 2) {
- mOwner->addDriftTimeUs(lastDurationUs - wallClockDurationUs);
+ int64_t driftTimeUs = 0;
+ if (meta_data->findInt64(kKeyDriftTime, &driftTimeUs)) {
+ mOwner->setDriftTimeUs(driftTimeUs);
}
- lastWallClockTimeUs = wallClockTimeUs;
}
if (isSync != 0) {
@@ -1851,10 +1840,10 @@ void MPEG4Writer::Track::logStatisticalData(bool isAudio) {
}
}
-void MPEG4Writer::addDriftTimeUs(int64_t driftTimeUs) {
- LOGV("addDriftTimeUs: %lld us", driftTimeUs);
+void MPEG4Writer::setDriftTimeUs(int64_t driftTimeUs) {
+ LOGV("setDriftTimeUs: %lld us", driftTimeUs);
Mutex::Autolock autolock(mLock);
- mDriftTimeUs += driftTimeUs;
+ mDriftTimeUs = driftTimeUs;
}
int64_t MPEG4Writer::getDriftTimeUs() {
diff --git a/media/libstagefright/codecs/aacenc/AACEncoder.cpp b/media/libstagefright/codecs/aacenc/AACEncoder.cpp
index 052c354..c05e3e5 100644
--- a/media/libstagefright/codecs/aacenc/AACEncoder.cpp
+++ b/media/libstagefright/codecs/aacenc/AACEncoder.cpp
@@ -208,6 +208,8 @@ status_t AACEncoder::read(
MediaBuffer *buffer;
CHECK_EQ(mBufferGroup->acquire_buffer(&buffer), OK);
uint8_t *outPtr = (uint8_t *)buffer->data();
+ bool readFromSource = false;
+ int64_t wallClockTimeUs = 0;
if (mFrameCount == 0) {
memcpy(outPtr, mAudioSpecificConfigData, 2);
@@ -238,9 +240,14 @@ status_t AACEncoder::read(
CHECK_EQ(align, 0);
int64_t timeUs;
+ CHECK(mInputBuffer->meta_data()->findInt64(kKeyDriftTime, &timeUs));
+ wallClockTimeUs = timeUs;
if (mInputBuffer->meta_data()->findInt64(kKeyTime, &timeUs)) {
mAnchorTimeUs = timeUs;
}
+ readFromSource = true;
+ } else {
+ readFromSource = false;
}
size_t copy =
(kNumSamplesPerFrame - mNumInputSamples) * sizeof(int16_t);
@@ -288,9 +295,13 @@ status_t AACEncoder::read(
CHECK(outputData.Length != 0);
buffer->set_range(0, outputData.Length);
- int64_t timestampUs = ((mFrameCount - 1) * 1000000LL * kNumSamplesPerFrame) / mSampleRate;
+ int64_t mediaTimeUs =
+ ((mFrameCount - 1) * 1000000LL * kNumSamplesPerFrame) / mSampleRate;
+ buffer->meta_data()->setInt64(kKeyTime, mAnchorTimeUs + mediaTimeUs);
+ if (readFromSource) {
+ buffer->meta_data()->setInt64(kKeyDriftTime, mediaTimeUs - wallClockTimeUs);
+ }
++mFrameCount;
- buffer->meta_data()->setInt64(kKeyTime, timestampUs);
*out = buffer;
return OK;
diff --git a/media/libstagefright/codecs/amrnb/enc/AMRNBEncoder.cpp b/media/libstagefright/codecs/amrnb/enc/AMRNBEncoder.cpp
index c875426..dab1390 100644
--- a/media/libstagefright/codecs/amrnb/enc/AMRNBEncoder.cpp
+++ b/media/libstagefright/codecs/amrnb/enc/AMRNBEncoder.cpp
@@ -147,6 +147,8 @@ status_t AMRNBEncoder::read(
int64_t seekTimeUs;
ReadOptions::SeekMode mode;
CHECK(options == NULL || !options->getSeekTo(&seekTimeUs, &mode));
+ bool readFromSource = false;
+ int64_t wallClockTimeUs = 0;
while (mNumInputSamples < kNumSamplesPerFrame) {
if (mInputBuffer == NULL) {
@@ -166,12 +168,16 @@ status_t AMRNBEncoder::read(
size_t align = mInputBuffer->range_length() % sizeof(int16_t);
CHECK_EQ(align, 0);
+ readFromSource = true;
int64_t timeUs;
+ CHECK(mInputBuffer->meta_data()->findInt64(kKeyDriftTime, &timeUs));
+ wallClockTimeUs = timeUs;
if (mInputBuffer->meta_data()->findInt64(kKeyTime, &timeUs)) {
mAnchorTimeUs = timeUs;
- mNumFramesOutput = 0;
}
+ } else {
+ readFromSource = false;
}
size_t copy =
@@ -217,8 +223,14 @@ status_t AMRNBEncoder::read(
buffer->set_range(0, res);
// Each frame of 160 samples is 20ms long.
+ int64_t mediaTimeUs = mNumFramesOutput * 20000LL;
buffer->meta_data()->setInt64(
- kKeyTime, mAnchorTimeUs + mNumFramesOutput * 20000);
+ kKeyTime, mAnchorTimeUs + mediaTimeUs);
+
+ if (readFromSource) {
+ buffer->meta_data()->setInt64(kKeyDriftTime,
+ mediaTimeUs - wallClockTimeUs);
+ }
++mNumFramesOutput;
diff --git a/media/libstagefright/codecs/amrwbenc/AMRWBEncoder.cpp b/media/libstagefright/codecs/amrwbenc/AMRWBEncoder.cpp
index 93304d0..b62eb5b 100644
--- a/media/libstagefright/codecs/amrwbenc/AMRWBEncoder.cpp
+++ b/media/libstagefright/codecs/amrwbenc/AMRWBEncoder.cpp
@@ -198,6 +198,8 @@ status_t AMRWBEncoder::read(
int64_t seekTimeUs;
ReadOptions::SeekMode mode;
CHECK(options == NULL || !options->getSeekTo(&seekTimeUs, &mode));
+ bool readFromSource = false;
+ int64_t wallClockTimeUs = 0;
while (mNumInputSamples < kNumSamplesPerFrame) {
if (mInputBuffer == NULL) {
@@ -219,9 +221,14 @@ status_t AMRWBEncoder::read(
CHECK_EQ(align, 0);
int64_t timeUs;
+ CHECK(mInputBuffer->meta_data()->findInt64(kKeyDriftTime, &timeUs));
+ wallClockTimeUs = timeUs;
if (mInputBuffer->meta_data()->findInt64(kKeyTime, &timeUs)) {
mAnchorTimeUs = timeUs;
}
+ readFromSource = true;
+ } else {
+ readFromSource = false;
}
size_t copy =
@@ -276,10 +283,11 @@ status_t AMRWBEncoder::read(
buffer->set_range(0, outputData.Length);
++mNumFramesOutput;
- // XXX: fix timestamp calculation
- int64_t timestampUs = mNumFramesOutput * 20000LL;
-
- buffer->meta_data()->setInt64(kKeyTime, timestampUs);
+ int64_t mediaTimeUs = mNumFramesOutput * 20000LL;
+ buffer->meta_data()->setInt64(kKeyTime, mAnchorTimeUs + mediaTimeUs);
+ if (readFromSource) {
+ buffer->meta_data()->setInt64(kKeyDriftTime, mediaTimeUs - wallClockTimeUs);
+ }
*out = buffer;
return OK;