Calculate audio media drift time from AudioSource

The problem was that the time to receive an output buffer
from an audio encoder is different because the encoder does not
need to read from the source for all output buffers. This leads
to large fluctuation in terms of wall clock duration between two
neighboring audio sample outputs from the audio encoder. As a
result, the media time for the video track after adjustment using
the drifting changes wildly sometimes.

This patch addresses this issue by only updating the media drift
time when an audio source input buffer is read. the wall clock
for the audio track is also calculated at the same time when
the input audio buffer is read at AudioSource.

bug - 2959800

Change-Id: I3174aa182f744784b540f0a7198524d4eee8bd7b

3 files changed, 39 insertions, 8 deletions

diff --git a/media/libstagefright/codecs/aacenc/AACEncoder.cpp b/media/libstagefright/codecs/aacenc/AACEncoder.cpp
index 052c354..c05e3e5 100644
--- a/media/libstagefright/codecs/aacenc/AACEncoder.cpp
+++ b/media/libstagefright/codecs/aacenc/AACEncoder.cpp
@@ -208,6 +208,8 @@ status_t AACEncoder::read(
     MediaBuffer *buffer;
     CHECK_EQ(mBufferGroup->acquire_buffer(&buffer), OK);
     uint8_t *outPtr = (uint8_t *)buffer->data();
+    bool readFromSource = false;
+    int64_t wallClockTimeUs = 0;
 
     if (mFrameCount == 0) {
         memcpy(outPtr, mAudioSpecificConfigData, 2);
@@ -238,9 +240,14 @@ status_t AACEncoder::read(
             CHECK_EQ(align, 0);
 
             int64_t timeUs;
+            CHECK(mInputBuffer->meta_data()->findInt64(kKeyDriftTime, &timeUs));
+            wallClockTimeUs = timeUs;
             if (mInputBuffer->meta_data()->findInt64(kKeyTime, &timeUs)) {
                 mAnchorTimeUs = timeUs;
             }
+            readFromSource = true;
+        } else {
+            readFromSource = false;
         }
         size_t copy =
             (kNumSamplesPerFrame - mNumInputSamples) * sizeof(int16_t);
@@ -288,9 +295,13 @@ status_t AACEncoder::read(
     CHECK(outputData.Length != 0);
     buffer->set_range(0, outputData.Length);
 
-    int64_t timestampUs = ((mFrameCount - 1) * 1000000LL * kNumSamplesPerFrame) / mSampleRate;
+    int64_t mediaTimeUs =
+        ((mFrameCount - 1) * 1000000LL * kNumSamplesPerFrame) / mSampleRate;
+    buffer->meta_data()->setInt64(kKeyTime, mAnchorTimeUs + mediaTimeUs);
+    if (readFromSource) {
+        buffer->meta_data()->setInt64(kKeyDriftTime, mediaTimeUs - wallClockTimeUs);
+    }
     ++mFrameCount;
-    buffer->meta_data()->setInt64(kKeyTime, timestampUs);
 
     *out = buffer;
     return OK;
diff --git a/media/libstagefright/codecs/amrnb/enc/AMRNBEncoder.cpp b/media/libstagefright/codecs/amrnb/enc/AMRNBEncoder.cpp
index c875426..dab1390 100644
--- a/media/libstagefright/codecs/amrnb/enc/AMRNBEncoder.cpp
+++ b/media/libstagefright/codecs/amrnb/enc/AMRNBEncoder.cpp
@@ -147,6 +147,8 @@ status_t AMRNBEncoder::read(
     int64_t seekTimeUs;
     ReadOptions::SeekMode mode;
     CHECK(options == NULL || !options->getSeekTo(&seekTimeUs, &mode));
+    bool readFromSource = false;
+    int64_t wallClockTimeUs = 0;
 
     while (mNumInputSamples < kNumSamplesPerFrame) {
         if (mInputBuffer == NULL) {
@@ -166,12 +168,16 @@ status_t AMRNBEncoder::read(
 
             size_t align = mInputBuffer->range_length() % sizeof(int16_t);
             CHECK_EQ(align, 0);
+            readFromSource = true;
 
             int64_t timeUs;
+            CHECK(mInputBuffer->meta_data()->findInt64(kKeyDriftTime, &timeUs));
+            wallClockTimeUs = timeUs;
             if (mInputBuffer->meta_data()->findInt64(kKeyTime, &timeUs)) {
                 mAnchorTimeUs = timeUs;
-                mNumFramesOutput = 0;
             }
+        } else {
+            readFromSource = false;
         }
 
         size_t copy =
@@ -217,8 +223,14 @@ status_t AMRNBEncoder::read(
     buffer->set_range(0, res);
 
     // Each frame of 160 samples is 20ms long.
+    int64_t mediaTimeUs = mNumFramesOutput * 20000LL;
     buffer->meta_data()->setInt64(
-            kKeyTime, mAnchorTimeUs + mNumFramesOutput * 20000);
+            kKeyTime, mAnchorTimeUs + mediaTimeUs);
+
+    if (readFromSource) {
+        buffer->meta_data()->setInt64(kKeyDriftTime,
+            mediaTimeUs - wallClockTimeUs);
+    }
 
     ++mNumFramesOutput;
 
diff --git a/media/libstagefright/codecs/amrwbenc/AMRWBEncoder.cpp b/media/libstagefright/codecs/amrwbenc/AMRWBEncoder.cpp
index 93304d0..b62eb5b 100644
--- a/media/libstagefright/codecs/amrwbenc/AMRWBEncoder.cpp
+++ b/media/libstagefright/codecs/amrwbenc/AMRWBEncoder.cpp
@@ -198,6 +198,8 @@ status_t AMRWBEncoder::read(
     int64_t seekTimeUs;
     ReadOptions::SeekMode mode;
     CHECK(options == NULL || !options->getSeekTo(&seekTimeUs, &mode));
+    bool readFromSource = false;
+    int64_t wallClockTimeUs = 0;
 
     while (mNumInputSamples < kNumSamplesPerFrame) {
         if (mInputBuffer == NULL) {
@@ -219,9 +221,14 @@ status_t AMRWBEncoder::read(
             CHECK_EQ(align, 0);
 
             int64_t timeUs;
+            CHECK(mInputBuffer->meta_data()->findInt64(kKeyDriftTime, &timeUs));
+            wallClockTimeUs = timeUs;
             if (mInputBuffer->meta_data()->findInt64(kKeyTime, &timeUs)) {
                 mAnchorTimeUs = timeUs;
             }
+            readFromSource = true;
+        } else {
+            readFromSource = false;
         }
 
         size_t copy =
@@ -276,10 +283,11 @@ status_t AMRWBEncoder::read(
     buffer->set_range(0, outputData.Length);
     ++mNumFramesOutput;
 
-    // XXX: fix timestamp calculation
-    int64_t timestampUs = mNumFramesOutput * 20000LL;
-
-    buffer->meta_data()->setInt64(kKeyTime, timestampUs);
+    int64_t mediaTimeUs = mNumFramesOutput * 20000LL;
+    buffer->meta_data()->setInt64(kKeyTime, mAnchorTimeUs + mediaTimeUs);
+    if (readFromSource) {
+        buffer->meta_data()->setInt64(kKeyDriftTime, mediaTimeUs - wallClockTimeUs);
+    }
 
     *out = buffer;
     return OK;