Support audio and video track interleaving in the recorded mp4 file

Change-Id: Ifa27eb23ee265f84fe06773b29b0eb2b0b075b60

4 files changed, 148 insertions, 48 deletions

diff --git a/media/libmediaplayerservice/StagefrightRecorder.cpp b/media/libmediaplayerservice/StagefrightRecorder.cpp
index 682ff3a..57db7e4 100644
--- a/media/libmediaplayerservice/StagefrightRecorder.cpp
+++ b/media/libmediaplayerservice/StagefrightRecorder.cpp
@@ -213,25 +213,31 @@ status_t StagefrightRecorder::setParamVideoEncodingBitRate(int32_t bitRate) {
     return OK;
 }
 
-status_t StagefrightRecorder::setMaxDurationOrFileSize(int32_t limit, bool limit_is_duration) {
-    LOGV("setMaxDurationOrFileSize: limit (%d) for %s",
+status_t StagefrightRecorder::setParamMaxDurationOrFileSize(int32_t limit,
+        bool limit_is_duration) {
+    LOGV("setParamMaxDurationOrFileSize: limit (%d) for %s",
             limit, limit_is_duration?"duration":"size");
     return OK;
 }
 
+status_t StagefrightRecorder::setParamInterleaveDuration(int32_t durationUs) {
+    LOGV("setParamInterleaveDuration: %d", durationUs);
+    mInterleaveDurationUs = durationUs;
+    return OK;
+}
 status_t StagefrightRecorder::setParameter(
         const String8 &key, const String8 &value) {
     LOGV("setParameter: key (%s) => value (%s)", key.string(), value.string());
     if (key == "max-duration") {
         int32_t max_duration_ms;
         if (safe_strtoi64(value.string(), &max_duration_ms)) {
-            return setMaxDurationOrFileSize(
+            return setParamMaxDurationOrFileSize(
                     max_duration_ms, true /* limit_is_duration */);
         }
     } else if (key == "max-filesize") {
         int32_t max_filesize_bytes;
         if (safe_strtoi64(value.string(), &max_filesize_bytes)) {
-            return setMaxDurationOrFileSize(
+            return setParamMaxDurationOrFileSize(
                     max_filesize_bytes, false /* limit is filesize */);
         }
     } else if (key == "audio-param-sampling-rate") {
@@ -254,6 +260,11 @@ status_t StagefrightRecorder::setParameter(
         if (safe_strtoi64(value.string(), &video_bitrate)) {
             return setParamVideoEncodingBitRate(video_bitrate);
         }
+    } else if (key == "param-interleave-duration-us") {
+        int32_t durationUs;
+        if (safe_strtoi64(value.string(), &durationUs)) {
+            return setParamInterleaveDuration(durationUs);
+        }
     } else {
         LOGE("setParameter: failed to find key %s", key.string());
         return BAD_VALUE;
@@ -480,6 +491,7 @@ status_t StagefrightRecorder::startMPEG4Recording() {
         mWriter->addSource(encoder);
     }
 
+    ((MPEG4Writer *)mWriter.get())->setInterleaveDuration(mInterleaveDurationUs);
     mWriter->start();
     return OK;
 }
diff --git a/media/libmediaplayerservice/StagefrightRecorder.h b/media/libmediaplayerservice/StagefrightRecorder.h
index ad1153c..3b99e91 100644
--- a/media/libmediaplayerservice/StagefrightRecorder.h
+++ b/media/libmediaplayerservice/StagefrightRecorder.h
@@ -74,6 +74,7 @@ private:
     int32_t mAudioBitRate;
     int32_t mAudioChannels;
     int32_t mSampleRate;
+    int32_t mInterleaveDurationUs;
 
     String8 mParams;
     int mOutputFd;
@@ -87,7 +88,8 @@ private:
     status_t setParamAudioEncodingBitRate(int32_t bitRate);
     status_t setParamAudioNumberOfChannels(int32_t channles);
     status_t setParamAudioSamplingRate(int32_t sampleRate);
-    status_t setMaxDurationOrFileSize(int32_t limit, bool limit_is_duration);
+    status_t setParamInterleaveDuration(int32_t durationUs);
+    status_t setParamMaxDurationOrFileSize(int32_t limit, bool limit_is_duration);
 
     StagefrightRecorder(const StagefrightRecorder &);
     StagefrightRecorder &operator=(const StagefrightRecorder &);
diff --git a/media/libstagefright/CameraSource.cpp b/media/libstagefright/CameraSource.cpp
index b07bd0e..b046a9c 100644
--- a/media/libstagefright/CameraSource.cpp
+++ b/media/libstagefright/CameraSource.cpp
@@ -130,8 +130,9 @@ CameraSource::CameraSource(const sp<Camera> &camera)
       mHeight(0),
       mFirstFrameTimeUs(0),
       mLastFrameTimestampUs(0),
-      mNumFrames(0),
-      mNumFramesReleased(0),
+      mNumFramesReceived(0),
+      mNumFramesEncoded(0),
+      mNumFramesDropped(0),
       mStarted(false) {
     String8 s = mCamera->getParameters();
     printf("params: \"%s\"\n", s.string());
@@ -178,9 +179,11 @@ status_t CameraSource::stop() {
     mCamera->stopRecording();
 
     releaseQueuedFrames();
-    LOGI("Frames received/released: %d/%d, timestamp (us) last/first: %lld/%lld",
-            mNumFrames, mNumFramesReleased,
+    LOGI("Frames received/encoded/dropped: %d/%d/%d, timestamp (us) last/first: %lld/%lld",
+            mNumFramesReceived, mNumFramesEncoded, mNumFramesDropped,
             mLastFrameTimestampUs, mFirstFrameTimeUs);
+
+    CHECK_EQ(mNumFramesReceived, mNumFramesEncoded + mNumFramesDropped);
     return OK;
 }
 
@@ -190,7 +193,7 @@ void CameraSource::releaseQueuedFrames() {
         it = mFrames.begin();
         mCamera->releaseRecordingFrame(*it);
         mFrames.erase(it);
-        ++mNumFramesReleased;
+        ++mNumFramesDropped;
     }
 }
 
@@ -231,7 +234,7 @@ status_t CameraSource::read(
 
         frameTime = *mFrameTimes.begin();
         mFrameTimes.erase(mFrameTimes.begin());
-        ++mNumFramesReleased;
+        ++mNumFramesEncoded;
     }
 
     *buffer = new MediaBuffer(frame->size());
@@ -252,15 +255,15 @@ void CameraSource::dataCallbackTimestamp(int64_t timestampUs,
     Mutex::Autolock autoLock(mLock);
     if (!mStarted) {
         mCamera->releaseRecordingFrame(data);
-        ++mNumFrames;
-        ++mNumFramesReleased;
+        ++mNumFramesReceived;
+        ++mNumFramesDropped;
         return;
     }
 
-    if (mNumFrames == 0) {
+    if (mNumFramesReceived == 0) {
         mFirstFrameTimeUs = timestampUs;
     }
-    ++mNumFrames;
+    ++mNumFramesReceived;
 
     mFrames.push_back(data);
     mFrameTimes.push_back(timestampUs - mFirstFrameTimeUs);
diff --git a/media/libstagefright/MPEG4Writer.cpp b/media/libstagefright/MPEG4Writer.cpp
index e6336e7..29ec876 100644
--- a/media/libstagefright/MPEG4Writer.cpp
+++ b/media/libstagefright/MPEG4Writer.cpp
@@ -57,10 +57,24 @@ private:
 
     struct SampleInfo {
         size_t size;
-        off_t offset;
         int64_t timestamp;
     };
-    List<SampleInfo> mSampleInfos;
+    List<SampleInfo>    mSampleInfos;
+    List<MediaBuffer *> mChunkSamples;
+    List<off_t>         mChunkOffsets;
+
+    struct StscTableEntry {
+
+        StscTableEntry(uint32_t chunk, uint32_t samples, uint32_t id)
+            : firstChunk(chunk),
+              samplesPerChunk(samples),
+              sampleDescriptionId(id) {}
+
+        uint32_t firstChunk;
+        uint32_t samplesPerChunk;
+        uint32_t sampleDescriptionId;
+    };
+    List<StscTableEntry> mStscTableEntries;
 
     List<int32_t> mStssTableEntries;
 
@@ -75,6 +89,7 @@ private:
 
     status_t makeAVCCodecSpecificData(
             const uint8_t *data, size_t size);
+    void writeOneChunk(bool isAvc);
 
     Track(const Track &);
     Track &operator=(const Track &);
@@ -85,14 +100,16 @@ private:
 MPEG4Writer::MPEG4Writer(const char *filename)
     : mFile(fopen(filename, "wb")),
       mOffset(0),
-      mMdatOffset(0) {
+      mMdatOffset(0),
+      mInterleaveDurationUs(500000) {
     CHECK(mFile != NULL);
 }
 
 MPEG4Writer::MPEG4Writer(int fd)
     : mFile(fdopen(fd, "wb")),
       mOffset(0),
-      mMdatOffset(0) {
+      mMdatOffset(0),
+      mInterleaveDurationUs(500000) {
     CHECK(mFile != NULL);
 }
 
@@ -213,9 +230,20 @@ void MPEG4Writer::stop() {
     mFile = NULL;
 }
 
-off_t MPEG4Writer::addSample(MediaBuffer *buffer) {
-    Mutex::Autolock autoLock(mLock);
+status_t MPEG4Writer::setInterleaveDuration(uint32_t durationUs) {
+    mInterleaveDurationUs = durationUs;
+    return OK;
+}
+
+void MPEG4Writer::lock() {
+    mLock.lock();
+}
+
+void MPEG4Writer::unlock() {
+    mLock.unlock();
+}
 
+off_t MPEG4Writer::addSample_l(MediaBuffer *buffer) {
     off_t old_offset = mOffset;
 
     fwrite((const uint8_t *)buffer->data() + buffer->range_offset(),
@@ -240,9 +268,7 @@ static void StripStartcode(MediaBuffer *buffer) {
     }
 }
 
-off_t MPEG4Writer::addLengthPrefixedSample(MediaBuffer *buffer) {
-    Mutex::Autolock autoLock(mLock);
-
+off_t MPEG4Writer::addLengthPrefixedSample_l(MediaBuffer *buffer) {
     StripStartcode(buffer);
 
     off_t old_offset = mOffset;
@@ -532,13 +558,17 @@ void MPEG4Writer::Track::threadEntry() {
                     !strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AAC);
     bool is_avc = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC);
     int32_t count = 0;
+    const int64_t interleaveDurationUs = mOwner->interleaveDuration();
+    int64_t chunkTimestampUs = 0;
+    int32_t nChunks = 0;
+    int32_t nZeroLengthFrames = 0;
 
     MediaBuffer *buffer;
     while (!mDone && mSource->read(&buffer) == OK) {
         if (buffer->range_length() == 0) {
             buffer->release();
             buffer = NULL;
-
+            ++nZeroLengthFrames;
             continue;
         }
 
@@ -661,20 +691,14 @@ void MPEG4Writer::Track::threadEntry() {
             continue;
         }
 
-        off_t offset = is_avc ? mOwner->addLengthPrefixedSample(buffer)
-                              : mOwner->addSample(buffer);
-
         SampleInfo info;
         info.size = is_avc
 #if USE_NALLEN_FOUR
-            ? buffer->range_length() + 4
+                ? buffer->range_length() + 4
 #else
-            ? buffer->range_length() + 2
+                ? buffer->range_length() + 2
 #endif
-            : buffer->range_length();
-
-        info.offset = offset;
-
+                : buffer->range_length();
 
         bool is_audio = !strncasecmp(mime, "audio/", 6);
 
@@ -687,12 +711,42 @@ void MPEG4Writer::Track::threadEntry() {
 
         // Our timestamp is in ms.
         info.timestamp = (timestampUs + 500) / 1000;
-
         mSampleInfos.push_back(info);
 
+////////////////////////////////////////////////////////////////////////////////
+        // Make a deep copy of the MediaBuffer less Metadata
+        MediaBuffer *copy = new MediaBuffer(buffer->range_length());
+        memcpy(copy->data(), (uint8_t *)buffer->data() + buffer->range_offset(),
+                buffer->range_length());
+        copy->set_range(0, buffer->range_length());
+
+        mChunkSamples.push_back(copy);
+        if (interleaveDurationUs == 0) {
+            StscTableEntry stscEntry(++nChunks, 1, 1);
+            mStscTableEntries.push_back(stscEntry);
+            writeOneChunk(is_avc);
+        } else {
+            if (chunkTimestampUs == 0) {
+                chunkTimestampUs = timestampUs;
+            } else {
+                if (timestampUs - chunkTimestampUs > interleaveDurationUs) {
+                    ++nChunks;
+                    if (nChunks == 1 ||  // First chunk
+                        (--(mStscTableEntries.end()))->samplesPerChunk !=
+                         mChunkSamples.size()) {
+                        StscTableEntry stscEntry(nChunks,
+                                mChunkSamples.size(), 1);
+                        mStscTableEntries.push_back(stscEntry);
+                    }
+                    writeOneChunk(is_avc);
+                    chunkTimestampUs = timestampUs;
+                }
+            }
+        }
+
         int32_t isSync = false;
-        buffer->meta_data()->findInt32(kKeyIsSyncFrame, &isSync);
-        if (isSync) {
+        if (buffer->meta_data()->findInt32(kKeyIsSyncFrame, &isSync) &&
+            isSync != 0) {
             mStssTableEntries.push_back(mSampleInfos.size());
         }
         // Our timestamp is in ms.
@@ -700,7 +754,37 @@ void MPEG4Writer::Track::threadEntry() {
         buffer = NULL;
     }
 
+    // Last chunk
+    if (!mChunkSamples.empty()) {
+        ++nChunks;
+        StscTableEntry stscEntry(nChunks, mChunkSamples.size(), 1);
+        mStscTableEntries.push_back(stscEntry);
+        writeOneChunk(is_avc);
+    }
+
     mReachedEOS = true;
+    LOGI("Received total/0-length (%d/%d) buffers and encoded %d frames",
+            count, nZeroLengthFrames, mSampleInfos.size());
+}
+
+void MPEG4Writer::Track::writeOneChunk(bool isAvc) {
+    mOwner->lock();
+    for (List<MediaBuffer *>::iterator it = mChunkSamples.begin();
+         it != mChunkSamples.end(); ++it) {
+        off_t offset = isAvc? mOwner->addLengthPrefixedSample_l(*it)
+                            : mOwner->addSample_l(*it);
+        if (it == mChunkSamples.begin()) {
+            mChunkOffsets.push_back(offset);
+        }
+    }
+    mOwner->unlock();
+    while (!mChunkSamples.empty()) {
+        List<MediaBuffer *>::iterator it = mChunkSamples.begin();
+        (*it)->release();
+        (*it) = NULL;
+        mChunkSamples.erase(it);
+    }
+    mChunkSamples.clear();
 }
 
 int64_t MPEG4Writer::Track::getDurationUs() const {
@@ -1018,22 +1102,21 @@ void MPEG4Writer::Track::writeTrackHeader(int32_t trackID) {
 
           mOwner->beginBox("stsc");
             mOwner->writeInt32(0);  // version=0, flags=0
-            mOwner->writeInt32(mSampleInfos.size());
-            int32_t n = 1;
-            for (List<SampleInfo>::iterator it = mSampleInfos.begin();
-                 it != mSampleInfos.end(); ++it, ++n) {
-                mOwner->writeInt32(n);
-                mOwner->writeInt32(1);
-                mOwner->writeInt32(1);
+            mOwner->writeInt32(mStscTableEntries.size());
+            for (List<StscTableEntry>::iterator it = mStscTableEntries.begin();
+                 it != mStscTableEntries.end(); ++it) {
+                mOwner->writeInt32(it->firstChunk);
+                mOwner->writeInt32(it->samplesPerChunk);
+                mOwner->writeInt32(it->sampleDescriptionId);
             }
           mOwner->endBox();  // stsc
 
           mOwner->beginBox("co64");
             mOwner->writeInt32(0);  // version=0, flags=0
-            mOwner->writeInt32(mSampleInfos.size());
-            for (List<SampleInfo>::iterator it = mSampleInfos.begin();
-                 it != mSampleInfos.end(); ++it) {
-                mOwner->writeInt64((*it).offset);
+            mOwner->writeInt32(mChunkOffsets.size());
+            for (List<off_t>::iterator it = mChunkOffsets.begin();
+                 it != mChunkOffsets.end(); ++it) {
+                mOwner->writeInt64((*it));
             }
           mOwner->endBox();  // co64