support for CEA-608 closed caption

Bug: 15470448

Change-Id: Ic6a527f5c35a8ee0a08a5b043336e4d193216083

6 files changed, 383 insertions, 3 deletions

diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
index dc69f73..b333043 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
@@ -375,14 +375,24 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
                 inbandTracks = mSource->getTrackCount();
             }
 
+            size_t ccTracks = 0;
+            if (mCCDecoder != NULL) {
+                ccTracks = mCCDecoder->getTrackCount();
+            }
+
             // total track count
-            reply->writeInt32(inbandTracks);
+            reply->writeInt32(inbandTracks + ccTracks);
 
             // write inband tracks
             for (size_t i = 0; i < inbandTracks; ++i) {
                 writeTrackInfo(reply, mSource->getTrackInfo(i));
             }
 
+            // write CC track
+            for (size_t i = 0; i < ccTracks; ++i) {
+                writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
+            }
+
             sp<AMessage> response = new AMessage;
             response->postReply(replyID);
             break;
@@ -404,9 +414,19 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
             if (mSource != NULL) {
                 inbandTracks = mSource->getTrackCount();
             }
+            size_t ccTracks = 0;
+            if (mCCDecoder != NULL) {
+                ccTracks = mCCDecoder->getTrackCount();
+            }
 
             if (trackIndex < inbandTracks) {
                 err = mSource->selectTrack(trackIndex, select);
+            } else {
+                trackIndex -= inbandTracks;
+
+                if (trackIndex < ccTracks) {
+                    err = mCCDecoder->selectTrack(trackIndex, select);
+                }
             }
 
             sp<AMessage> response = new AMessage;
@@ -870,6 +890,12 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
             break;
         }
 
+        case kWhatClosedCaptionNotify:
+        {
+            onClosedCaptionNotify(msg);
+            break;
+        }
+
         default:
             TRESPASS();
             break;
@@ -933,6 +959,9 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
         AString mime;
         CHECK(format->findString("mime", &mime));
         mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
+
+        sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id());
+        mCCDecoder = new CCDecoder(ccNotify);
     }
 
     sp<AMessage> notify =
@@ -1073,6 +1102,10 @@ status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
          mediaTimeUs / 1E6);
 #endif
 
+    if (!audio) {
+        mCCDecoder->decode(accessUnit);
+    }
+
     reply->setBuffer("buffer", accessUnit);
     reply->post();
 
@@ -1101,14 +1134,15 @@ void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
     sp<ABuffer> buffer;
     CHECK(msg->findBuffer("buffer", &buffer));
 
+    int64_t mediaTimeUs;
+    CHECK(buffer->meta()->findInt64("timeUs", &mediaTimeUs));
+
     int64_t &skipUntilMediaTimeUs =
         audio
             ? mSkipRenderingAudioUntilMediaTimeUs
             : mSkipRenderingVideoUntilMediaTimeUs;
 
     if (skipUntilMediaTimeUs >= 0) {
-        int64_t mediaTimeUs;
-        CHECK(buffer->meta()->findInt64("timeUs", &mediaTimeUs));
 
         if (mediaTimeUs < skipUntilMediaTimeUs) {
             ALOGV("dropping %s buffer at time %lld as requested.",
@@ -1122,6 +1156,10 @@ void NuPlayer::renderBuffer(bool audio, const sp<AMessage> &msg) {
         skipUntilMediaTimeUs = -1;
     }
 
+    if (!audio && mCCDecoder->isSelected()) {
+        mCCDecoder->display(mediaTimeUs);
+    }
+
     mRenderer->queueBuffer(audio, buffer, reply);
 }
 
@@ -1510,6 +1548,39 @@ void NuPlayer::onSourceNotify(const sp<AMessage> &msg) {
     }
 }
 
+void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
+    int32_t what;
+    CHECK(msg->findInt32("what", &what));
+
+    switch (what) {
+        case NuPlayer::CCDecoder::kWhatClosedCaptionData:
+        {
+            sp<ABuffer> buffer;
+            CHECK(msg->findBuffer("buffer", &buffer));
+
+            size_t inbandTracks = 0;
+            if (mSource != NULL) {
+                inbandTracks = mSource->getTrackCount();
+            }
+
+            sendSubtitleData(buffer, inbandTracks);
+            break;
+        }
+
+        case NuPlayer::CCDecoder::kWhatTrackAdded:
+        {
+            notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
+
+            break;
+        }
+
+        default:
+            TRESPASS();
+    }
+
+
+}
+
 void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
     int32_t trackIndex;
     int64_t timeUs, durationUs;
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h
index f95cc11..5be71fb 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h
@@ -76,6 +76,7 @@ public:
 
 private:
     struct Decoder;
+    struct CCDecoder;
     struct GenericSource;
     struct HTTPLiveSource;
     struct Renderer;
@@ -98,6 +99,7 @@ private:
         kWhatScanSources                = 'scan',
         kWhatVideoNotify                = 'vidN',
         kWhatAudioNotify                = 'audN',
+        kWhatClosedCaptionNotify        = 'capN',
         kWhatRendererNotify             = 'renN',
         kWhatReset                      = 'rset',
         kWhatSeek                       = 'seek',
@@ -119,6 +121,7 @@ private:
     sp<Decoder> mVideoDecoder;
     bool mVideoIsAVC;
     sp<Decoder> mAudioDecoder;
+    sp<CCDecoder> mCCDecoder;
     sp<Renderer> mRenderer;
 
     List<sp<Action> > mDeferredActions;
@@ -186,6 +189,7 @@ private:
     void performSetSurface(const sp<NativeWindowWrapper> &wrapper);
 
     void onSourceNotify(const sp<AMessage> &msg);
+    void onClosedCaptionNotify(const sp<AMessage> &msg);
 
     void queueDecoderShutdown(
             bool audio, bool video, const sp<AMessage> &reply);
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
index cfbf282..5abfb71 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
@@ -22,6 +22,7 @@
 #include "NuPlayerDecoder.h"
 
 #include <media/ICrypto.h>
+#include <media/stagefright/foundation/ABitReader.h>
 #include <media/stagefright/foundation/ABuffer.h>
 #include <media/stagefright/foundation/ADebug.h>
 #include <media/stagefright/foundation/AMessage.h>
@@ -535,5 +536,272 @@ bool NuPlayer::Decoder::supportsSeamlessFormatChange(const sp<AMessage> &targetF
     return seamless;
 }
 
+struct NuPlayer::CCDecoder::CCData {
+    CCData(uint8_t type, uint8_t data1, uint8_t data2)
+        : mType(type), mData1(data1), mData2(data2) {
+    }
+
+    uint8_t mType;
+    uint8_t mData1;
+    uint8_t mData2;
+};
+
+NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> &notify)
+    : mNotify(notify),
+      mTrackCount(0),
+      mSelectedTrack(-1) {
+}
+
+size_t NuPlayer::CCDecoder::getTrackCount() const {
+    return mTrackCount;
+}
+
+sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const {
+    CHECK(index == 0);
+
+    sp<AMessage> format = new AMessage();
+
+    format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE);
+    format->setString("language", "und");
+    format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608);
+    format->setInt32("auto", 1);
+    format->setInt32("default", 1);
+    format->setInt32("forced", 0);
+
+    return format;
+}
+
+status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) {
+    CHECK(index < mTrackCount);
+
+    if (select) {
+        if (mSelectedTrack == (ssize_t)index) {
+            ALOGE("track %zu already selected", index);
+            return BAD_VALUE;
+        }
+        ALOGV("selected track %zu", index);
+        mSelectedTrack = index;
+    } else {
+        if (mSelectedTrack != (ssize_t)index) {
+            ALOGE("track %zu is not selected", index);
+            return BAD_VALUE;
+        }
+        ALOGV("unselected track %zu", index);
+        mSelectedTrack = -1;
+    }
+
+    return OK;
+}
+
+bool NuPlayer::CCDecoder::isSelected() const {
+    return mSelectedTrack >= 0 && mSelectedTrack < (int32_t)mTrackCount;
+}
+
+bool NuPlayer::CCDecoder::isNullPad(CCData *cc) const {
+    return cc->mData1 < 0x10 && cc->mData2 < 0x10;
+}
+
+void NuPlayer::CCDecoder::dumpBytePair(const sp<ABuffer> &ccBuf) const {
+    size_t offset = 0;
+    AString out;
+
+    while (offset < ccBuf->size()) {
+        char tmp[128];
+
+        CCData *cc = (CCData *) (ccBuf->data() + offset);
+
+        if (isNullPad(cc)) {
+            // 1 null pad or XDS metadata, ignore
+            offset += sizeof(CCData);
+            continue;
+        }
+
+        if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) {
+            // 2 basic chars
+            sprintf(tmp, "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2);
+        } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
+                 && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) {
+            // 1 special char
+            sprintf(tmp, "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2);
+        } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A)
+                 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
+            // 1 Spanish/French char
+            sprintf(tmp, "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2);
+        } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B)
+                 && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
+            // 1 Portuguese/German/Danish char
+            sprintf(tmp, "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2);
+        } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
+                 && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){
+            // Mid-Row Codes (Table 69)
+            sprintf(tmp, "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2);
+        } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c)
+                  && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f)
+                  ||
+                   ((cc->mData1 == 0x17 || cc->mData1 == 0x1f)
+                  && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){
+            // Misc Control Codes (Table 70)
+            sprintf(tmp, "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2);
+        } else if ((cc->mData1 & 0x70) == 0x10
+                && (cc->mData2 & 0x40) == 0x40
+                && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) {
+            // Preamble Address Codes (Table 71)
+            sprintf(tmp, "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2);
+        } else {
+            sprintf(tmp, "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2);
+        }
+
+        if (out.size() > 0) {
+            out.append(", ");
+        }
+
+        out.append(tmp);
+
+        offset += sizeof(CCData);
+    }
+
+    ALOGI("%s", out.c_str());
+}
+
+bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) {
+    int64_t timeUs;
+    CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
+
+    sp<ABuffer> sei;
+    if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) {
+        return false;
+    }
+
+    bool hasCC = false;
+
+    ABitReader br(sei->data() + 1, sei->size() - 1);
+    // sei_message()
+    while (br.numBitsLeft() >= 16) { // at least 16-bit for sei_message()
+        uint32_t payload_type = 0;
+        size_t payload_size = 0;
+        uint8_t last_byte;
+
+        do {
+            last_byte = br.getBits(8);
+            payload_type += last_byte;
+        } while (last_byte == 0xFF);
+
+        do {
+            last_byte = br.getBits(8);
+            payload_size += last_byte;
+        } while (last_byte == 0xFF);
+
+        // sei_payload()
+        if (payload_type == 4) {
+            // user_data_registered_itu_t_t35()
+
+            // ATSC A/72: 6.4.2
+            uint8_t itu_t_t35_country_code = br.getBits(8);
+            uint16_t itu_t_t35_provider_code = br.getBits(16);
+            uint32_t user_identifier = br.getBits(32);
+            uint8_t user_data_type_code = br.getBits(8);
+
+            payload_size -= 1 + 2 + 4 + 1;
+
+            if (itu_t_t35_country_code == 0xB5
+                    && itu_t_t35_provider_code == 0x0031
+                    && user_identifier == 'GA94'
+                    && user_data_type_code == 0x3) {
+                hasCC = true;
+
+                // MPEG_cc_data()
+                // ATSC A/53 Part 4: 6.2.3.1
+                br.skipBits(1); //process_em_data_flag
+                bool process_cc_data_flag = br.getBits(1);
+                br.skipBits(1); //additional_data_flag
+                size_t cc_count = br.getBits(5);
+                br.skipBits(8); // em_data;
+                payload_size -= 2;
+
+                if (process_cc_data_flag) {
+                    AString out;
+
+                    sp<ABuffer> ccBuf = new ABuffer(cc_count * sizeof(CCData));
+                    ccBuf->setRange(0, 0);
+
+                    for (size_t i = 0; i < cc_count; i++) {
+                        uint8_t marker = br.getBits(5);
+                        CHECK_EQ(marker, 0x1f);
+
+                        bool cc_valid = br.getBits(1);
+                        uint8_t cc_type = br.getBits(2);
+                        // remove odd parity bit
+                        uint8_t cc_data_1 = br.getBits(8) & 0x7f;
+                        uint8_t cc_data_2 = br.getBits(8) & 0x7f;
+
+                        if (cc_valid
+                                && (cc_type == 0 || cc_type == 1)) {
+                            CCData cc(cc_type, cc_data_1, cc_data_2);
+                            if (!isNullPad(&cc)) {
+                                memcpy(ccBuf->data() + ccBuf->size(),
+                                        (void *)&cc, sizeof(cc));
+                                ccBuf->setRange(0, ccBuf->size() + sizeof(CCData));
+                            }
+                        }
+                    }
+                    payload_size -= cc_count * 3;
+
+                    mCCMap.add(timeUs, ccBuf);
+                    break;
+                }
+            } else {
+                ALOGV("Malformed SEI payload type 4");
+            }
+        } else {
+            ALOGV("Unsupported SEI payload type %d", payload_type);
+        }
+
+        // skipping remaining bits of this payload
+        br.skipBits(payload_size * 8);
+    }
+
+    return hasCC;
+}
+
+void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) {
+    if (extractFromSEI(accessUnit) && mTrackCount == 0) {
+        mTrackCount++;
+
+        ALOGI("Found CEA-608 track");
+        sp<AMessage> msg = mNotify->dup();
+        msg->setInt32("what", kWhatTrackAdded);
+        msg->post();
+    }
+    // TODO: extract CC from other sources
+}
+
+void NuPlayer::CCDecoder::display(int64_t timeUs) {
+    ssize_t index = mCCMap.indexOfKey(timeUs);
+    if (index < 0) {
+        ALOGV("cc for timestamp %" PRId64 " not found", timeUs);
+        return;
+    }
+
+    sp<ABuffer> &ccBuf = mCCMap.editValueAt(index);
+
+    if (ccBuf->size() > 0) {
+#if 0
+        dumpBytePair(ccBuf);
+#endif
+
+        ccBuf->meta()->setInt32("trackIndex", mSelectedTrack);
+        ccBuf->meta()->setInt64("timeUs", timeUs);
+        ccBuf->meta()->setInt64("durationUs", 0ll);
+
+        sp<AMessage> msg = mNotify->dup();
+        msg->setInt32("what", kWhatClosedCaptionData);
+        msg->setBuffer("buffer", ccBuf);
+        msg->post();
+    }
+
+    // remove all entries before timeUs
+    mCCMap.removeItemsAt(0, index + 1);
+}
+
 }  // namespace android
 
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h
index 2892584..1a4f4ab 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h
@@ -101,6 +101,36 @@ private:
     DISALLOW_EVIL_CONSTRUCTORS(Decoder);
 };
 
+struct NuPlayer::CCDecoder : public RefBase {
+    enum {
+        kWhatClosedCaptionData,
+        kWhatTrackAdded,
+    };
+
+    CCDecoder(const sp<AMessage> &notify);
+
+    size_t getTrackCount() const;
+    sp<AMessage> getTrackInfo(size_t index) const;
+    status_t selectTrack(size_t index, bool select);
+    bool isSelected() const;
+    void decode(const sp<ABuffer> &accessUnit);
+    void display(int64_t timeUs);
+
+private:
+    struct CCData;
+
+    sp<AMessage> mNotify;
+    KeyedVector<int64_t, sp<ABuffer> > mCCMap;
+    size_t mTrackCount;
+    int32_t mSelectedTrack;
+
+    bool isNullPad(CCData *cc) const;
+    void dumpBytePair(const sp<ABuffer> &ccBuf) const;
+    bool extractFromSEI(const sp<ABuffer> &accessUnit);
+
+    DISALLOW_EVIL_CONSTRUCTORS(CCDecoder);
+};
+
 }  // namespace android
 
 #endif  // NUPLAYER_DECODER_H_
diff --git a/media/libstagefright/MediaDefs.cpp b/media/libstagefright/MediaDefs.cpp
index f38729e..d48dd84 100644
--- a/media/libstagefright/MediaDefs.cpp
+++ b/media/libstagefright/MediaDefs.cpp
@@ -59,5 +59,6 @@ const char *MEDIA_MIMETYPE_CONTAINER_WVM = "video/wvm";
 const char *MEDIA_MIMETYPE_TEXT_3GPP = "text/3gpp-tt";
 const char *MEDIA_MIMETYPE_TEXT_SUBRIP = "application/x-subrip";
 const char *MEDIA_MIMETYPE_TEXT_VTT = "text/vtt";
+const char *MEDIA_MIMETYPE_TEXT_CEA_608 = "text/cea-608";
 
 }  // namespace android
diff --git a/media/libstagefright/mpeg2ts/ESQueue.cpp b/media/libstagefright/mpeg2ts/ESQueue.cpp
index f7abf01..3c8f03e 100644
--- a/media/libstagefright/mpeg2ts/ESQueue.cpp
+++ b/media/libstagefright/mpeg2ts/ESQueue.cpp
@@ -777,6 +777,12 @@ sp<ABuffer> ElementaryStreamQueue::dequeueAccessUnitH264() {
 
                 unsigned nalType = mBuffer->data()[pos.nalOffset] & 0x1f;
 
+                if (nalType == 6) {
+                    sp<ABuffer> sei = new ABuffer(pos.nalSize);
+                    memcpy(sei->data(), mBuffer->data() + pos.nalOffset, pos.nalSize);
+                    accessUnit->meta()->setBuffer("sei", sei);
+                }
+
 #if !LOG_NDEBUG
                 char tmp[128];
                 sprintf(tmp, "0x%02x", nalType);