In "special" mode we now establish a UDP RTCP channel in addition to the

TCP RTP channel and provide feedback on the latency of arriving packets from
the sink back to the source. This information is then used to throttle
video bitrate.

Change-Id: Ic589a3cb65e4893a3ff67de947da6063d32a1c6e

10 files changed, 135 insertions, 26 deletions

diff --git a/media/libstagefright/wifi-display/ANetworkSession.cpp b/media/libstagefright/wifi-display/ANetworkSession.cpp
index df20ae2..88ca1cc 100644
--- a/media/libstagefright/wifi-display/ANetworkSession.cpp
+++ b/media/libstagefright/wifi-display/ANetworkSession.cpp
@@ -565,7 +565,7 @@ status_t ANetworkSession::Session::writeMore() {
         mSawSendFailure = true;
     }
 
-#if 1
+#if 0
     int numBytesQueued;
     int res = ioctl(mSocket, SIOCOUTQ, &numBytesQueued);
     if (res == 0 && numBytesQueued > 50 * 1024) {
@@ -576,7 +576,7 @@ status_t ANetworkSession::Session::writeMore() {
         int64_t nowUs = ALooper::GetNowUs();
 
         if (mLastStallReportUs < 0ll
-                || nowUs > mLastStallReportUs + 500000ll) {
+                || nowUs > mLastStallReportUs + 100000ll) {
             sp<AMessage> msg = mNotify->dup();
             msg->setInt32("sessionID", mSessionID);
             msg->setInt32("reason", kWhatNetworkStall);
diff --git a/media/libstagefright/wifi-display/MediaSender.cpp b/media/libstagefright/wifi-display/MediaSender.cpp
index 123bc1c..33af66d 100644
--- a/media/libstagefright/wifi-display/MediaSender.cpp
+++ b/media/libstagefright/wifi-display/MediaSender.cpp
@@ -85,10 +85,11 @@ ssize_t MediaSender::addTrack(const sp<AMessage> &format, uint32_t flags) {
 
 status_t MediaSender::initAsync(
         ssize_t trackIndex,
-        RTPSender::TransportMode transportMode,
         const char *remoteHost,
         int32_t remoteRTPPort,
+        RTPSender::TransportMode rtpMode,
         int32_t remoteRTCPPort,
+        RTPSender::TransportMode rtcpMode,
         int32_t *localRTPPort) {
     if (trackIndex < 0) {
         if (mMode != MODE_UNDEFINED) {
@@ -126,12 +127,9 @@ status_t MediaSender::initAsync(
             err = mTSSender->initAsync(
                     remoteHost,
                     remoteRTPPort,
-                    transportMode,  // rtpMode
+                    rtpMode,
                     remoteRTCPPort,
-                    (transportMode == RTPSender::TRANSPORT_UDP
-                        && remoteRTCPPort >= 0)
-                        ? transportMode
-                        : RTPSender::TRANSPORT_NONE,  // rtcpMode
+                    rtcpMode,
                     localRTPPort);
 
             if (err != OK) {
@@ -180,11 +178,9 @@ status_t MediaSender::initAsync(
     status_t err = info->mSender->initAsync(
             remoteHost,
             remoteRTPPort,
-            transportMode,  // rtpMode
+            rtpMode,
             remoteRTCPPort,
-            (transportMode == RTPSender::TRANSPORT_UDP && remoteRTCPPort >= 0)
-                ? transportMode
-                : RTPSender::TRANSPORT_NONE,  // rtcpMode
+            rtcpMode,
             localRTPPort);
 
     if (err != OK) {
@@ -345,6 +341,22 @@ void MediaSender::onSenderNotify(const sp<AMessage> &msg) {
             break;
         }
 
+        case kWhatInformSender:
+        {
+            int64_t avgLatencyUs;
+            CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs));
+
+            int64_t maxLatencyUs;
+            CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs));
+
+            sp<AMessage> notify = mNotify->dup();
+            notify->setInt32("what", kWhatInformSender);
+            notify->setInt64("avgLatencyUs", avgLatencyUs);
+            notify->setInt64("maxLatencyUs", maxLatencyUs);
+            notify->post();
+            break;
+        }
+
         default:
             TRESPASS();
     }
diff --git a/media/libstagefright/wifi-display/MediaSender.h b/media/libstagefright/wifi-display/MediaSender.h
index 447abf7..04538ea 100644
--- a/media/libstagefright/wifi-display/MediaSender.h
+++ b/media/libstagefright/wifi-display/MediaSender.h
@@ -43,6 +43,7 @@ struct MediaSender : public AHandler {
         kWhatInitDone,
         kWhatError,
         kWhatNetworkStall,
+        kWhatInformSender,
     };
 
     MediaSender(
@@ -59,10 +60,11 @@ struct MediaSender : public AHandler {
     // If trackIndex == -1, initialize for transport stream muxing.
     status_t initAsync(
             ssize_t trackIndex,
-            RTPSender::TransportMode transportMode,
             const char *remoteHost,
             int32_t remoteRTPPort,
+            RTPSender::TransportMode rtpMode,
             int32_t remoteRTCPPort,
+            RTPSender::TransportMode rtcpMode,
             int32_t *localRTPPort);
 
     status_t queueAccessUnit(
diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.cpp b/media/libstagefright/wifi-display/rtp/RTPSender.cpp
index c686e01..9eeeabd 100644
--- a/media/libstagefright/wifi-display/rtp/RTPSender.cpp
+++ b/media/libstagefright/wifi-display/rtp/RTPSender.cpp
@@ -91,8 +91,8 @@ status_t RTPSender::initAsync(
     CHECK_NE(rtpMode, TRANSPORT_TCP_INTERLEAVED);
     CHECK_NE(rtcpMode, TRANSPORT_TCP_INTERLEAVED);
 
-    if (rtcpMode == TRANSPORT_NONE && remoteRTCPPort >= 0
-            || rtcpMode != TRANSPORT_NONE && remoteRTCPPort < 0) {
+    if ((rtcpMode == TRANSPORT_NONE && remoteRTCPPort >= 0)
+            || (rtcpMode != TRANSPORT_NONE && remoteRTCPPort < 0)) {
         return INVALID_OPERATION;
     }
 
@@ -616,6 +616,7 @@ status_t RTPSender::onRTCPData(const sp<ABuffer> &buffer) {
                 break;
 
             case 204:  // APP
+                parseAPP(data, headerLength);
                 break;
 
             case 205:  // TSFB (transport layer specific feedback)
@@ -721,6 +722,21 @@ status_t RTPSender::parseTSFB(const uint8_t *data, size_t size) {
     return OK;
 }
 
+status_t RTPSender::parseAPP(const uint8_t *data, size_t size) {
+    if (!memcmp("late", &data[8], 4)) {
+        int64_t avgLatencyUs = (int64_t)U64_AT(&data[12]);
+        int64_t maxLatencyUs = (int64_t)U64_AT(&data[20]);
+
+        sp<AMessage> notify = mNotify->dup();
+        notify->setInt32("what", kWhatInformSender);
+        notify->setInt64("avgLatencyUs", avgLatencyUs);
+        notify->setInt64("maxLatencyUs", maxLatencyUs);
+        notify->post();
+    }
+
+    return OK;
+}
+
 void RTPSender::notifyInitDone(status_t err) {
     sp<AMessage> notify = mNotify->dup();
     notify->setInt32("what", kWhatInitDone);
diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.h b/media/libstagefright/wifi-display/rtp/RTPSender.h
index 8409b8d..3a926ea 100644
--- a/media/libstagefright/wifi-display/rtp/RTPSender.h
+++ b/media/libstagefright/wifi-display/rtp/RTPSender.h
@@ -37,6 +37,7 @@ struct RTPSender : public RTPBase, public AHandler {
         kWhatInitDone,
         kWhatError,
         kWhatNetworkStall,
+        kWhatInformSender,
     };
     RTPSender(
             const sp<ANetworkSession> &netSession,
@@ -105,6 +106,7 @@ private:
     status_t onRTCPData(const sp<ABuffer> &data);
     status_t parseReceiverReport(const uint8_t *data, size_t size);
     status_t parseTSFB(const uint8_t *data, size_t size);
+    status_t parseAPP(const uint8_t *data, size_t size);
 
     void notifyInitDone(status_t err);
     void notifyError(status_t err);
diff --git a/media/libstagefright/wifi-display/source/Converter.cpp b/media/libstagefright/wifi-display/source/Converter.cpp
index bb8c387..d41e1e6 100644
--- a/media/libstagefright/wifi-display/source/Converter.cpp
+++ b/media/libstagefright/wifi-display/source/Converter.cpp
@@ -622,6 +622,7 @@ status_t Converter::feedEncoderInputBuffers() {
 }
 
 status_t Converter::doMoreWork() {
+#if 0
     if (mIsVideo) {
         int32_t videoBitrate = getBitrate("media.wfd.video-bitrate", 5000000);
         if (videoBitrate != mPrevVideoBitrate) {
@@ -633,6 +634,7 @@ status_t Converter::doMoreWork() {
             mPrevVideoBitrate = videoBitrate;
         }
     }
+#endif
 
     status_t err;
 
@@ -708,4 +710,19 @@ void Converter::dropAFrame() {
     (new AMessage(kWhatDropAFrame, id()))->post();
 }
 
+int32_t Converter::getVideoBitrate() const {
+    return mPrevVideoBitrate;
+}
+
+void Converter::setVideoBitrate(int32_t bitRate) {
+    if (mIsVideo && mEncoder != NULL && bitRate != mPrevVideoBitrate) {
+        sp<AMessage> params = new AMessage;
+        params->setInt32("videoBitrate", bitRate);
+
+        mEncoder->setParameters(params);
+
+        mPrevVideoBitrate = bitRate;
+    }
+}
+
 }  // namespace android
diff --git a/media/libstagefright/wifi-display/source/Converter.h b/media/libstagefright/wifi-display/source/Converter.h
index a418f69..538f10a 100644
--- a/media/libstagefright/wifi-display/source/Converter.h
+++ b/media/libstagefright/wifi-display/source/Converter.h
@@ -70,6 +70,9 @@ struct Converter : public AHandler {
 
     void shutdownAsync();
 
+    int32_t getVideoBitrate() const;
+    void setVideoBitrate(int32_t bitrate);
+
 protected:
     virtual ~Converter();
     virtual void onMessageReceived(const sp<AMessage> &msg);
diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.cpp b/media/libstagefright/wifi-display/source/PlaybackSession.cpp
index a3b6542..68aa9cb 100644
--- a/media/libstagefright/wifi-display/source/PlaybackSession.cpp
+++ b/media/libstagefright/wifi-display/source/PlaybackSession.cpp
@@ -362,8 +362,11 @@ WifiDisplaySource::PlaybackSession::PlaybackSession(
 }
 
 status_t WifiDisplaySource::PlaybackSession::init(
-        const char *clientIP, int32_t clientRtp, int32_t clientRtcp,
-        RTPSender::TransportMode transportMode,
+        const char *clientIP,
+        int32_t clientRtp,
+        RTPSender::TransportMode rtpMode,
+        int32_t clientRtcp,
+        RTPSender::TransportMode rtcpMode,
         bool enableAudio,
         bool usePCMAudio,
         bool enableVideo,
@@ -385,10 +388,11 @@ status_t WifiDisplaySource::PlaybackSession::init(
     if (err == OK) {
         err = mMediaSender->initAsync(
                 -1 /* trackIndex */,
-                transportMode,
                 clientIP,
                 clientRtp,
+                rtpMode,
                 clientRtcp,
+                rtcpMode,
                 &mLocalRTPPort);
     }
 
@@ -548,6 +552,8 @@ void WifiDisplaySource::PlaybackSession::onMessageReceived(
                         converter->dropAFrame();
                     }
                 }
+            } else if (what == MediaSender::kWhatInformSender) {
+                onSinkFeedback(msg);
             } else {
                 TRESPASS();
             }
@@ -643,6 +649,46 @@ void WifiDisplaySource::PlaybackSession::onMessageReceived(
     }
 }
 
+void WifiDisplaySource::PlaybackSession::onSinkFeedback(const sp<AMessage> &msg) {
+    int64_t avgLatencyUs;
+    CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs));
+
+    int64_t maxLatencyUs;
+    CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs));
+
+    ALOGI("sink reports avg. latency of %lld ms (max %lld ms)",
+          avgLatencyUs / 1000ll,
+          maxLatencyUs / 1000ll);
+
+    if (mVideoTrackIndex >= 0) {
+        const sp<Track> &videoTrack = mTracks.valueFor(mVideoTrackIndex);
+        sp<Converter> converter = videoTrack->converter();
+        if (converter != NULL) {
+            int32_t videoBitrate = converter->getVideoBitrate();
+
+            if (avgLatencyUs > 300000ll) {
+                videoBitrate *= 0.6;
+
+                if (videoBitrate < 500000) {
+                    videoBitrate = 500000;  // cap at 500kbit/sec
+                }
+            } else if (avgLatencyUs < 100000ll) {
+                videoBitrate *= 1.1;
+
+                if (videoBitrate > 10000000) {
+                    videoBitrate = 10000000;  // cap at 10Mbit/sec
+                }
+            }
+
+            if (videoBitrate != converter->getVideoBitrate()) {
+                ALOGI("setting video bitrate to %d bps", videoBitrate);
+
+                converter->setVideoBitrate(videoBitrate);
+            }
+        }
+    }
+}
+
 status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer(
         bool enableAudio, bool enableVideo) {
     DataSource::RegisterDefaultSniffers();
diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.h b/media/libstagefright/wifi-display/source/PlaybackSession.h
index da207e2..39086a1 100644
--- a/media/libstagefright/wifi-display/source/PlaybackSession.h
+++ b/media/libstagefright/wifi-display/source/PlaybackSession.h
@@ -44,8 +44,11 @@ struct WifiDisplaySource::PlaybackSession : public AHandler {
             const char *path = NULL);
 
     status_t init(
-            const char *clientIP, int32_t clientRtp, int32_t clientRtcp,
-            RTPSender::TransportMode transportMode,
+            const char *clientIP,
+            int32_t clientRtp,
+            RTPSender::TransportMode rtpMode,
+            int32_t clientRtcp,
+            RTPSender::TransportMode rtcpMode,
             bool enableAudio,
             bool usePCMAudio,
             bool enableVideo,
@@ -149,6 +152,8 @@ private:
     void schedulePullExtractor();
     void onPullExtractor();
 
+    void onSinkFeedback(const sp<AMessage> &msg);
+
     DISALLOW_EVIL_CONSTRUCTORS(PlaybackSession);
 };
 
diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp
index 5167cb3..f2e659a 100644
--- a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp
+++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp
@@ -1159,7 +1159,7 @@ status_t WifiDisplaySource::onSetupRequest(
         return ERROR_MALFORMED;
     }
 
-    RTPSender::TransportMode transportMode = RTPSender::TRANSPORT_UDP;
+    RTPSender::TransportMode rtpMode = RTPSender::TRANSPORT_UDP;
 
     int clientRtp, clientRtcp;
     if (transport.startsWith("RTP/AVP/TCP;")) {
@@ -1168,7 +1168,7 @@ status_t WifiDisplaySource::onSetupRequest(
                     transport.c_str(), "interleaved", &interleaved)
                 && sscanf(interleaved.c_str(), "%d-%d",
                           &clientRtp, &clientRtcp) == 2) {
-            transportMode = RTPSender::TRANSPORT_TCP_INTERLEAVED;
+            rtpMode = RTPSender::TRANSPORT_TCP_INTERLEAVED;
         } else {
             bool badRequest = false;
 
@@ -1190,7 +1190,7 @@ status_t WifiDisplaySource::onSetupRequest(
                 return ERROR_MALFORMED;
             }
 
-            transportMode = RTPSender::TRANSPORT_TCP;
+            rtpMode = RTPSender::TRANSPORT_TCP;
         }
     } else if (transport.startsWith("RTP/AVP;unicast;")
             || transport.startsWith("RTP/AVP/UDP;unicast;")) {
@@ -1249,11 +1249,17 @@ status_t WifiDisplaySource::onSetupRequest(
         return ERROR_MALFORMED;
     }
 
+    RTPSender::TransportMode rtcpMode = RTPSender::TRANSPORT_UDP;
+    if (clientRtcp < 0) {
+        rtcpMode = RTPSender::TRANSPORT_NONE;
+    }
+
     status_t err = playbackSession->init(
             mClientInfo.mRemoteIP.c_str(),
             clientRtp,
+            rtpMode,
             clientRtcp,
-            transportMode,
+            rtcpMode,
             mSinkSupportsAudio,
             mUsingPCMAudio,
             mSinkSupportsVideo,
@@ -1282,7 +1288,7 @@ status_t WifiDisplaySource::onSetupRequest(
     AString response = "RTSP/1.0 200 OK\r\n";
     AppendCommonResponse(&response, cseq, playbackSessionID);
 
-    if (transportMode == RTPSender::TRANSPORT_TCP_INTERLEAVED) {
+    if (rtpMode == RTPSender::TRANSPORT_TCP_INTERLEAVED) {
         response.append(
                 StringPrintf(
                     "Transport: RTP/AVP/TCP;interleaved=%d-%d;",
@@ -1291,7 +1297,7 @@ status_t WifiDisplaySource::onSetupRequest(
         int32_t serverRtp = playbackSession->getRTPPort();
 
         AString transportString = "UDP";
-        if (transportMode == RTPSender::TRANSPORT_TCP) {
+        if (rtpMode == RTPSender::TRANSPORT_TCP) {
             transportString = "TCP";
         }