Implement proper multihoming support, second attempt.

This patch makes our HTTP client multihoming-aware, so if one server fails for
whatever reason (including timeout), we'll fall back to the next. It's a bit
more complex than the first attempt, but we're hopefully not breaking SSL
connection (incl. checkin) anymore.

Also includes one patch from upstream, in that timeouts are converted from
Java's exception hierarchy to our own exceptions.

Here's an example tcpdump from a fake checkin server with both AAAA and A records,
where the IPv6 connectivity is deliberately broken to demonstrate the effects of
this patch:

  11:49:28.202620 IP6 2620:0:105f:a:223:76ff:fe8d:3a3c.37109 > 2001:700:300:1880::2.80: S 24035192:24035192(0) win 5760 <mss 1440,sackOK,timestamp 1110775 0,[|tcp]>
  11:49:31.211370 IP6 2620:0:105f:a:223:76ff:fe8d:3a3c.37109 > 2001:700:300:1880::2.80: S 24035192:24035192(0) win 5760 <mss 1440,sackOK,timestamp 1111075 0,[|tcp]>
  11:49:37.211186 IP6 2620:0:105f:a:223:76ff:fe8d:3a3c.37109 > 2001:700:300:1880::2.80: S 24035192:24035192(0) win 5760 <mss 1440,sackOK,timestamp 1111675 0,[|tcp]>
  11:49:48.216299 IP 74.125.57.33.58205 > 129.241.93.35.80: S 2632654863:2632654863(0) win 5840 <mss 1372,sackOK,timestamp 1112775 0,nop,wscale 1>
  11:49:48.216324 IP 129.241.93.35.80 > 74.125.57.33.58205: S 3149921981:3149921981(0) ack 2632654864 win 5792 <mss 1460,sackOK,timestamp 62633484 1112775,nop,wscale 8>
  (...)

and then the HTTP connection proceeds as usual.

I intend to push this fix upstream once we get it reviewed and committed locally.

2 files changed, 54 insertions, 12 deletions

diff --git a/src/org/apache/http/conn/scheme/PlainSocketFactory.java b/src/org/apache/http/conn/scheme/PlainSocketFactory.java
index acc13f7..b15df44 100644
--- a/src/org/apache/http/conn/scheme/PlainSocketFactory.java
+++ b/src/org/apache/http/conn/scheme/PlainSocketFactory.java
@@ -35,7 +35,9 @@ import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.Socket;
+import java.net.SocketTimeoutException;
 
+import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.params.HttpConnectionParams;
 import org.apache.http.params.HttpParams;
 
@@ -113,9 +115,11 @@ public final class PlainSocketFactory implements SocketFactory {
         } else {
             remoteAddress = new InetSocketAddress(host, port);            
         }
-        
-        sock.connect(remoteAddress, timeout);
-
+        try {
+            sock.connect(remoteAddress, timeout);
+        } catch (SocketTimeoutException ex) {
+            throw new ConnectTimeoutException("Connect to " + remoteAddress + " timed out");
+        }
         return sock;
 
     } // connectSocket
diff --git a/src/org/apache/http/impl/conn/DefaultClientConnectionOperator.java b/src/org/apache/http/impl/conn/DefaultClientConnectionOperator.java
index 41488e1..854b2b0 100644
--- a/src/org/apache/http/impl/conn/DefaultClientConnectionOperator.java
+++ b/src/org/apache/http/impl/conn/DefaultClientConnectionOperator.java
@@ -44,7 +44,9 @@ import org.apache.http.protocol.HttpContext;
 import org.apache.http.conn.HttpHostConnectException;
 import org.apache.http.conn.OperatedClientConnection;
 import org.apache.http.conn.ClientConnectionOperator;
+import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.conn.scheme.LayeredSocketFactory;
+import org.apache.http.conn.scheme.PlainSocketFactory;
 import org.apache.http.conn.scheme.Scheme;
 import org.apache.http.conn.scheme.SchemeRegistry;
 import org.apache.http.conn.scheme.SocketFactory;
@@ -67,6 +69,7 @@ import org.apache.http.conn.scheme.SocketFactory;
 public class DefaultClientConnectionOperator
     implements ClientConnectionOperator {
 
+    private static final PlainSocketFactory staticPlainSocketFactory = new PlainSocketFactory();
 
     /** The scheme registry for looking up socket factories. */
     protected SchemeRegistry schemeRegistry;
@@ -121,19 +124,55 @@ public class DefaultClientConnectionOperator
 
         final Scheme schm = schemeRegistry.getScheme(target.getSchemeName());
         final SocketFactory sf = schm.getSocketFactory();
+        final SocketFactory plain_sf;
+        final LayeredSocketFactory layered_sf;
+        if (sf instanceof LayeredSocketFactory) {
+            plain_sf = staticPlainSocketFactory;
+            layered_sf = (LayeredSocketFactory)sf;
+        } else {
+            plain_sf = sf;
+            layered_sf = null;
+        }
+        InetAddress[] addresses = InetAddress.getAllByName(target.getHostName());
 
-        Socket sock = sf.createSocket();
-        conn.opening(sock, target);
+        for (int i = 0; i < addresses.length; ++i) {
+            Socket sock = plain_sf.createSocket();
+            conn.opening(sock, target);
 
-        try {
-            sock = sf.connectSocket(sock, target.getHostName(),
+            try {
+                Socket connsock = plain_sf.connectSocket(sock,
+                    addresses[i].getHostAddress(),
                     schm.resolvePort(target.getPort()),
                     local, 0, params);
-        } catch (ConnectException ex) {
-            throw new HttpHostConnectException(target, ex);
+                if (sock != connsock) {
+                    sock = connsock;
+                    conn.opening(sock, target);
+                }
+                if (layered_sf != null) {
+                    Socket layeredsock = layered_sf.createSocket(sock,
+                        target.getHostName(),
+                        schm.resolvePort(target.getPort()),
+                        true);
+                    if (layeredsock != sock) {
+                        conn.opening(layeredsock, target);
+                    }
+                    prepareSocket(layeredsock, context, params);
+                    conn.openCompleted(sf.isSecure(layeredsock), params);
+                } else {
+                    prepareSocket(sock, context, params);
+                    conn.openCompleted(sf.isSecure(sock), params);
+                }
+                break;
+            } catch (ConnectException ex) {
+                if (i == addresses.length - 1) {
+                    throw new HttpHostConnectException(target, ex);
+                }
+            } catch (ConnectTimeoutException ex) {
+                if (i == addresses.length - 1) {
+                    throw ex;
+                }
+            }
         }
-        prepareSocket(sock, context, params);
-        conn.openCompleted(sf.isSecure(sock), params);
     } // openConnection
 
 
@@ -213,4 +252,3 @@ public class DefaultClientConnectionOperator
 
 
 } // class DefaultClientConnectionOperator
-