Implement proper multihoming support, so if one server fails for whatever reason

(including timeout), we'll fall back to the next.

Also includes one patch from upstream, in that timeouts are converted from
Java's exception hierarchy to our own exceptions.

Here's an example tcpdump from a fake checkin server with both AAAA and A records,
where the IPv6 connectivity is deliberately broken to demonstrate the effects of
this patch:

  11:49:28.202620 IP6 2620:0:105f:a:223:76ff:fe8d:3a3c.37109 > 2001:700:300:1880::2.80: S 24035192:24035192(0) win 5760 <mss 1440,sackOK,timestamp 1110775 0,[|tcp]>
  11:49:31.211370 IP6 2620:0:105f:a:223:76ff:fe8d:3a3c.37109 > 2001:700:300:1880::2.80: S 24035192:24035192(0) win 5760 <mss 1440,sackOK,timestamp 1111075 0,[|tcp]>
  11:49:37.211186 IP6 2620:0:105f:a:223:76ff:fe8d:3a3c.37109 > 2001:700:300:1880::2.80: S 24035192:24035192(0) win 5760 <mss 1440,sackOK,timestamp 1111675 0,[|tcp]>
  11:49:48.216299 IP 74.125.57.33.58205 > 129.241.93.35.80: S 2632654863:2632654863(0) win 5840 <mss 1372,sackOK,timestamp 1112775 0,nop,wscale 1>
  11:49:48.216324 IP 129.241.93.35.80 > 74.125.57.33.58205: S 3149921981:3149921981(0) ack 2632654864 win 5792 <mss 1460,sackOK,timestamp 62633484 1112775,nop,wscale 8>
  (...)

and then the HTTP connection proceeds as usual.

I intend to push this fix upstream once we get it reviewed and committed locally.

2 files changed, 34 insertions, 16 deletions

diff --git a/src/org/apache/http/conn/scheme/PlainSocketFactory.java b/src/org/apache/http/conn/scheme/PlainSocketFactory.java
index acc13f7..b15df44 100644
--- a/src/org/apache/http/conn/scheme/PlainSocketFactory.java
+++ b/src/org/apache/http/conn/scheme/PlainSocketFactory.java
@@ -35,7 +35,9 @@ import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.Socket;
+import java.net.SocketTimeoutException;
 
+import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.params.HttpConnectionParams;
 import org.apache.http.params.HttpParams;
 
@@ -113,9 +115,11 @@ public final class PlainSocketFactory implements SocketFactory {
         } else {
             remoteAddress = new InetSocketAddress(host, port);            
         }
-        
-        sock.connect(remoteAddress, timeout);
-
+        try {
+            sock.connect(remoteAddress, timeout);
+        } catch (SocketTimeoutException ex) {
+            throw new ConnectTimeoutException("Connect to " + remoteAddress + " timed out");
+        }
         return sock;
 
     } // connectSocket
diff --git a/src/org/apache/http/impl/conn/DefaultClientConnectionOperator.java b/src/org/apache/http/impl/conn/DefaultClientConnectionOperator.java
index 41488e1..0fc1725 100644
--- a/src/org/apache/http/impl/conn/DefaultClientConnectionOperator.java
+++ b/src/org/apache/http/impl/conn/DefaultClientConnectionOperator.java
@@ -44,6 +44,7 @@ import org.apache.http.protocol.HttpContext;
 import org.apache.http.conn.HttpHostConnectException;
 import org.apache.http.conn.OperatedClientConnection;
 import org.apache.http.conn.ClientConnectionOperator;
+import org.apache.http.conn.ConnectTimeoutException;
 import org.apache.http.conn.scheme.LayeredSocketFactory;
 import org.apache.http.conn.scheme.Scheme;
 import org.apache.http.conn.scheme.SchemeRegistry;
@@ -121,19 +122,33 @@ public class DefaultClientConnectionOperator
 
         final Scheme schm = schemeRegistry.getScheme(target.getSchemeName());
         final SocketFactory sf = schm.getSocketFactory();
-
-        Socket sock = sf.createSocket();
-        conn.opening(sock, target);
-
-        try {
-            sock = sf.connectSocket(sock, target.getHostName(),
-                    schm.resolvePort(target.getPort()),
-                    local, 0, params);
-        } catch (ConnectException ex) {
-            throw new HttpHostConnectException(target, ex);
+        InetAddress[] addresses = InetAddress.getAllByName(target.getHostName());
+
+        for (int i = 0; i < addresses.length; ++i) {
+            Socket sock = sf.createSocket();
+            conn.opening(sock, target);
+
+            try {
+                Socket connsock = sf.connectSocket(sock, addresses[i].getHostAddress(),
+                        schm.resolvePort(target.getPort()),
+                        local, 0, params);
+                if (sock != connsock) {
+                    sock = connsock;
+                    conn.opening(sock, target);
+                }
+                prepareSocket(sock, context, params);
+                conn.openCompleted(sf.isSecure(sock), params);
+                break;
+            } catch (ConnectException ex) {
+                if (i == addresses.length - 1) {
+                    throw new HttpHostConnectException(target, ex);
+                }
+            } catch (ConnectTimeoutException ex) {
+                if (i == addresses.length - 1) {
+                    throw ex;
+                }
+            }
         }
-        prepareSocket(sock, context, params);
-        conn.openCompleted(sf.isSecure(sock), params);
     } // openConnection
 
 
@@ -213,4 +228,3 @@ public class DefaultClientConnectionOperator
 
 
 } // class DefaultClientConnectionOperator
-