Fix server side SSLEngine ServerKeyExchange signature.

Code using SSLEngine for non-blocking SSL can't talk with openssl as a client,
since the signature is computed on different content (and openssl checks it,
unlike java). The fix is to use strip the 0x00 prefix when signing - like
it is done when generating the message, refactored both to use a common
method. We also include the length in the signature, it was also missing.

2 files changed, 30 insertions, 26 deletions

diff --git a/x-net/src/main/java/org/apache/harmony/xnet/provider/jsse/ServerHandshakeImpl.java b/x-net/src/main/java/org/apache/harmony/xnet/provider/jsse/ServerHandshakeImpl.java
index 782bb39..b76c42f 100644
--- a/x-net/src/main/java/org/apache/harmony/xnet/provider/jsse/ServerHandshakeImpl.java
+++ b/x-net/src/main/java/org/apache/harmony/xnet/provider/jsse/ServerHandshakeImpl.java
@@ -575,27 +575,31 @@ public class ServerHandshakeImpl extends HandshakeProtocol {
                     byte[] tmpLength = new byte[2];
 //FIXME 1_byte==0x00                    
                     if (cipher_suite.keyExchange == CipherSuite.KeyExchange_RSA_EXPORT) {
-                        tmp = rsakey.getModulus().toByteArray();
+                        tmp = ServerKeyExchange.toUnsignedByteArray(rsakey.getModulus());
                         tmpLength[0] = (byte) ((tmp.length & 0xFF00) >>> 8);
                         tmpLength[1] = (byte) (tmp.length & 0xFF);
                         ds.update(tmpLength);
                         ds.update(tmp);
-                        tmp = rsakey.getPublicExponent().toByteArray();
+                        tmp = ServerKeyExchange.toUnsignedByteArray(rsakey.getPublicExponent());
                         tmpLength[0] = (byte) ((tmp.length & 0xFF00) >>> 8);
                         tmpLength[1] = (byte) (tmp.length & 0xFF);
+                        ds.update(tmpLength);
                         ds.update(tmp);
                     } else {
-                        tmp = dhkeySpec.getP().toByteArray();
+                        tmp = ServerKeyExchange.toUnsignedByteArray(dhkeySpec.getP());
                         tmpLength[0] = (byte) ((tmp.length & 0xFF00) >>> 8);
                         tmpLength[1] = (byte) (tmp.length & 0xFF);
+                        ds.update(tmpLength);
                         ds.update(tmp);
-                        tmp = dhkeySpec.getG().toByteArray();
+                        tmp = ServerKeyExchange.toUnsignedByteArray(dhkeySpec.getG());
                         tmpLength[0] = (byte) ((tmp.length & 0xFF00) >>> 8);
                         tmpLength[1] = (byte) (tmp.length & 0xFF);
+                        ds.update(tmpLength);
                         ds.update(tmp);
-                        tmp = dhkeySpec.getY().toByteArray();
+                        tmp = ServerKeyExchange.toUnsignedByteArray(dhkeySpec.getY());
                         tmpLength[0] = (byte) ((tmp.length & 0xFF00) >>> 8);
                         tmpLength[1] = (byte) (tmp.length & 0xFF);
+                        ds.update(tmpLength);
                         ds.update(tmp);
                     }
                     hash = ds.sign();
diff --git a/x-net/src/main/java/org/apache/harmony/xnet/provider/jsse/ServerKeyExchange.java b/x-net/src/main/java/org/apache/harmony/xnet/provider/jsse/ServerKeyExchange.java
index 446b7b4..af056a3 100644
--- a/x-net/src/main/java/org/apache/harmony/xnet/provider/jsse/ServerKeyExchange.java
+++ b/x-net/src/main/java/org/apache/harmony/xnet/provider/jsse/ServerKeyExchange.java
@@ -65,22 +65,9 @@ public class ServerKeyExchange extends Message {
         this.par3 = par3;
         this.hash = hash;
 
-        byte[] bb = this.par1.toByteArray();
-        if (bb[0] == 0) {
-// XXX check for par1 == 0 or bb.length > 1
-            bytes1 = new byte[bb.length - 1];
-            System.arraycopy(bb, 1, bytes1, 0, bytes1.length);
-        } else {
-            bytes1 = bb;
-        }
-
-        bb = this.par2.toByteArray();
-        if (bb[0] == 0) {
-            bytes2 = new byte[bb.length - 1];
-            System.arraycopy(bb, 1, bytes2, 0, bytes2.length);
-        } else {
-            bytes2 = bb;
-        }
+        bytes1 = toUnsignedByteArray(this.par1);
+   
+        bytes2 = toUnsignedByteArray(this.par2);
 
         length = 4 + bytes1.length + bytes2.length;
         if (hash != null) {
@@ -90,14 +77,27 @@ public class ServerKeyExchange extends Message {
             bytes3 = null;
             return;
         }
-        bb = this.par3.toByteArray();
+        bytes3 = toUnsignedByteArray(this.par3);
+        length += 2 + bytes3.length;
+    }
+    
+    /**
+     * Remove first byte if 0. Needed because BigInteger.toByteArray() sometimes
+     * returns a zero prefix.
+     */
+    public static byte[] toUnsignedByteArray(BigInteger bi) {
+        if (bi == null) {
+            return null;
+        }
+        byte[] bb = bi.toByteArray();
+        // bb is not null, and has at least 1 byte - ZERO is represented as [0]
         if (bb[0] == 0) {
-            bytes3 = new byte[bb.length - 1];
-            System.arraycopy(bb, 1, bytes3, 0, bytes3.length);
+            byte[] noZero = new byte[bb.length - 1];
+            System.arraycopy(bb, 1, noZero, 0, noZero.length);
+            return noZero;
         } else {
-            bytes3 = bb;
+            return bb;
         }
-        length += 2 + bytes3.length;
     }
 
     /**