correctly throw SocketTimeoutException from WolfSSLSession.connect() for JSSE WolfSSLSocket.startHandshake() to handle timeout

Author: cconlon

native/com_wolfssl_WolfSSL.h

@@ -554,7 +554,7 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_getFd
 /* enum values used in socketSelect() */
 enum {
     WOLFJNI_SELECT_FAIL = -10,
-    WOLFJNI_TIMEOUT     = -11,
+    WOLFJNI_TIMEOUT     = -11,  /* also in WolfSSL.java */
     WOLFJNI_RECV_READY  = -12,
     WOLFJNI_SEND_READY  = -13,
     WOLFJNI_ERROR_READY = -14
@@ -623,7 +623,7 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_connect
     (void)jcl;
 
     if (jenv == NULL || ssl == NULL) {
-        return SSL_FATAL_ERROR;
+        return SSL_FAILURE;
     }
 
     /* make sure we don't have any outstanding exceptions pending */
@@ -674,8 +674,12 @@ JNIEXPORT jint JNICALL Java_com_wolfssl_WolfSSLSession_connect
             if (ret == WOLFJNI_RECV_READY || ret == WOLFJNI_SEND_READY) {
                 /* I/O ready, continue handshake and try again */
                 continue;
+            } else if (ret == WOLFJNI_TIMEOUT) {
+                /* Java will throw SocketTimeoutException */
+                break;
             } else {
-                /* error or timeout */
+                /* error */
+                ret = SSL_FAILURE;
                 break;
             }
         }

native/com_wolfssl_WolfSSLSession.c

@@ -56,6 +56,11 @@ public static enum TLS_VERSION {
     /** Session unavailable */
     public final static int JNI_SESSION_UNAVAILABLE = -10001;
 
+    /**
+     * Socket timed out, matches com_wolfssl_WolfSSLSession.c socketSelect()
+     * return value */
+    public final static int WOLFJNI_TIMEOUT = -11;
+
     /* ----------------------- wolfSSL codes ---------------------------- */
 
     /** Error code: no error */

src/java/com/wolfssl/WolfSSL.java

@@ -508,16 +508,26 @@ public int getFd()
      * before calling <code>newSSL()</code>, though it's not recommended.
      *
      * @return <code>SSL_SUCCESS</code> if successful, otherwise
-     *         <code>SSL_FATAL_ERROR</code> if an error occurred. To get
+     *         <code>SSL_FAILURE</code> if an error occurred. To get
      *         a more detailed error code, call <code>getError()</code>.
      * @throws IllegalStateException WolfSSLContext has been freed
+     * @throws SocketTimeoutException if underlying socket timed out
      */
-    public int connect() throws IllegalStateException {
+    public int connect() throws IllegalStateException, SocketTimeoutException {
+
+        int ret = 0;
 
         if (this.active == false)
             throw new IllegalStateException("Object has been freed");
 
-        return connect(getSessionPtr(), 0);
+        ret = connect(getSessionPtr(), 0);
+
+        if (ret == WolfSSL.WOLFJNI_TIMEOUT) {
+            throw new SocketTimeoutException(
+                    "Native socket timed out during SSL_connect()");
+        }
+
+        return ret;
     }
 
     /**

src/java/com/wolfssl/WolfSSLSession.java

@@ -697,6 +697,7 @@ protected int doHandshake(int isSSLEngine, int timeout)
             if (this.clientMode) {
                 WolfSSLDebug.log(getClass(), WolfSSLDebug.INFO,
                         "calling native wolfSSL_connect()");
+                /* may throw SocketTimeoutException on socket timeout */
                 ret = this.ssl.connect(timeout);
 
             } else {

src/java/com/wolfssl/provider/jsse/WolfSSLEngineHelper.java

@@ -29,6 +29,7 @@
 import java.net.Socket;
 import java.net.UnknownHostException;
 import java.net.ConnectException;
+import java.net.SocketTimeoutException;
 
 import com.wolfssl.WolfSSL;
 import com.wolfssl.WolfSSLContext;
@@ -459,6 +460,10 @@ public void test_WolfSSLSession_UseAfterFree() {
         } catch (IllegalStateException ise) {
             System.out.println("\t\t... passed");
             return;
+        } catch (SocketTimeoutException e) {
+            System.out.println("\t\t... failed");
+            e.printStackTrace();
+            return;
         }
 
         /* fail here means WolfSSLSession was used after free without