JSSE: unset native verify callback when SSLEngine is finished, allows garbage collection

Author: cconlon

src/java/com/wolfssl/provider/jsse/WolfSSLEngine.java

@@ -606,7 +606,12 @@ public synchronized SSLEngineResult wrap(ByteBuffer[] in, int ofst, int len,
          */
         if (produced >= 0 &&
             (!outBoundOpen || (!inBoundOpen && this.closeNotifySent))) {
+            /* Mark SSLEngine status as CLOSED */
             status = SSLEngineResult.Status.CLOSED;
+            /* Handshake has finished and SSLEngine is closed, release
+             * global JNI verify callback pointer */
+            this.EngineHelper.unsetVerifyCallback();
+
             try {
                 ClosingConnection();
             } catch (SocketException e) {
@@ -962,7 +967,11 @@ else if (hs == SSLEngineResult.HandshakeStatus.NEED_WRAP &&
             if (outBoundOpen == false) {
                 try {
                     if (ClosingConnection() == WolfSSL.SSL_SUCCESS) {
+                        /* Mark SSLEngine status as CLOSED */
                         status = SSLEngineResult.Status.CLOSED;
+                        /* Handshake has finished and SSLEngine is closed,
+                         * release, global JNI verify callback pointer */
+                        this.EngineHelper.unsetVerifyCallback();
                     }
                 } catch (SocketException e) {
                     throw new SSLException(e);
@@ -1030,7 +1039,11 @@ else if (hs == SSLEngineResult.HandshakeStatus.NEED_WRAP &&
                 }
 
                 if (outBoundOpen == false || this.closeNotifySent) {
+                    /* Mark SSLEngine status as CLOSED */
                     status = SSLEngineResult.Status.CLOSED;
+                    /* Handshake has finished and SSLEngine is closed,
+                     * release, global JNI verify callback pointer */
+                    this.EngineHelper.unsetVerifyCallback();
                 }
 
                 int err = ssl.getError(ret);
@@ -1773,7 +1786,7 @@ protected synchronized void finalize() throws Throwable {
             this.ssl.freeSSL();
             this.ssl = null;
         }
-        EngineHelper = null;
+        this.EngineHelper = null;
         super.finalize();
     }
 }

src/java/com/wolfssl/provider/jsse/WolfSSLEngineHelper.java

@@ -87,6 +87,9 @@ public class WolfSSLEngineHelper {
     /* Has setUseClientMode() been called on this object */
     private boolean modeSet = false;
 
+    /* wolfSSL verification mode, set inside setLocalAuth() */
+    private int verifyMask = WolfSSL.SSL_VERIFY_PEER;
+
     /* Internal Java verify callback, used when user/app is not using
      * com.wolfssl.provider.jsse.WolfSSLTrustX509 and instead using their
      * own TrustManager to perform verification via checkClientTrusted()
@@ -805,7 +808,7 @@ private void setLocalAuth(SSLSocket socket, SSLEngine engine) {
              * Algorithm has been set. To get this callback to be called,
              * native wolfSSL should be compiled with the following define:
              * WOLFSSL_ALWAYS_VERIFY_CB */
-            this.ssl.setVerify(mask, wicb);
+            this.verifyMask = mask;
 
         } else {
             /* not our own TrustManager, set up callback so JSSE can use
@@ -814,8 +817,10 @@ private void setLocalAuth(SSLSocket socket, SSLEngine engine) {
                 "X509TrustManager is not of type WolfSSLTrustX509");
             WolfSSLDebug.log(getClass(), WolfSSLDebug.INFO,
                 "Using checkClientTrusted/ServerTrusted() for verification");
-            this.ssl.setVerify(WolfSSL.SSL_VERIFY_PEER, wicb);
+            this.verifyMask = WolfSSL.SSL_VERIFY_PEER;
         }
+
+        this.ssl.setVerify(this.verifyMask, wicb);
     }
 
 
@@ -1330,6 +1335,35 @@ else if (peerAddr != null) {
         return ret;
     }
 
+    /**
+     * Unset the native verify callback and reset internal verify
+     * callback state.
+     *
+     * This helper method is called by SSLEngine to reset the native
+     * wolfSSL verify callback back to null. Since a pointer to that verify
+     * callback is stored as a global JNI variable, it can prevent garbage
+     * collection from being done. This helper can be called when an SSLEngine
+     * or SSLSocket is closed/done to reset the verify callback.
+     *
+     * The verify callback will be set again if needed when
+     * initHandshake() is called.
+     */
+    protected synchronized void unsetVerifyCallback() {
+        /* Set native callback to null, releases JNI global and allows for
+         * garbage collection if needed */
+        if (this.ssl != null) {
+            this.ssl.setVerify(this.verifyMask, null);
+        }
+
+        /* Reset internal state of WolfSSLInternalVerifyCallback, removes
+         * references to SSLSocket/SSLEngine to allow garbage collection if
+         * needed */
+        if (this.wicb != null) {
+            this.wicb.clearInternalVars();
+            this.wicb = null;
+        }
+    }
+
     /**
      * Saves session on connection close for resumption
      *
@@ -1357,6 +1391,7 @@ protected synchronized void finalize() throws Throwable {
          * may be used by wrapper object to WolfSSLEngineHelper and should
          * be freed there */
         this.ssl = null;
+        this.wicb = null;
 
         this.session = null;
         this.params = null;

src/java/com/wolfssl/provider/jsse/WolfSSLInternalVerifyCb.java

@@ -68,6 +68,16 @@ public WolfSSLInternalVerifyCb(X509TrustManager xtm, boolean client,
         this.params = params;
     }
 
+    /**
+     * Reset internal variables back to null/default.
+     */
+    protected void clearInternalVars() {
+        this.callingSocket = null;
+        this.callingEngine = null;
+        this.params = null;
+        this.tm = null;
+    }
+
     /**
      * Verify hostname of provided peer certificate using
      * Endpoint Identification Algorithm if set in SSLParameters.
@@ -349,5 +359,15 @@ else if ((preverify_ok == 1) && (x509certs.length == 0) &&
         /* Continue handshake, verification succeeded */
         return 1;
     }
+
+    @SuppressWarnings("deprecation")
+    @Override
+    protected void finalize() throws Throwable {
+        this.callingSocket = null;
+        this.callingEngine = null;
+        this.tm = null;
+        this.params = null;
+        super.finalize();
+    }
 }