Merge pull request #180 from cconlon/SSLEngineGCFix

Release native verify callback with SSLEngine is closed

Author: douzzer

examples/provider/ClientJSSE.java

@@ -87,6 +87,10 @@ public void run(String[] args) throws Exception {
         boolean putEnabledProtocols  = false; /* set enabled protocols */
         boolean sendGET = false;              /* send HTTP GET */
 
+        /* Sleep 10 seconds before and after execution of main example,
+         * to allow profilers like VisualVM to be attached. */
+        boolean profileSleep = false;
+
         boolean resumeSession = false;        /* try one session resumption */
         byte[] firstSessionId = null;         /* sess ID of first session */
         byte[] resumeSessionId = null;        /* sess ID of resumed session */
@@ -180,6 +184,9 @@ public void run(String[] args) throws Exception {
             } else if (arg.equals("-r")) {
                 resumeSession = true;
 
+            } else if (arg.equals("-profile")) {
+                profileSleep = true;
+
             } else {
                 printUsage();
             }
@@ -197,6 +204,12 @@ public void run(String[] args) throws Exception {
                 return;
         }
 
+        if (profileSleep) {
+            System.out.println(
+                "Sleeping 10 seconds to allow profiler to attach");
+            Thread.sleep(10000);
+        }
+
         /* X509TrustManager that trusts all peer certificates. Used if peer
          * authentication (-d) has been passed in */
         TrustManager[] trustAllCerts = new TrustManager[] {
@@ -322,6 +335,25 @@ public java.security.cert.X509Certificate[] getAcceptedIssuers() {
             System.out.println("Server message : " + new String(back));
             sock.close();
         }
+
+        if (profileSleep) {
+            /* Remove provider and set variables to null to help garbage
+             * collector for profiling */
+            Security.removeProvider("wolfJSSE");
+            sock = null;
+            sf = null;
+            ctx = null;
+            km = null;
+            tm = null;
+
+            /* Try and kick start garbage collector before profiling
+             * heap dump */
+            System.gc();
+
+            System.out.println(
+                "Sleeping 10 seconds to allow profiler to dump heap");
+            Thread.sleep(10000);
+        }
     }
 
     private void showPeer(SSLSocket sock) {
@@ -385,6 +417,8 @@ private void printUsage() {
         System.out.println("-A <file>:<password>\tCertificate/key CA JKS file,\tdefault " +
                 "../provider/ca-server.jks:wolfSSL test");
         System.out.println("-r Resume session");
+        System.out.println("-profile\tSleep for 10 sec before/after running " +
+                "to allow profilers to attach");
         System.exit(1);
     }
 }

examples/provider/MultiThreadedSSLClient.java

@@ -78,6 +78,10 @@ public class MultiThreadedSSLClient
     int successClientConnections = 0; /* successful client connections */
     int failedClientConnections = 0;  /* failed client connections */
 
+    /* Sleep 10 seconds before and after execution of main example,
+     * to allow profilers like VisualVM to be attached. */
+    boolean profileSleep = false;
+
     long totalConnectionTimeMs = 0;   /* total handshake time, across clients */
     final Object timeLock = new Object();
 
@@ -156,10 +160,6 @@ public MultiThreadedSSLClient(String[] args) {
         String jkspass = "wolfSSL test";
         char[] passArr = jkspass.toCharArray();
 
-        if (args.length != 2) {
-            printUsage();
-        }
-
         /* pull in command line options from user */
         for (int i = 0; i < args.length; i++)
         {
@@ -170,12 +170,22 @@ public MultiThreadedSSLClient(String[] args) {
                     printUsage();
                 numClientConnections = Integer.parseInt(args[++i]);
 
+            } else if (arg.equals("-profile")) {
+                profileSleep = true;
+
             } else {
                 printUsage();
             }
         }
 
         try {
+
+            if (profileSleep) {
+                System.out.println(
+                    "Sleeping 10 seconds to allow profiler to attach");
+                Thread.sleep(10000);
+            }
+
             List<ClientThread> clientList = new ArrayList<ClientThread>();
             CountDownLatch latch = new CountDownLatch(numClientConnections);
 
@@ -209,12 +219,22 @@ public MultiThreadedSSLClient(String[] args) {
             latch.await();
             executor.shutdown();
 
+            Security.removeProvider("wolfJSSE");
+
+            if (profileSleep) {
+                /* Try and kick start garbage collector before profiling
+                 * heap dump */
+                System.gc();
+
+                System.out.println(
+                    "Sleeping 10 seconds to allow profiler to dump heap");
+                Thread.sleep(10000);
+            }
+
         } catch (Exception e) {
             e.printStackTrace();
         }
 
-        Security.removeProvider("wolfJSSE");
-
         System.out.println("================================================");
         System.out.println("All Client Connections Finished");
         System.out.println("Successful = " + successClientConnections);
@@ -234,6 +254,8 @@ public static void main(String[] args) {
     private void printUsage() {
         System.out.println("Java wolfJSSE example threaded client usage:");
         System.out.println("-n <num>\tNumber of client connections");
+        System.out.println("-profile\tSleep for 10 sec before/after running " +
+                "to allow profilers to attach");
         System.exit(1);
     }
 }

examples/provider/ServerJSSE.java

@@ -57,8 +57,12 @@ public void run(String[] args) {
         boolean verifyPeer = true;            /* verify peer by default */
         boolean useEnvVar  = false;           /* load cert/key from enviornment variable */
         boolean listSuites = false;           /* list all supported cipher suites */
-        boolean listEnabledProtocols = false;  /* show enabled protocols */
-        boolean putEnabledProtocols  = false;  /* set enabled protocols */
+        boolean listEnabledProtocols = false; /* show enabled protocols */
+        boolean putEnabledProtocols  = false; /* set enabled protocols */
+
+        /* Sleep 10 seconds before and after execution of main example,
+         * to allow profilers like VisualVM to be attached. */
+        boolean profileSleep = false;
 
         /* cert info */
         String serverJKS  = "../provider/server.jks";
@@ -86,6 +90,12 @@ public void run(String[] args) {
 
             /* load WolfSSLprovider */
             Security.addProvider(new WolfSSLProvider());
+            if (Security.getProvider("wolfJSSE") == null) {
+                System.out.println("Can't find wolfJSSE provider");
+            }
+            else {
+                System.out.println("Registered wolfJSSE provider");
+            }
 
             /* pull in command line options from user */
             for (int i = 0; i < args.length; i++)
@@ -151,6 +161,9 @@ public void run(String[] args) {
                     protocols = args[++i].split(" ");
                     sslVersion = -1;
 
+                } else if (arg.equals("-profile")) {
+                    profileSleep = true;
+
                 } else {
                     printUsage();
                 }
@@ -181,6 +194,12 @@ public void run(String[] args) {
                     System.exit(1);
             }
 
+            if (profileSleep) {
+                System.out.println(
+                    "Sleeping 10 seconds to allow profiler to attach");
+                Thread.sleep(10000);
+            }
+
             /* set up keystore and truststore */
             KeyStore keystore = KeyStore.getInstance("JKS");
             keystore.load(new FileInputStream(serverJKS),
@@ -258,6 +277,32 @@ public void run(String[] args) {
                 sock.getOutputStream().write(msg.getBytes());
 
                 sock.close();
+
+                if (profileSleep) {
+                    /* If profiling, only loop once */
+                    sock = null;
+                    break;
+                }
+            }
+
+            ss.close();
+
+            if (profileSleep) {
+                /* Remove provider and set variables to null to help
+                 * garbage collector for profiling */
+                Security.removeProvider("wolfJSSE");
+                ss = null;
+                ctx = null;
+                km = null;
+                tm = null;
+
+                /* Try and kick start garbage collector before profiling
+                 * heap dump */
+                System.gc();
+
+                System.out.println(
+                    "Sleeping 10 seconds to allow profiler to dump heap");
+                Thread.sleep(10000);
             }
 
         } catch (Exception e) {
@@ -298,6 +343,8 @@ private void printUsage() {
                 "../provider/server.jks:\"wolfSSL test\"");
         System.out.println("-A <file>:<password>\tCertificate/key CA JKS file,\tdefault " +
                 "../provider/ca-client.jks:\"wolfSSL test\"");
+        System.out.println("-profile\tSleep for 10 sec before/after running " +
+                "to allow profilers to attach");
         System.exit(1);
     }
 

native/com_wolfssl_WolfSSLSession.c

@@ -3912,11 +3912,24 @@ JNIEXPORT void JNICALL Java_com_wolfssl_WolfSSLSession_setVerify
         return;
     }
 
+    /* Release global reference if already set, before setting again */
+    appData = (SSLAppData*)wolfSSL_get_app_data(ssl);
+    if (appData != NULL) {
+        verifyCb = appData->g_verifySSLCbIfaceObj;
+        if (verifyCb != NULL) {
+            (*jenv)->DeleteGlobalRef(jenv, (jobject)(*verifyCb));
+            XFREE(verifyCb, NULL, DYNAMIC_TYPE_TMP_BUFFER);
+            verifyCb = NULL;
+            appData->g_verifySSLCbIfaceObj = NULL;
+        }
+    }
+
+    /* Set verify callback to NULL (reset), or passed in callback */
     if (!callbackIface) {
         wolfSSL_set_verify(ssl, mode, NULL);
     }
     else {
-        /* get app data to store verify callback jobject */
+        /* Get app data to store verify callback jobject */
         appData = (SSLAppData*)wolfSSL_get_app_data(ssl);
         if (appData == NULL) {
             printf("Error getting app data from WOLFSSL\n");

src/java/com/wolfssl/provider/jsse/WolfSSLEngine.java

@@ -606,7 +606,12 @@ public synchronized SSLEngineResult wrap(ByteBuffer[] in, int ofst, int len,
          */
         if (produced >= 0 &&
             (!outBoundOpen || (!inBoundOpen && this.closeNotifySent))) {
+            /* Mark SSLEngine status as CLOSED */
             status = SSLEngineResult.Status.CLOSED;
+            /* Handshake has finished and SSLEngine is closed, release
+             * global JNI verify callback pointer */
+            this.EngineHelper.unsetVerifyCallback();
+
             try {
                 ClosingConnection();
             } catch (SocketException e) {
@@ -962,7 +967,11 @@ else if (hs == SSLEngineResult.HandshakeStatus.NEED_WRAP &&
             if (outBoundOpen == false) {
                 try {
                     if (ClosingConnection() == WolfSSL.SSL_SUCCESS) {
+                        /* Mark SSLEngine status as CLOSED */
                         status = SSLEngineResult.Status.CLOSED;
+                        /* Handshake has finished and SSLEngine is closed,
+                         * release, global JNI verify callback pointer */
+                        this.EngineHelper.unsetVerifyCallback();
                     }
                 } catch (SocketException e) {
                     throw new SSLException(e);
@@ -1030,7 +1039,11 @@ else if (hs == SSLEngineResult.HandshakeStatus.NEED_WRAP &&
                 }
 
                 if (outBoundOpen == false || this.closeNotifySent) {
+                    /* Mark SSLEngine status as CLOSED */
                     status = SSLEngineResult.Status.CLOSED;
+                    /* Handshake has finished and SSLEngine is closed,
+                     * release, global JNI verify callback pointer */
+                    this.EngineHelper.unsetVerifyCallback();
                 }
 
                 int err = ssl.getError(ret);
@@ -1773,7 +1786,7 @@ protected synchronized void finalize() throws Throwable {
             this.ssl.freeSSL();
             this.ssl = null;
         }
-        EngineHelper = null;
+        this.EngineHelper = null;
         super.finalize();
     }
 }

src/java/com/wolfssl/provider/jsse/WolfSSLEngineHelper.java

@@ -87,6 +87,9 @@ public class WolfSSLEngineHelper {
     /* Has setUseClientMode() been called on this object */
     private boolean modeSet = false;
 
+    /* wolfSSL verification mode, set inside setLocalAuth() */
+    private int verifyMask = WolfSSL.SSL_VERIFY_PEER;
+
     /* Internal Java verify callback, used when user/app is not using
      * com.wolfssl.provider.jsse.WolfSSLTrustX509 and instead using their
      * own TrustManager to perform verification via checkClientTrusted()
@@ -805,7 +808,7 @@ private void setLocalAuth(SSLSocket socket, SSLEngine engine) {
              * Algorithm has been set. To get this callback to be called,
              * native wolfSSL should be compiled with the following define:
              * WOLFSSL_ALWAYS_VERIFY_CB */
-            this.ssl.setVerify(mask, wicb);
+            this.verifyMask = mask;
 
         } else {
             /* not our own TrustManager, set up callback so JSSE can use
@@ -814,8 +817,10 @@ private void setLocalAuth(SSLSocket socket, SSLEngine engine) {
                 "X509TrustManager is not of type WolfSSLTrustX509");
             WolfSSLDebug.log(getClass(), WolfSSLDebug.INFO,
                 "Using checkClientTrusted/ServerTrusted() for verification");
-            this.ssl.setVerify(WolfSSL.SSL_VERIFY_PEER, wicb);
+            this.verifyMask = WolfSSL.SSL_VERIFY_PEER;
         }
+
+        this.ssl.setVerify(this.verifyMask, wicb);
     }
 
 
@@ -1330,6 +1335,35 @@ else if (peerAddr != null) {
         return ret;
     }
 
+    /**
+     * Unset the native verify callback and reset internal verify
+     * callback state.
+     *
+     * This helper method is called by SSLEngine to reset the native
+     * wolfSSL verify callback back to null. Since a pointer to that verify
+     * callback is stored as a global JNI variable, it can prevent garbage
+     * collection from being done. This helper can be called when an SSLEngine
+     * or SSLSocket is closed/done to reset the verify callback.
+     *
+     * The verify callback will be set again if needed when
+     * initHandshake() is called.
+     */
+    protected synchronized void unsetVerifyCallback() {
+        /* Set native callback to null, releases JNI global and allows for
+         * garbage collection if needed */
+        if (this.ssl != null) {
+            this.ssl.setVerify(this.verifyMask, null);
+        }
+
+        /* Reset internal state of WolfSSLInternalVerifyCallback, removes
+         * references to SSLSocket/SSLEngine to allow garbage collection if
+         * needed */
+        if (this.wicb != null) {
+            this.wicb.clearInternalVars();
+            this.wicb = null;
+        }
+    }
+
     /**
      * Saves session on connection close for resumption
      *
@@ -1357,6 +1391,7 @@ protected synchronized void finalize() throws Throwable {
          * may be used by wrapper object to WolfSSLEngineHelper and should
          * be freed there */
         this.ssl = null;
+        this.wicb = null;
 
         this.session = null;
         this.params = null;