Allow users to supply a custom HttpClientBuilder and HttpClientConnectionManager (#352)

Resolves #310 by allowing users to pass in an existing HttpClientBuilder and HttpClientConnectionManager using MantaConnectionFactoryConfigurator. USAGE.md, changelog, and JavaDocs have been update to explain expectations around this feature.

Author: tjcelaya

CHANGELOG.md

@@ -9,6 +9,11 @@ This project aims to adhere to [Semantic Versioning](http://semver.org/).
    directory listings
  - Added [connection request timeout](https://github.com/joyent/java-manta/issues/347)
    configuration parameter: `manta.connection_request_timeout` / `MANTA_CONNECTION_REQUEST_TIMEOUT`
+ - `MantaClient` will now accept an externally-created `HttpClientBuilder` in order to
+   allow for customization now available through standard configuration parameters. Users are expected to provide a
+   `MantaConnectionFactoryConfigurator` containing their custom instance to the
+   `MantaClient(ConfigContext, MantaConnectionFactoryConfigurator)` constructor. Constructor documentation
+   explains the benefits and trade offs.
 ### Fixed
  - Clarify version history of `MantaInputStreamEntity`
  - MPU parts which were missing an ETag in their response were

USAGE.md

@@ -335,4 +335,12 @@ and you wish to debug the establishment and leasing of HTTP connections:
 
 Please note that the Commons Logger adaptor is not a dependency of `java-manta-client-unshaded` and it is the user's
 responsibility to add their own dependency if they wish to collect Apache HttpClient logs. For more information on log
-bridging in SLF4J please review [this page](https://www.slf4j.org/legacy.html).
+bridging in SLF4J please review [this page](https://www.slf4j.org/legacy.html).
+
+### Customizing the client further
+
+It is possible to supply an `HttpClientBuilder` in order to further customize the behavior of a `MantaClient` instance.
+Users leveraging this feature should be comfortable with the internals of the Apache HttpClient library and
+familiarity with the
+[`MantaConnectionFactory`](/java-manta-client-unshaded/src/main/java/com/joyent/manta/http/MantaConnectionFactory.java)
+class is recommended.

java-manta-client-unshaded/src/main/java/com/joyent/manta/client/MantaClient.java

@@ -29,6 +29,7 @@
 import com.joyent.manta.http.HttpHelper;
 import com.joyent.manta.http.MantaApacheHttpClientContext;
 import com.joyent.manta.http.MantaConnectionFactory;
+import com.joyent.manta.http.MantaConnectionFactoryConfigurator;
 import com.joyent.manta.http.MantaContentTypes;
 import com.joyent.manta.http.MantaHttpHeaders;
 import com.joyent.manta.http.StandardHttpHelper;
@@ -39,7 +40,6 @@
 import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.BooleanUtils;
-import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.Validate;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpHeaders;
@@ -199,27 +199,49 @@ public class MantaClient implements AutoCloseable {
      * @param config The configuration context that provides all of the configuration values.
      */
     public MantaClient(final ConfigContext config) {
+        this(config, null);
+    }
+
 
+    /**
+     * Creates a new instance of the Manta client based on user-provided connection objects. This allows for a higher
+     * degree of customization at the cost of more involvement from the consumer.
+     *
+     * Users opting into advanced configuration (i.e. not passing {@code null} as the second parameter)
+     * should be comfortable with the internals of {@link CloseableHttpClient} and accept that we can only make a
+     * best effort to support all possible use-cases. For example, uses may pass in a builder which is wired to a
+     * {@link org.apache.http.impl.conn.BasicHttpClientConnectionManager} and effectively make the client
+     * single-threaded by eliminating the connection pool. Bug or feature? You decide!
+     *
+     * @param config The configuration context that provides all of the configuration values
+     * @param connectionFactoryConfigurator pre-configured objects for use with a MantaConnectionFactory
+     */
+    public MantaClient(final ConfigContext config,
+                       final MantaConnectionFactoryConfigurator connectionFactoryConfigurator) {
         dumpConfig(config);
 
         ConfigContext.validate(config);
 
-        this.config = config;
         this.url = config.getMantaURL();
+        this.config = config;
         this.home = ConfigContext.deriveHomeDirectoryFromUser(config.getMantaUser());
+        this.beanSupervisor = new MantaMBeanSupervisor();
 
         final KeyPair keyPair = new KeyPairFactory(config).createKeyPair();
 
         final Signer.Builder builder = new Signer.Builder(keyPair);
-        if (ObjectUtils.firstNonNull(
-                config.disableNativeSignatures(),
-                DefaultsConfigContext.DEFAULT_DISABLE_NATIVE_SIGNATURES)) {
+        if (BooleanUtils.isTrue(config.disableNativeSignatures())) {
             builder.providerCode("stdlib");
         }
         final ThreadLocalSigner signer = new ThreadLocalSigner(builder);
         this.signerRef = new WeakReference<>(signer);
 
-        final MantaConnectionFactory connectionFactory = new MantaConnectionFactory(config, keyPair, signer);
+        final MantaConnectionFactory connectionFactory = new MantaConnectionFactory(
+                config,
+                keyPair,
+                signer,
+                connectionFactoryConfigurator);
+
         final MantaApacheHttpClientContext connectionContext = new MantaApacheHttpClientContext(connectionFactory);
 
         if (BooleanUtils.isTrue(config.isClientEncryptionEnabled())) {
@@ -230,48 +252,14 @@ public MantaClient(final ConfigContext config) {
 
         this.uriSigner = new UriSigner(this.config, keyPair, signer);
 
-        this.beanSupervisor = new MantaMBeanSupervisor();
-
         beanSupervisor.expose(this.config);
         beanSupervisor.expose(connectionFactory);
     }
 
-    /**
-     * Creates a new instance of the Manta client with the specified connection factory.
-     * This method is typically used by unit tests to inject connection factory
-     * customizations so we can simulate different network events.
-     *
-     * @param config The configuration context that provides all of the configuration values
-     * @param keyPair cryptographic signing key pair used for HTTP signatures
-     * @param connectionFactory connection factory instance used to tune connection settings
-     */
-    MantaClient(final ConfigContext config,
-                final KeyPair keyPair,
-                final MantaConnectionFactory connectionFactory,
-                final ThreadLocalSigner signer) {
-        dumpConfig(config);
-
-        ConfigContext.validate(config);
-
-        this.url = config.getMantaURL();
-        this.config = config;
-        this.home = ConfigContext.deriveHomeDirectoryFromUser(config.getMantaUser());
-
-        this.signerRef = new WeakReference<>(signer);
-
-        final MantaApacheHttpClientContext connectionContext =
-                new MantaApacheHttpClientContext(connectionFactory);
-
-        if (BooleanUtils.isTrue(config.isClientEncryptionEnabled())) {
-            this.httpHelper = new EncryptionHttpHelper(connectionContext, config);
-        } else {
-            this.httpHelper = new StandardHttpHelper(connectionContext, config);
-        }
-
-        this.uriSigner = new UriSigner(this.config, keyPair, signer);
+    /* ======================================================================
+     * Constructor Helpers
+     * ====================================================================== */
 
-        this.beanSupervisor = new MantaMBeanSupervisor();
-    }
 
     /**
      * Dumps the configuration that is used to load a {@link MantaClient} if
@@ -295,6 +283,10 @@ private static void dumpConfig(final ConfigContext context) {
         }
     }
 
+    /* ======================================================================
+     * Object Access
+     * ====================================================================== */
+
     /**
      * Deletes an object from Manta.
      *
@@ -2414,6 +2406,10 @@ protected Stream<String> responseAsStream(final HttpResponse response)
         return stream;
     }
 
+    /* ======================================================================
+     * Lifecyle Methods
+     * ====================================================================== */
+
     /**
      * Flag indicating if the client is closed or is in the process of being
      * closed.

java-manta-client-unshaded/src/main/java/com/joyent/manta/http/MantaConnectionFactory.java

@@ -15,6 +15,7 @@
 import com.joyent.manta.config.DefaultsConfigContext;
 import com.joyent.manta.exception.ConfigurationException;
 import com.joyent.manta.util.MantaVersion;
+import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.Validate;
 import org.apache.http.Header;
@@ -118,36 +119,40 @@ public class MantaConnectionFactory implements Closeable, MantaMBeanable {
     /**
      * Create new instance using the passed configuration.
      *
-     * @param config        configuration of the connection parameters
-     * @param keyPair       cryptographic signing key pair used for HTTP signatures
-     * @param signer        Signer configured to use the given keyPair
+     * @param config    configuration of the connection parameters
+     * @param keyPair   cryptographic signing key pair used for HTTP signatures
+     * @param signer    Signer configured to use the given keyPair
      */
     public MantaConnectionFactory(final ConfigContext config,
                                   final KeyPair keyPair,
                                   final ThreadLocalSigner signer) {
-        Validate.notNull(config, "Configuration context must not be null");
+        this(config, keyPair, signer, null);
+    }
 
+    /**
+     * Create a new instance based on a shared {@link HttpClientBuilder} and {@link HttpClientConnectionManager}.
+     *
+     * @param config                        configuration of the connection parameters
+     * @param keyPair                       cryptographic signing key pair used for HTTP signatures
+     * @param signer                        Signer configured to use the given keyPair
+     * @param connectionFactoryConfigurator existing HttpClient objects to reuse
+     */
+    public MantaConnectionFactory(final ConfigContext config,
+                                  final KeyPair keyPair,
+                                  final ThreadLocalSigner signer,
+                                  final MantaConnectionFactoryConfigurator connectionFactoryConfigurator) {
+        Validate.notNull(config, "Configuration context must not be null");
         this.config = config;
 
-        this.connectionManager = buildConnectionManager();
-
-        this.httpClientBuilder = createBuilder();
-
-        final boolean authDisabled = ObjectUtils.firstNonNull(
-                config.noAuth(),
-                DefaultsConfigContext.DEFAULT_NO_AUTH);
-
-        if (!authDisabled) {
-            Validate.notNull(keyPair, "KeyPair must not be null if authentication is enabled");
-            Validate.notNull(signer, "Signer must not be null if authentication is enabled");
-
-            // pass true directly to the constructor because auth is enabled
-            final HttpRequestInterceptor authInterceptor = new HttpSignatureRequestInterceptor(
-                            new HttpSignatureAuthScheme(keyPair, signer),
-                            new UsernamePasswordCredentials(config.getMantaUser(), null),
-                            true);
-            this.httpClientBuilder.addInterceptorLast(authInterceptor);
+        if (connectionFactoryConfigurator != null) {
+            this.connectionManager = null;
+            this.httpClientBuilder = connectionFactoryConfigurator.getHttpClientBuilder();
+        } else {
+            this.connectionManager = buildConnectionManager();
+            this.httpClientBuilder = createStandardBuilder();
         }
+
+        configureHttpClientBuilderDefaults(keyPair, signer);
     }
 
     /**
@@ -186,6 +191,7 @@ protected SocketConfig buildSocketConfig() {
 
     /**
      * Builds and configures a {@link ConnectionConfig} instance.
+     *
      * @return fully configured instance
      */
     protected ConnectionConfig buildConnectionConfig() {
@@ -204,7 +210,7 @@ protected ConnectionConfig buildConnectionConfig() {
      *
      * @return fully configured connection manager
      */
-    protected PoolingHttpClientConnectionManager buildConnectionManager() {
+    protected HttpClientConnectionManager buildConnectionManager() {
         final int maxConns = ObjectUtils.firstNonNull(
                 config.getMaximumConnections(),
                 DefaultsConfigContext.DEFAULT_MAX_CONNS);
@@ -241,7 +247,7 @@ protected PoolingHttpClientConnectionManager buildConnectionManager() {
      *
      * @return configured instance
      */
-    protected HttpClientBuilder createBuilder() {
+    protected HttpClientBuilder createStandardBuilder() {
         final int maxConns = ObjectUtils.firstNonNull(
                 config.getMaximumConnections(),
                 DefaultsConfigContext.DEFAULT_MAX_CONNS);
@@ -264,7 +270,6 @@ protected HttpClientBuilder createBuilder() {
         final HttpClientBuilder builder = HttpClients.custom()
                 .disableAuthCaching()
                 .disableCookieManagement()
-                .setDefaultHeaders(HEADERS)
                 .setUserAgent(USER_AGENT)
                 .setConnectionReuseStrategy(new DefaultConnectionReuseStrategy())
                 .setMaxConnTotal(maxConns)
@@ -274,24 +279,51 @@ protected HttpClientBuilder createBuilder() {
                 .setRequestExecutor(new MantaHttpRequestExecutor())
                 .setConnectionBackoffStrategy(new DefaultBackoffStrategy());
 
+        final HttpHost proxyHost = findProxyServer();
+
+        if (proxyHost != null) {
+            builder.setProxy(proxyHost);
+        }
+
+        return builder;
+    }
+
+    /**
+     * Apply required configuration to an HttpClientBuilder that may have been created by us or provided externally.
+     *
+     * @param keyPair the keypair to use with signature authentication
+     * @param signer  Signer configured to use the given keyPair
+     */
+    private void configureHttpClientBuilderDefaults(final KeyPair keyPair,
+                                                    final ThreadLocalSigner signer) {
         if (config.getRetries() > 0) {
-            builder.setRetryHandler(new MantaHttpRequestRetryHandler(config))
-                   .setServiceUnavailableRetryStrategy(new MantaServiceUnavailableRetryStrategy(config));
+            httpClientBuilder.setRetryHandler(new MantaHttpRequestRetryHandler(config));
+            httpClientBuilder.setServiceUnavailableRetryStrategy(new MantaServiceUnavailableRetryStrategy(config));
         } else {
             LOGGER.info("Retry of failed requests is disabled");
-            builder.disableAutomaticRetries();
+            httpClientBuilder.disableAutomaticRetries();
         }
 
-        final HttpHost proxyHost = findProxyServer();
-
-        if (proxyHost != null) {
-            builder.setProxy(proxyHost);
+        // attach the connection manager if it was created by us
+        // users providing a custom HttpClientBuilder are expected to wire this up themselves
+        if (this.connectionManager != null) {
+            httpClientBuilder.setConnectionManager(this.connectionManager);
         }
 
-        builder.addInterceptorFirst(new RequestIdInterceptor());
-        builder.setConnectionManager(this.connectionManager);
+        httpClientBuilder.setDefaultHeaders(HEADERS);
+        httpClientBuilder.addInterceptorFirst(new RequestIdInterceptor());
 
-        return builder;
+        if (BooleanUtils.isNotTrue(config.noAuth())) {
+            Validate.notNull(keyPair, "KeyPair must not be null if authentication is enabled");
+            Validate.notNull(signer, "Signer must not be null if authentication is enabled");
+
+            // pass true directly to the constructor because auth is enabled
+            final HttpRequestInterceptor authInterceptor = new HttpSignatureRequestInterceptor(
+                    new HttpSignatureAuthScheme(keyPair, signer),
+                    new UsernamePasswordCredentials(config.getMantaUser(), null),
+                    true);
+            this.httpClientBuilder.addInterceptorLast(authInterceptor);
+        }
     }
 
     /**
@@ -343,16 +375,6 @@ public CloseableHttpClient createConnection() {
         return httpClientBuilder.build();
     }
 
-    /**
-     * package-private method for building a {@link MantaHttpRequestFactory} from this object's
-     * config. Should be removed with the deprecated constructor for {@link MantaApacheHttpClientContext}.
-     *
-     * @return a request factory pointed at the same url as {@code this}
-     */
-    ConfigContext getConfig() {
-        return config;
-    }
-
     @Override
     public DynamicMBean toMBean() {
         if (!(connectionManager instanceof PoolingHttpClientConnectionManager)) {
@@ -365,8 +387,10 @@ public DynamicMBean toMBean() {
     @Override
     public void close() throws IOException {
         if (connectionManager == null) {
+            // user provided their own connectionManager in the httpClientBuilder
             return;
         }
+
         connectionManager.shutdown();
     }
 }