Skip to content

Commit e36725e

Browse files
authored
Fixes #266 - MDC logging of the load balancer address now logs the proper address (#355)
* Fixes #266 This introduces a new implementation of a HttpRequestExecutor that logs the load balancer IP address and wraps Apache HTTP Client exceptions with exceptions that provide additional logging.
1 parent ed31c89 commit e36725e

9 files changed

Lines changed: 208 additions & 8 deletions

File tree

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ This project aims to adhere to [Semantic Versioning](http://semver.org/).
1919
leave the default Apache HttpClient [retry behavior](https://hc.apache.org/httpcomponents-client-4.5.x/tutorial/html/fundamentals.html#d5e316).
2020
- [Content type is set for file object in directory listing when it isn't available](https://github.com/joyent/java-manta/issues/341)
2121
- [Fixes validation guard clauses that are not validating anything](https://github.com/joyent/java-manta/issues/346)
22+
- [MDC logging of the load balancer address now logs the proper address](https://github.com/joyent/java-manta/issues/266)
2223
### Changed
2324
- Validation of paths passed to `MantaClient` is now more consistently strict.
2425
More useful errors should be thrown sooner for invalid paths, without any

java-manta-client-unshaded/src/main/java/com/joyent/manta/client/MantaClient.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import com.joyent.manta.exception.MantaException;
2323
import com.joyent.manta.exception.MantaIOException;
2424
import com.joyent.manta.exception.MantaJobException;
25+
import com.joyent.manta.exception.MantaNoHttpResponseException;
2526
import com.joyent.manta.exception.OnCloseAggregateException;
2627
import com.joyent.manta.http.ContentTypeLookup;
2728
import com.joyent.manta.http.EncryptionHttpHelper;
@@ -1756,7 +1757,7 @@ public UUID createJob(final MantaJob job) throws IOException {
17561757
try {
17571758
return httpHelper.executeAndCloseRequest(post,
17581759
jobIdFunction, "POST {} response [{}] {} ", path);
1759-
} catch (NoHttpResponseException e) {
1760+
} catch (NoHttpResponseException | MantaNoHttpResponseException e) {
17601761
lastException = e;
17611762
LOG.warn("Error posting createJob. Retrying.", e);
17621763
}

java-manta-client-unshaded/src/main/java/com/joyent/manta/client/MantaObjectOutputStream.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,10 @@ public synchronized void close() throws IOException {
310310
} catch (InterruptedException e) {
311311
// continue execution if interrupted
312312
} catch (ExecutionException e) {
313-
MantaIOException mioe = new MantaIOException(e);
313+
/* We wrap the cause because the stack trace for the
314+
* ExecutionException offers nothing useful and is just a wrapper
315+
* for exceptions that are thrown within a Future. */
316+
MantaIOException mioe = new MantaIOException(e.getCause());
314317

315318
if (this.objectResponse != null) {
316319
final String requestId = this.objectResponse.getHeaderAsString(
Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Copyright (c) 2017, Joyent, Inc. All rights reserved.
3+
*
4+
* This Source Code Form is subject to the terms of the Mozilla Public
5+
* License, v. 2.0. If a copy of the MPL was not distributed with this
6+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
7+
*/
8+
package com.joyent.manta.exception;
9+
10+
/**
11+
* Signals that the target server failed to respond with a valid HTTP response.
12+
*
13+
* This exception typically wraps {@link org.apache.http.NoHttpResponseException}.
14+
*
15+
* @author <a href="https://github.com/dekobon">Elijah Zupancic</a>
16+
* @since 3.1.7
17+
*/
18+
public class MantaNoHttpResponseException extends MantaIOException {
19+
/**
20+
* Constructs an instance with {@code null}
21+
* as its error detail message.
22+
*/
23+
public MantaNoHttpResponseException() {
24+
}
25+
26+
/**
27+
* Constructs an instance with the specified detail message.
28+
*
29+
* @param message The detail message (which is saved for later retrieval
30+
* by the {@link #getMessage()} method)
31+
*/
32+
public MantaNoHttpResponseException(final String message) {
33+
super(message);
34+
}
35+
36+
/**
37+
* Constructs an instance with the specified detail message
38+
* and cause.
39+
*
40+
* <p>Note that the detail message associated with {@code cause} is
41+
* <i>not</i> automatically incorporated into this exception's detail
42+
* message.</p>
43+
*
44+
* @param message The detail message (which is saved for later retrieval
45+
* by the {@link #getMessage()} method)
46+
* @param cause The cause (which is saved for later retrieval by the
47+
* {@link #getCause()} method). (A null value is permitted,
48+
* and indicates that the cause is nonexistent or unknown.)
49+
*/
50+
public MantaNoHttpResponseException(final String message, final Throwable cause) {
51+
super(message, cause);
52+
}
53+
54+
/**
55+
* Constructs an instance with the specified cause and a
56+
* detail message of {@code (cause==null ? null : cause.toString())}
57+
* (which typically contains the class and detail message of {@code cause}).
58+
* This constructor is useful for IO exceptions that are little more
59+
* than wrappers for other throwables.
60+
*
61+
* @param cause The cause (which is saved for later retrieval by the
62+
* {@link #getCause()} method). (A null value is permitted,
63+
* and indicates that the cause is nonexistent or unknown.)
64+
*/
65+
public MantaNoHttpResponseException(final Throwable cause) {
66+
super(cause);
67+
}
68+
}

java-manta-client-unshaded/src/main/java/com/joyent/manta/http/MantaConnectionFactory.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import org.slf4j.Logger;
4949
import org.slf4j.LoggerFactory;
5050

51+
import javax.management.DynamicMBean;
5152
import java.io.Closeable;
5253
import java.io.IOException;
5354
import java.net.InetSocketAddress;
@@ -58,7 +59,6 @@
5859
import java.util.Arrays;
5960
import java.util.Collection;
6061
import java.util.List;
61-
import javax.management.DynamicMBean;
6262

6363
/**
6464
* Factory class that creates instances of
@@ -271,6 +271,7 @@ protected HttpClientBuilder createBuilder() {
271271
.setKeepAliveStrategy(new DefaultConnectionKeepAliveStrategy())
272272
.setDefaultRequestConfig(requestConfig)
273273
.setConnectionManagerShared(false)
274+
.setRequestExecutor(new MantaHttpRequestExecutor())
274275
.setConnectionBackoffStrategy(new DefaultBackoffStrategy());
275276

276277
if (config.getRetries() > 0) {
Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
/*
2+
* Copyright (c) 2017, Joyent, Inc. All rights reserved.
3+
*
4+
* This Source Code Form is subject to the terms of the Mozilla Public
5+
* License, v. 2.0. If a copy of the MPL was not distributed with this
6+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
7+
*/
8+
package com.joyent.manta.http;
9+
10+
import com.joyent.manta.exception.MantaIOException;
11+
import com.joyent.manta.exception.MantaNoHttpResponseException;
12+
import org.apache.commons.lang3.StringUtils;
13+
import org.apache.http.HttpClientConnection;
14+
import org.apache.http.HttpException;
15+
import org.apache.http.HttpRequest;
16+
import org.apache.http.HttpResponse;
17+
import org.apache.http.NoHttpResponseException;
18+
import org.apache.http.protocol.HttpContext;
19+
import org.apache.http.protocol.HttpRequestExecutor;
20+
import org.slf4j.MDC;
21+
22+
import java.io.IOException;
23+
24+
/**
25+
* Extended implementation of {@link HttpRequestExecutor} with Manta specific
26+
* extensions for logging and exception handling.
27+
*
28+
* @author <a href="https://github.com/dekobon">Elijah Zupancic</a>
29+
* @since 3.1.7
30+
*/
31+
public class MantaHttpRequestExecutor extends HttpRequestExecutor {
32+
/**
33+
* Creates new instance of HttpRequestExecutor.
34+
*
35+
* @param waitForContinue Maximum time in milliseconds to wait for a 100-continue response
36+
*/
37+
public MantaHttpRequestExecutor(final int waitForContinue) {
38+
super(waitForContinue);
39+
}
40+
41+
/**
42+
* Creates new instance of HttpRequestExecutor.
43+
*/
44+
public MantaHttpRequestExecutor() {
45+
}
46+
47+
/**
48+
* Adds a context value for the Manta load balancer associated with the
49+
* request to the MDC object with the key <code>mantaLoadBalancerAddress</code>
50+
* and proxies the parent class
51+
* {@link HttpRequestExecutor#doSendRequest(HttpRequest, HttpClientConnection, HttpContext)}
52+
* method.
53+
*
54+
* {@inheritDoc}
55+
*/
56+
@Override
57+
protected HttpResponse doSendRequest(final HttpRequest request,
58+
final HttpClientConnection conn,
59+
final HttpContext context) throws IOException, HttpException {
60+
MDC.put("mantaLoadBalancerAddress", extractLoadBalancerAddress(conn));
61+
return super.doSendRequest(request, conn, context);
62+
}
63+
64+
/**
65+
* Proxies the parent class
66+
* {@link HttpRequestExecutor#doReceiveResponse(HttpRequest, HttpClientConnection, HttpContext)}
67+
* method and catches {@link IOException} instances thrown. Those exceptions
68+
* are then wrapped in a {@link MantaIOException} or
69+
* {@link MantaNoHttpResponseException} instance in order to provide
70+
* detailed information for debugging.
71+
*
72+
* {@inheritDoc}
73+
*/
74+
@Override
75+
protected HttpResponse doReceiveResponse(
76+
final HttpRequest request,
77+
final HttpClientConnection conn,
78+
final HttpContext context) throws HttpException, IOException {
79+
HttpResponse response = null;
80+
81+
try {
82+
response = super.doReceiveResponse(request, conn, context);
83+
84+
/* We catch all IOExceptions and wrap then in a MantaIOException because
85+
* this allows us to capture key information like the request id and
86+
* load balancer address directly in the exception message. */
87+
} catch (IOException e) {
88+
final MantaIOException mioe;
89+
90+
/* If the source exception is NoHttpResponseException we create
91+
* a MantaNoHttpResponseException, so that we can act upon that
92+
* exception type directly within Manta. */
93+
if (e instanceof NoHttpResponseException) {
94+
mioe = new MantaNoHttpResponseException(e);
95+
} else {
96+
mioe = new MantaIOException(e);
97+
}
98+
99+
HttpHelper.annotateContextedException(mioe, request, response);
100+
101+
if (request.getFirstHeader(MantaHttpHeaders.REQUEST_ID) != null) {
102+
mioe.setContextValue("requestId",
103+
request.getFirstHeader(MantaHttpHeaders.REQUEST_ID).getValue());
104+
}
105+
106+
mioe.setContextValue("loadBalancerAddress", extractLoadBalancerAddress(conn));
107+
108+
throw mioe;
109+
}
110+
111+
return response;
112+
}
113+
114+
/**
115+
* Extracts the remote load balancer IP address from the toString() method
116+
* of a {@link HttpClientConnection}.
117+
*
118+
* @param conn connection to extract IP information from
119+
* @return IP address string or null if connection is null
120+
*/
121+
private static String extractLoadBalancerAddress(final HttpClientConnection conn) {
122+
if (conn == null) {
123+
return null;
124+
}
125+
126+
return StringUtils.substringBetween(conn.toString(), "<->", ":");
127+
}
128+
}

java-manta-client-unshaded/src/main/java/com/joyent/manta/http/ShufflingDnsResolver.java

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
package com.joyent.manta.http;
99

1010
import org.apache.http.conn.DnsResolver;
11-
import org.slf4j.MDC;
1211

1312
import java.net.InetAddress;
1413
import java.net.UnknownHostException;
@@ -28,8 +27,6 @@ public InetAddress[] resolve(final String host) throws UnknownHostException {
2827
final InetAddress[] addresses = InetAddress.getAllByName(host);
2928
shuffle(addresses);
3029

31-
MDC.put("mantaLoadBalancerAddress", addresses[0].getHostAddress());
32-
3330
return addresses;
3431
}
3532

java-manta-client-unshaded/src/test/java/com/joyent/manta/client/MantaClientConnectionFailuresIT.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import com.joyent.manta.config.ConfigContext;
1313
import com.joyent.manta.config.KeyPairFactory;
1414
import com.joyent.manta.config.TestConfigContext;
15+
import com.joyent.manta.exception.MantaNoHttpResponseException;
1516
import com.joyent.manta.http.MantaConnectionFactory;
1617
import org.apache.http.HttpException;
1718
import org.apache.http.HttpResponse;
@@ -125,7 +126,7 @@ public void canRetryOnNoHttpResponseException() throws IOException {
125126
boolean thrown = false;
126127
try {
127128
mantaClient.head(testPathPrefix);
128-
} catch (NoHttpResponseException e) {
129+
} catch (NoHttpResponseException | MantaNoHttpResponseException e) {
129130
thrown = true;
130131
} finally {
131132
mantaClient.closeWithWarning();

java-manta-it/src/test/resources/logback-test.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<!-- encoders are assigned the type
44
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
55
<encoder>
6-
<pattern>[%thread] %-5level %logger [%X{mantaRequestId}] - %msg%n</pattern>
6+
<pattern>[%thread] %-5level %logger [%X{mantaLoadBalancerAddress} %X{mantaRequestId}] - %msg%n</pattern>
77
</encoder>
88
</appender>
99

0 commit comments

Comments
 (0)