Skip to content

Commit 3ad1860

Browse files
authored
Port putDirectory tunable to 3.2.x (#381)
Resolve #371 by adding new configuration parameter: manta.skip_directory_depth
1 parent 9c65a25 commit 3ad1860

14 files changed

Lines changed: 376 additions & 28 deletions

File tree

USAGE.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ Below is a table of available configuration parameters followed by detailed desc
9393
| manta.tcp_socket_timeout | MANTA_TCP_SOCKET_TIMEOUT | 10000 | |
9494
| manta.connection_request_timeout | MANTA_CONNECTION_REQUEST_TIMEOUT | 1000 | |
9595
| manta.upload_buffer_size | MANTA_UPLOAD_BUFFER_SIZE | 16384 | |
96+
| manta.skip_directory_depth | MANTA_SKIP_DIRECTORY_DEPTH | | |
9697
| manta.client_encryption | MANTA_CLIENT_ENCRYPTION | false | |
9798
| manta.encryption_key_id | MANTA_CLIENT_ENCRYPTION_KEY_ID | | |
9899
| manta.encryption_algorithm | MANTA_ENCRYPTION_ALGORITHM | AES128/CTR/NoPadding | |
@@ -148,6 +149,12 @@ Note: Dynamic Updates marked with an asterisk (*) are enabled by the `AuthAwareC
148149
The initial amount of bytes to attempt to load into memory when uploading a stream. If the
149150
entirety of the stream fits within the number of bytes of this value, then the
150151
contents of the buffer are directly uploaded to Manta in a retryable form.
152+
* `manta.skip_directory_depth` (**MANTA_SKIP_DIRECTORY_DEPTH**)
153+
Integer indicating the number of directory levels to attempt to skip when performing a recursive `putDirectory`
154+
operation. Set to 0 to disable the optimization entirely. Irrelevant when the depth of the recursive `putDirectory`
155+
call is less than the setting. When creating a directory with more levels than the setting, the client will attempt
156+
to skip this many non-system directories from the root. Will return to normal directory creation procedure if
157+
the skipped `PUT` fails or proceed creating all directories between the skip depth and the child on success.
151158
* `manta.client_encryption` (**MANTA_CLIENT_ENCRYPTION**)
152159
Boolean indicating if client-side encryption is enabled.
153160
* `manta.encryption_key_id` (**MANTA_CLIENT_ENCRYPTION_KEY_ID**)

checkstyle.xml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,9 @@
8282
<property name="excludeScope" value="package"/>
8383
</module>
8484
<module name="JavadocType" />
85-
<module name="JavadocVariable"/>
85+
<module name="JavadocVariable">
86+
<property name="ignoreNamePattern" value="LOG|LOGGER"/>
87+
</module>
8688
<module name="JavadocStyle" />
8789

8890
<!-- Checks for Naming Conventions. -->

java-manta-client-unshaded/src/main/java/com/joyent/manta/client/MantaClient.java

Lines changed: 11 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,6 @@
7979
import java.nio.charset.StandardCharsets;
8080
import java.nio.file.Files;
8181
import java.nio.file.Path;
82-
import java.nio.file.Paths;
8382
import java.nio.file.StandardCopyOption;
8483
import java.time.Instant;
8584
import java.time.temporal.TemporalAmount;
@@ -1576,13 +1575,14 @@ public void putDirectory(final String path, final boolean recursive)
15761575
/**
15771576
* Creates a directory in Manta.
15781577
*
1579-
* @param rawPath The fully qualified path of the Manta directory.
1578+
* @param rawPath The fully qualified path of the Manta directory.
15801579
* @param recursive recursive create all of the directories specified in the path
1581-
* @param headers Optional {@link MantaHttpHeaders}. Consult the Manta api for more header information.
1582-
* @throws IOException If an IO exception has occurred.
1580+
* @param headers Optional {@link MantaHttpHeaders}. Consult the Manta api for more header information.
1581+
* @throws IOException If an IO exception has occurred.
15831582
* @throws MantaClientHttpResponseException If a http status code {@literal > 300} is returned.
15841583
*/
1585-
public void putDirectory(final String rawPath, final boolean recursive,
1584+
public void putDirectory(final String rawPath,
1585+
final boolean recursive,
15861586
final MantaHttpHeaders headers)
15871587
throws IOException {
15881588
Validate.notBlank(rawPath, "rawPath must not be blank");
@@ -1592,23 +1592,12 @@ public void putDirectory(final String rawPath, final boolean recursive,
15921592
return;
15931593
}
15941594

1595-
final String[] parts = rawPath.split(SEPARATOR);
1596-
final Iterator<Path> itr = Paths.get("", parts).iterator();
1597-
final StringBuilder sb = new StringBuilder(SEPARATOR);
1598-
1599-
for (int i = 0; itr.hasNext(); i++) {
1600-
final String part = itr.next().toString();
1601-
sb.append(part);
1602-
1603-
// This means we aren't in the home nor in the reserved
1604-
// directory path (stor, public, jobs, etc)
1605-
if (i > 1) {
1606-
putDirectory(sb.toString(), headers);
1607-
}
1608-
1609-
if (itr.hasNext()) {
1610-
sb.append(SEPARATOR);
1611-
}
1595+
final Integer skipDepth = config.getSkipDirectoryDepth();
1596+
final RecursiveDirectoryCreationStrategy directoryCreationStrategy;
1597+
if (skipDepth != null && 0 < skipDepth) {
1598+
RecursiveDirectoryCreationStrategy.createWithSkipDepth(this, rawPath, headers, skipDepth);
1599+
} else {
1600+
RecursiveDirectoryCreationStrategy.createCompletely(this, rawPath, headers);
16121601
}
16131602
}
16141603

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*
2+
* Copyright (c) 2017, Joyent, Inc. All rights reserved.
3+
*
4+
* This Source Code Form is subject to the terms of the Mozilla Public
5+
* License, v. 2.0. If a copy of the MPL was not distributed with this
6+
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
7+
*/
8+
package com.joyent.manta.client;
9+
10+
import com.joyent.manta.exception.MantaClientHttpResponseException;
11+
import com.joyent.manta.exception.MantaErrorCode;
12+
import com.joyent.manta.http.MantaHttpHeaders;
13+
import org.slf4j.Logger;
14+
import org.slf4j.LoggerFactory;
15+
16+
import java.io.IOException;
17+
18+
import static com.joyent.manta.util.MantaUtils.writeablePrefixPaths;
19+
20+
/**
21+
* Utility class for recursive directory creation strategies.
22+
*
23+
* @author <a href="https://github.com/tjcelaya">Tomas Celayac</a>
24+
* @since 3.1.7
25+
*/
26+
final class RecursiveDirectoryCreationStrategy {
27+
28+
private static final Logger LOG = LoggerFactory.getLogger(RecursiveDirectoryCreationStrategy.class);
29+
30+
private RecursiveDirectoryCreationStrategy() {
31+
}
32+
33+
static long createWithSkipDepth(final MantaClient client,
34+
final String rawPath,
35+
final MantaHttpHeaders headers,
36+
final int skipDepth) throws IOException {
37+
final String[] paths = writeablePrefixPaths(rawPath);
38+
39+
if (paths.length <= skipDepth) {
40+
return createCompletely(client, rawPath, headers);
41+
}
42+
43+
final String assumedExistingDirectory = paths[skipDepth - 1];
44+
final String maybeNewDirectory = paths[skipDepth];
45+
46+
LOG.debug("ASSUME {}", assumedExistingDirectory);
47+
48+
final Boolean redundantPut = createNewDirectory(client, maybeNewDirectory, headers, rawPath);
49+
long ops = 1;
50+
51+
if (redundantPut == null) {
52+
LOG.debug("FAILED {}", maybeNewDirectory);
53+
54+
// failed to create directory at the skip depth, proceed normally
55+
return ops + createCompletely(client, rawPath, headers);
56+
}
57+
58+
for (int idx = skipDepth + 1; idx < paths.length; idx++) {
59+
client.putDirectory(paths[idx], headers);
60+
ops++;
61+
}
62+
63+
return ops;
64+
}
65+
66+
static long createCompletely(final MantaClient client,
67+
final String rawPath,
68+
final MantaHttpHeaders headers) throws IOException {
69+
long ops = 0;
70+
for (final String path : writeablePrefixPaths(rawPath)) {
71+
client.putDirectory(path, headers);
72+
ops++;
73+
}
74+
75+
return ops;
76+
}
77+
78+
/**
79+
* Try to create a directory and unpack the error. The Boolean is intentional and acts as a tri-state variable.
80+
*
81+
* NULL = creation failed.
82+
* TRUE = a new directory was actually created
83+
* FALSE = the directory already existed
84+
*
85+
* @return whether or not the directory was actually new, or null if it failed to be created
86+
*/
87+
private static Boolean createNewDirectory(final MantaClient client,
88+
final String path,
89+
final MantaHttpHeaders headers,
90+
final String targetPath) throws IOException {
91+
try {
92+
return client.putDirectory(path, headers);
93+
} catch (final MantaClientHttpResponseException mchre) {
94+
if (mchre.getServerCode().equals(MantaErrorCode.DIRECTORY_DOES_NOT_EXIST_ERROR)) {
95+
return null;
96+
} else {
97+
mchre.setContextValue("recursiveDirectoryCreationTarget", targetPath);
98+
throw mchre;
99+
}
100+
}
101+
}
102+
}

java-manta-client-unshaded/src/main/java/com/joyent/manta/config/BaseChainedConfigContext.java

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,11 @@ public abstract class BaseChainedConfigContext implements SettableConfigContext<
105105
*/
106106
private volatile Boolean verifyUploads;
107107

108+
/**
109+
* Number of directories to assume exist when recursively creating directories.
110+
*/
111+
private Integer skipDirectoryDepth;
112+
108113
/**
109114
* Number of bytes to read into memory for a streaming upload before
110115
* deciding if we want to load it in memory before send it.
@@ -262,6 +267,11 @@ public Boolean verifyUploads() {
262267
return verifyUploads;
263268
}
264269

270+
@Override
271+
public Integer getSkipDirectoryDepth() {
272+
return this.skipDirectoryDepth;
273+
}
274+
265275
@Override
266276
public Integer getUploadBufferSize() {
267277
return uploadBufferSize;
@@ -406,6 +416,10 @@ public void overwriteWithContext(final ConfigContext context) {
406416
this.uploadBufferSize = context.getUploadBufferSize();
407417
}
408418

419+
if (context.getSkipDirectoryDepth() != null) {
420+
this.skipDirectoryDepth = context.getSkipDirectoryDepth();
421+
}
422+
409423
if (context.isClientEncryptionEnabled() != null) {
410424
this.clientEncryptionEnabled = context.isClientEncryptionEnabled();
411425
}
@@ -510,6 +524,10 @@ protected void overwriteWithDefaultContext(final DefaultsConfigContext context)
510524
this.uploadBufferSize = context.getUploadBufferSize();
511525
}
512526

527+
if (this.skipDirectoryDepth == null) {
528+
this.skipDirectoryDepth = context.getSkipDirectoryDepth();
529+
}
530+
513531
if (this.clientEncryptionEnabled == null) {
514532
this.clientEncryptionEnabled = context.isClientEncryptionEnabled();
515533
}
@@ -686,6 +704,13 @@ public BaseChainedConfigContext setUploadBufferSize(final Integer size) {
686704
return this;
687705
}
688706

707+
@Override
708+
public BaseChainedConfigContext setSkipDirectoryDepth(final Integer depth) {
709+
this.skipDirectoryDepth = depth;
710+
711+
return this;
712+
}
713+
689714
@Override
690715
public BaseChainedConfigContext setClientEncryptionEnabled(final Boolean clientEncryptionEnabled) {
691716
this.clientEncryptionEnabled = clientEncryptionEnabled;
@@ -774,6 +799,7 @@ public boolean equals(final Object other) {
774799
&& Objects.equals(connectionRequestTimeout, that.connectionRequestTimeout)
775800
&& Objects.equals(verifyUploads, that.verifyUploads)
776801
&& Objects.equals(uploadBufferSize, that.uploadBufferSize)
802+
&& Objects.equals(skipDirectoryDepth, that.skipDirectoryDepth)
777803
&& Objects.equals(clientEncryptionEnabled, that.clientEncryptionEnabled)
778804
&& Objects.equals(encryptionKeyId, that.encryptionKeyId)
779805
&& Objects.equals(encryptionAlgorithm, that.encryptionAlgorithm)
@@ -790,6 +816,7 @@ public int hashCode() {
790816
httpBufferSize, httpsProtocols, httpsCipherSuites, noAuth,
791817
disableNativeSignatures, tcpSocketTimeout, connectionRequestTimeout,
792818
verifyUploads, uploadBufferSize,
819+
skipDirectoryDepth,
793820
clientEncryptionEnabled, encryptionKeyId,
794821
encryptionAlgorithm, permitUnencryptedDownloads,
795822
encryptionAuthenticationMode, encryptionPrivateKeyPath,

java-manta-client-unshaded/src/main/java/com/joyent/manta/config/ConfigContext.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,11 @@ default String getMantaHomeDirectory() {
130130
*/
131131
Integer getUploadBufferSize();
132132

133+
/**
134+
* @return number of directories of depth to assume would exist when creating directories
135+
*/
136+
Integer getSkipDirectoryDepth();
137+
133138
/**
134139
* @return true when client-side encryption is enabled.
135140
*/
@@ -218,6 +223,7 @@ static String toString(final ConfigContext context) {
218223
sb.append(", connectionRequestTimeout=").append(context.getConnectionRequestTimeout());
219224
sb.append(", verifyUploads=").append(context.verifyUploads());
220225
sb.append(", uploadBufferSize=").append(context.getUploadBufferSize());
226+
sb.append(", skipDirectoryDepth=").append(context.getSkipDirectoryDepth());
221227
sb.append(", clientEncryptionEnabled=").append(context.isClientEncryptionEnabled());
222228
sb.append(", permitUnencryptedDownloads=").append(context.permitUnencryptedDownloads());
223229
sb.append(", encryptionAuthenticationMode=").append(context.getEncryptionAuthenticationMode());
@@ -294,6 +300,10 @@ static void validate(final ConfigContext config) {
294300
}
295301
}
296302

303+
if (config.getSkipDirectoryDepth() != null && config.getSkipDirectoryDepth() < 0) {
304+
failureMessages.add("Manta skip directory depth must be 0 or greater");
305+
}
306+
297307
if (BooleanUtils.isTrue(config.isClientEncryptionEnabled())) {
298308
encryptionSettings(config, failureMessages);
299309
}
@@ -493,6 +503,9 @@ static Object getAttributeFromContext(final String attribute, final ConfigContex
493503
case MapConfigContext.MANTA_UPLOAD_BUFFER_SIZE_KEY:
494504
case EnvVarConfigContext.MANTA_UPLOAD_BUFFER_SIZE_ENV_KEY:
495505
return config.getUploadBufferSize();
506+
case MapConfigContext.MANTA_SKIP_DIRECTORY_DEPTH_KEY:
507+
case EnvVarConfigContext.MANTA_SKIP_DIRECTORY_DEPTH_ENV_KEY:
508+
return config.getSkipDirectoryDepth();
496509
case MapConfigContext.MANTA_PERMIT_UNENCRYPTED_DOWNLOADS_KEY:
497510
case EnvVarConfigContext.MANTA_PERMIT_UNENCRYPTED_DOWNLOADS_ENV_KEY:
498511
return config.permitUnencryptedDownloads();

java-manta-client-unshaded/src/main/java/com/joyent/manta/config/ConfigContextMBean.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,10 @@ private MBeanAttributeInfo[] beanAttributeInfoBuilder() {
132132
Integer.class.getName(),
133133
"The size of pre-streaming upload buffers",
134134
true, this.isSettable, false),
135+
new MBeanAttributeInfo(MapConfigContext.MANTA_SKIP_DIRECTORY_DEPTH_KEY,
136+
Integer.class.getName(),
137+
"The depth of directories to skip when attempting creating directories recursively",
138+
true, this.isSettable, false),
135139
new MBeanAttributeInfo(MapConfigContext.MANTA_CLIENT_ENCRYPTION_ENABLED_KEY,
136140
Boolean.class.getName(),
137141
"Flag indicating client-side encryption is enabled",

java-manta-client-unshaded/src/main/java/com/joyent/manta/config/DefaultsConfigContext.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,11 @@ public Integer getUploadBufferSize() {
217217
return DEFAULT_UPLOAD_BUFFER_SIZE;
218218
}
219219

220+
@Override
221+
public Integer getSkipDirectoryDepth() {
222+
return null;
223+
}
224+
220225
@Override
221226
public Boolean isClientEncryptionEnabled() {
222227
return false;

java-manta-client-unshaded/src/main/java/com/joyent/manta/config/EnvVarConfigContext.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,11 @@ public class EnvVarConfigContext implements ConfigContext {
109109
*/
110110
public static final String MANTA_CONNECTION_REQUEST_TIMEOUT_ENV_KEY = "MANTA_CONNECTION_REQUEST_TIMEOUT";
111111

112+
/**
113+
* Environment variable for setting the depth of directories to assume exists.
114+
*/
115+
public static final String MANTA_SKIP_DIRECTORY_DEPTH_ENV_KEY = "MANTA_SKIP_DIRECTORY_DEPTH";
116+
112117
/**
113118
* Environment variable for flag indicating when client-side encryption is enabled.
114119
*/
@@ -162,6 +167,7 @@ public class EnvVarConfigContext implements ConfigContext {
162167
MANTA_CONNECTION_REQUEST_TIMEOUT_ENV_KEY,
163168
MANTA_VERIFY_UPLOADS_ENV_KEY,
164169
MANTA_UPLOAD_BUFFER_SIZE_ENV_KEY,
170+
MANTA_SKIP_DIRECTORY_DEPTH_ENV_KEY,
165171
MANTA_CLIENT_ENCRYPTION_ENABLED_ENV_KEY,
166172
MANTA_ENCRYPTION_KEY_ID_ENV_KEY,
167173
MANTA_PERMIT_UNENCRYPTED_DOWNLOADS_ENV_KEY,
@@ -287,6 +293,11 @@ public Boolean verifyUploads() {
287293
return MantaUtils.parseBooleanOrNull(verify);
288294
}
289295

296+
@Override
297+
public Integer getSkipDirectoryDepth() {
298+
return MantaUtils.parseIntegerOrNull(getEnv(MANTA_SKIP_DIRECTORY_DEPTH_ENV_KEY));
299+
}
300+
290301
@Override
291302
public Integer getUploadBufferSize() {
292303
String buffString = getEnv(MANTA_UPLOAD_BUFFER_SIZE_ENV_KEY);

0 commit comments

Comments
 (0)