Skip to content

Commit

Permalink
Add generic AWS S3 domain support (#1470)
Browse files Browse the repository at this point in the history 
Signed-off-by: Bala.FA <[email protected]>
  • Loading branch information
@balamurugana
balamurugana authored Aug 17, 2023
1 parent 0a04661 commit 6f1a144
Show file tree
Hide file tree
Showing 5 changed files with 293 additions and 197 deletions.
38 changes: 22 additions & 16 deletions api/src/main/java/io/minio/BucketArgs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@

package io.minio;

import io.minio.http.HttpUtils;
import io.minio.org.apache.commons.validator.routines.InetAddressValidator;
import java.util.Objects;
import java.util.regex.Pattern;

/** Base argument class holds bucket name and region. */
public abstract class BucketArgs extends BaseArgs {
Expand All @@ -34,32 +37,35 @@ public String region() {
/** Base argument builder class for {@link BucketArgs}. */
public abstract static class Builder<B extends Builder<B, A>, A extends BucketArgs>
extends BaseArgs.Builder<B, A> {
private static final Pattern BUCKET_NAME_REGEX =
Pattern.compile("^[a-z0-9][a-z0-9\\.\\-]{1,61}[a-z0-9]$");

protected void validateBucketName(String name) {
validateNotNull(name, "bucket name");

// Bucket names cannot be no less than 3 and no more than 63 characters long.
if (name.length() < 3 || name.length() > 63) {
if (!BUCKET_NAME_REGEX.matcher(name).find()) {
throw new IllegalArgumentException(
name + " : " + "bucket name must be at least 3 and no more than 63 characters long");
"bucket name '"
+ name
+ "' does not follow Amazon S3 standards. For more information refer "
+ "https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html");
}
// Successive periods in bucket names are not allowed.
if (name.contains("..")) {
String msg =
"bucket name cannot contain successive periods. For more information refer "
+ "http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html";
throw new IllegalArgumentException(name + " : " + msg);

if (InetAddressValidator.getInstance().isValidInet4Address(name)) {
throw new IllegalArgumentException(
"bucket name '" + name + "' must not be formatted as an IP address");
}
// Bucket names should be dns compatible.
if (!name.matches("^[a-z0-9][a-z0-9\\.\\-]+[a-z0-9]$")) {
String msg =
"bucket name does not follow Amazon S3 standards. For more information refer "
+ "http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html";
throw new IllegalArgumentException(name + " : " + msg);

if (name.contains("..") || name.contains(".-") || name.contains("-.")) {
throw new IllegalArgumentException(
"bucket name '" + name + "' cannot contain successive characters '..', '.-' and '-.'");
}
}

private void validateRegion(String region) {
validateNullOrNotEmptyString(region, "region");
if (region != null && !HttpUtils.REGION_REGEX.matcher(region).find()) {
throw new IllegalArgumentException("invalid region " + region);
}
}

@Override
Expand Down
148 changes: 65 additions & 83 deletions api/src/main/java/io/minio/MinioAsyncClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
import java.nio.file.StandardOpenOption;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.LinkedList;
Expand All @@ -75,6 +76,7 @@
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.ExecutionException;
import java.util.regex.Matcher;
import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
import okhttp3.Request;
Expand Down Expand Up @@ -131,22 +133,20 @@
public class MinioAsyncClient extends S3Base {
private MinioAsyncClient(
HttpUrl baseUrl,
String region,
boolean isAwsHost,
boolean isFipsHost,
boolean isAccelerateHost,
boolean isDualStackHost,
String awsS3Prefix,
String awsDomainSuffix,
boolean awsDualstack,
boolean useVirtualStyle,
String region,
Provider provider,
OkHttpClient httpClient) {
super(
baseUrl,
region,
isAwsHost,
isFipsHost,
isAccelerateHost,
isDualStackHost,
awsS3Prefix,
awsDomainSuffix,
awsDualstack,
useVirtualStyle,
region,
provider,
httpClient);
}
Expand Down Expand Up @@ -3213,87 +3213,66 @@ public static Builder builder() {
/** Argument builder of {@link MinioClient}. */
public static final class Builder {
private HttpUrl baseUrl;
private String region;
private boolean isAwsHost;
private boolean isFipsHost;
private boolean isAccelerateHost;
private boolean isDualStackHost;
private String awsS3Prefix;
private String awsDomainSuffix;
private boolean awsDualstack;
private boolean useVirtualStyle;

private String region;
private Provider provider;
private OkHttpClient httpClient;

private boolean isAwsChinaHost;
private String regionInUrl;
private void setAwsInfo(String host, boolean https) {
this.awsS3Prefix = null;
this.awsDomainSuffix = null;
this.awsDualstack = false;

private boolean isAwsFipsEndpoint(String endpoint) {
return endpoint.startsWith("s3-fips.");
}
if (!HttpUtils.HOSTNAME_REGEX.matcher(host).find()) return;

private boolean isAwsAccelerateEndpoint(String endpoint) {
return endpoint.startsWith("s3-accelerate.");
}
if (HttpUtils.AWS_ELB_ENDPOINT_REGEX.matcher(host).find()) {
String[] tokens = host.split("\\.elb\\.amazonaws\\.com", 1)[0].split("\\.");
this.region = tokens[tokens.length - 1];
return;
}

private boolean isAwsEndpoint(String endpoint) {
return (endpoint.startsWith("s3.")
|| isAwsFipsEndpoint(endpoint)
|| isAwsAccelerateEndpoint(endpoint))
&& (endpoint.endsWith(".amazonaws.com") || endpoint.endsWith(".amazonaws.com.cn"));
}
if (!HttpUtils.AWS_ENDPOINT_REGEX.matcher(host).find()) return;

private boolean isAwsDualStackEndpoint(String endpoint) {
return endpoint.contains(".dualstack.");
}
if (!HttpUtils.AWS_S3_ENDPOINT_REGEX.matcher(host).find()) {
throw new IllegalArgumentException("invalid Amazon AWS host " + host);
}

Matcher matcher = HttpUtils.AWS_S3_PREFIX_REGEX.matcher(host);
matcher.lookingAt();
int end = matcher.end();

this.awsS3Prefix = host.substring(0, end);
if (this.awsS3Prefix.contains("s3-accesspoint") && !https) {
throw new IllegalArgumentException("use HTTPS scheme for host " + host);
}

/**
* Extracts region from AWS endpoint if available. Region is placed at second token normal
* endpoints and third token for dualstack endpoints.
*
* <p>Region is marked in square brackets in below examples.
* <pre>
* https://s3.[us-east-2].amazonaws.com
* https://s3.dualstack.[ca-central-1].amazonaws.com
* https://s3.[cn-north-1].amazonaws.com.cn
* https://s3.dualstack.[cn-northwest-1].amazonaws.com.cn
*/
private String extractRegion(String endpoint) {
String[] tokens = endpoint.split("\\.");
String token = tokens[1];

// If token is "dualstack", then region might be in next token.
if (token.equals("dualstack")) {
token = tokens[2];
String[] tokens = host.substring(end).split("\\.");
awsDualstack = "dualstack".equals(tokens[0]);
if (awsDualstack) tokens = Arrays.copyOfRange(tokens, 1, tokens.length);
String regionInHost = null;
if (!tokens[0].equals("vpce") && !tokens[0].equals("amazonaws")) {
regionInHost = tokens[0];
tokens = Arrays.copyOfRange(tokens, 1, tokens.length);
}
this.awsDomainSuffix = String.join(".", tokens);

// If token is equal to "amazonaws", region is not passed in the endpoint.
if (token.equals("amazonaws")) {
return null;
if (host.equals("s3-external-1.amazonaws.com")) regionInHost = "us-east-1";
if (host.equals("s3-us-gov-west-1.amazonaws.com")
|| host.equals("s3-fips-us-gov-west-1.amazonaws.com")) {
regionInHost = "us-gov-west-1";
}

// Return token as region.
return token;
if (regionInHost != null) this.region = regionInHost;
}

private void setBaseUrl(HttpUrl url) {
String host = url.host();

this.isAwsHost = isAwsEndpoint(host);
this.isAwsChinaHost = false;
if (this.isAwsHost) {
this.isAwsChinaHost = host.endsWith(".cn");
url =
url.newBuilder()
.host(this.isAwsChinaHost ? "amazonaws.com.cn" : "amazonaws.com")
.build();
this.isFipsHost = isAwsFipsEndpoint(host);
this.isAccelerateHost = isAwsAccelerateEndpoint(host);
this.isDualStackHost = isAwsDualStackEndpoint(host);
this.regionInUrl = extractRegion(host);
this.useVirtualStyle = true;
} else {
this.useVirtualStyle = host.endsWith("aliyuncs.com");
}

this.baseUrl = url;
this.setAwsInfo(url.host(), url.isHttps());
this.useVirtualStyle = this.awsDomainSuffix != null || url.host().endsWith("aliyuncs.com");
}

public Builder endpoint(String endpoint) {
Expand Down Expand Up @@ -3325,8 +3304,9 @@ public Builder endpoint(HttpUrl url) {
}

public Builder region(String region) {
HttpUtils.validateNullOrNotEmptyString(region, "region");
this.regionInUrl = region;
if (region != null && !HttpUtils.REGION_REGEX.matcher(region).find()) {
throw new IllegalArgumentException("invalid region " + region);
}
this.region = region;
return this;
}
Expand All @@ -3349,7 +3329,11 @@ public Builder httpClient(OkHttpClient httpClient) {

public MinioAsyncClient build() {
HttpUtils.validateNotNull(this.baseUrl, "endpoint");
if (this.isAwsChinaHost && this.regionInUrl == null && this.region == null) {

if (this.awsDomainSuffix != null
&& this.awsDomainSuffix.endsWith(".cn")
&& !this.awsS3Prefix.endsWith("s3-accelerate.")
&& this.region == null) {
throw new IllegalArgumentException(
"Region missing in Amazon S3 China endpoint " + this.baseUrl);
}
Expand All @@ -3358,17 +3342,15 @@ public MinioAsyncClient build() {
this.httpClient =
HttpUtils.newDefaultHttpClient(
DEFAULT_CONNECTION_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT);
if (this.region == null) this.region = regionInUrl;
}

return new MinioAsyncClient(
baseUrl,
region,
isAwsHost,
isFipsHost,
isAccelerateHost,
isDualStackHost,
awsS3Prefix,
awsDomainSuffix,
awsDualstack,
useVirtualStyle,
region,
provider,
httpClient);
}
Expand Down
19 changes: 17 additions & 2 deletions api/src/main/java/io/minio/MinioClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2405,12 +2405,22 @@ public void traceOff() throws IOException {
asyncClient.traceOff();
}

/** Enables accelerate endpoint for Amazon S3 endpoint. */
/**
* Enables accelerate endpoint for Amazon S3 endpoint.
*
* @deprecated This method is no longer supported.
*/
@Deprecated
public void enableAccelerateEndpoint() {
asyncClient.enableAccelerateEndpoint();
}

/** Disables accelerate endpoint for Amazon S3 endpoint. */
/**
* Disables accelerate endpoint for Amazon S3 endpoint.
*
* @deprecated This method is no longer supported.
*/
@Deprecated
public void disableAccelerateEndpoint() {
asyncClient.disableAccelerateEndpoint();
}
Expand All @@ -2435,6 +2445,11 @@ public void disableVirtualStyleEndpoint() {
asyncClient.disableVirtualStyleEndpoint();
}

/** Sets AWS S3 domain prefix. */
public void setAwsS3Prefix(String awsS3Prefix) {
asyncClient.setAwsS3Prefix(awsS3Prefix);
}

public static Builder builder() {
return new Builder();
}
Expand Down
Loading

0 comments on commit 6f1a144

Please sign in to comment.