Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public void createResource(CreateResourceStmt stmt) throws DdlException {
Resource resource = Resource.fromStmt(stmt);
if (createResource(resource, stmt.isIfNotExists())) {
Env.getCurrentEnv().getEditLog().logCreateResource(resource);
LOG.info("Create resource success. Resource: {}", resource);
LOG.info("Create resource success. Resource: {}", resource.getName());
}
}

Expand Down
36 changes: 16 additions & 20 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
Original file line number Diff line number Diff line change
Expand Up @@ -109,18 +109,15 @@ protected void setProperties(Map<String, String> properties) throws DdlException
if (needCheck) {
String bucketName = properties.get(S3Properties.BUCKET);
String rootPath = properties.get(S3Properties.ROOT_PATH);
boolean available = pingS3(credential, bucketName, rootPath, properties);
if (!available) {
throw new DdlException("S3 can't use, please check your properties");
}
pingS3(credential, bucketName, rootPath, properties);
}
// optional
S3Properties.optionalS3Property(properties);
this.properties = properties;
}

private static boolean pingS3(CloudCredentialWithEndpoint credential, String bucketName, String rootPath,
Map<String, String> properties) {
private static void pingS3(CloudCredentialWithEndpoint credential, String bucketName, String rootPath,
Map<String, String> properties) throws DdlException {
String bucket = "s3://" + bucketName + "/";
Map<String, String> propertiesPing = new HashMap<>();
propertiesPing.put(S3Properties.Env.ACCESS_KEY, credential.getAccessKey());
Expand All @@ -134,24 +131,27 @@ private static boolean pingS3(CloudCredentialWithEndpoint credential, String buc
String testFile = bucket + rootPath + "/test-object-valid.txt";
String content = "doris will be better";
if (FeConstants.runningUnitTest) {
return true;
return;
}
Status status = Status.OK;
try {
Status status = fileSystem.directUpload(content, testFile);
status = fileSystem.directUpload(content, testFile);
if (status != Status.OK) {
LOG.warn("ping update file status: {}, properties: {}", status, propertiesPing);
return false;
throw new DdlException(
"ping s3 failed(upload), status: " + status + ", properties: " + new PrintableMap<>(
propertiesPing, "=", true, false, true, false));
}
} finally {
Status delete = fileSystem.delete(testFile);
if (delete != Status.OK) {
LOG.warn("ping delete file status: {}, properties: {}", delete, propertiesPing);
return false;
if (status.ok()) {
Status delete = fileSystem.delete(testFile);
if (delete != Status.OK) {
LOG.warn("delete test file failed, status: {}, properties: {}", delete, new PrintableMap<>(
propertiesPing, "=", true, false, true, false));
}
}
}

LOG.info("success to ping s3");
return true;
}

@Override
Expand All @@ -178,11 +178,7 @@ public void modifyProperties(Map<String, String> properties) throws DdlException
String rootPath = properties.getOrDefault(S3Properties.ROOT_PATH,
this.properties.get(S3Properties.ROOT_PATH));

boolean available = pingS3(getS3PingCredentials(changedProperties),
bucketName, rootPath, changedProperties);
if (!available) {
throw new DdlException("S3 can't use, please check your properties");
}
pingS3(getS3PingCredentials(changedProperties), bucketName, rootPath, changedProperties);
}

// modify properties
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ public static S3Client buildS3Client(URI endpoint, String region, CloudCredentia
// use virtual hosted-style access
.serviceConfiguration(S3Configuration.builder()
.chunkedEncodingEnabled(false)
.pathStyleAccessEnabled(false)
.pathStyleAccessEnabled(true)
.build())
.build();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ suite("create_policy") {
"AWS_MAX_CONNECTIONS" = "50",
"AWS_REQUEST_TIMEOUT_MS" = "3000",
"AWS_CONNECTION_TIMEOUT_MS" = "1000",
"AWS_BUCKET" = "test-bucket",
"AWS_BUCKET" = "test-bucket"
);
"""
// errCode = 2, detailMessage = Missing [s3_validity_check] in properties.
Expand Down