From f23b4226a0df11df09ef877254e428cf846ab0f7 Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Mon, 6 Mar 2023 18:40:14 +0800 Subject: [PATCH 1/3] add s3 validity checker for alter resource. --- .../java/org/apache/doris/catalog/S3Resource.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index 94520d3d1649be..35d99eba5ba248 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -201,6 +201,20 @@ public void modifyProperties(Map properties) throws DdlException throw new DdlException("current not support modify property : " + any.get()); } } + + boolean needCheck = !this.properties.containsKey(S3_VALIDITY_CHECK) + || Boolean.parseBoolean(this.properties.get(S3_VALIDITY_CHECK)); + if (properties.containsKey(S3_VALIDITY_CHECK)) { + needCheck = Boolean.parseBoolean(properties.get(S3_VALIDITY_CHECK)); + } + LOG.debug("s3 info need check validity : {}", needCheck); + if (needCheck) { + boolean available = pingS3(); + if (!available) { + throw new DdlException("S3 can't use, please check your properties"); + } + } + // modify properties writeLock(); for (Map.Entry kv : properties.entrySet()) { From e3176acced78b44a372ef6bc526537f6e1bc88ec Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Mon, 6 Mar 2023 19:02:21 +0800 Subject: [PATCH 2/3] add s3 validity checker for alter resource. --- .../org/apache/doris/catalog/S3Resource.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index 35d99eba5ba248..547801703e4044 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -137,7 +137,7 @@ protected void setProperties(Map properties) throws DdlException || Boolean.parseBoolean(properties.get(S3_VALIDITY_CHECK)); LOG.debug("s3 info need check validity : {}", needCheck); if (needCheck) { - boolean available = pingS3(); + boolean available = pingS3(this.properties); if (!available) { throw new DdlException("S3 can't use, please check your properties"); } @@ -149,7 +149,7 @@ protected void setProperties(Map properties) throws DdlException checkOptionalProperty(S3_CONNECTION_TIMEOUT_MS, DEFAULT_S3_CONNECTION_TIMEOUT_MS); } - private boolean pingS3() { + private boolean pingS3(Map properties) { String bucket = "s3://" + properties.getOrDefault(S3_BUCKET, "") + "/"; Map propertiesPing = new HashMap<>(); propertiesPing.put("AWS_ACCESS_KEY", properties.getOrDefault(S3_ACCESS_KEY, "")); @@ -209,7 +209,18 @@ public void modifyProperties(Map properties) throws DdlException } LOG.debug("s3 info need check validity : {}", needCheck); if (needCheck) { - boolean available = pingS3(); + Map s3Properties = new HashMap<>(); + s3Properties.put(S3_BUCKET, properties.containsKey(S3_BUCKET) ? properties.get(S3_BUCKET) : + this.properties.getOrDefault(S3_BUCKET, "")); + s3Properties.put(S3_ACCESS_KEY, properties.containsKey(S3_ACCESS_KEY) ? properties.get(S3_ACCESS_KEY) : + this.properties.getOrDefault(S3_ACCESS_KEY, "")); + s3Properties.put(S3_SECRET_KEY, properties.containsKey(S3_SECRET_KEY) ? properties.get(S3_SECRET_KEY) : + this.properties.getOrDefault(S3_SECRET_KEY, "")); + s3Properties.put(S3_ENDPOINT, properties.containsKey(S3_ENDPOINT) ? properties.get(S3_ENDPOINT) : + this.properties.getOrDefault(S3_ENDPOINT, "")); + s3Properties.put(S3_REGION, properties.containsKey(S3_REGION) ? properties.get(S3_REGION) : + this.properties.getOrDefault(S3_REGION, "")); + boolean available = pingS3(s3Properties); if (!available) { throw new DdlException("S3 can't use, please check your properties"); } From 744851e807c36f23a1d51cc34b0a90fa8381dfff Mon Sep 17 00:00:00 2001 From: pengxianyu Date: Mon, 6 Mar 2023 19:12:37 +0800 Subject: [PATCH 3/3] add s3 validity checker for alter resource. --- .../src/main/java/org/apache/doris/catalog/S3Resource.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index 547801703e4044..4c21cdac6e9fba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -220,6 +220,8 @@ public void modifyProperties(Map properties) throws DdlException this.properties.getOrDefault(S3_ENDPOINT, "")); s3Properties.put(S3_REGION, properties.containsKey(S3_REGION) ? properties.get(S3_REGION) : this.properties.getOrDefault(S3_REGION, "")); + s3Properties.put(S3_ROOT_PATH, properties.containsKey(S3_ROOT_PATH) ? properties.get(S3_ROOT_PATH) : + this.properties.getOrDefault(S3_ROOT_PATH, "")); boolean available = pingS3(s3Properties); if (!available) { throw new DdlException("S3 can't use, please check your properties");