From b51cfeecb274c153b21f496b7312dfddd907b64d Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 1 Jun 2024 18:27:21 +0800 Subject: [PATCH 1/5] 1 --- .../main/java/org/apache/doris/fs/remote/S3FileSystem.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java index 3869824de55d64..a1aa23af4ab2db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java @@ -60,7 +60,9 @@ protected FileSystem nativeFileSystem(String remotePath) throws UserException { if (dfsFileSystem == null) { Configuration conf = new Configuration(); System.setProperty("com.amazonaws.services.s3.enableV4", "true"); - PropertyConverter.convertToHadoopFSProperties(properties).forEach(conf::set); + PropertyConverter.convertToHadoopFSProperties(properties).entrySet().stream() + .filter(entry -> entry.getKey() != null && entry.getValue() != null) + .forEach(entry -> conf.set(entry.getKey(), entry.getValue())); try { dfsFileSystem = FileSystem.get(new Path(remotePath).toUri(), conf); } catch (Exception e) { From 8d425707eca1a2453dcba7e7b1ff428d704428cb Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 1 Jun 2024 18:43:44 +0800 Subject: [PATCH 2/5] 2 --- .../src/main/java/org/apache/doris/catalog/S3Resource.java | 6 +++--- .../main/java/org/apache/doris/fs/remote/S3FileSystem.java | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index a26038970477a6..c8cb23b4998ce2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -121,7 +121,7 @@ protected void setProperties(Map properties) throws DdlException private static void pingS3(CloudCredentialWithEndpoint credential, String bucketName, String rootPath, Map properties) throws DdlException { String bucket = "s3://" + bucketName + "/"; - Map propertiesPing = new HashMap<>(); + Map propertiesPing = Maps.newHashMap(properties); propertiesPing.put(S3Properties.Env.ACCESS_KEY, credential.getAccessKey()); propertiesPing.put(S3Properties.Env.SECRET_KEY, credential.getSecretKey()); propertiesPing.put(S3Properties.Env.TOKEN, credential.getSessionToken()); @@ -129,8 +129,8 @@ private static void pingS3(CloudCredentialWithEndpoint credential, String bucket propertiesPing.put(S3Properties.Env.REGION, credential.getRegion()); propertiesPing.put(PropertyConverter.USE_PATH_STYLE, properties.getOrDefault(PropertyConverter.USE_PATH_STYLE, "false")); - properties.putAll(propertiesPing); - S3FileSystem fileSystem = new S3FileSystem(properties); + // properties.putAll(propertiesPing); + S3FileSystem fileSystem = new S3FileSystem(propertiesPing); String testFile = bucket + rootPath + "/test-object-valid.txt"; String content = "doris will be better"; if (FeConstants.runningUnitTest) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java index a1aa23af4ab2db..2b94d2195da6bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/S3FileSystem.java @@ -60,6 +60,7 @@ protected FileSystem nativeFileSystem(String remotePath) throws UserException { if (dfsFileSystem == null) { Configuration conf = new Configuration(); System.setProperty("com.amazonaws.services.s3.enableV4", "true"); + // the entry value in properties may be null, and PropertyConverter.convertToHadoopFSProperties(properties).entrySet().stream() .filter(entry -> entry.getKey() != null && entry.getValue() != null) .forEach(entry -> conf.set(entry.getKey(), entry.getValue())); From d015f5e99e07345ddd199d3bf892acd03f1326a2 Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 1 Jun 2024 19:12:16 +0800 Subject: [PATCH 3/5] 3 --- .../org/apache/doris/catalog/S3Resource.java | 16 ++++------------ .../tablefunction/S3TableValuedFunction.java | 7 +++++-- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index c8cb23b4998ce2..1e20d85752cade 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -121,16 +121,7 @@ protected void setProperties(Map properties) throws DdlException private static void pingS3(CloudCredentialWithEndpoint credential, String bucketName, String rootPath, Map properties) throws DdlException { String bucket = "s3://" + bucketName + "/"; - Map propertiesPing = Maps.newHashMap(properties); - propertiesPing.put(S3Properties.Env.ACCESS_KEY, credential.getAccessKey()); - propertiesPing.put(S3Properties.Env.SECRET_KEY, credential.getSecretKey()); - propertiesPing.put(S3Properties.Env.TOKEN, credential.getSessionToken()); - propertiesPing.put(S3Properties.Env.ENDPOINT, credential.getEndpoint()); - propertiesPing.put(S3Properties.Env.REGION, credential.getRegion()); - propertiesPing.put(PropertyConverter.USE_PATH_STYLE, - properties.getOrDefault(PropertyConverter.USE_PATH_STYLE, "false")); - // properties.putAll(propertiesPing); - S3FileSystem fileSystem = new S3FileSystem(propertiesPing); + S3FileSystem fileSystem = new S3FileSystem(properties); String testFile = bucket + rootPath + "/test-object-valid.txt"; String content = "doris will be better"; if (FeConstants.runningUnitTest) { @@ -142,14 +133,14 @@ private static void pingS3(CloudCredentialWithEndpoint credential, String bucket if (status != Status.OK) { throw new DdlException( "ping s3 failed(upload), status: " + status + ", properties: " + new PrintableMap<>( - propertiesPing, "=", true, false, true, false)); + properties, "=", true, false, true, false)); } } finally { if (status.ok()) { Status delete = fileSystem.delete(testFile); if (delete != Status.OK) { LOG.warn("delete test file failed, status: {}, properties: {}", delete, new PrintableMap<>( - propertiesPing, "=", true, false, true, false)); + properties, "=", true, false, true, false)); } } } @@ -250,3 +241,4 @@ protected void getProcNodeData(BaseProcResult result) { readUnlock(); } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index 98b35de7d3e34e..8476f1c978b3fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -73,8 +73,10 @@ public S3TableValuedFunction(Map properties) throws AnalysisExce S3URI s3uri = getS3Uri(uriStr, Boolean.parseBoolean(usePathStyle.toLowerCase()), Boolean.parseBoolean(forceParsingByStandardUri.toLowerCase())); - String endpoint = getOrDefaultAndRemove(otherProps, S3Properties.ENDPOINT, s3uri.getEndpoint().orElseThrow(() -> - new AnalysisException(String.format("Properties '%s' is required.", S3Properties.ENDPOINT)))); + String endpoint = getOrDefaultAndRemove(otherProps, S3Properties.ENDPOINT, s3uri.getEndpoint().orElse("")); + if (Strings.isNullOrEmpty(endpoint)) { + throw new AnalysisException(String.format("Properties '%s' is required.", S3Properties.ENDPOINT)); + } if (!otherProps.containsKey(S3Properties.REGION)) { String region = s3uri.getRegion().orElseThrow(() -> new AnalysisException(String.format("Properties '%s' is required.", S3Properties.REGION))); @@ -151,3 +153,4 @@ public String getTableName() { return "S3TableValuedFunction"; } } + From 908d3bda32f79e481acee8e4d512fe6d96284a6d Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 1 Jun 2024 23:20:22 +0800 Subject: [PATCH 4/5] 1 --- .../src/main/java/org/apache/doris/catalog/S3Resource.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index 1e20d85752cade..8b9b5f6af3703d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -23,7 +23,6 @@ import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.util.PrintableMap; import org.apache.doris.datasource.credentials.CloudCredentialWithEndpoint; -import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.fs.remote.S3FileSystem; From d4a03d38addadff3d86d7d160339c2a07841d246 Mon Sep 17 00:00:00 2001 From: morningman Date: Sat, 1 Jun 2024 23:22:04 +0800 Subject: [PATCH 5/5] fix --- .../doris/nereids/trees/expressions/functions/table/Hdfs.java | 3 +-- .../doris/nereids/trees/expressions/functions/table/Local.java | 3 +-- .../doris/nereids/trees/expressions/functions/table/S3.java | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java index 6c32de50eea2a2..5f8651ed61c826 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Hdfs.java @@ -45,8 +45,7 @@ protected TableValuedFunctionIf toCatalogFunction() { Map arguments = getTVFProperties().getMap(); return new HdfsTableValuedFunction(arguments); } catch (Throwable t) { - throw new AnalysisException("Can not build HdfsTableValuedFunction by " - + this + ": " + t.getMessage(), t); + throw new AnalysisException("Can not build hdfs(): " + t.getMessage(), t); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Local.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Local.java index d45a4c939433f5..4330980ee86d24 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Local.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/Local.java @@ -46,8 +46,7 @@ protected TableValuedFunctionIf toCatalogFunction() { Map arguments = getTVFProperties().getMap(); return new LocalTableValuedFunction(arguments); } catch (Throwable t) { - throw new AnalysisException("Can not build LocalTableValuedFunction by " - + this + ": " + t.getMessage(), t); + throw new AnalysisException("Can not build local(): " + t.getMessage(), t); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java index 1f3d7cb805e26a..d2623d8fd3c6bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java @@ -44,8 +44,7 @@ protected TableValuedFunctionIf toCatalogFunction() { Map arguments = getTVFProperties().getMap(); return new S3TableValuedFunction(arguments); } catch (Throwable t) { - throw new AnalysisException("Can not build S3TableValuedFunction by " - + this + ": " + t.getMessage(), t); + throw new AnalysisException("Can not build s3(): " + t.getMessage(), t); } }