From e9ea8a3536d250ee87ab5af4d32c7754d2f6ee51 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 23 Apr 2025 14:43:13 -0700 Subject: [PATCH 1/2] [fix](create-resource) fix potential concurrent modification exception when craeting resource --- .../src/main/java/org/apache/doris/catalog/JdbcResource.java | 2 +- .../org/apache/doris/nereids/parser/LogicalPlanBuilder.java | 2 +- .../trees/plans/commands/info/CreateResourceInfo.java | 5 +++-- .../trees/plans/commands/CreateResourceCommandTest.java | 4 +--- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java index c3c020c54a7423..0309c0676b96d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java @@ -197,7 +197,7 @@ public void checkProperties(Map properties) throws AnalysisExcep protected void setProperties(Map properties) throws DdlException { Preconditions.checkState(properties != null); validateProperties(properties); - configs = properties; + configs = Maps.newHashMap(properties); applyDefaultProperties(); String currentDateTime = LocalDateTime.now(ZoneId.systemDefault()).toString().replace("T", " "); configs.put(CREATE_TIME, currentDateTime); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 85a443ae3441f1..99dcd83fcce217 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -6771,7 +6771,7 @@ public UserDesc visitGrantUserIdentify(DorisParser.GrantUserIdentifyContext ctx) @Override public LogicalPlan visitCreateResource(DorisParser.CreateResourceContext ctx) { String resourceName = visitIdentifierOrText(ctx.name); - Map properties = new HashMap<>(visitPropertyClause(ctx.properties)); + ImmutableMap properties = ImmutableMap.copyOf(visitPropertyClause(ctx.properties)); CreateResourceInfo createResourceInfo = new CreateResourceInfo( ctx.EXTERNAL() != null, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateResourceInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateResourceInfo.java index 5ccba3e690422a..39d7ba3037b3eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateResourceInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateResourceInfo.java @@ -31,6 +31,7 @@ import org.apache.doris.qe.ConnectContext; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import java.util.Map; @@ -42,14 +43,14 @@ public class CreateResourceInfo { private final boolean isExternal; private final boolean ifNotExists; private final String resourceName; - private final Map properties; + private ImmutableMap properties; private ResourceType resourceType; /** * CreateResourceInfo */ public CreateResourceInfo(boolean isExternal, boolean ifNotExists, String resourceName, - Map properties) { + com.google.common.collect.ImmutableMap properties) { this.isExternal = isExternal; this.ifNotExists = ifNotExists; this.resourceName = resourceName; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CreateResourceCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CreateResourceCommandTest.java index 6245c28b919f6b..1a70820e168631 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CreateResourceCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/CreateResourceCommandTest.java @@ -33,8 +33,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import java.util.Map; - public class CreateResourceCommandTest extends TestWithFeService { @Test public void testValidate(@Mocked Env env, @Mocked AccessControllerManager accessManager) { @@ -48,7 +46,7 @@ public void testValidate(@Mocked Env env, @Mocked AccessControllerManager access }; // test validate normal - Map properties = ImmutableMap.of("type", "spark", "host", "http://127.0.0.1:29200"); + ImmutableMap properties = ImmutableMap.of("type", "spark", "host", "http://127.0.0.1:29200"); CreateResourceInfo info = new CreateResourceInfo(true, false, "test", properties); CreateResourceCommand createResourceCommand = new CreateResourceCommand(info); Assertions.assertDoesNotThrow(() -> createResourceCommand.getInfo().validate()); From 30338255b729f2d50121432b2ce1cdbd29114a60 Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 25 Apr 2025 12:45:59 -0700 Subject: [PATCH 2/2] fix --- .../doris/analysis/CreateResourceStmt.java | 7 +-- .../analysis/CreateStorageVaultStmt.java | 7 +-- .../apache/doris/catalog/AzureResource.java | 25 ++++----- .../org/apache/doris/catalog/EsResource.java | 3 +- .../org/apache/doris/catalog/HMSResource.java | 3 +- .../apache/doris/catalog/HdfsResource.java | 7 +-- .../doris/catalog/HdfsStorageVault.java | 3 +- .../apache/doris/catalog/JdbcResource.java | 7 +-- .../doris/catalog/OdbcCatalogResource.java | 5 +- .../org/apache/doris/catalog/Resource.java | 3 +- .../org/apache/doris/catalog/S3Resource.java | 8 +-- .../apache/doris/catalog/S3StorageVault.java | 3 +- .../apache/doris/catalog/SparkResource.java | 3 +- .../apache/doris/catalog/StorageVault.java | 3 +- .../commands/info/CreateResourceInfo.java | 4 +- .../catalog/OdbcCatalogResourceTest.java | 14 ++--- .../apache/doris/catalog/S3ResourceTest.java | 52 ++++++++++--------- .../cloud/catalog/HdfsStorageVaultTest.java | 30 ++++++----- 18 files changed, 105 insertions(+), 82 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java index 3feccbba9baedb..6ea190977a7db1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java @@ -31,6 +31,7 @@ import org.apache.doris.qe.ConnectContext; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import java.util.Map; @@ -42,7 +43,7 @@ public class CreateResourceStmt extends DdlStmt implements NotFallbackInParser { private final boolean isExternal; private final boolean ifNotExists; private final String resourceName; - private final Map properties; + private final ImmutableMap properties; private ResourceType resourceType; public CreateResourceStmt(boolean isExternal, boolean ifNotExists, String resourceName, @@ -50,7 +51,7 @@ public CreateResourceStmt(boolean isExternal, boolean ifNotExists, String resour this.isExternal = isExternal; this.ifNotExists = ifNotExists; this.resourceName = resourceName; - this.properties = properties; + this.properties = ImmutableMap.copyOf(properties); this.resourceType = ResourceType.UNKNOWN; } @@ -62,7 +63,7 @@ public String getResourceName() { return resourceName; } - public Map getProperties() { + public ImmutableMap getProperties() { return properties; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java index 6e414db931486d..dcc846fe72315c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateStorageVaultStmt.java @@ -32,6 +32,7 @@ import org.apache.doris.qe.ConnectContext; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import java.util.Map; @@ -48,7 +49,7 @@ public class CreateStorageVaultStmt extends DdlStmt implements NotFallbackInPars private final boolean ifNotExists; private final String vaultName; - private final Map properties; + private final ImmutableMap properties; private boolean setAsDefault; private int pathVersion = 0; private int numShard = 0; @@ -57,7 +58,7 @@ public class CreateStorageVaultStmt extends DdlStmt implements NotFallbackInPars public CreateStorageVaultStmt(boolean ifNotExists, String vaultName, Map properties) { this.ifNotExists = ifNotExists; this.vaultName = vaultName; - this.properties = properties; + this.properties = ImmutableMap.copyOf(properties); this.vaultType = vaultType.UNKNOWN; } @@ -81,7 +82,7 @@ public int getPathVersion() { return pathVersion; } - public Map getProperties() { + public ImmutableMap getProperties() { return properties; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/AzureResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/AzureResource.java index fb04e25ad9eed1..c24c26ee22c89e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/AzureResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/AzureResource.java @@ -27,6 +27,7 @@ import org.apache.doris.fs.obj.RemoteObjects; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; @@ -52,32 +53,32 @@ public AzureResource(String name) { } @Override - protected void setProperties(Map newProperties) throws DdlException { + protected void setProperties(ImmutableMap newProperties) throws DdlException { Preconditions.checkState(newProperties != null); + this.properties = Maps.newHashMap(newProperties); // check properties - S3Properties.requiredS3PingProperties(newProperties); + S3Properties.requiredS3PingProperties(this.properties); // default need check resource conf valid, so need fix ut and regression case - boolean needCheck = isNeedCheck(newProperties); + boolean needCheck = isNeedCheck(this.properties); if (LOG.isDebugEnabled()) { LOG.debug("azure info need check validity : {}", needCheck); } // the endpoint for ping need add uri scheme. - String pingEndpoint = newProperties.get(S3Properties.ENDPOINT); + String pingEndpoint = this.properties.get(S3Properties.ENDPOINT); if (!pingEndpoint.startsWith("http://")) { - pingEndpoint = "http://" + newProperties.get(S3Properties.ENDPOINT); - newProperties.put(S3Properties.ENDPOINT, pingEndpoint); - newProperties.put(S3Properties.Env.ENDPOINT, pingEndpoint); + pingEndpoint = "http://" + this.properties.get(S3Properties.ENDPOINT); + this.properties.put(S3Properties.ENDPOINT, pingEndpoint); + this.properties.put(S3Properties.Env.ENDPOINT, pingEndpoint); } if (needCheck) { - String bucketName = newProperties.get(S3Properties.BUCKET); - String rootPath = newProperties.get(S3Properties.ROOT_PATH); - pingAzure(bucketName, rootPath, newProperties); + String bucketName = this.properties.get(S3Properties.BUCKET); + String rootPath = this.properties.get(S3Properties.ROOT_PATH); + pingAzure(bucketName, rootPath, this.properties); } // optional - S3Properties.optionalS3Property(newProperties); - this.properties = newProperties; + S3Properties.optionalS3Property(this.properties); } protected static void pingAzure(String bucketName, String rootPath, diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java index 8203a051129faa..697bd4ac2ae576 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/EsResource.java @@ -21,6 +21,7 @@ import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.datasource.es.EsUtil; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -96,7 +97,7 @@ public void modifyProperties(Map properties) throws DdlException } @Override - protected void setProperties(Map properties) throws DdlException { + protected void setProperties(ImmutableMap properties) throws DdlException { valid(properties, false); this.properties = processCompatibleProperties(properties); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java index 0a3b422fc1a005..007cb4660e8976 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java @@ -22,6 +22,7 @@ import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.constants.HMSProperties; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -63,7 +64,7 @@ public void modifyProperties(Map properties) throws DdlException } @Override - protected void setProperties(Map properties) throws DdlException { + protected void setProperties(ImmutableMap properties) throws DdlException { for (String field : HMSProperties.REQUIRED_FIELDS) { if (!properties.containsKey(field)) { throw new DdlException("Missing [" + field + "] in properties."); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java index c9cb77fbd93ff8..e1b482bd1b352e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsResource.java @@ -23,6 +23,7 @@ import org.apache.doris.thrift.THdfsConf; import org.apache.doris.thrift.THdfsParams; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -73,13 +74,13 @@ public void modifyProperties(Map properties) throws DdlException } @Override - protected void setProperties(Map properties) throws DdlException { + protected void setProperties(ImmutableMap newProperties) throws DdlException { // `dfs.client.read.shortcircuit` and `dfs.domain.socket.path` should be both set to enable short circuit read. // We should disable short circuit read if they are not both set because it will cause performance down. + this.properties = Maps.newHashMap(newProperties); if (!(enableShortCircuitRead(properties))) { - properties.put(HADOOP_SHORT_CIRCUIT, "false"); + this.properties.put(HADOOP_SHORT_CIRCUIT, "false"); } - this.properties = properties; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java index 498170c09887d2..50c686a88d01bf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java @@ -27,6 +27,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -90,7 +91,7 @@ public HdfsStorageVault(String name, boolean ifNotExists, boolean setAsDefault) } @Override - public void modifyProperties(Map newProperties) throws DdlException { + public void modifyProperties(ImmutableMap newProperties) throws DdlException { for (Map.Entry kv : newProperties.entrySet()) { replaceIfEffectiveValue(this.properties, kv.getKey(), kv.getValue()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java index 0309c0676b96d4..0ff0e74a90e1a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java @@ -29,6 +29,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -194,10 +195,10 @@ public void checkProperties(Map properties) throws AnalysisExcep } @Override - protected void setProperties(Map properties) throws DdlException { + protected void setProperties(ImmutableMap properties) throws DdlException { Preconditions.checkState(properties != null); - validateProperties(properties); - configs = Maps.newHashMap(properties); + this.configs = Maps.newHashMap(properties); + validateProperties(this.configs); applyDefaultProperties(); String currentDateTime = LocalDateTime.now(ZoneId.systemDefault()).toString().replace("T", " "); configs.put(CREATE_TIME, currentDateTime); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcCatalogResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcCatalogResource.java index dbe03ce4db047e..60fb9f9555e720 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcCatalogResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OdbcCatalogResource.java @@ -22,6 +22,7 @@ import org.apache.doris.common.proc.BaseProcResult; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -167,10 +168,10 @@ public int getSignature(int signatureVersion) { } @Override - protected void setProperties(Map properties) throws DdlException { + protected void setProperties(ImmutableMap properties) throws DdlException { Preconditions.checkState(properties != null); - configs = properties; + configs = Maps.newHashMap(properties); checkProperties(HOST); checkProperties(PORT); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java index 5c384d7783ba06..359ebab0b31eef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java @@ -31,6 +31,7 @@ import org.apache.doris.persist.gson.GsonUtils; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; import org.apache.logging.log4j.LogManager; @@ -242,7 +243,7 @@ protected void replaceIfEffectiveValue(Map properties, String ke /** * Set and check the properties in child resources */ - protected abstract void setProperties(Map properties) throws DdlException; + protected abstract void setProperties(ImmutableMap properties) throws DdlException; public abstract Map getCopiedProperties(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index 392b73d2280fd0..22df2c9082f5cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -28,6 +28,7 @@ import org.apache.doris.fs.obj.S3ObjStorage; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -85,8 +86,10 @@ public String getProperty(String propertyKey) { } @Override - protected void setProperties(Map properties) throws DdlException { - Preconditions.checkState(properties != null); + protected void setProperties(ImmutableMap newProperties) throws DdlException { + Preconditions.checkState(newProperties != null); + this.properties = Maps.newHashMap(newProperties); + // check properties S3Properties.requiredS3PingProperties(properties); // default need check resource conf valid, so need fix ut and regression case @@ -112,7 +115,6 @@ protected void setProperties(Map properties) throws DdlException } // optional S3Properties.optionalS3Property(properties); - this.properties = properties; } protected static void pingS3(String bucketName, String rootPath, Map newProperties) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java index 3cc8ffb3f7e142..4ff47c03ff0b41 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3StorageVault.java @@ -22,6 +22,7 @@ import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.constants.S3Properties; +import com.google.common.collect.ImmutableMap; import com.google.gson.annotations.SerializedName; import java.util.Arrays; @@ -77,7 +78,7 @@ public S3StorageVault(String name, boolean ifNotExists, } @Override - public void modifyProperties(Map properties) throws DdlException { + public void modifyProperties(ImmutableMap properties) throws DdlException { resource.setProperties(properties); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SparkResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SparkResource.java index 2af2a9b4a90df2..4e85b8208a78a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SparkResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SparkResource.java @@ -29,6 +29,7 @@ import org.apache.doris.load.loadv2.SparkYarnConfigFiles; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -282,7 +283,7 @@ private void updateProperties(Map properties) throws DdlExceptio } @Override - protected void setProperties(Map properties) throws DdlException { + protected void setProperties(ImmutableMap properties) throws DdlException { Preconditions.checkState(properties != null); // get spark configs diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java index 192f007329dde9..7b31bd813a25eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java @@ -27,6 +27,7 @@ import org.apache.doris.qe.ShowResultSetMetaData; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import com.google.protobuf.TextFormat; import java.util.ArrayList; @@ -172,7 +173,7 @@ public StorageVaultType getType() { * @param properties * @throws DdlException */ - public abstract void modifyProperties(Map properties) throws DdlException; + public abstract void modifyProperties(ImmutableMap properties) throws DdlException; /** * Check properties in child resources diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateResourceInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateResourceInfo.java index 39d7ba3037b3eb..722b80f116362f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateResourceInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateResourceInfo.java @@ -50,7 +50,7 @@ public class CreateResourceInfo { * CreateResourceInfo */ public CreateResourceInfo(boolean isExternal, boolean ifNotExists, String resourceName, - com.google.common.collect.ImmutableMap properties) { + ImmutableMap properties) { this.isExternal = isExternal; this.ifNotExists = ifNotExists; this.resourceName = resourceName; @@ -122,7 +122,7 @@ public String getResourceName() { return resourceName; } - public Map getProperties() { + public ImmutableMap getProperties() { return properties; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java index a1c2e679829162..88b5d9aa26f524 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OdbcCatalogResourceTest.java @@ -29,6 +29,7 @@ import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import mockit.Expectations; import mockit.Injectable; @@ -42,7 +43,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.HashMap; import java.util.Map; public class OdbcCatalogResourceTest { @@ -125,11 +125,13 @@ public void testSerialization() throws Exception { OdbcCatalogResource odbcCatalogResource1 = new OdbcCatalogResource("odbc1"); odbcCatalogResource1.write(dos); - Map configs = new HashMap<>(); - configs.put("host", "host"); - configs.put("port", "port"); - configs.put("user", "user"); - configs.put("password", "password"); + ImmutableMap configs = ImmutableMap.of( + "host", "host", + "port", "port", + "user", "user", + "password", "password" + ); + OdbcCatalogResource odbcCatalogResource2 = new OdbcCatalogResource("odbc2"); odbcCatalogResource2.setProperties(configs); odbcCatalogResource2.write(dos); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java index 5f2daf94cc32e2..b7d14ab70171b5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java @@ -31,6 +31,7 @@ import org.apache.doris.qe.ConnectContext; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import mockit.Expectations; import mockit.Injectable; import mockit.Mocked; @@ -172,14 +173,15 @@ public void testSerialization() throws Exception { S3Resource s3Resource1 = new S3Resource("s3_1"); s3Resource1.write(s3Dos); - Map properties = new HashMap<>(); - properties.put("AWS_ENDPOINT", "aaa"); - properties.put("AWS_REGION", "bbb"); - properties.put("AWS_ROOT_PATH", "/path/to/root"); - properties.put("AWS_ACCESS_KEY", "xxx"); - properties.put("AWS_SECRET_KEY", "yyy"); - properties.put("AWS_BUCKET", "test-bucket"); - properties.put("s3_validity_check", "false"); + ImmutableMap properties = ImmutableMap.of( + "AWS_ENDPOINT", "aaa", + "AWS_REGION", "bbb", + "AWS_ROOT_PATH", "/path/to/root", + "AWS_ACCESS_KEY", "xxx", + "AWS_SECRET_KEY", "yyy", + "AWS_BUCKET", "test-bucket", + "s3_validity_check", "false" + ); S3Resource s3Resource2 = new S3Resource("s3_2"); s3Resource2.setProperties(properties); s3Resource2.write(s3Dos); @@ -211,14 +213,15 @@ public void testSerialization() throws Exception { @Test public void testModifyProperties() throws Exception { - Map properties = new HashMap<>(); - properties.put("AWS_ENDPOINT", "aaa"); - properties.put("AWS_REGION", "bbb"); - properties.put("AWS_ROOT_PATH", "/path/to/root"); - properties.put("AWS_ACCESS_KEY", "xxx"); - properties.put("AWS_SECRET_KEY", "yyy"); - properties.put("AWS_BUCKET", "test-bucket"); - properties.put("s3_validity_check", "false"); + ImmutableMap properties = ImmutableMap.of( + "AWS_ENDPOINT", "aaa", + "AWS_REGION", "bbb", + "AWS_ROOT_PATH", "/path/to/root", + "AWS_ACCESS_KEY", "xxx", + "AWS_SECRET_KEY", "yyy", + "AWS_BUCKET", "test-bucket", + "s3_validity_check", "false" + ); S3Resource s3Resource = new S3Resource("t_source"); s3Resource.setProperties(properties); FeConstants.runningUnitTest = true; @@ -231,14 +234,15 @@ public void testModifyProperties() throws Exception { @Test public void testHttpScheme() throws DdlException { // if https:// is set, it should be replaced with http:// - Map properties = new HashMap<>(); - properties.put("AWS_ENDPOINT", "https://aaa"); - properties.put("AWS_REGION", "bbb"); - properties.put("AWS_ROOT_PATH", "/path/to/root"); - properties.put("AWS_ACCESS_KEY", "xxx"); - properties.put("AWS_SECRET_KEY", "yyy"); - properties.put("AWS_BUCKET", "test-bucket"); - properties.put("s3_validity_check", "false"); + ImmutableMap properties = ImmutableMap.of( + "AWS_ENDPOINT", "https://aaa", + "AWS_REGION", "bbb", + "AWS_ROOT_PATH", "/path/to/root", + "AWS_ACCESS_KEY", "xxx", + "AWS_SECRET_KEY", "yyy", + "AWS_BUCKET", "test-bucket", + "s3_validity_check", "false" + ); S3Resource s3Resource = new S3Resource("s3_2"); s3Resource.setProperties(properties); Assert.assertEquals(s3Resource.getProperty(S3Properties.ENDPOINT), "https://aaa"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java index 32d6eb559b27c8..ee84aaa665a541 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java @@ -46,7 +46,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -237,10 +236,11 @@ public void testCheckConnectivity() { Assumptions.assumeTrue(!Strings.isNullOrEmpty(hadoopFsName), "HADOOP_FS_NAME isNullOrEmpty."); Assumptions.assumeTrue(!Strings.isNullOrEmpty(hadoopUser), "HADOOP_USER isNullOrEmpty."); - Map properties = new HashMap<>(); - properties.put(HdfsStorageVault.HADOOP_FS_NAME, hadoopFsName); - properties.put(AuthenticationConfig.HADOOP_USER_NAME, hadoopUser); - properties.put(HdfsStorageVault.VAULT_PATH_PREFIX, "testCheckConnectivityUtPrefix"); + ImmutableMap properties = ImmutableMap.of( + HdfsStorageVault.HADOOP_FS_NAME, hadoopFsName, + AuthenticationConfig.HADOOP_USER_NAME, hadoopUser, + HdfsStorageVault.VAULT_PATH_PREFIX, "testCheckConnectivityUtPrefix" + ); HdfsStorageVault vault = new HdfsStorageVault("testHdfsVault", false, false); vault.modifyProperties(properties); @@ -252,10 +252,11 @@ public void testCheckConnectivity() { @Test public void testCheckConnectivityException() { - Map properties = new HashMap<>(); - properties.put(HdfsStorageVault.HADOOP_FS_NAME, "hdfs://localhost:10000"); - properties.put(AuthenticationConfig.HADOOP_USER_NAME, "notExistUser"); - properties.put(HdfsStorageVault.VAULT_PATH_PREFIX, "testCheckConnectivityUtPrefix"); + ImmutableMap properties = ImmutableMap.of( + HdfsStorageVault.HADOOP_FS_NAME, "hdfs://localhost:10000", + AuthenticationConfig.HADOOP_USER_NAME, "notExistUser", + HdfsStorageVault.VAULT_PATH_PREFIX, "testCheckConnectivityUtPrefix" + ); HdfsStorageVault vault = new HdfsStorageVault("testHdfsVault", false, false); Assertions.assertThrows(DdlException.class, () -> { @@ -265,11 +266,12 @@ public void testCheckConnectivityException() { @Test public void testIgnoreCheckConnectivity() throws DdlException { - Map properties = new HashMap<>(); - properties.put(HdfsStorageVault.HADOOP_FS_NAME, "hdfs://localhost:10000"); - properties.put(AuthenticationConfig.HADOOP_USER_NAME, "notExistUser"); - properties.put(HdfsStorageVault.VAULT_PATH_PREFIX, "testCheckConnectivityUtPrefix"); - properties.put(S3Properties.VALIDITY_CHECK, "false"); + ImmutableMap properties = ImmutableMap.of( + HdfsStorageVault.HADOOP_FS_NAME, "hdfs://localhost:10000", + AuthenticationConfig.HADOOP_USER_NAME, "notExistUser", + HdfsStorageVault.VAULT_PATH_PREFIX, "testCheckConnectivityUtPrefix", + S3Properties.VALIDITY_CHECK, "false" + ); HdfsStorageVault vault = new HdfsStorageVault("testHdfsVault", false, false); vault.modifyProperties(properties);