From d8b34b54eb6e03d76431581406c4aa8b4d2a1597 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Thu, 24 Apr 2025 11:12:26 +0800 Subject: [PATCH 01/48] [feat](refactor-param)Integrate New Storage System Support for BACKUP/RESTORE/LOAD/TVF #### Key Changes Enhanced core functionalities (BACKUP, RESTORE, LOAD, TVF) to support new storage parameters Unified storage path and parameter parsing logic across modules to enable compatibility with multiple storage backends (S3, OSS, COS, etc.). #### Details Refactored the storage parameter handling logic to automatically detect the schema and route to the corresponding storage system implementation. Each operation (BACKUP/RESTORE/LOAD/TVF) now uses a unified file system interface to ensure consistent behavior and extensibility. Maintains full backward compatibility with existing storage formats such as HDFS and local file systems. #### Tests Added comprehensive unit and integration tests covering: Storage parameter parsing across different systems Execution flow for each operation under new storage systems Edge cases including invalid parameters, permission errors, and non-existent paths --- fe/fe-core/pom.xml | 6 + .../org/apache/doris/analysis/BrokerDesc.java | 52 ++- .../org/apache/doris/analysis/LoadStmt.java | 97 +----- .../apache/doris/analysis/OutFileClause.java | 81 +---- .../apache/doris/analysis/StorageBackend.java | 55 +-- .../apache/doris/analysis/StorageDesc.java | 17 + .../apache/doris/backup/BackupHandler.java | 65 +++- .../org/apache/doris/backup/BackupJob.java | 5 +- .../org/apache/doris/backup/Repository.java | 129 ++++--- .../apache/doris/backup/RepositoryMgr.java | 7 +- .../org/apache/doris/backup/RestoreJob.java | 5 +- .../apache/doris/common/util/BrokerUtil.java | 12 +- .../property/ConnectionProperties.java | 3 +- .../metastore/AliyunDLFProperties.java | 3 +- .../property/metastore/HMSProperties.java | 3 +- .../AbstractS3CompatibleProperties.java | 4 +- .../property/storage/AzureProperties.java | 2 +- .../property/storage/COSProperties.java | 2 +- .../property/storage/HdfsProperties.java | 2 +- .../property/storage/OBSProperties.java | 4 +- .../property/storage/OSSHdfsProperties.java | 2 +- .../property/storage/S3Properties.java | 6 + .../property/storage/S3PropertyUtils.java | 20 +- .../property/storage/StorageProperties.java | 2 +- .../trees/plans/commands/LoadCommand.java | 5 +- .../apache/doris/persist/gson/GsonUtils.java | 17 + .../org/apache/doris/planner/ExportSink.java | 2 +- .../ExternalFileTableValuedFunction.java | 2 + .../HdfsTableValuedFunction.java | 36 +- .../tablefunction/S3TableValuedFunction.java | 120 +------ .../apache/doris/backup/BackupJobTest.java | 9 +- .../apache/doris/backup/RepositoryTest.java | 12 +- .../apache/doris/backup/RestoreJobTest.java | 9 +- .../apache/doris/fs/obj/S3FileSystemTest.java | 10 +- fe/pom.xml | 6 + regression-test/conf/regression-conf.groovy | 1 + .../backup_restore_azure.groovy | 182 ++++++++++ .../backup_restore_object_storage.groovy | 291 ++++++++++++++++ .../hdfs_all_test.groovy | 318 ++++++++++++++++++ .../refactor_storage_param_p0/s3_load.groovy | 293 ++++++++++++++++ .../test_outfile_s3_storage.groovy | 215 ++++++++++++ .../test_s3_tvf_s3_storage.groovy | 261 ++++++++++++++ 42 files changed, 1870 insertions(+), 503 deletions(-) create mode 100644 regression-test/suites/refactor_storage_param_p0/backup_restore_azure.groovy create mode 100644 regression-test/suites/refactor_storage_param_p0/backup_restore_object_storage.groovy create mode 100644 regression-test/suites/refactor_storage_param_p0/hdfs_all_test.groovy create mode 100644 regression-test/suites/refactor_storage_param_p0/s3_load.groovy create mode 100644 regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy create mode 100644 regression-test/suites/refactor_storage_param_p0/test_s3_tvf_s3_storage.groovy diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index f7048b9f03d073..1e41df15e39e49 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -102,6 +102,10 @@ under the License. commons-pool commons-pool + + com.google.re2j + re2j + org.apache.commons commons-text @@ -1189,6 +1193,8 @@ under the License. maven-compiler-plugin 8 + 9 + 9 diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java index 4755159379f5fe..5c129db0ab66c4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java @@ -20,17 +20,17 @@ import org.apache.doris.analysis.StorageBackend.StorageType; import org.apache.doris.catalog.Env; import org.apache.doris.common.FeMetaVersion; +import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.datasource.property.S3ClientBEProperties; -import org.apache.doris.datasource.property.constants.BosProperties; -import org.apache.doris.fs.PersistentFileSystem; +import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.thrift.TFileType; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -53,6 +53,7 @@ public class BrokerDesc extends StorageDesc implements Writable { // just for multi load public static final String MULTI_LOAD_BROKER = "__DORIS_MULTI_LOAD_BROKER__"; public static final String MULTI_LOAD_BROKER_BACKEND_KEY = "__DORIS_MULTI_LOAD_BROKER_BACKEND__"; + @Deprecated @SerializedName("cts3") private boolean convertedToS3 = false; @@ -70,6 +71,15 @@ public BrokerDesc(String name) { } public BrokerDesc(String name, Map properties) { + this.name = name; + this.properties = Maps.newHashMap(); + if (properties != null) { + this.properties.putAll(properties); + } + this.storageProperties = StorageProperties.createPrimary(properties); + // we should use storage properties? + this.properties.putAll(storageProperties.getBackendConfigProperties()); + this.storageType = StorageBackend.StorageType.convertToStorageType(storageProperties.getStorageName()); this.name = name; this.properties = Maps.newHashMap(); if (properties != null) { @@ -77,13 +87,6 @@ public BrokerDesc(String name, Map properties) { } if (isMultiLoadBroker()) { this.storageType = StorageBackend.StorageType.LOCAL; - } else { - this.storageType = StorageBackend.StorageType.BROKER; - } - this.properties.putAll(S3ClientBEProperties.getBeFSProperties(this.properties)); - this.convertedToS3 = BosProperties.tryConvertBosToS3(this.properties, this.storageType); - if (this.convertedToS3) { - this.storageType = StorageBackend.StorageType.S3; } } @@ -93,24 +96,23 @@ public BrokerDesc(String name, StorageBackend.StorageType storageType, Map PROVIDERS = - new ArrayList<>(Arrays.asList("cos", "oss", "s3", "obs", "bos", "azure")); - // mini load params public static final String KEY_IN_PARAM_COLUMNS = "columns"; public static final String KEY_IN_PARAM_SET = "set"; @@ -522,27 +516,6 @@ public void analyze(Analyzer analyzer) throws UserException { user = ConnectContext.get().getQualifiedUser(); } - - private String getProviderFromEndpoint() { - Map properties = brokerDesc.getProperties(); - for (Map.Entry entry : properties.entrySet()) { - if (entry.getKey().equalsIgnoreCase(S3Properties.PROVIDER)) { - // S3 Provider properties should be case insensitive. - return entry.getValue().toUpperCase(); - } - } - return S3Properties.S3_PROVIDER; - } - - private String getBucketFromFilePath(String filePath) throws Exception { - String[] parts = filePath.split("\\/\\/"); - if (parts.length < 2) { - throw new Exception("filePath is not valid"); - } - String buckt = parts[1].split("\\/")[0]; - return buckt; - } - public String getComment() { return comment; } @@ -630,21 +603,16 @@ private void checkEndpoint(String endpoint) throws UserException { } public void checkS3Param() throws UserException { - Map brokerDescProperties = brokerDesc.getProperties(); - if (brokerDescProperties.containsKey(S3Properties.Env.ENDPOINT) - && brokerDescProperties.containsKey(S3Properties.Env.ACCESS_KEY) - && brokerDescProperties.containsKey(S3Properties.Env.SECRET_KEY) - && brokerDescProperties.containsKey(S3Properties.Env.REGION)) { - String endpoint = brokerDescProperties.get(S3Properties.Env.ENDPOINT); - endpoint = endpoint.replaceFirst("^http://", ""); - endpoint = endpoint.replaceFirst("^https://", ""); - brokerDescProperties.put(S3Properties.Env.ENDPOINT, endpoint); + if (brokerDesc.getFileType() != null && brokerDesc.getFileType().equals(TFileType.FILE_S3)) { + + ObjectStorageProperties storageProperties = (ObjectStorageProperties) brokerDesc.getStorageProperties(); + String endpoint = storageProperties.getEndpoint(); checkWhiteList(endpoint); - if (AzureProperties.checkAzureProviderPropertyExist(brokerDescProperties)) { - return; + //should add connectivity test + boolean connectivityTest = FileSystemFactory.get(brokerDesc.getStorageProperties()).connectivityTest(); + if (!connectivityTest) { + throw new UserException("Failed to access object storage, message=connectivity test failed"); } - checkEndpoint(endpoint); - checkAkSk(); } } @@ -657,47 +625,6 @@ public void checkWhiteList(String endpoint) throws UserException { } } - private void checkAkSk() throws UserException { - RemoteBase remote = null; - ObjectInfo objectInfo = null; - String curFile = null; - try { - Map brokerDescProperties = brokerDesc.getProperties(); - String provider = getProviderFromEndpoint(); - for (DataDescription dataDescription : dataDescriptions) { - for (String filePath : dataDescription.getFilePaths()) { - curFile = filePath; - String bucket = getBucketFromFilePath(filePath); - objectInfo = new ObjectInfo(ObjectStoreInfoPB.Provider.valueOf(provider.toUpperCase()), - brokerDescProperties.get(S3Properties.Env.ACCESS_KEY), - brokerDescProperties.get(S3Properties.Env.SECRET_KEY), - bucket, brokerDescProperties.get(S3Properties.Env.ENDPOINT), - brokerDescProperties.get(S3Properties.Env.REGION), ""); - remote = RemoteBase.newInstance(objectInfo); - // RemoteBase#headObject does not throw exception if key does not exist. - remote.headObject("1"); - remote.listObjects(null); - remote.close(); - } - } - } catch (Exception e) { - LOG.warn("Failed to access object storage, file={}, proto={}, err={}", curFile, objectInfo, e.toString()); - String msg; - if (e instanceof UserException) { - msg = ((UserException) e).getDetailMessage(); - } else { - msg = e.getMessage(); - } - throw new UserException(InternalErrorCode.GET_REMOTE_DATA_ERROR, - "Failed to access object storage, message=" + msg, e); - } finally { - if (remote != null) { - remote.close(); - } - } - - } - @Override public StmtType stmtType() { return StmtType.LOAD; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java index 744442955c8ab9..12034426ecf77b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java @@ -32,8 +32,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.ParseUtil; import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.datasource.property.PropertyConverter; -import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.datasource.property.fileformat.FileFormatProperties; import org.apache.doris.thrift.TFileFormatType; @@ -47,14 +45,12 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import org.apache.hadoop.fs.Path; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -114,10 +110,7 @@ public class OutFileClause { } public static final String LOCAL_FILE_PREFIX = "file:///"; - private static final String S3_FILE_PREFIX = "S3://"; private static final String HDFS_FILE_PREFIX = "hdfs://"; - private static final String HADOOP_FS_PROP_PREFIX = "dfs."; - private static final String HADOOP_PROP_PREFIX = "hadoop."; private static final String BROKER_PROP_PREFIX = "broker."; public static final String PROP_BROKER_NAME = "broker.name"; public static final String PROP_COLUMN_SEPARATOR = "column_separator"; @@ -541,12 +534,6 @@ private void analyzeProperties() throws UserException { copiedProps.remove(PROP_SUCCESS_FILE_NAME); } - // For Azure compatibility, this is temporarily added to the map without further processing. - // The validity of each provider's value will be checked later in S3Properties' check. - if (copiedProps.containsKey(S3Properties.PROVIDER)) { - copiedProps.remove(S3Properties.PROVIDER); - } - if (fileFormatProperties.getFileFormatType() == TFileFormatType.FORMAT_PARQUET) { getParquetProperties(copiedProps); } @@ -554,10 +541,11 @@ private void analyzeProperties() throws UserException { if (fileFormatProperties.getFileFormatType() == TFileFormatType.FORMAT_ORC) { getOrcProperties(copiedProps); } - - if (!copiedProps.isEmpty()) { + //fixme really need this? + /* if (!copiedProps.isEmpty()) { throw new AnalysisException("Unknown properties: " + copiedProps.keySet()); - } + }*/ + } /** @@ -566,63 +554,8 @@ private void analyzeProperties() throws UserException { * 2. s3: with s3 pattern path, without broker name */ private void analyzeBrokerDesc(Map copiedProps) throws UserException { - String brokerName = copiedProps.get(PROP_BROKER_NAME); - StorageBackend.StorageType storageType; - if (copiedProps.containsKey(PROP_BROKER_NAME)) { - copiedProps.remove(PROP_BROKER_NAME); - storageType = StorageBackend.StorageType.BROKER; - } else if (filePath.toUpperCase().startsWith(S3_FILE_PREFIX)) { - brokerName = StorageBackend.StorageType.S3.name(); - storageType = StorageBackend.StorageType.S3; - } else if (filePath.toUpperCase().startsWith(HDFS_FILE_PREFIX.toUpperCase())) { - brokerName = StorageBackend.StorageType.HDFS.name(); - storageType = StorageBackend.StorageType.HDFS; - } else { - return; - } - - Map brokerProps = Maps.newHashMap(); - Iterator> iterator = copiedProps.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - if (entry.getKey().startsWith(BROKER_PROP_PREFIX) && !entry.getKey().equals(PROP_BROKER_NAME)) { - brokerProps.put(entry.getKey().substring(BROKER_PROP_PREFIX.length()), entry.getValue()); - iterator.remove(); - } else if (entry.getKey().toLowerCase().startsWith(S3Properties.S3_PREFIX) - || entry.getKey().toUpperCase().startsWith(S3Properties.Env.PROPERTIES_PREFIX)) { - brokerProps.put(entry.getKey(), entry.getValue()); - iterator.remove(); - } else if (entry.getKey().contains(HdfsResource.HADOOP_FS_NAME) - && storageType == StorageBackend.StorageType.HDFS) { - brokerProps.put(entry.getKey(), entry.getValue()); - iterator.remove(); - } else if ((entry.getKey().startsWith(HADOOP_FS_PROP_PREFIX) - || entry.getKey().startsWith(HADOOP_PROP_PREFIX)) - && storageType == StorageBackend.StorageType.HDFS) { - brokerProps.put(entry.getKey(), entry.getValue()); - iterator.remove(); - } - } - - if (storageType == StorageBackend.StorageType.S3) { - if (copiedProps.containsKey(PropertyConverter.USE_PATH_STYLE)) { - brokerProps.put(PropertyConverter.USE_PATH_STYLE, copiedProps.get(PropertyConverter.USE_PATH_STYLE)); - copiedProps.remove(PropertyConverter.USE_PATH_STYLE); - } - S3Properties.requiredS3Properties(brokerProps); - } else if (storageType == StorageBackend.StorageType.HDFS) { - if (!brokerProps.containsKey(HdfsResource.HADOOP_FS_NAME)) { - brokerProps.put(HdfsResource.HADOOP_FS_NAME, getFsName(filePath)); - } - } - brokerDesc = new BrokerDesc(brokerName, storageType, brokerProps); - } - - public static String getFsName(String path) { - Path hdfsPath = new Path(path); - String fullPath = hdfsPath.toUri().toString(); - String filePath = hdfsPath.toUri().getPath(); - return fullPath.replace(filePath, ""); + String brokerName = properties.get(PROP_BROKER_NAME); + brokerDesc = new BrokerDesc(brokerName, properties); } /** @@ -760,7 +693,7 @@ public TResultFileSinkOptions toSinkOptions() { sinkOptions.setWithBom(withBom); if (brokerDesc != null) { - sinkOptions.setBrokerProperties(brokerDesc.getProperties()); + sinkOptions.setBrokerProperties(brokerDesc.getStorageProperties().getBackendConfigProperties()); // broker_addresses of sinkOptions will be set in Coordinator. // Because we need to choose the nearest broker with the result sink node. } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java index 67a76cec450e36..3b5b4a4ede2357 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java @@ -22,7 +22,6 @@ import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.UserException; import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.common.util.URI; import org.apache.doris.datasource.property.constants.BosProperties; import org.apache.doris.thrift.TStorageBackendType; @@ -41,46 +40,9 @@ public static void checkPath(String path, StorageBackend.StorageType type, Strin if (Strings.isNullOrEmpty(path)) { throw new AnalysisException(exceptionMsg == null ? "No destination path specified." : exceptionMsg); } - checkUri(URI.create(path), type); + //checkUri(URI.create(path), type); } - public static void checkUri(URI uri, StorageBackend.StorageType type) throws AnalysisException { - String schema = uri.getScheme(); - if (schema == null) { - throw new AnalysisException( - "Invalid export path, there is no schema of URI found. please check your path."); - } - if (type == StorageBackend.StorageType.BROKER) { - if (!schema.equalsIgnoreCase("bos") - && !schema.equalsIgnoreCase("afs") - && !schema.equalsIgnoreCase("hdfs") - && !schema.equalsIgnoreCase("viewfs") - && !schema.equalsIgnoreCase("ofs") - && !schema.equalsIgnoreCase("obs") - && !schema.equalsIgnoreCase("oss") - && !schema.equalsIgnoreCase("s3a") - && !schema.equalsIgnoreCase("cosn") - && !schema.equalsIgnoreCase("gfs") - && !schema.equalsIgnoreCase("jfs") - && !schema.equalsIgnoreCase("azure") - && !schema.equalsIgnoreCase("gs")) { - throw new AnalysisException( - "Invalid broker path " + uri.toString() + ". please use valid 'hdfs://', 'viewfs://', 'afs://'," - + " 'bos://', 'ofs://', 'obs://', 'oss://', 's3a://', 'cosn://', 'gfs://', 'gs://'" - + " or 'jfs://' path."); - } - } else if (type == StorageBackend.StorageType.S3 && !schema.equalsIgnoreCase("s3")) { - throw new AnalysisException("Invalid export path " + uri.toString() + ". please use valid 's3://' path."); - } else if (type == StorageBackend.StorageType.AZURE && !schema.equalsIgnoreCase("azure")) { - throw new AnalysisException("Invalid export path. please use valid 'azure://' path."); - } else if (type == StorageBackend.StorageType.HDFS && !schema.equalsIgnoreCase("hdfs") - && !schema.equalsIgnoreCase("viewfs")) { - throw new AnalysisException("Invalid export path. please use valid 'HDFS://' or 'viewfs://' path."); - } else if (type == StorageBackend.StorageType.LOCAL && !schema.equalsIgnoreCase("file")) { - throw new AnalysisException( - "Invalid export path. please use valid '" + OutFileClause.LOCAL_FILE_PREFIX + "' path."); - } - } public StorageBackend(String storageName, String location, StorageType storageType, Map properties) { @@ -185,6 +147,21 @@ public TStorageBackendType toThrift() { return TStorageBackendType.BROKER; } } + + public static StorageType convertToStorageType(String storageName) { + switch (storageName.toLowerCase()) { + case "hdfs": + return StorageType.HDFS; + case "s3": + return StorageType.S3; + case "jfs": + return StorageType.JFS; + case "local": + return StorageType.LOCAL; + default: + throw new IllegalArgumentException("Invalid storage type: " + storageName); + } + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java index 902f5d24b91820..a5fad0808f159c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java @@ -17,7 +17,11 @@ package org.apache.doris.analysis; +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.property.storage.StorageProperties; + import com.google.gson.annotations.SerializedName; +import lombok.Getter; import java.util.Map; @@ -32,16 +36,29 @@ * The broker's StorageBackend.StorageType desc */ public class StorageDesc extends ResourceDesc { + + @Deprecated @SerializedName("st") protected StorageBackend.StorageType storageType; + @Getter + protected StorageProperties storageProperties; + public StorageDesc() { } public StorageDesc(String name, StorageBackend.StorageType storageType, Map properties) { this.name = name; this.storageType = storageType; + this.storageProperties = StorageProperties.createPrimary(properties); + this.properties = properties; + } + + public StorageDesc(String name, Map properties) throws UserException { + this.name = name; this.properties = properties; + this.storageProperties = StorageProperties.createPrimary(properties); + this.storageType = StorageBackend.StorageType.convertToStorageType(storageProperties.getStorageName()); } public void setName(String name) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index b3511d9d6e9910..7b87736c173267 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -45,15 +45,17 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.MasterDaemon; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; -import org.apache.doris.fs.FileSystemFactory; -import org.apache.doris.fs.remote.AzureFileSystem; -import org.apache.doris.fs.remote.RemoteFileSystem; -import org.apache.doris.fs.remote.S3FileSystem; +import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.fsv2.FileSystemFactory; +import org.apache.doris.fsv2.remote.AzureFileSystem; +import org.apache.doris.fsv2.remote.RemoteFileSystem; +import org.apache.doris.fsv2.remote.S3FileSystem; import org.apache.doris.persist.BarrierLog; import org.apache.doris.task.DirMoveTask; import org.apache.doris.task.DownloadTask; @@ -214,8 +216,12 @@ public void createRepository(CreateRepositoryStmt stmt) throws DdlException { "broker does not exist: " + stmt.getBrokerName()); } - RemoteFileSystem fileSystem = FileSystemFactory.get(stmt.getBrokerName(), stmt.getStorageType(), - stmt.getProperties()); + RemoteFileSystem fileSystem; + try { + fileSystem = FileSystemFactory.get(stmt.getProperties()); + } catch (UserException e) { + throw new DdlException("Failed to initialize remote file system: " + e.getMessage()); + } long repoId = env.getNextId(); Repository repo = new Repository(repoId, stmt.getName(), stmt.isReadOnly(), stmt.getLocation(), fileSystem); @@ -231,9 +237,41 @@ public void createRepository(CreateRepositoryStmt stmt) throws DdlException { } public void alterRepository(AlterRepositoryStmt stmt) throws DdlException { - alterRepositoryInternal(stmt.getName(), stmt.getProperties()); + tryLock(); + try { + Repository repo = repoMgr.getRepo(stmt.getName()); + if (repo == null) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Repository does not exist"); + } + Map allProperties = new HashMap<>(repo.getRemoteFileSystem().getProperties()); + allProperties.putAll(stmt.getProperties()); + StorageProperties newStorageProperties = StorageProperties.createPrimary(allProperties); + RemoteFileSystem fileSystem = FileSystemFactory.get(newStorageProperties); + + Repository newRepo = new Repository(repo.getId(), repo.getName(), repo.isReadOnly(), + repo.getLocation(), fileSystem); + if (!newRepo.ping()) { + LOG.warn("Failed to connect repository {}. msg: {}", repo.getName(), repo.getErrorMsg()); + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Repo can not ping with new storage properties"); + } + + Status st = repoMgr.alterRepo(newRepo, false /* not replay */); + if (!st.ok()) { + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Failed to alter repository: " + st.getErrMsg()); + } + for (AbstractJob job : getAllCurrentJobs()) { + if (!job.isDone() && job.getRepoId() == repo.getId()) { + job.updateRepo(newRepo); + } + } + } finally { + seqlock.unlock(); + } } + public void alterRepositoryInternal(String repoName, Map properties) throws DdlException { tryLock(); try { @@ -249,14 +287,7 @@ public void alterRepositoryInternal(String repoName, Map propert if (!status.ok()) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, status.getErrMsg()); } - RemoteFileSystem fileSystem = null; - if (repo.getRemoteFileSystem() instanceof S3FileSystem) { - fileSystem = FileSystemFactory.get(repo.getRemoteFileSystem().getName(), - StorageBackend.StorageType.S3, oldProperties); - } else if (repo.getRemoteFileSystem() instanceof AzureFileSystem) { - fileSystem = FileSystemFactory.get(repo.getRemoteFileSystem().getName(), - StorageBackend.StorageType.AZURE, oldProperties); - } + RemoteFileSystem fileSystem = FileSystemFactory.get(oldProperties); Repository newRepo = new Repository(repo.getId(), repo.getName(), repo.isReadOnly(), repo.getLocation(), fileSystem); @@ -280,6 +311,10 @@ public void alterRepositoryInternal(String repoName, Map propert ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Only support alter s3 or azure repository"); } + } catch (UserException e) { + LOG.warn("Failed to alter repository {}. msg: {}", repoName, e.getMessage()); + ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, + "Failed to alter repository: " + e.getMessage()); } finally { seqlock.unlock(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index b530d3b35e0a68..92e25ecc341503 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -39,7 +39,6 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.TimeUtils; -import org.apache.doris.datasource.property.S3ClientBEProperties; import org.apache.doris.persist.BarrierLog; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; @@ -388,7 +387,7 @@ public synchronized Status updateRepo(Repository repo) { continue; } ((UploadTask) task).updateBrokerProperties( - S3ClientBEProperties.getBeFSProperties(repo.getRemoteFileSystem().getProperties())); + repo.getRemoteFileSystem().getStorageProperties().getBackendConfigProperties()); AgentTaskQueue.updateTask(beId, TTaskType.UPLOAD, signature, task); } LOG.info("finished to update upload job properties. {}", this); @@ -783,7 +782,7 @@ private void uploadSnapshot() { long signature = env.getNextId(); UploadTask task = new UploadTask(null, beId, signature, jobId, dbId, srcToDest, brokers.get(0), - S3ClientBEProperties.getBeFSProperties(repo.getRemoteFileSystem().getProperties()), + repo.getRemoteFileSystem().getStorageProperties().getBackendConfigProperties(), repo.getRemoteFileSystem().getStorageType(), repo.getLocation()); batchTask.addTask(task); unfinishedTaskIds.put(signature, beId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java index 1450ef9a03459f..e05e753e8d4777 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java @@ -25,19 +25,19 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.property.constants.S3Properties; -import org.apache.doris.fs.FileSystemFactory; -import org.apache.doris.fs.PersistentFileSystem; -import org.apache.doris.fs.remote.AzureFileSystem; -import org.apache.doris.fs.remote.BrokerFileSystem; -import org.apache.doris.fs.remote.RemoteFile; -import org.apache.doris.fs.remote.RemoteFileSystem; -import org.apache.doris.fs.remote.S3FileSystem; -import org.apache.doris.fs.remote.dfs.DFSFileSystem; +import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.fsv2.FileSystemFactory; +import org.apache.doris.fsv2.PersistentFileSystem; +import org.apache.doris.fsv2.remote.BrokerFileSystem; +import org.apache.doris.fsv2.remote.RemoteFile; +import org.apache.doris.fsv2.remote.RemoteFileSystem; +import org.apache.doris.fsv2.remote.S3FileSystem; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.system.Backend; @@ -203,17 +203,41 @@ public static Repository read(DataInput in) throws IOException { } } + public Status alterRepositoryS3Properties(Map properties) { + if (this.fileSystem instanceof S3FileSystem) { + Map oldProperties = new HashMap<>(this.getRemoteFileSystem().getProperties()); + oldProperties.remove(S3Properties.ACCESS_KEY); + oldProperties.remove(S3Properties.SECRET_KEY); + oldProperties.remove(S3Properties.SESSION_TOKEN); + oldProperties.remove(S3Properties.Env.ACCESS_KEY); + oldProperties.remove(S3Properties.Env.SECRET_KEY); + oldProperties.remove(S3Properties.Env.TOKEN); + for (Map.Entry entry : properties.entrySet()) { + if (Objects.equals(entry.getKey(), S3Properties.ACCESS_KEY) + || Objects.equals(entry.getKey(), S3Properties.Env.ACCESS_KEY)) { + oldProperties.putIfAbsent(S3Properties.ACCESS_KEY, entry.getValue()); + } + if (Objects.equals(entry.getKey(), S3Properties.SECRET_KEY) + || Objects.equals(entry.getKey(), S3Properties.Env.SECRET_KEY)) { + oldProperties.putIfAbsent(S3Properties.SECRET_KEY, entry.getValue()); + } + if (Objects.equals(entry.getKey(), S3Properties.SESSION_TOKEN) + || Objects.equals(entry.getKey(), S3Properties.Env.TOKEN)) { + oldProperties.putIfAbsent(S3Properties.SESSION_TOKEN, entry.getValue()); + } + } + properties.clear(); + properties.putAll(oldProperties); + return Status.OK; + } else { + return new Status(ErrCode.COMMON_ERROR, "Only support alter s3 repository"); + } + } + @Override public void gsonPostProcess() { - StorageBackend.StorageType type = StorageBackend.StorageType.BROKER; - if (this.fileSystem.properties.containsKey(PersistentFileSystem.STORAGE_TYPE)) { - type = StorageBackend.StorageType.valueOf( - this.fileSystem.properties.get(PersistentFileSystem.STORAGE_TYPE)); - this.fileSystem.properties.remove(PersistentFileSystem.STORAGE_TYPE); - } - this.fileSystem = FileSystemFactory.get(this.fileSystem.getName(), - type, - this.fileSystem.getProperties()); + StorageProperties storageProperties = StorageProperties.createPrimary(this.fileSystem.properties); + this.fileSystem = FileSystemFactory.get(storageProperties); } public long getId() { @@ -229,7 +253,14 @@ public boolean isReadOnly() { } public String getLocation() { - return location; + if (null == fileSystem) { + return location; + } + try { + return fileSystem.getStorageProperties().validateAndNormalizeUri(location); + } catch (UserException e) { + throw new RuntimeException(e); + } } public String getErrorMsg() { @@ -277,7 +308,7 @@ public Status initRepository() { if (name.compareTo((String) root.get("name")) != 0) { return new Status(ErrCode.COMMON_ERROR, "Invalid repository __repo_info, expected repo '" + name + "', but get name '" - + (String) root.get("name") + "' from " + repoInfoFilePath); + + (String) root.get("name") + "' from " + repoInfoFilePath); } name = (String) root.get("name"); createTime = TimeUtils.timeStringToLong((String) root.get("create_time")); @@ -307,54 +338,23 @@ public Status initRepository() { } } - public Status alterRepositoryS3Properties(Map properties) { - if (fileSystem instanceof S3FileSystem) { - Map oldProperties = new HashMap<>(this.getRemoteFileSystem().getProperties()); - oldProperties.remove(S3Properties.ACCESS_KEY); - oldProperties.remove(S3Properties.SECRET_KEY); - oldProperties.remove(S3Properties.SESSION_TOKEN); - oldProperties.remove(S3Properties.Env.ACCESS_KEY); - oldProperties.remove(S3Properties.Env.SECRET_KEY); - oldProperties.remove(S3Properties.Env.TOKEN); - for (Map.Entry entry : properties.entrySet()) { - if (Objects.equals(entry.getKey(), S3Properties.ACCESS_KEY) - || Objects.equals(entry.getKey(), S3Properties.Env.ACCESS_KEY)) { - oldProperties.putIfAbsent(S3Properties.ACCESS_KEY, entry.getValue()); - } - if (Objects.equals(entry.getKey(), S3Properties.SECRET_KEY) - || Objects.equals(entry.getKey(), S3Properties.Env.SECRET_KEY)) { - oldProperties.putIfAbsent(S3Properties.SECRET_KEY, entry.getValue()); - } - if (Objects.equals(entry.getKey(), S3Properties.SESSION_TOKEN) - || Objects.equals(entry.getKey(), S3Properties.Env.TOKEN)) { - oldProperties.putIfAbsent(S3Properties.SESSION_TOKEN, entry.getValue()); - } - } - properties.clear(); - properties.putAll(oldProperties); - return Status.OK; - } else { - return new Status(ErrCode.COMMON_ERROR, "Only support alter s3 repository"); - } - } - // eg: location/__palo_repository_repo_name/__repo_info public String assembleRepoInfoFilePath() { - return Joiner.on(PATH_DELIMITER).join(location, + return Joiner.on(PATH_DELIMITER).join(getLocation(), joinPrefix(PREFIX_REPO, name), FILE_REPO_INFO); } // eg: location/__palo_repository_repo_name/__my_sp1/__meta public String assembleMetaInfoFilePath(String label) { - return Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name), + return Joiner.on(PATH_DELIMITER).join(getLocation(), joinPrefix(PREFIX_REPO, name), joinPrefix(PREFIX_SNAPSHOT_DIR, label), FILE_META_INFO); } // eg: location/__palo_repository_repo_name/__my_sp1/__info_2018-01-01-08-00-00 public String assembleJobInfoFilePath(String label, long createTime) { - return Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name), + return Joiner.on(PATH_DELIMITER).join(getLocation(), joinPrefix(PREFIX_REPO, name), joinPrefix(PREFIX_SNAPSHOT_DIR, label), jobInfoFileNameWithTimestamp(createTime)); } @@ -362,7 +362,7 @@ public String assembleJobInfoFilePath(String label, long createTime) { // eg: // __palo_repository_repo_name/__ss_my_ss1/__ss_content/__db_10001/__tbl_10020/__part_10031/__idx_10020/__10022/ public String getRepoTabletPathBySnapshotInfo(String label, SnapshotInfo info) { - String path = Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name), + String path = Joiner.on(PATH_DELIMITER).join(getLocation(), joinPrefix(PREFIX_REPO, name), joinPrefix(PREFIX_SNAPSHOT_DIR, label), DIR_SNAPSHOT_CONTENT, joinPrefix(PREFIX_DB, info.getDbId()), @@ -381,7 +381,7 @@ public String getRepoTabletPathBySnapshotInfo(String label, SnapshotInfo info) { } public String getRepoPath(String label, String childPath) { - String path = Joiner.on(PATH_DELIMITER).join(location, joinPrefix(PREFIX_REPO, name), + String path = Joiner.on(PATH_DELIMITER).join(getLocation(), joinPrefix(PREFIX_REPO, name), joinPrefix(PREFIX_SNAPSHOT_DIR, label), DIR_SNAPSHOT_CONTENT, childPath); @@ -568,7 +568,7 @@ public Status upload(String localFilePath, String remoteFilePath) { if (!st.ok()) { return st; } - } else if (fileSystem instanceof S3FileSystem || fileSystem instanceof AzureFileSystem) { + } else { if (LOG.isDebugEnabled()) { LOG.debug("get md5sum of file: {}. final remote path: {}", localFilePath, finalRemotePath); } @@ -577,27 +577,12 @@ public Status upload(String localFilePath, String remoteFilePath) { return st; } - // upload final file - st = fileSystem.upload(localFilePath, finalRemotePath); - if (!st.ok()) { - return st; - } - } else if (fileSystem instanceof DFSFileSystem) { - if (LOG.isDebugEnabled()) { - LOG.debug("hdfs get md5sum of file: {}. final remote path: {}", localFilePath, finalRemotePath); - } - st = fileSystem.delete(finalRemotePath); - if (!st.ok()) { - return st; - } - // upload final file st = fileSystem.upload(localFilePath, finalRemotePath); if (!st.ok()) { return st; } } - LOG.info("finished to upload local file {} to remote file: {}", localFilePath, finalRemotePath); return st; } @@ -637,7 +622,7 @@ public Status download(String remoteFilePath, String localFilePath) { // 2. download status = fileSystem.downloadWithFileSize(remoteFilePathWithChecksum, localFilePath, - remoteFiles.get(0).getSize()); + remoteFiles.get(0).getSize()); if (!status.ok()) { return status; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java index 853c1841449046..d57593c5443098 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RepositoryMgr.java @@ -23,8 +23,8 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.Daemon; -import org.apache.doris.fs.remote.AzureFileSystem; -import org.apache.doris.fs.remote.S3FileSystem; +import org.apache.doris.fsv2.remote.AzureFileSystem; +import org.apache.doris.fsv2.remote.S3FileSystem; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; @@ -104,6 +104,9 @@ public Repository getRepo(long repoId) { return repoIdMap.get(repoId); } + /** + * todo: why not support alter other file system like hdfs + */ public Status alterRepo(Repository newRepo, boolean isReplay) { lock.lock(); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 617733fad03db2..0a02147c62634e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -68,7 +68,6 @@ import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.datasource.property.S3ClientBEProperties; import org.apache.doris.persist.ColocatePersistInfo; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; @@ -437,7 +436,7 @@ public synchronized Status updateRepo(Repository repo) { continue; } ((DownloadTask) task).updateBrokerProperties( - S3ClientBEProperties.getBeFSProperties(repo.getRemoteFileSystem().getProperties())); + repo.getRemoteFileSystem().getStorageProperties().getBackendConfigProperties()); AgentTaskQueue.updateTask(beId, TTaskType.DOWNLOAD, signature, task); } LOG.info("finished to update download job properties. {}", this); @@ -1864,7 +1863,7 @@ private void downloadRemoteSnapshots() { long signature = env.getNextId(); DownloadTask task = new DownloadTask(null, beId, signature, jobId, dbId, srcToDest, brokerAddrs.get(0), - S3ClientBEProperties.getBeFSProperties(repo.getRemoteFileSystem().getProperties()), + repo.getRemoteFileSystem().getStorageProperties().getBackendConfigProperties(), repo.getRemoteFileSystem().getStorageType(), repo.getLocation()); batchTask.addTask(task); unfinishedSignatureToId.put(signature, beId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java index 617d346ba3daa3..b452d1cfeabc90 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java @@ -28,9 +28,9 @@ import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.datasource.hive.HiveMetaStoreCache; -import org.apache.doris.fs.FileSystemFactory; -import org.apache.doris.fs.remote.RemoteFile; -import org.apache.doris.fs.remote.RemoteFileSystem; +import org.apache.doris.fsv2.FileSystemFactory; +import org.apache.doris.fsv2.remote.RemoteFile; +import org.apache.doris.fsv2.remote.RemoteFileSystem; import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.TBrokerCheckPathExistRequest; import org.apache.doris.thrift.TBrokerCheckPathExistResponse; @@ -85,8 +85,7 @@ public class BrokerUtil { public static void parseFile(String path, BrokerDesc brokerDesc, List fileStatuses) throws UserException { List rfiles = new ArrayList<>(); - try (RemoteFileSystem fileSystem = FileSystemFactory.get( - brokerDesc.getName(), brokerDesc.getStorageType(), brokerDesc.getProperties())) { + try (RemoteFileSystem fileSystem = FileSystemFactory.get(brokerDesc.getStorageProperties())) { Status st = fileSystem.globList(path, rfiles, false); if (!st.ok()) { throw new UserException(st.getErrMsg()); @@ -107,8 +106,7 @@ public static void parseFile(String path, BrokerDesc brokerDesc, List getBackendConfigProperties() { @Override - protected void initNormalizeAndCheckProps() throws UserException { + protected void initNormalizeAndCheckProps() { super.initNormalizeAndCheckProps(); setEndpointIfNotSet(); if (!isValidEndpoint(getEndpoint())) { @@ -184,7 +184,7 @@ private boolean isValidEndpoint(String endpoint) { return endpointPattern().matcher(endpoint).matches(); } - private void setEndpointIfNotSet() throws UserException { + private void setEndpointIfNotSet() { if (StringUtils.isNotBlank(getEndpoint())) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AzureProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AzureProperties.java index 512ed92cca9054..9d85c3657668c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AzureProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AzureProperties.java @@ -100,7 +100,7 @@ public AzureProperties(Map origProps) { private static final String AZURE_ENDPOINT_SUFFIX = ".blob.core.windows.net"; @Override - protected void initNormalizeAndCheckProps() throws UserException { + protected void initNormalizeAndCheckProps() { super.initNormalizeAndCheckProps(); //check endpoint if (!endpoint.endsWith(AZURE_ENDPOINT_SUFFIX)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java index 6dc72e1b27cc97..e382015a6a3713 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java @@ -45,7 +45,7 @@ public class COSProperties extends AbstractS3CompatibleProperties { protected String region = ""; @Getter - @ConnectorProperty(names = {"cos.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key"}, + @ConnectorProperty(names = {"cos.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key", "s3.access_key"}, description = "The access key of COS.") protected String accessKey = ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java index d4722a3c1ad0f5..6da1d2fd0ef207 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java @@ -98,7 +98,7 @@ public static boolean guessIsMe(Map props) { } @Override - protected void initNormalizeAndCheckProps() throws UserException { + protected void initNormalizeAndCheckProps() { super.initNormalizeAndCheckProps(); extractUserOverriddenHdfsConfig(origProps); initHadoopConfiguration(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java index 87ad9b5761c7ab..9192aef87eaec6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java @@ -38,12 +38,12 @@ public class OBSProperties extends AbstractS3CompatibleProperties { protected String endpoint = ""; @Getter - @ConnectorProperty(names = {"obs.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key"}, + @ConnectorProperty(names = {"obs.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key", "s3.access_key"}, description = "The access key of OBS.") protected String accessKey = ""; @Getter - @ConnectorProperty(names = {"obs.secret_key", "secret_key", "s3.secret_key"}, + @ConnectorProperty(names = {"obs.secret_key", "s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY"}, description = "The secret key of OBS.") protected String secretKey = ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java index e0e0def53f6d01..0a09a8602532dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java @@ -82,7 +82,7 @@ protected void checkRequiredProperties() { } @Override - protected void initNormalizeAndCheckProps() throws UserException { + protected void initNormalizeAndCheckProps() { super.initNormalizeAndCheckProps(); initConfigurationParams(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index 478f45ee3d06b5..f4fad1c87d31ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -53,6 +53,12 @@ public class S3Properties extends AbstractS3CompatibleProperties { description = "The access key of S3.") protected String accessKey = ""; + @Getter + @ConnectorProperty(names = {"s3.session_token", "session_token"}, + required = false, + description = "The session token of S3.") + protected String sessionToken = ""; + @Getter @ConnectorProperty(names = {"s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY"}, description = "The secret key of S3.") diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java index 745838438dd012..a5db0e56a570b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java @@ -43,14 +43,20 @@ public class S3PropertyUtils { */ public static String constructEndpointFromUrl(Map props, String stringUsePathStyle, - String stringForceParsingByStandardUri) throws UserException { + String stringForceParsingByStandardUri) { String uri = props.get(URI_KEY); if (uri == null || uri.isEmpty()) { return null; } boolean usePathStyle = Boolean.parseBoolean(stringUsePathStyle); boolean forceParsingByStandardUri = Boolean.parseBoolean(stringForceParsingByStandardUri); - S3URI s3uri = S3URI.create(uri, usePathStyle, forceParsingByStandardUri); + S3URI s3uri; + try { + s3uri = S3URI.create(uri, usePathStyle, forceParsingByStandardUri); + } catch (UserException e) { + throw new IllegalArgumentException("Invalid S3 URI: " + uri + ",usePathStyle: " + usePathStyle + + " forceParsingByStandardUri: " + forceParsingByStandardUri, e); + } return s3uri.getEndpoint().orElse(null); } @@ -68,14 +74,20 @@ public static String constructEndpointFromUrl(Map props, */ public static String constructRegionFromUrl(Map props, String stringUsePathStyle, - String stringForceParsingByStandardUri) throws UserException { + String stringForceParsingByStandardUri) { String uri = props.get(URI_KEY); if (uri == null || uri.isEmpty()) { return null; } boolean usePathStyle = Boolean.parseBoolean(stringUsePathStyle); boolean forceParsingByStandardUri = Boolean.parseBoolean(stringForceParsingByStandardUri); - S3URI s3uri = S3URI.create(uri, usePathStyle, forceParsingByStandardUri); + S3URI s3uri = null; + try { + s3uri = S3URI.create(uri, usePathStyle, forceParsingByStandardUri); + } catch (UserException e) { + throw new IllegalArgumentException("Invalid S3 URI: " + uri + ",usePathStyle: " + usePathStyle + + " forceParsingByStandardUri: " + forceParsingByStandardUri, e); + } return s3uri.getRegion().orElse(null); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java index 58cdedde869a5e..c8784b3a377ed3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java @@ -98,7 +98,7 @@ public static List createAll(Map origProps) t * @return a StorageProperties instance for the primary storage type * @throws RuntimeException if no supported storage type is found */ - public static StorageProperties createPrimary(Map origProps) throws UserException { + public static StorageProperties createPrimary(Map origProps) { for (Function, StorageProperties> func : PROVIDERS) { StorageProperties p = func.apply(origProps); if (p != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java index 3b810d72172057..78fe220fa23556 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java @@ -30,7 +30,6 @@ import org.apache.doris.common.util.FileFormatConstants; import org.apache.doris.common.util.FileFormatUtils; import org.apache.doris.common.util.TimeUtils; -import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.job.base.JobExecuteType; import org.apache.doris.job.base.JobExecutionConfiguration; import org.apache.doris.job.extensions.insert.InsertJob; @@ -469,10 +468,8 @@ private static Map getTvfProperties(BulkLoadDataDesc dataDesc, B // TODO: support multi location by union String listFilePath = filePaths.get(0); if (bulkStorageDesc.getStorageType() == BulkStorageDesc.StorageType.S3) { - S3Properties.convertToStdProperties(tvfProperties); - tvfProperties.keySet().removeIf(S3Properties.Env.FS_KEYS::contains); // TODO: check file path by s3 fs list status - tvfProperties.put(S3TableValuedFunction.PROP_URI, listFilePath); + tvfProperties.put("uri", listFilePath); } final Map dataDescProps = dataDesc.getProperties(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index c699ddaca3fd85..5e16ffc8f76d20 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -579,6 +579,22 @@ public class GsonUtils { .registerSubtype(S3FileSystem.class, S3FileSystem.class.getSimpleName()) .registerSubtype(AzureFileSystem.class, AzureFileSystem.class.getSimpleName()); + private static RuntimeTypeAdapterFactory + remoteFileSystemTypeAdapterFactoryV2 + = RuntimeTypeAdapterFactory.of(org.apache.doris.fsv2.PersistentFileSystem.class, "clazz") + .registerSubtype(org.apache.doris.fsv2.remote.dfs.DFSFileSystem.class, + org.apache.doris.fsv2.remote.dfs.DFSFileSystem.class.getSimpleName()) + .registerSubtype(org.apache.doris.fsv2.remote.dfs.JFSFileSystem.class, + org.apache.doris.fsv2.remote.dfs.JFSFileSystem.class.getSimpleName()) + .registerSubtype(org.apache.doris.fsv2.remote.dfs.OFSFileSystem.class, + org.apache.doris.fsv2.remote.dfs.OFSFileSystem.class.getSimpleName()) + .registerSubtype(org.apache.doris.fsv2.remote.ObjFileSystem.class, + org.apache.doris.fsv2.remote.ObjFileSystem.class.getSimpleName()) + .registerSubtype(org.apache.doris.fsv2.remote.S3FileSystem.class, + org.apache.doris.fsv2.remote.S3FileSystem.class.getSimpleName()) + .registerSubtype(org.apache.doris.fsv2.remote.AzureFileSystem.class, + org.apache.doris.fsv2.remote.AzureFileSystem.class.getSimpleName()); + private static RuntimeTypeAdapterFactory jobBackupTypeAdapterFactory = RuntimeTypeAdapterFactory.of(org.apache.doris.backup.AbstractJob.class, "clazz") @@ -643,6 +659,7 @@ public class GsonUtils { .registerTypeAdapterFactory(routineLoadTypeAdapterFactory) .registerTypeAdapterFactory(routineLoadJobTypeAdapterFactory) .registerTypeAdapterFactory(remoteFileSystemTypeAdapterFactory) + .registerTypeAdapterFactory(remoteFileSystemTypeAdapterFactoryV2) .registerTypeAdapterFactory(jobBackupTypeAdapterFactory) .registerTypeAdapterFactory(loadJobTypeAdapterFactory) .registerTypeAdapterFactory(partitionItemTypeAdapterFactory) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExportSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExportSink.java index 0fd2535b62bff7..9f307a376b2f01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExportSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExportSink.java @@ -82,7 +82,7 @@ protected TDataSink toThrift() { tExportSink.addToBrokerAddresses(new TNetworkAddress(broker.host, broker.port)); } } - tExportSink.setProperties(brokerDesc.getProperties()); + tExportSink.setProperties(brokerDesc.getStorageProperties().getBackendConfigProperties()); tExportSink.setHeader(header); result.setExportSink(tExportSink); return result; diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index d47b7f082569d2..752d581229eea5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -42,6 +42,7 @@ import org.apache.doris.common.util.Util; import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.datasource.property.fileformat.FileFormatProperties; +import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.datasource.tvf.source.TVFScanNode; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.planner.PlanNodeId; @@ -106,6 +107,7 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio protected List fileStatuses = Lists.newArrayList(); protected Map locationProperties = Maps.newHashMap(); + protected StorageProperties storageProperties; protected String filePath; protected Optional resourceName = Optional.empty(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java index 80149e3d1380ae..b6caa0345d0d1a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java @@ -18,15 +18,13 @@ package org.apache.doris.tablefunction; import org.apache.doris.analysis.BrokerDesc; -import org.apache.doris.analysis.StorageBackend; import org.apache.doris.analysis.StorageBackend.StorageType; -import org.apache.doris.catalog.HdfsResource; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; -import org.apache.doris.common.util.URI; +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.thrift.TFileType; -import com.google.common.base.Strings; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -39,7 +37,6 @@ public class HdfsTableValuedFunction extends ExternalFileTableValuedFunction { public static final Logger LOG = LogManager.getLogger(HdfsTableValuedFunction.class); public static final String NAME = "hdfs"; - private static final String PROP_URI = "uri"; public HdfsTableValuedFunction(Map properties) throws AnalysisException { init(properties); @@ -47,28 +44,15 @@ public HdfsTableValuedFunction(Map properties) throws AnalysisEx private void init(Map properties) throws AnalysisException { // 1. analyze common properties - Map otherProps = super.parseCommonProperties(properties); - + Map props = super.parseCommonProperties(properties); // 2. analyze uri - String uriStr = getOrDefaultAndRemove(otherProps, PROP_URI, null); - if (Strings.isNullOrEmpty(uriStr)) { - throw new AnalysisException(String.format("Properties '%s' is required.", PROP_URI)); - } - URI uri = URI.create(uriStr); - StorageBackend.checkUri(uri, StorageType.HDFS); - filePath = uri.getScheme() + "://" + uri.getAuthority() + uri.getPath(); - - // 3. analyze other properties - for (String key : otherProps.keySet()) { - if (HdfsResource.HADOOP_FS_NAME.equalsIgnoreCase(key)) { - locationProperties.put(HdfsResource.HADOOP_FS_NAME, otherProps.get(key)); - } else { - locationProperties.put(key, otherProps.get(key)); - } - } - // If the user does not specify the HADOOP_FS_NAME, we will use the uri's scheme and authority - if (!locationProperties.containsKey(HdfsResource.HADOOP_FS_NAME)) { - locationProperties.put(HdfsResource.HADOOP_FS_NAME, uri.getScheme() + "://" + uri.getAuthority()); + try { + this.storageProperties = StorageProperties.createPrimary(props); + locationProperties.putAll(storageProperties.getBackendConfigProperties()); + String uri = storageProperties.validateAndGetUri(props); + filePath = storageProperties.validateAndNormalizeUri(uri); + } catch (UserException e) { + throw new AnalysisException("Failed check storage props", e); } if (!FeConstants.runningUnitTest) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index 56c438c303e31d..d4767cd74b0a13 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -18,22 +18,12 @@ package org.apache.doris.tablefunction; import org.apache.doris.analysis.BrokerDesc; -import org.apache.doris.analysis.StorageBackend.StorageType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.UserException; -import org.apache.doris.common.credentials.CloudCredentialWithEndpoint; -import org.apache.doris.common.util.S3URI; -import org.apache.doris.datasource.property.PropertyConverter; -import org.apache.doris.datasource.property.S3ClientBEProperties; -import org.apache.doris.datasource.property.constants.AzureProperties; -import org.apache.doris.datasource.property.constants.S3Properties; -import org.apache.doris.fs.FileSystemFactory; +import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.thrift.TFileType; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableSet; - import java.util.Map; /** @@ -49,112 +39,28 @@ */ public class S3TableValuedFunction extends ExternalFileTableValuedFunction { public static final String NAME = "s3"; - public static final String PROP_URI = "uri"; - - private static final ImmutableSet DEPRECATED_KEYS = - ImmutableSet.of("access_key", "secret_key", "session_token", "region", - "ACCESS_KEY", "SECRET_KEY", "SESSION_TOKEN", "REGION"); public S3TableValuedFunction(Map properties) throws AnalysisException { // 1. analyze common properties - Map otherProps = super.parseCommonProperties(properties); - - // 2. analyze uri and other properties - String uriStr = getOrDefaultAndRemove(otherProps, PROP_URI, null); - if (Strings.isNullOrEmpty(uriStr)) { - throw new AnalysisException(String.format("Properties '%s' is required.", PROP_URI)); - } - forwardCompatibleDeprecatedKeys(otherProps); - - String usePathStyle = getOrDefaultAndRemove(otherProps, PropertyConverter.USE_PATH_STYLE, - PropertyConverter.USE_PATH_STYLE_DEFAULT_VALUE); - String forceParsingByStandardUri = getOrDefaultAndRemove(otherProps, - PropertyConverter.FORCE_PARSING_BY_STANDARD_URI, - PropertyConverter.FORCE_PARSING_BY_STANDARD_URI_DEFAULT_VALUE); - - S3URI s3uri = getS3Uri(uriStr, Boolean.parseBoolean(usePathStyle.toLowerCase()), - Boolean.parseBoolean(forceParsingByStandardUri.toLowerCase())); - - // get endpoint first from properties, if not present, get it from s3 uri. - // If endpoint is missing, exception will be thrown. - String endpoint = constructEndpoint(otherProps, s3uri); - if (!otherProps.containsKey(S3Properties.REGION)) { - String region; - if (AzureProperties.checkAzureProviderPropertyExist(properties)) { - // Azure could run without region - region = s3uri.getRegion().orElse("DUMMY-REGION"); - } else { - region = s3uri.getRegion().orElseThrow(() -> new AnalysisException( - String.format("Properties '%s' is required.", S3Properties.REGION))); - } - otherProps.put(S3Properties.REGION, region); - } - checkNecessaryS3Properties(otherProps); - CloudCredentialWithEndpoint credential = new CloudCredentialWithEndpoint(endpoint, - getOrDefaultAndRemove(otherProps, S3Properties.REGION, ""), - getOrDefaultAndRemove(otherProps, S3Properties.ACCESS_KEY, ""), - getOrDefaultAndRemove(otherProps, S3Properties.SECRET_KEY, "")); - if (otherProps.containsKey(S3Properties.SESSION_TOKEN)) { - credential.setSessionToken(getOrDefaultAndRemove(otherProps, S3Properties.SESSION_TOKEN, "")); - } - - locationProperties = S3Properties.credentialToMap(credential); - locationProperties.put(PropertyConverter.USE_PATH_STYLE, usePathStyle); - if (AzureProperties.checkAzureProviderPropertyExist(properties)) { - // For Azure's compatibility, we need bucket to connect to the blob storage's container - locationProperties.put(S3Properties.BUCKET, s3uri.getBucket()); + Map props = super.parseCommonProperties(properties); + try { + //todo rename locationProperties + this.storageProperties = StorageProperties.createPrimary(props); + locationProperties.putAll(storageProperties.getBackendConfigProperties()); + String uri = storageProperties.validateAndGetUri(props); + filePath = storageProperties.validateAndNormalizeUri(uri); + } catch (UserException e) { + throw new RuntimeException(e); } - locationProperties.putAll(S3ClientBEProperties.getBeFSProperties(locationProperties)); - locationProperties.putAll(otherProps); - - filePath = NAME + S3URI.SCHEME_DELIM + s3uri.getBucket() + S3URI.PATH_DELIM + s3uri.getKey(); - if (FeConstants.runningUnitTest) { // Just check - FileSystemFactory.getS3FileSystem(locationProperties); + // Fixme wait to be done #50320 + // FileSystemFactory.get(storageProperties); } else { parseFile(); } } - private String constructEndpoint(Map properties, S3URI s3uri) throws AnalysisException { - // get endpoint first from properties, if not present, get it from s3 uri. - String endpoint = getOrDefaultAndRemove(properties, S3Properties.ENDPOINT, s3uri.getEndpoint().orElse("")); - if (AzureProperties.checkAzureProviderPropertyExist(properties)) { - String accountName = properties.getOrDefault(S3Properties.ACCESS_KEY, ""); - if (accountName.isEmpty()) { - throw new AnalysisException(String.format("Properties '%s' is required.", S3Properties.ACCESS_KEY)); - } - endpoint = AzureProperties.formatAzureEndpoint(endpoint, accountName); - } else if (Strings.isNullOrEmpty(endpoint)) { - throw new AnalysisException(String.format("Properties '%s' is required.", S3Properties.ENDPOINT)); - } - return endpoint; - } - - private void forwardCompatibleDeprecatedKeys(Map props) { - for (String deprecatedKey : DEPRECATED_KEYS) { - String value = props.remove(deprecatedKey); - if (!Strings.isNullOrEmpty(value)) { - props.put("s3." + deprecatedKey.toLowerCase(), value); - } - } - } - - private void checkNecessaryS3Properties(Map props) throws AnalysisException { - if (Strings.isNullOrEmpty(props.get(S3Properties.REGION))) { - throw new AnalysisException(String.format("Properties '%s' is required.", S3Properties.REGION)); - } - // do not check ak and sk, because we can read them from system environment. - } - - private S3URI getS3Uri(String uri, boolean isPathStyle, boolean forceParsingStandardUri) throws AnalysisException { - try { - return S3URI.create(uri, isPathStyle, forceParsingStandardUri); - } catch (UserException e) { - throw new AnalysisException("parse s3 uri failed, uri = " + uri, e); - } - } // =========== implement abstract methods of ExternalFileTableValuedFunction ================= @Override @@ -170,7 +76,7 @@ public String getFilePath() { @Override public BrokerDesc getBrokerDesc() { - return new BrokerDesc("S3TvfBroker", StorageType.S3, locationProperties); + return new BrokerDesc("S3TvfBroker", locationProperties); } // =========== implement abstract methods of TableValuedFunctionIf ================= diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java index cbee4a8bcecd5b..2dac48d4afe0b7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java @@ -18,7 +18,6 @@ package org.apache.doris.backup; import org.apache.doris.analysis.BackupStmt; -import org.apache.doris.analysis.StorageBackend; import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TableRef; import org.apache.doris.backup.BackupJob.BackupJobState; @@ -29,10 +28,11 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.util.UnitTestUtil; import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.fs.FileSystemFactory; +import org.apache.doris.fsv2.FileSystemFactory; import org.apache.doris.persist.EditLog; import org.apache.doris.task.AgentBatchTask; import org.apache.doris.task.AgentTask; @@ -96,6 +96,9 @@ public class BackupJobTest { private MockRepositoryMgr repoMgr; + public BackupJobTest() throws UserException { + } + // Thread is not mockable in Jmockit, use subclass instead private final class MockBackupHandler extends BackupHandler { public MockBackupHandler(Env env) { @@ -124,7 +127,7 @@ public Repository getRepo(long repoId) { private EditLog editLog; private Repository repo = new Repository(repoId, "repo", false, "my_repo", - FileSystemFactory.get("broker", StorageBackend.StorageType.BROKER, Maps.newHashMap())); + FileSystemFactory.get(Maps.newHashMap())); @BeforeClass public static void start() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java index a49d7e4328e94b..e2e42563702f69 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java @@ -18,13 +18,13 @@ package org.apache.doris.backup; import org.apache.doris.analysis.ShowRepositoriesStmt; -import org.apache.doris.analysis.StorageBackend; import org.apache.doris.catalog.BrokerMgr; import org.apache.doris.catalog.FsBroker; import org.apache.doris.common.AnalysisException; -import org.apache.doris.fs.FileSystemFactory; -import org.apache.doris.fs.remote.RemoteFile; -import org.apache.doris.fs.remote.RemoteFileSystem; +import org.apache.doris.common.UserException; +import org.apache.doris.fsv2.FileSystemFactory; +import org.apache.doris.fsv2.remote.RemoteFile; +import org.apache.doris.fsv2.remote.RemoteFileSystem; import org.apache.doris.service.FrontendOptions; import com.google.common.collect.Lists; @@ -318,12 +318,12 @@ public Status list(String remotePath, List result) { } @Test - public void testPersist() { + public void testPersist() throws UserException { Map properties = Maps.newHashMap(); properties.put("bos_endpoint", "http://gz.bcebos.com"); properties.put("bos_accesskey", "a"); properties.put("bos_secret_accesskey", "b"); - RemoteFileSystem fs = FileSystemFactory.get(brokerName, StorageBackend.StorageType.BROKER, properties); + RemoteFileSystem fs = FileSystemFactory.get(properties); repo = new Repository(10000, "repo", false, location, fs); File file = new File("./Repository"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java index d47be652861c44..7588d94203f1ac 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java @@ -17,7 +17,6 @@ package org.apache.doris.backup; -import org.apache.doris.analysis.StorageBackend; import org.apache.doris.backup.BackupJobInfo.BackupIndexInfo; import org.apache.doris.backup.BackupJobInfo.BackupOlapTableInfo; import org.apache.doris.backup.BackupJobInfo.BackupPartitionInfo; @@ -38,9 +37,10 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.MarkedCountDownLatch; +import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.InternalCatalog; -import org.apache.doris.fs.FileSystemFactory; +import org.apache.doris.fsv2.FileSystemFactory; import org.apache.doris.persist.EditLog; import org.apache.doris.resource.Tag; import org.apache.doris.system.SystemInfoService; @@ -91,6 +91,9 @@ public class RestoreJobTest { private MockRepositoryMgr repoMgr; + public RestoreJobTest() throws UserException { + } + // Thread is not mockable in Jmockit, use subclass instead private final class MockBackupHandler extends BackupHandler { public MockBackupHandler(Env env) { @@ -122,7 +125,7 @@ public Repository getRepo(long repoId) { @Injectable private Repository repo = new Repository(repoId, "repo", false, "bos://my_repo", - FileSystemFactory.get("broker", StorageBackend.StorageType.BROKER, Maps.newHashMap())); + FileSystemFactory.get(Maps.newHashMap())); private BackupMeta backupMeta; diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java index 442883573ce49a..e0cfbd4789a8fc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java @@ -22,9 +22,9 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.S3URI; import org.apache.doris.datasource.property.PropertyConverter; -import org.apache.doris.fs.FileSystemFactory; -import org.apache.doris.fs.remote.RemoteFile; -import org.apache.doris.fs.remote.S3FileSystem; +import org.apache.doris.fsv2.FileSystemFactory; +import org.apache.doris.fsv2.remote.RemoteFile; +import org.apache.doris.fsv2.remote.S3FileSystem; import mockit.Mock; import mockit.MockUp; @@ -105,7 +105,7 @@ public S3Client getClient() throws UserException { S3ObjStorage mockedStorage = new S3ObjStorage(properties); Assertions.assertTrue(mockedStorage.getClient() instanceof MockedS3Client); // inject storage to file system. - fileSystem = new S3FileSystem(mockedStorage); + fileSystem = (S3FileSystem) FileSystemFactory.get(properties); new MockUp(S3FileSystem.class) { @Mock public Status globList(String remotePath, List result, boolean fileNameOnly) { @@ -124,7 +124,7 @@ public Status globList(String remotePath, List result, boolean fileN }; } else { // can also real file system to test. - fileSystem = (S3FileSystem) FileSystemFactory.getS3FileSystem(properties); + fileSystem = (S3FileSystem) FileSystemFactory.get(properties); } testFile = bucket + basePath + "/Ode_to_the_West_Wind"; Assertions.assertEquals(Status.OK, fileSystem.directUpload(content, testFile)); diff --git a/fe/pom.xml b/fe/pom.xml index 55d2763ac52303..4b70f93382efd4 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -340,6 +340,7 @@ under the License. 1.12.669 3.0.9 3.3.6 + 1.8 1.2.0 1.1.1 2.4.9 @@ -628,6 +629,11 @@ under the License. + + com.google.re2j + re2j + ${re2j.version} + org.apache.hadoop hadoop-client-api diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 69aa7bb6ef8541..84431bfdfa2221 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -248,6 +248,7 @@ enableTrinoConnectorTest = false enableKerberosTest=false kerberosHmsPort=9883 kerberosHdfsPort=8820 +enableRefactorParamsHdfsTest=true // LakeSoul catalog test config diff --git a/regression-test/suites/refactor_storage_param_p0/backup_restore_azure.groovy b/regression-test/suites/refactor_storage_param_p0/backup_restore_azure.groovy new file mode 100644 index 00000000000000..822c5fe72036cf --- /dev/null +++ b/regression-test/suites/refactor_storage_param_p0/backup_restore_azure.groovy @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import org.awaitility.Awaitility; +import static java.util.concurrent.TimeUnit.SECONDS; +import static groovy.test.GroovyAssert.shouldFail + +suite("refactor_storage_backup_restore_azure") { + + String enabled = context.config.otherConfigs.get("enableAzureBackupRestoreTest") + if (enabled == null || enabled.equalsIgnoreCase("false")) { + return ; + } + String objPrefix = "azure" + String container = context.config.otherConfigs.get("azure.container") + String account =context.config.otherConfigs.get("azure.account") + String s3_endpoint = "${account}.blob.core.windows.net" + String ak = context.config.otherConfigs.get("azure.ak") + String sk = context.config.otherConfigs.get("azure.sk") + + def s3table = "test_backup_restore_azure"; + + def databaseQueryResult = sql """ + select database(); + """ + println databaseQueryResult + def currentDBName = databaseQueryResult.get(0).get(0) + println currentDBName + // cos + + def createDBAndTbl = { String dbName -> + + sql """ + drop database if exists ${dbName} + """ + + sql """ + create database ${dbName} + """ + + sql """ + use ${dbName} + """ + sql """ + CREATE TABLE ${s3table}( + user_id BIGINT NOT NULL COMMENT "user id", + name VARCHAR(20) COMMENT "name", + age INT COMMENT "age" + ) + DUPLICATE KEY(user_id) + DISTRIBUTED BY HASH(user_id) BUCKETS 10 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + insert into ${s3table} values (1, 'a', 10); + """ + + def insertResult = sql """ + SELECT count(1) FROM ${s3table} + """ + + println "insertResult: ${insertResult}" + + assert insertResult.get(0).get(0) == 1 + } + + def createRepository = { String repoName, String endpointName, String endpoint, String regionName, String region, String accessKeyName, String accessKey, String secretKeyName, String secretKey, String usePathStyle, String location -> + try { + sql """ + drop repository ${repoName}; + """ + } catch (Exception e) { + // ignore exception, repo may not exist + } + + sql """ + CREATE REPOSITORY ${repoName} + WITH S3 + ON LOCATION "${location}" + PROPERTIES ( + "${endpointName}" = "${endpoint}", + "${regionName}" = "${region}", + "${accessKeyName}" = "${accessKey}", + "${secretKeyName}" = "${secretKey}", + "provider"="azure", + "use_path_style" = "${usePathStyle}" + ); + """ + } + + def backupAndRestore = { String repoName, String dbName, String tableName, String backupLabel -> + sql """ + BACKUP SNAPSHOT ${dbName}.${backupLabel} + TO ${repoName} + ON (${tableName}) + """ + Awaitility.await().atMost(60, SECONDS).pollInterval(5, SECONDS).until( + { + def backupResult = sql """ + show backup from ${dbName} where SnapshotName = '${backupLabel}'; + """ + println "backupResult: ${backupResult}" + return backupResult.get(0).get(3) == "FINISHED" + }) + + def querySnapshotResult = sql """ + SHOW SNAPSHOT ON ${repoName} WHERE SNAPSHOT = '${backupLabel}'; + """ + println querySnapshotResult + def snapshotTimes = querySnapshotResult.get(0).get(1).split('\n') + def snapshotTime = snapshotTimes[0] + + sql """ + drop table if exists ${tableName}; + """ + + sql """ + RESTORE SNAPSHOT ${dbName}.${backupLabel} + FROM ${repoName} + ON (`${tableName}`) + PROPERTIES + ( + "backup_timestamp"="${snapshotTime}", + "replication_num" = "1" + ); + """ + Awaitility.await().atMost(60, SECONDS).pollInterval(5, SECONDS).until( + { + try { + + sql """ + use ${dbName} + """ + def restoreResult = sql """ + SELECT count(1) FROM ${tableName} + """ + println "restoreResult: ${restoreResult}" + def count = restoreResult.get(0).get(0) + println "count: ${count}" + return restoreResult.get(0).get(0) == 1 + } catch (Exception e) { + // tbl not found + println "tbl not found" + e.getMessage() + return false + } + }) + } + + + def s3repoName1 = "azure_repo_1" + createRepository("${s3repoName1}", "s3.endpoint", s3_endpoint, "s3.region", "", "s3.access_key", ak, "s3.secret_key", sk, "true", "s3://${container}/test_" + System.currentTimeMillis()) + + def dbName1 = currentDBName + "${objPrefix}_1" + createDBAndTbl("${dbName1}") + backupAndRestore("${s3repoName1}", dbName1, s3table, "backup_${s3repoName1}_test") + def s3repoName2 = "${objPrefix}_repo_2" + createRepository("${s3repoName2}", "s3.endpoint", s3_endpoint, "s3.region", "", "s3.access_key", ak, "s3.secret_key", sk, "true", "https://${s3_endpoint}/${container}/test_" + System.currentTimeMillis()) + def dbName2 = currentDBName + "${objPrefix}_2" + createDBAndTbl("${dbName2}") + backupAndRestore("${s3repoName2}", dbName2, s3table, "backup_${s3repoName2}_test") + String failedRepoName = "azure_failed_repo" + shouldFail { + createRepository("${failedRepoName}", "s3.endpoint", s3_endpoint, "s3.region", "", "s3.access_key", ak, "s3.secret_key", sk, "false", "https://${s3_endpoint}/${container}/test_" + System.currentTimeMillis()) + } + + +} \ No newline at end of file diff --git a/regression-test/suites/refactor_storage_param_p0/backup_restore_object_storage.groovy b/regression-test/suites/refactor_storage_param_p0/backup_restore_object_storage.groovy new file mode 100644 index 00000000000000..e03f8f725632b5 --- /dev/null +++ b/regression-test/suites/refactor_storage_param_p0/backup_restore_object_storage.groovy @@ -0,0 +1,291 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import org.awaitility.Awaitility; +import static java.util.concurrent.TimeUnit.SECONDS; +import static groovy.test.GroovyAssert.shouldFail + +suite("refactor_storage_backup_restore_object_storage", "p0") { + def s3table = "test_backup_restore"; + + def databaseQueryResult = sql """ + select database(); + """ + println databaseQueryResult + def currentDBName = databaseQueryResult.get(0).get(0) + println currentDBName + // cos + + def createDBAndTbl = { String dbName -> + + sql """ + drop database if exists ${dbName} + """ + + sql """ + create database ${dbName} + """ + + sql """ + use ${dbName} + """ + sql """ + CREATE TABLE ${s3table}( + user_id BIGINT NOT NULL COMMENT "user id", + name VARCHAR(20) COMMENT "name", + age INT COMMENT "age" + ) + DUPLICATE KEY(user_id) + DISTRIBUTED BY HASH(user_id) BUCKETS 10 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + insert into ${s3table} values (1, 'a', 10); + """ + + def insertResult = sql """ + SELECT count(1) FROM ${s3table} + """ + + println "insertResult: ${insertResult}" + + assert insertResult.get(0).get(0) == 1 + } + + def createRepository = { String repoName, String endpointName, String endpoint, String regionName, String region, String accessKeyName, String accessKey, String secretKeyName, String secretKey, String usePathStyle, String location -> + try { + sql """ + drop repository ${repoName}; + """ + } catch (Exception e) { + // ignore exception, repo may not exist + } + + sql """ + CREATE REPOSITORY ${repoName} + WITH S3 + ON LOCATION "${location}" + PROPERTIES ( + "${endpointName}" = "${endpoint}", + "${regionName}" = "${region}", + "${accessKeyName}" = "${accessKey}", + "${secretKeyName}" = "${secretKey}", + "use_path_style" = "${usePathStyle}" + ); + """ + } + + def backupAndRestore = { String repoName, String dbName, String tableName, String backupLabel -> + sql """ + BACKUP SNAPSHOT ${dbName}.${backupLabel} + TO ${repoName} + ON (${tableName}) + """ + Awaitility.await().atMost(60, SECONDS).pollInterval(5, SECONDS).until( + { + def backupResult = sql """ + show backup from ${dbName} where SnapshotName = '${backupLabel}'; + """ + println "backupResult: ${backupResult}" + return backupResult.get(0).get(3) == "FINISHED" + }) + + def querySnapshotResult = sql """ + SHOW SNAPSHOT ON ${repoName} WHERE SNAPSHOT = '${backupLabel}'; + """ + println querySnapshotResult + def snapshotTimes = querySnapshotResult.get(0).get(1).split('\n') + def snapshotTime = snapshotTimes[0] + + sql """ + drop table if exists ${tableName}; + """ + + sql """ + RESTORE SNAPSHOT ${dbName}.${backupLabel} + FROM ${repoName} + ON (`${tableName}`) + PROPERTIES + ( + "backup_timestamp"="${snapshotTime}", + "replication_num" = "1" + ); + """ + Awaitility.await().atMost(60, SECONDS).pollInterval(5, SECONDS).until( + { + try { + + sql """ + use ${dbName} + """ + def restoreResult = sql """ + SELECT count(1) FROM ${tableName} + """ + println "restoreResult: ${restoreResult}" + def count = restoreResult.get(0).get(0) + println "count: ${count}" + return restoreResult.get(0).get(0) == 1 + } catch (Exception e) { + // tbl not found + println "tbl not found"+e.getMessage() + return false + } + }) + } + + + + def test_backup_restore= {String ak,String sk,String s3_endpoint,String region,String bucket,String objPrefix -> + def s3repoName1 = "${objPrefix}_repo_1" + createRepository("${s3repoName1}", "s3.endpoint", s3_endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "true", "s3://${bucket}/test_" + System.currentTimeMillis()) + + def dbName1 = currentDBName + "${objPrefix}_1" + createDBAndTbl("${dbName1}") + backupAndRestore("${s3repoName1}", dbName1, s3table, "backup_${s3repoName1}_test") + def s3repoName2 = "${objPrefix}_repo_2" + createRepository("${s3repoName2}", "s3.endpoint", s3_endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "false", "s3://${bucket}/test_" + System.currentTimeMillis()) + def dbName2 = currentDBName + "${objPrefix}_2" + createDBAndTbl("${dbName2}") + backupAndRestore("${s3repoName2}", dbName2, s3table, "backup_${s3repoName2}_test") + + def s3repoName3 = "${objPrefix}_repo_3" + createRepository("${s3repoName3}", "s3.endpoint", s3_endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "", "s3://${bucket}/test_" + System.currentTimeMillis()) + def dbName3 = currentDBName + "${objPrefix}_3" + createDBAndTbl("${dbName3}") + backupAndRestore("${s3repoName3}", dbName3, s3table, "backup_${s3repoName3}_test") + + def s3repoName4 = "${objPrefix}_s3_repo_4" + createRepository("${s3repoName4}", "s3.endpoint", s3_endpoint, "s3.region", region, "AWS_ACCESS_KEY", ak, "AWS_SECRET_KEY", sk, "true", "s3://${bucket}/test_" + System.currentTimeMillis()) + def dbName4 = currentDBName + "${objPrefix}_4" + createDBAndTbl("${dbName4}") + backupAndRestore("${s3repoName4}", dbName4, s3table, "backup_${s3repoName4}_test") + def s3repoName5 = "${objPrefix}_s3_repo_5" + createRepository("${s3repoName5}", "s3.endpoint", s3_endpoint, "s3.region", region, "AWS_ACCESS_KEY", ak, "AWS_SECRET_KEY", sk, "false", "s3://${bucket}/test_" + System.currentTimeMillis()) + def dbName5 = currentDBName + "${objPrefix}_5" + createDBAndTbl("${dbName5}") + backupAndRestore("${s3repoName5}", dbName5, s3table, "backup_${s3repoName5}_test") + def s3repoName6 = "${objPrefix}_s3_repo_6" + createRepository("${s3repoName6}", "AWS_ENDPOINT", s3_endpoint, "AWS_REGION", region, "AWS_ACCESS_KEY", ak, "AWS_SECRET_KEY", sk, "false", "s3://${bucket}/test_" + System.currentTimeMillis()) + def dbName6 = currentDBName + "${objPrefix}_6" + createDBAndTbl("${dbName6}") + backupAndRestore("${s3repoName6}", dbName6, s3table, "backup_${s3repoName6}_test") + def s3repoName7 = "${objPrefix}_s3_repo_7" + createRepository("${s3repoName7}", "s3.endpoint", s3_endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "", "https://${bucket}/test_" + System.currentTimeMillis()) + def dbName7 = currentDBName + "${objPrefix}_7" + + createDBAndTbl("${dbName7}") + backupAndRestore("${s3repoName7}", dbName7, s3table, "backup_${s3repoName7}_test") + def failedRepoName = "s3_repo_failed" + // wrong address + shouldFail { + createRepository("${failedRepoName}", "s3.endpoint", s3_endpoint, "s3.region", region, "AWS_ACCESS_KEY", ak, "AWS_SECRET_KEY", sk, "true", "s3://ck/" + System.currentTimeMillis()) + } + //endpoint is empty + shouldFail { + createRepository("${failedRepoName}", "s3.endpoint", "", "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "", "s3://${bucket}/test_" + System.currentTimeMillis()) + } + //region is empty + shouldFail { + createRepository("${failedRepoName}", "s3.endpoint", "", "s3.region", "", "s3.access_key", ak, "s3.secret_key", sk, "", "s3://${bucket}/test_" + System.currentTimeMillis()) + } + } + /*-------------AWS S3--------------------------------*/ + String ak = context.config.otherConfigs.get("AWSAK") + String sk = context.config.otherConfigs.get("AWSSK") + String s3_endpoint = "s3.ap-northeast-1.amazonaws.com" + String region = "ap-northeast-1" + String bucket = "selectdb-qa-datalake-test" + String objPrefix="s3" + //test_backup_restore(ak,sk,s3_endpoint,region,bucket,objPrefix) + /*-----------------Tencent COS----------------*/ + ak = context.config.otherConfigs.get("txYunAk") + sk = context.config.otherConfigs.get("txYunSk") + s3_endpoint = "cos.ap-beijing.myqcloud.com" + region = "ap-beijing" + bucket = "doris-build-1308700295"; + + objPrefix="cos" + test_backup_restore(ak,sk,s3_endpoint,region,bucket,objPrefix) + /* cos_url */ + def cos_repoName1 = "${objPrefix}_repo_cos_prefix_1" + // url is : cos://bucket/prefix/ + createRepository("${cos_repoName1}", "cos.endpoint", s3_endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "true", "cos://${bucket}/test_" + System.currentTimeMillis()) + + def cosDbName1 = currentDBName + "${objPrefix}_cos_1" + createDBAndTbl("${cosDbName1}") + backupAndRestore("${cos_repoName1}", cosDbName1, s3table, "backup_${cos_repoName1}_test") + def cos_repoName2 = "${objPrefix}_repo_cos_prefix_2" + // url is : cos://bucket/prefix/ + createRepository("${cos_repoName2}", "cos.endpoint", s3_endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "false", "https://${bucket}.${s3_endpoint}/test_" + System.currentTimeMillis()) + + def cosDbName2 = currentDBName + "${objPrefix}_cos_2" + createDBAndTbl("${cosDbName2}") + backupAndRestore("${cos_repoName2}", cosDbName2, s3table, "backup_${cos_repoName1}_test") + + + + /*-----------------Huawei OBS----------------*/ + ak = context.config.otherConfigs.get("hwYunAk") + sk = context.config.otherConfigs.get("hwYunSk") + s3_endpoint = "obs.cn-north-4.myhuaweicloud.com" + region = "cn-north-4" + bucket = "doris-build"; + objPrefix="obs" + test_backup_restore(ak,sk,s3_endpoint,region,bucket,objPrefix) + def obs_repoName1 = "${objPrefix}_repo_obs_prefix_1" + // url is : cos://bucket/prefix/ + createRepository("${obs_repoName1}", "obs.endpoint", s3_endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "true", "obs://${bucket}/test_" + System.currentTimeMillis()) + + def obsDbName1 = currentDBName + "${objPrefix}_obs_1" + createDBAndTbl("${obsDbName1}") + backupAndRestore("${obs_repoName1}", obsDbName1, s3table, "backup_${obs_repoName1}_test") + def obs_repoName2 = "${objPrefix}_repo_obs_prefix_2" + // url is : cos://bucket/prefix/ + createRepository("${obs_repoName2}", "obs.endpoint", s3_endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "false", "https://${bucket}.${s3_endpoint}/test_" + System.currentTimeMillis()) + + def obsDbName2 = currentDBName + "${objPrefix}_obs_2" + createDBAndTbl("${obsDbName2}") + backupAndRestore("${obs_repoName2}", obsDbName2, s3table, "backup_${obs_repoName1}_test") + + + /*-----------------Aliyun OSS----------------*/ + ak = context.config.otherConfigs.get("aliYunAk") + sk = context.config.otherConfigs.get("aliYunSk") + s3_endpoint = "oss-cn-hongkong.aliyuncs.com" + region = "oss-cn-hongkong" + bucket = "doris-regression-hk"; + objPrefix="oss" + // oss has some problem, so we comment it. + //test_backup_restore(ak,sk,s3_endpoint,region,bucket,objPrefix) + def oss_repoName1 = "${objPrefix}_repo_oss_prefix_1" + // url is : cos://bucket/prefix/ + createRepository("${oss_repoName1}", "oss.endpoint", s3_endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "false", "oss://${bucket}/test_" + System.currentTimeMillis()) + + def ossDbName1 = currentDBName + "${objPrefix}_oss_1" + createDBAndTbl("${ossDbName1}") + backupAndRestore("${oss_repoName1}", ossDbName1, s3table, "backup_${oss_repoName1}_test") + def oss_repoName2 = "${objPrefix}_repo_oss_prefix_2" + // url is : cos://bucket/prefix/ + createRepository("${oss_repoName2}", "oss.endpoint", s3_endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "false", "https://${bucket}.${s3_endpoint}/test_" + System.currentTimeMillis()) + + def ossDbName2 = currentDBName + "${objPrefix}_oss_2" + createDBAndTbl("${ossDbName2}") + backupAndRestore("${oss_repoName2}", ossDbName2, s3table, "backup_${oss_repoName1}_test") + + +} \ No newline at end of file diff --git a/regression-test/suites/refactor_storage_param_p0/hdfs_all_test.groovy b/regression-test/suites/refactor_storage_param_p0/hdfs_all_test.groovy new file mode 100644 index 00000000000000..bddefc51d1d963 --- /dev/null +++ b/regression-test/suites/refactor_storage_param_p0/hdfs_all_test.groovy @@ -0,0 +1,318 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import org.awaitility.Awaitility; +import static java.util.concurrent.TimeUnit.SECONDS; +import static groovy.test.GroovyAssert.shouldFail + +suite("refactor_params_hdfs_all_test", "p0,external,kerberos,external_docker,external_docker_kerberos") { + String enabled = context.config.otherConfigs.get("enableRefactorParamsHdfsTest") + if (enabled == null || enabled.equalsIgnoreCase("false")) { + return + } + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + def table = "hdfs_all_test"; + + def databaseQueryResult = sql """ + select database(); + """ + println databaseQueryResult + def currentDBName = databaseQueryResult.get(0).get(0) + println currentDBName + // cos + + def createDBAndTbl = { String dbName -> + + sql """ + drop database if exists ${dbName} + """ + + sql """ + create database ${dbName} + """ + + sql """ + use ${dbName} + """ + sql """ + CREATE TABLE ${table}( + user_id BIGINT NOT NULL COMMENT "user id", + name VARCHAR(20) COMMENT "name", + age INT COMMENT "age" + ) + DUPLICATE KEY(user_id) + DISTRIBUTED BY HASH(user_id) BUCKETS 10 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + insert into ${table} values (1, 'a', 10); + """ + + def insertResult = sql """ + SELECT count(1) FROM ${table} + """ + + println "insertResult: ${insertResult}" + + assert insertResult.get(0).get(0) == 1 + } + + def hdfsNonXmlParams = "\"fs.defaultFS\" = \"hdfs://${externalEnvIp}:8520\",\n" + + "\"hadoop.kerberos.min.seconds.before.relogin\" = \"5\",\n" + + "\"hadoop.security.authentication\" = \"kerberos\",\n" + + "\"hadoop.kerberos.principal\"=\"hive/presto-master.docker.cluster@LABS.TERADATA.COM\",\n" + + "\"hadoop.kerberos.keytab\" = \"/mnt/disk1/gq/keytabs/keytabs/hive-presto-master.keytab\",\n" + + "\"hive.metastore.sasl.enabled \" = \"true\",\n" + + "\"hadoop.security.auth_to_local\" = \"RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*//\n" + + " RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*//\n" + + " RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*//\n" + + " DEFAULT\"" + + def createRepository = { String repoName, String location, String hdfsParams -> + try { + sql """ + drop repository ${repoName}; + """ + } catch (Exception e) { + // ignore exception, repo may not exist + } + + sql """ + CREATE REPOSITORY ${repoName} + WITH HDFS + ON LOCATION "${location}" + PROPERTIES ( + ${hdfsParams} + ); + """ + } + + def backupAndRestore = { String repoName, String dbName, String tableName, String backupLabel -> + sql """ + BACKUP SNAPSHOT ${dbName}.${backupLabel} + TO ${repoName} + ON (${tableName}) + """ + Awaitility.await().atMost(60, SECONDS).pollInterval(5, SECONDS).until( + { + def backupResult = sql """ + show backup from ${dbName} where SnapshotName = '${backupLabel}'; + """ + println "backupResult: ${backupResult}" + return backupResult.get(0).get(3) == "FINISHED" + }) + + def querySnapshotResult = sql """ + SHOW SNAPSHOT ON ${repoName} WHERE SNAPSHOT = '${backupLabel}'; + """ + println querySnapshotResult + def snapshotTimes = querySnapshotResult.get(0).get(1).split('\n') + def snapshotTime = snapshotTimes[0] + + sql """ + drop table if exists ${tableName}; + """ + + sql """ + RESTORE SNAPSHOT ${dbName}.${backupLabel} + FROM ${repoName} + ON (`${tableName}`) + PROPERTIES + ( + "backup_timestamp"="${snapshotTime}", + "replication_num" = "1" + ); + """ + Awaitility.await().atMost(60, SECONDS).pollInterval(5, SECONDS).until( + { + try { + + sql """ + use ${dbName} + """ + def restoreResult = sql """ + SELECT count(1) FROM ${tableName} + """ + println "restoreResult: ${restoreResult}" + def count = restoreResult.get(0).get(0) + println "count: ${count}" + return restoreResult.get(0).get(0) == 1 + } catch (Exception e) { + // tbl not found + println "tbl not found" + e.getMessage() + return false + } + }) + } + def hdfs_tvf = { filePath, hdfsParam -> + + def hdfs_tvf_sql = sql """ + select * from hdfs + + ( + 'uri'='${filePath}', + "format" = "csv", + ${hdfsParam} + ); + """ + } + def export_hdfs = { defaultFs, hdfsParams -> + def exportPath = defaultFs + "/test/_export/" + System.currentTimeMillis() + def exportLabel = "export_" + System.currentTimeMillis(); + sql """ + EXPORT TABLE ${table} + TO "${exportPath}" + PROPERTIES + ( + "label"="${exportLabel}", + "line_delimiter" = "," + ) + with HDFS + ( + + ${hdfsParams} + ); + """ + + databaseQueryResult = sql """ + select database(); + """ + currentDBName = databaseQueryResult.get(0).get(0) + Awaitility.await().atMost(60, SECONDS).pollInterval(5, SECONDS).until({ + def exportResult = sql """ + SHOW EXPORT FROM ${currentDBName} WHERE LABEL = "${exportLabel}"; + + """ + + println exportResult + + if (null == exportResult || exportResult.isEmpty() || null == exportResult.get(0) || exportResult.get(0).size() < 3) { + return false; + } + if (exportResult.get(0).get(2) == 'CANCELLED' || exportResult.get(0).get(2) == 'FAILED') { + throw new RuntimeException("load failed") + } + + return exportResult.get(0).get(2) == 'FINISHED' + }) + + } + def outfile_to_hdfs = { defaultFs, hdfsParams -> + def outFilePath = "${defaultFs}/outfile_different_hdfs/exp_" + // select ... into outfile ... + def res = sql """ + SELECT * FROM ${table} ORDER BY user_id + INTO OUTFILE "${outFilePath}" + FORMAT AS CSV + PROPERTIES ( + ${hdfsParams} + ); + """ + return res[0][3] + } + def hdfsLoad = { filePath, hdfsParams -> + databaseQueryResult = sql """ + select database(); + """ + println databaseQueryResult + def dataCountResult = sql """ + SELECT count(*) FROM ${table} + """ + def dataCount = dataCountResult[0][0] + def label = "hdfs_load_label_" + System.currentTimeMillis() + def load = sql """ + LOAD LABEL `${label}` ( + data infile ("${filePath}") + into table ${table} + COLUMNS TERMINATED BY "\\\t" + FORMAT AS "CSV" + ( + user_id, + name, + age + )) + with hdfs + ( + ${hdfsParams} + ) + PROPERTIES + ( + "timeout" = "3600" + ); + """ + Awaitility.await().atMost(60, SECONDS).pollInterval(5, SECONDS).until({ + def loadResult = sql """ + show load where label = '${label}'; + """ + println 'test' + println loadResult + + if (null == loadResult || loadResult.isEmpty() || null == loadResult.get(0) || loadResult.get(0).size() < 3) { + return false; + } + if (loadResult.get(0).get(2) == 'CANCELLED' || loadResult.get(0).get(2) == 'FAILED') { + throw new RuntimeException("load failed") + } + + return loadResult.get(0).get(2) == 'FINISHED' + }) + + + def expectedCount = dataCount + 1 + Awaitility.await().atMost(60, SECONDS).pollInterval(5, SECONDS).until({ + def loadResult = sql """ + select count(*) from ${table} + """ + println "loadResult: ${loadResult} " + return loadResult.get(0).get(0) == expectedCount + }) + + } + def defaultFs = 'hdfs://172.20.32.136:8520' + def repoName = 'hdfs_non_xml_repo'; + // create repo + createRepository(repoName,"${defaultFs}/test_repo",hdfsNonXmlParams); + def dbName1 = currentDBName + "${repoName}_1" + createDBAndTbl(dbName1) + def backupLabel=repoName+System.currentTimeMillis() + //backup and restore + backupAndRestore(repoName,dbName1,table,backupLabel) + def failedRepoName='failedRepo' + shouldFail { + createRepository(failedRepoName,"s3://172.20.32.136:8520",hdfsNonXmlParams); + } + shouldFail { + createRepository(failedRepoName," ",hdfsNonXmlParams); + } + + //outfile + dbName1 = currentDBName + 'outfile_test_1' + createDBAndTbl(dbName1) + def outfile = outfile_to_hdfs(defaultFs, hdfsNonXmlParams); + println outfile + //hdfs tvf + def hdfsTvfResult = hdfs_tvf(outfile, hdfsNonXmlParams) + println hdfsTvfResult + + //hdfsLoad(outfile,hdfsNonXmlParams) + + //export + export_hdfs(defaultFs, hdfsNonXmlParams) + + +} \ No newline at end of file diff --git a/regression-test/suites/refactor_storage_param_p0/s3_load.groovy b/regression-test/suites/refactor_storage_param_p0/s3_load.groovy new file mode 100644 index 00000000000000..27814356220075 --- /dev/null +++ b/regression-test/suites/refactor_storage_param_p0/s3_load.groovy @@ -0,0 +1,293 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import org.awaitility.Awaitility + +import static groovy.test.GroovyAssert.shouldFail; +import static java.util.concurrent.TimeUnit.SECONDS; + +suite("refactor_storage_param_load") { + + String ak = context.config.otherConfigs.get("AWSAK") + String sk = context.config.otherConfigs.get("AWSSK") + String endpoint = "s3.ap-northeast-1.amazonaws.com" + String region = "ap-northeast-1" + String bucket = "selectdb-qa-datalake-test" + + def s3table = "test_s3load"; + sql """ + drop table if exists ${s3table}; + """ + sql """ + CREATE TABLE ${s3table}( + user_id BIGINT NOT NULL COMMENT "user id", + name VARCHAR(20) COMMENT "name", + age INT COMMENT "age" + ) + DUPLICATE KEY(user_id) + DISTRIBUTED BY HASH(user_id) BUCKETS 10 + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + insert into ${s3table} values (1, 'a', 10); + """ + + def insertResult = sql """ + SELECT count(1) FROM ${s3table} + """ + + println "insertResult: ${insertResult}" + assert insertResult.get(0).get(0) == 1 + + def outfile_to_S3 = { objBucket, objEndpoint, objRegion, objAk, objSk -> + def outFilePath = "${objBucket}/outfile_different_s3/exp_" + // select ... into outfile ... + def res = sql """ + SELECT * FROM ${s3table} t ORDER BY user_id + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS CSV + PROPERTIES ( + "s3.endpoint" = "${objEndpoint}", + "s3.region" = "${objRegion}", + "s3.secret_key"="${objSk}", + "s3.access_key" = "${objAk}" + ); + """ + return res[0][3] + } + def outfile_path = outfile_to_S3(bucket, endpoint, region, ak, sk); + def filePath = outfile_path.replace("s3://${bucket}", "") + + def s3Load = { String objFilePath, String objBucket, String objEndpointName, String objEndpoint, String objRegionName, String objRegion, String objAkName, String objAk, String objSkName, String objSk, String usePathStyle -> + + def dataCountResult = sql """ + SELECT count(*) FROM ${s3table} + """ + def dataCount = dataCountResult[0][0] + def label = "s3_load_label_" + System.currentTimeMillis() + def load = sql """ + LOAD LABEL `${label}` ( + data infile ("${objFilePath}") + into table ${s3table} + COLUMNS TERMINATED BY "\\\t" + FORMAT AS "CSV" + ( + user_id, + name, + age + )) + with s3 + ( + "${objEndpointName}" = "${objEndpoint}", + "${objRegionName}" = "${objRegion}", + "${objSkName}"="${objSk}", + "use_path_style" = "${usePathStyle}", + "${objAkName}" = "${objAk}" + ) + PROPERTIES + ( + "timeout" = "3600" + ); + """ + Awaitility.await().atMost(60, SECONDS).pollInterval(5, SECONDS).until({ + def loadResult = sql """ + show load where label = '${label}' + """ + if (loadResult.get(0).get(2) == 'CANCELLED' || loadResult.get(0).get(2) == 'FAILED') { + throw new RuntimeException("load failed") + } + return loadResult.get(0).get(2) == 'FINISHED' + }) + + + def expectedCount = dataCount + 1 + Awaitility.await().atMost(60, SECONDS).pollInterval(5, SECONDS).until({ + def loadResult = sql """ + select count(*) from ${s3table} + """ + println "loadResult: ${loadResult} " + return loadResult.get(0).get(0) == expectedCount + }) + + } + s3Load("s3://${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "true") + s3Load("s3://${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "false") + s3Load("s3://${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "") + s3Load("s3://${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "AWS_ACCESS_KEY", ak, "AWS_SECRET_KEY", sk, "") + s3Load("http://${bucket}.${endpoint}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "false") + s3Load("http://${bucket}.${endpoint}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "") + s3Load("https://${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "false") + shouldFail { + s3Load("https://${bucket}${filePath}", bucket, "", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "false") + } + + shouldFail { + s3Load("https://${bucket}${filePath}", bucket, "", endpoint, "s3.region", region, "s3.access_key", "", "s3.secret_key", sk, "false") + } + shouldFail { + s3Load("https://${bucket}/${endpoint}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "") + + } + shouldFail { + s3Load("https://${bucket}/${endpoint}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "true") + } + shouldFail { + s3Load("s3://${endpoint}/${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "false") + } + shouldFail { + s3Load("s3://${bucket}/${endpoint}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "false") + } + shouldFail { + s3Load("s3://${endpoint}/${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "false") + } + /*----------obs---------------*/ + ak = context.config.otherConfigs.get("hwYunAk") + sk = context.config.otherConfigs.get("hwYunSk") + endpoint = "obs.cn-north-4.myhuaweicloud.com" + region = "cn-north-4" + bucket = "doris-build"; + outfile_path = outfile_to_S3(bucket, endpoint, region, ak, sk); + filePath = outfile_path.replace("s3://${bucket}", "") + s3Load("s3://${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "true") + s3Load("s3://${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "false") + s3Load("s3://${bucket}${filePath}", bucket, "obs.endpoint", endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "true") + s3Load("s3://${bucket}${filePath}", bucket, "obs.endpoint", endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "false") + s3Load("obs://${bucket}${filePath}", bucket, "obs.endpoint", endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "true") + s3Load("obs://${bucket}${filePath}", bucket, "obs.endpoint", endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "false") + s3Load("obs://${bucket}${filePath}", bucket, "obs.endpoint", endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "") + s3Load("s3://${bucket}${filePath}", bucket, "obs.endpoint", endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "") + s3Load("http://${bucket}.${endpoint}${filePath}", bucket, "obs.endpoint", endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "") + s3Load("https://${bucket}.${endpoint}${filePath}", bucket, "obs.endpoint", endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "") + shouldFail { + s3Load("https://${bucket}${filePath}", bucket, "", endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "false") + } + + shouldFail { + s3Load("https://${bucket}${filePath}", bucket, "", endpoint, "obs.region", region, "obs.access_key", "", "obs.secret_key", sk, "false") + } + shouldFail { + s3Load("https://${bucket}/${endpoint}${filePath}", bucket, "obs.endpoint", endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "") + + } + shouldFail { + s3Load("https://${bucket}/${endpoint}${filePath}", bucket, "obs.endpoint", endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "true") + } + shouldFail { + s3Load("s3://${endpoint}/${bucket}${filePath}", bucket, "obs.endpoint", endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "false") + } + shouldFail { + s3Load("obs://${bucket}/${endpoint}${filePath}", bucket, "obs.endpoint", endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "false") + } + shouldFail { + s3Load("obs://${endpoint}/${bucket}${filePath}", bucket, "obs.endpoint", endpoint, "obs.region", region, "obs.access_key", ak, "obs.secret_key", sk, "false") + } + + /*-------------Tencent COS ----------*/ + ak = context.config.otherConfigs.get("txYunAk") + sk = context.config.otherConfigs.get("txYunSk") + endpoint = "cos.ap-beijing.myqcloud.com" + region = "ap-beijing" + bucket = "doris-build-1308700295"; + + outfile_path = outfile_to_S3(bucket, endpoint, region, ak, sk); + filePath = outfile_path.replace("s3://${bucket}", "") + s3Load("s3://${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "true") + s3Load("s3://${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "false") + s3Load("s3://${bucket}${filePath}", bucket, "cos.endpoint", endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "true") + s3Load("s3://${bucket}${filePath}", bucket, "cos.endpoint", endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "false") + s3Load("cos://${bucket}${filePath}", bucket, "cos.endpoint", endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "true") + s3Load("cos://${bucket}${filePath}", bucket, "cos.endpoint", endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "false") + s3Load("cos://${bucket}${filePath}", bucket, "cos.endpoint", endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "") + s3Load("s3://${bucket}${filePath}", bucket, "cos.endpoint", endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "") + s3Load("http://${bucket}.${endpoint}${filePath}", bucket, "cos.endpoint", endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "") + s3Load("https://${bucket}.${endpoint}${filePath}", bucket, "cos.endpoint", endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "") + s3Load("http://${bucket}.${endpoint}${filePath}", bucket, "cos.endpoint", endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "false") + shouldFail { + s3Load("https://${bucket}${filePath}", bucket, "", endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "false") + } + + shouldFail { + s3Load("https://${bucket}${filePath}", bucket, "", endpoint, "cos.region", region, "cos.access_key", "", "cos.secret_key", sk, "false") + } + shouldFail { + s3Load("https://${bucket}/${endpoint}${filePath}", bucket, "cos.endpoint", endpoint, "cos.region", region, "cos.access_key", ak, "obs.secret_key", sk, "") + + } + shouldFail { + s3Load("https://${bucket}/${endpoint}${filePath}", bucket, "cos.endpoint", endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "true") + } + shouldFail { + s3Load("s3://${endpoint}/${bucket}${filePath}", bucket, "cos.endpoint", endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "false") + } + shouldFail { + s3Load("cos://${bucket}/${endpoint}${filePath}", bucket, "cos.endpoint", endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "false") + } + shouldFail { + s3Load("cos://${endpoint}/${bucket}${filePath}", bucket, "cos.endpoint", endpoint, "cos.region", region, "cos.access_key", ak, "cos.secret_key", sk, "false") + } + /************ Aliyun OSS ************/ + /*-----------------Aliyun OSS----------------*/ +/* ak = context.config.otherConfigs.get("aliYunAk") + sk = context.config.otherConfigs.get("aliYunSk") + endpoint = "oss-cn-hongkong.aliyuncs.com" + region = "oss-cn-hongkong" + bucket = "doris-regression-hk"; + + outfile_path = outfile_to_S3(bucket, endpoint, region, ak, sk); + filePath = outfile_path.replace("s3://${bucket}", "") + s3Load("s3://${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "true") + s3Load("s3://${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "false") + s3Load("s3://${bucket}${filePath}", bucket, "oss.endpoint", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "true") + s3Load("s3://${bucket}${filePath}", bucket, "oss.endpoint", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "false") + s3Load("cos://${bucket}${filePath}", bucket, "oss.endpoint", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "true") + s3Load("cos://${bucket}${filePath}", bucket, "oss.endpoint", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "false") + s3Load("cos://${bucket}${filePath}", bucket, "oss.endpoint", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "") + s3Load("s3://${bucket}${filePath}", bucket, "oss.endpoint", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "") + s3Load("http://${bucket}${filePath}", bucket, "oss.endpoint", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "") + s3Load("https://${bucket}${filePath}", bucket, "oss.endpoint", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "") + s3Load("http://${bucket}${filePath}", bucket, "oss.endpoint", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "true") + s3Load("http://${bucket}${filePath}", bucket, "oss.endpoint", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "false") + shouldFail { + s3Load("https://${bucket}${filePath}", bucket, "", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "false") + } + + shouldFail { + s3Load("https://${bucket}${filePath}", bucket, "", endpoint, "oss.region", region, "oss.access_key", "", "oss.secret_key", sk, "false") + } + shouldFail { + s3Load("https://${bucket}/${endpoint}${filePath}", bucket, "oss.endpoint", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "") + + } + shouldFail { + s3Load("https://${bucket}/${endpoint}${filePath}", bucket, "oss.endpoint", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "true") + } + shouldFail { + s3Load("s3://${endpoint}/${bucket}${filePath}", bucket, "oss.endpoint", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "false") + } + shouldFail { + s3Load("oss://${bucket}/${endpoint}${filePath}", bucket, "oss.endpoint", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "false") + } + shouldFail { + s3Load("oss://${endpoint}/${bucket}${filePath}", bucket, "oss.endpoint", endpoint, "oss.region", region, "oss.access_key", ak, "oss.secret_key", sk, "false") + } + */ + + +} + + diff --git a/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy b/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy new file mode 100644 index 00000000000000..9e9b147e838ad8 --- /dev/null +++ b/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_outfile_s3_storage", "p0") { + + def export_table_name = "test_outfile_s3_storage" + + def s3_tvf = {bucket, s3_endpoint, region, ak, sk, path -> + // http schema + order_qt_s3_tvf_1_http """ SELECT * FROM S3 ( + "uri" = "http://${bucket}.${s3_endpoint}${path}0.parquet", + "s3.access_key"= "${ak}", + "s3.secret_key" = "${sk}", + "format" = "parquet", + "region" = "${region}" + ); + """ + } + + + sql """ DROP TABLE IF EXISTS ${export_table_name} """ + sql """ + CREATE TABLE IF NOT EXISTS ${export_table_name} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `Name` STRING COMMENT "用户年龄", + `Age` int(11) NULL + ) + DISTRIBUTED BY HASH(user_id) BUCKETS 3 + PROPERTIES("replication_num" = "1"); + """ + StringBuilder sb = new StringBuilder() + int i = 1 + for (; i < 10; i ++) { + sb.append(""" + (${i}, 'ftw-${i}', ${i + 18}), + """) + } + sb.append(""" + (${i}, NULL, NULL) + """) + sql """ INSERT INTO ${export_table_name} VALUES + ${sb.toString()} + """ + qt_select_export """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + + String ak = "" + String sk = "" + String s3_endpoint = "" + String region = "" + String bucket = "" + + /******************************************************************************************************* + ***************************** TEST AWS ***************************************************** + *******************************************************************************************************/ + try { + ak = getS3AK() + sk = getS3SK() + s3_endpoint = getS3Endpoint() + region = getS3Region() + bucket = context.config.otherConfigs.get("s3BucketName"); + + // 1. test s3 schema + def outFilePath = "${bucket}/test_outfile_s3_storage/exp_" + def res = sql """ + SELECT * FROM ${export_table_name} t ORDER BY user_id + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS parquet + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + def outfile_url = res[0][3]; + s3_tvf(bucket, s3_endpoint, region, ak, sk, outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)); + + // 2. test AWS_ENDPOINT + outFilePath = "${bucket}/test_outfile_s3_storage/exp_" + res = sql """ + SELECT * FROM ${export_table_name} t ORDER BY user_id + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS parquet + PROPERTIES ( + "AWS_ENDPOINT" = "${s3_endpoint}", + "AWS_REGION" = "${region}", + "AWS_SECRET_KEY"="${sk}", + "AWS_ACCESS_KEY" = "${ak}" + ); + """ + outfile_url = res[0][3]; + s3_tvf(bucket, s3_endpoint, region, ak, sk, outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)); + + } finally { + } + + /******************************************************************************************************* + ***************************** TEST COS & COSN ************************************************* + *******************************************************************************************************/ + try { + ak = context.config.otherConfigs.get("txYunAk") + sk = context.config.otherConfigs.get("txYunSk") + s3_endpoint = "cos.ap-beijing.myqcloud.com" + region = "ap-beijing" + bucket = "doris-build-1308700295"; + + // 1. test s3 schema + def outFilePath = "${bucket}/test_outfile_s3_storage/exp_" + def res = sql """ + SELECT * FROM ${export_table_name} t ORDER BY user_id + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS parquet + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + def outfile_url = res[0][3]; + s3_tvf(bucket, s3_endpoint, region, ak, sk, outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)); + + // 2. test AWS_ENDPOINT + outFilePath = "${bucket}/test_outfile_s3_storage/exp_" + res = sql """ + SELECT * FROM ${export_table_name} t ORDER BY user_id + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS parquet + PROPERTIES ( + "AWS_ENDPOINT" = "${s3_endpoint}", + "AWS_REGION" = "${region}", + "AWS_SECRET_KEY"="${sk}", + "AWS_ACCESS_KEY" = "${ak}" + ); + """ + outfile_url = res[0][3]; + s3_tvf(bucket, s3_endpoint, region, ak, sk, outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)); + + } finally { + } + + /******************************************************************************************************* + ***************************** TEST OSS ******************************************************** + *******************************************************************************************************/ + try { + ak = context.config.otherConfigs.get("aliYunAk") + sk = context.config.otherConfigs.get("aliYunSk") + s3_endpoint = "oss-cn-hongkong.aliyuncs.com" + region = "oss-cn-hongkong" + bucket = "doris-regression-hk"; + + // 1. test s3 schema + def outFilePath = "${bucket}/test_outfile_s3_storage/exp_" + def res = sql """ + SELECT * FROM ${export_table_name} t ORDER BY user_id + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS parquet + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + def outfile_url = res[0][3]; + s3_tvf(bucket, s3_endpoint, region, ak, sk, outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)); + + // 2. test AWS_ENDPOINT + outFilePath = "${bucket}/test_outfile_s3_storage/exp_" + res = sql """ + SELECT * FROM ${export_table_name} t ORDER BY user_id + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS parquet + PROPERTIES ( + "AWS_ENDPOINT" = "${s3_endpoint}", + "AWS_REGION" = "${region}", + "AWS_SECRET_KEY"="${sk}", + "AWS_ACCESS_KEY" = "${ak}" + ); + """ + outfile_url = res[0][3]; + s3_tvf(bucket, s3_endpoint, region, ak, sk, outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)); + } finally { + + } + + + /******************************************************************************************************* + ***************************** TEST OBS ******************************************************** + *******************************************************************************************************/ + try { + ak = context.config.otherConfigs.get("hwYunAk") + sk = context.config.otherConfigs.get("hwYunSk") + s3_endpoint = "obs.cn-north-4.myhuaweicloud.com" + region = "cn-north-4" + bucket = "doris-build"; + } finally { + } + +} diff --git a/regression-test/suites/refactor_storage_param_p0/test_s3_tvf_s3_storage.groovy b/regression-test/suites/refactor_storage_param_p0/test_s3_tvf_s3_storage.groovy new file mode 100644 index 00000000000000..c60ff1dbd2f6f3 --- /dev/null +++ b/regression-test/suites/refactor_storage_param_p0/test_s3_tvf_s3_storage.groovy @@ -0,0 +1,261 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +import static groovy.test.GroovyAssert.shouldFail + +suite("test_s3_tvf_s3_storage", "p0") { + + def export_table_name = "test_s3_tvf_s3_storage" + + def outfile_to_S3 = { bucket, s3_endpoint, region, ak, sk -> + def outFilePath = "${bucket}/outfile_different_s3/exp_" + // select ... into outfile ... + def res = sql """ + SELECT * FROM ${export_table_name} t ORDER BY user_id + INTO OUTFILE "s3://${outFilePath}" + FORMAT AS parquet + PROPERTIES ( + "s3.endpoint" = "${s3_endpoint}", + "s3.region" = "${region}", + "s3.secret_key"="${sk}", + "s3.access_key" = "${ak}" + ); + """ + return res[0][3] + } + + + sql """ DROP TABLE IF EXISTS ${export_table_name} """ + sql """ + CREATE TABLE IF NOT EXISTS ${export_table_name} ( + `user_id` LARGEINT NOT NULL COMMENT "用户id", + `Name` STRING COMMENT "用户年龄", + `Age` int(11) NULL + ) + DISTRIBUTED BY HASH(user_id) BUCKETS 3 + PROPERTIES("replication_num" = "1"); + """ + StringBuilder sb = new StringBuilder() + int i = 1 + for (; i < 10; i++) { + sb.append(""" + (${i}, 'ftw-${i}', ${i + 18}), + """) + } + sb.append(""" + (${i}, NULL, NULL) + """) + sql """ INSERT INTO ${export_table_name} VALUES + ${sb.toString()} + """ + qt_select_export """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + + + String ak = "" + String sk = "" + String s3_endpoint = "" + String region = "" + String bucket = "" + String outfile_url = "" + + def s3_tvf = { uri_prefix, endpoint_key, ak_key, sk_key, region_key, is_path_style -> + // http schema + order_qt_s3_tvf """ SELECT * FROM S3 ( + "uri" = "${uri_prefix}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.parquet", + "${endpoint_key}" = "${s3_endpoint}", + "${ak_key}"= "${ak}", + "${sk_key}" = "${sk}", + "${region_key}" = "${region}", + "use_path_style" = "${is_path_style}", + "format" = "parquet" + ); + """ + } + + + /******************************************************************************************************* + ***************************** TEST AWS ***************************************************** + *******************************************************************************************************/ + try { + ak = context.config.otherConfigs.get("AWSAK") + sk = context.config.otherConfigs.get("AWSSK") + s3_endpoint = "s3.ap-northeast-1.amazonaws.com" + region = "ap-northeast-1" + bucket = "selectdb-qa-datalake-test" + + outfile_url = outfile_to_S3(bucket, s3_endpoint, region, ak, sk) + + s3_tvf("http://${bucket}.${s3_endpoint}", "", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("http://${bucket}.${s3_endpoint}", "", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "region", "false"); + s3_tvf("http://${bucket}.${s3_endpoint}", "", "s3.access_key", "s3.secret_key", "s3.region", "false"); + + //s3_tvf("http://${bucket}.${s3_endpoint}", "", "s3.access_key" , "s3.secret_key", "region", "true"); + s3_tvf("http://${bucket}.${s3_endpoint}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("http://${s3_endpoint}/${bucket}", "", "s3.access_key", "s3.secret_key", "region", "true"); + //s3_tvf("http://${s3_endpoint}/${bucket}", "", "s3.access_key" , "s3.secret_key", "region", "false"); + // s3_tvf("s3://${s3_endpoint}/${bucket}", "", "s3.access_key" , "s3.secret_key", "region", "false"); + s3_tvf("s3://${bucket}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("s3://${bucket}", "s3.endpoint", "s3.access_key", "s3.secret_key", "s3.region", "true"); + s3_tvf("s3://${bucket}", "AWS_ENDPOINT", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "region", "false"); + s3_tvf("s3://${bucket}", "AWS_ENDPOINT", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "s3.region", "false"); + s3_tvf("s3://${bucket}", "AWS_ENDPOINT", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_REGION", "false"); + s3_tvf("s3://${bucket}", "s3.endpoint", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "region", "false"); + s3_tvf("cos://${bucket}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("cos://${bucket}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + + } finally { + } + + /******************************************************************************************************* + ***************************** TEST COS & COSN ************************************************* + *******************************************************************************************************/ + try { + ak = context.config.otherConfigs.get("txYunAk") + sk = context.config.otherConfigs.get("txYunSk") + s3_endpoint = "cos.ap-beijing.myqcloud.com" + region = "ap-beijing" + bucket = "doris-build-1308700295"; + + + outfile_url = outfile_to_S3(bucket, s3_endpoint, region, ak, sk) + + s3_tvf("http://${bucket}.${s3_endpoint}", "", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("http://${bucket}.${s3_endpoint}", "", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "region", "false"); + s3_tvf("http://${bucket}.${s3_endpoint}", "", "s3.access_key", "s3.secret_key", "s3.region", "false"); + s3_tvf("http://${bucket}.${s3_endpoint}", "", "cos.access_key", "cos.secret_key", "region", "false"); + s3_tvf("http://${bucket}.${s3_endpoint}", "", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("http://${bucket}.${s3_endpoint}", "cos.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("http://${bucket}.${s3_endpoint}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("http://${s3_endpoint}/${bucket}", "", "s3.access_key", "s3.secret_key", "region", "true"); + shouldFail { + s3_tvf("http://${s3_endpoint}/${bucket}", "", "s3.access_key", "s3.secret_key", "region", "false"); + } + + s3_tvf("s3://${bucket}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("s3://${bucket}", "s3.endpoint", "s3.access_key", "s3.secret_key", "s3.region", "true"); + s3_tvf("s3://${bucket}", "AWS_ENDPOINT", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "region", "false"); + s3_tvf("s3://${bucket}", "AWS_ENDPOINT", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "s3.region", "false"); + s3_tvf("s3://${bucket}", "AWS_ENDPOINT", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_REGION", "false"); + s3_tvf("s3://${bucket}", "s3.endpoint", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "region", "false"); + s3_tvf("s3://${bucket}", "s3.endpoint", "s3.access_key", "AWS_SECRET_KEY", "region", "false"); + s3_tvf("s3://${bucket}", "cos.endpoint", "cos.access_key", "cos.secret_key", "cos.region", "false"); + s3_tvf("s3://${bucket}", "s3.endpoint", "cos.access_key", "cos.secret_key", "cos.region", "false"); + s3_tvf("cos://${bucket}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("cos://${bucket}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + + } finally { + } + + /******************************************************************************************************* + ***************************** TEST OSS ******************************************************** + *******************************************************************************************************/ + try { + ak = context.config.otherConfigs.get("aliYunAk") + sk = context.config.otherConfigs.get("aliYunSk") + s3_endpoint = "oss-cn-hongkong.aliyuncs.com" + region = "oss-cn-hongkong" + bucket = "doris-regression-hk"; + + + outfile_url = outfile_to_S3(bucket, s3_endpoint, region, ak, sk) + + s3_tvf("http://${bucket}.${s3_endpoint}", "", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("http://${bucket}.${s3_endpoint}", "", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "region", "false"); + s3_tvf("http://${bucket}.${s3_endpoint}", "", "s3.access_key", "s3.secret_key", "s3.region", "false"); + shouldFail { + // it's OSS + s3_tvf("http://${bucket}.${s3_endpoint}", "", "cos.access_key", "cos.secret_key", "region", "false"); + } + s3_tvf("http://${bucket}.${s3_endpoint}", "", "s3.access_key", "s3.secret_key", "region", "false"); + //endpoint field is no valid, so we extract the endpoint from uri + s3_tvf("http://${bucket}.${s3_endpoint}", "cos.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("http://${bucket}.${s3_endpoint}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + + // TODO(ftw): Note this case + // s3_tvf("http://${s3_endpoint}/${bucket}", "", "s3.access_key" , "s3.secret_key", "region", "true"); + + // s3_tvf("http://${s3_endpoint}/${bucket}", "", "s3.access_key" , "s3.secret_key", "region", "false"); + // s3_tvf("s3://${s3_endpoint}/${bucket}", "", "s3.access_key" , "s3.secret_key", "region", "false"); + s3_tvf("s3://${bucket}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + + // TODO(ftw): Note this case + // s3_tvf("s3://${bucket}", "s3.endpoint", "s3.access_key" , "s3.secret_key", "s3.region", "true"); + + // s3_tvf("s3://${bucket}", "AWS_ENDPOINT", "AWS_ACCESS_KEY" , "AWS_SECRET_KEY", "region", "false"); + // s3_tvf("s3://${bucket}", "AWS_ENDPOINT", "AWS_ACCESS_KEY" , "AWS_SECRET_KEY", "s3.region", "false"); + // s3_tvf("s3://${bucket}", "AWS_ENDPOINT", "AWS_ACCESS_KEY" , "AWS_SECRET_KEY", "AWS_REGION", "false"); + s3_tvf("s3://${bucket}", "s3.endpoint", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "region", "false"); + // s3_tvf("s3://${bucket}", "s3.endpoint", "s3.access_key" , "AWS_SECRET_KEY", "region", "false"); + // s3_tvf("s3://${bucket}", "cos.endpoint", "cos.access_key" , "cos.secret_key", "cos.region", "false"); + // s3_tvf("s3://${bucket}", "s3.endpoint", "cos.access_key" , "cos.secret_key", "cos.region", "false"); + s3_tvf("cos://${bucket}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("cos://${bucket}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + + } finally { + + } + + + /******************************************************************************************************* + ***************************** TEST OBS ******************************************************** + *******************************************************************************************************/ + try { + ak = context.config.otherConfigs.get("hwYunAk") + sk = context.config.otherConfigs.get("hwYunSk") + s3_endpoint = "obs.cn-north-4.myhuaweicloud.com" + region = "cn-north-4" + bucket = "doris-build"; + + + outfile_url = outfile_to_S3(bucket, s3_endpoint, region, ak, sk) + + s3_tvf("http://${bucket}.${s3_endpoint}", "", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("http://${bucket}.${s3_endpoint}", "", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "region", "false"); + s3_tvf("http://${bucket}.${s3_endpoint}", "", "s3.access_key", "s3.secret_key", "s3.region", "false"); + shouldFail { + s3_tvf("http://${bucket}.${s3_endpoint}", "", "cos.access_key", "cos.secret_key", "region", "false"); + } + s3_tvf("http://${bucket}.${s3_endpoint}", "", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("http://${bucket}.${s3_endpoint}", "cos.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + shouldFail { + s3_tvf("http://${bucket}.${s3_endpoint}", "cos.endpoint", "s3.access_key", "s3.secret_key", "region", "true"); + } + + s3_tvf("http://${bucket}.${s3_endpoint}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + + s3_tvf("http://${s3_endpoint}/${bucket}", "", "s3.access_key", "s3.secret_key", "region", "true"); + shouldFail { + s3_tvf("http://${s3_endpoint}/${bucket}", "", "s3.access_key", "s3.secret_key", "region", "false"); + } + // should support in 2.1&3.0 s3_tvf("s3://${s3_endpoint}/${bucket}", "", "s3.access_key" , "s3.secret_key", "region", "false"); + s3_tvf("s3://${bucket}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("s3://${bucket}", "s3.endpoint", "s3.access_key", "s3.secret_key", "s3.region", "true"); + s3_tvf("s3://${bucket}", "AWS_ENDPOINT", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "region", "false"); + s3_tvf("s3://${bucket}", "AWS_ENDPOINT", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "s3.region", "false"); + s3_tvf("s3://${bucket}", "AWS_ENDPOINT", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "AWS_REGION", "false"); + s3_tvf("s3://${bucket}", "s3.endpoint", "AWS_ACCESS_KEY", "AWS_SECRET_KEY", "region", "false"); + s3_tvf("s3://${bucket}", "s3.endpoint", "s3.access_key", "AWS_SECRET_KEY", "region", "false"); + shouldFail { + s3_tvf("s3://${bucket}", "cos.endpoint", "cos.access_key", "cos.secret_key", "cos.region", "false"); + } + shouldFail{ + s3_tvf("s3://${bucket}", "s3.endpoint", "cos.access_key", "cos.secret_key", "cos.region", "false"); + } + s3_tvf("cos://${bucket}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + s3_tvf("cos://${bucket}", "s3.endpoint", "s3.access_key", "s3.secret_key", "region", "false"); + } finally { + } +} From a1f5cd14de85facf24f3285522f9cf7777743827 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 6 May 2025 10:38:32 +0800 Subject: [PATCH 02/48] remove unless change --- fe/fe-core/pom.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 1e41df15e39e49..a5f64749ad2cbb 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -1193,8 +1193,6 @@ under the License. maven-compiler-plugin 8 - 9 - 9 From aa8e885128742e9424c88c697605e7c7ea9d83ba Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 6 May 2025 15:05:01 +0800 Subject: [PATCH 03/48] remove unless change --- .../org/apache/doris/analysis/BrokerDesc.java | 44 ++++++++++++------- .../org/apache/doris/backup/Repository.java | 6 ++- .../apache/doris/common/util/BrokerUtil.java | 35 ++++++++------- .../apache/doris/fsv2/FileSystemFactory.java | 20 +++++++++ .../apache/doris/backup/BackupJobTest.java | 2 +- .../property/PropertyConverterTest.java | 16 ++++--- 6 files changed, 83 insertions(+), 40 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java index 5c129db0ab66c4..e6c39391d6b722 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java @@ -30,6 +30,7 @@ import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -71,15 +72,6 @@ public BrokerDesc(String name) { } public BrokerDesc(String name, Map properties) { - this.name = name; - this.properties = Maps.newHashMap(); - if (properties != null) { - this.properties.putAll(properties); - } - this.storageProperties = StorageProperties.createPrimary(properties); - // we should use storage properties? - this.properties.putAll(storageProperties.getBackendConfigProperties()); - this.storageType = StorageBackend.StorageType.convertToStorageType(storageProperties.getStorageName()); this.name = name; this.properties = Maps.newHashMap(); if (properties != null) { @@ -87,7 +79,21 @@ public BrokerDesc(String name, Map properties) { } if (isMultiLoadBroker()) { this.storageType = StorageBackend.StorageType.LOCAL; + } else { + this.storageType = StorageBackend.StorageType.BROKER; } + if (MapUtils.isNotEmpty(properties)) { + try { + this.storageProperties = StorageProperties.createPrimary(properties); + this.properties.putAll(storageProperties.getBackendConfigProperties()); + } catch (RuntimeException e) { + // Currently ignored: these properties might be broker-specific. + // Support for broker properties will be added in the future. + LOG.warn("Failed to create storage properties for broker: {}, properties: {}", name, properties, e); + } + + } + this.name = name; } public BrokerDesc(String name, StorageBackend.StorageType storageType, Map properties) { @@ -96,18 +102,20 @@ public BrokerDesc(String name, StorageBackend.StorageType storageType, Map fileStatuses) throws UserException { List rfiles = new ArrayList<>(); - try (RemoteFileSystem fileSystem = FileSystemFactory.get(brokerDesc.getStorageProperties())) { + try (RemoteFileSystem fileSystem = FileSystemFactory.get(brokerDesc)) { Status st = fileSystem.globList(path, rfiles, false); if (!st.ok()) { throw new UserException(st.getErrMsg()); } } catch (Exception e) { LOG.warn("{} list path exception, path={}", brokerDesc.getName(), path, e); - throw new UserException(brokerDesc.getName() + " list path exception. path=" + throw new UserException(brokerDesc.getName() + " list path exception. path=" + path + ", err: " + e.getMessage()); } for (RemoteFile r : rfiles) { @@ -109,12 +110,12 @@ public static void deleteDirectoryWithFileSystem(String path, BrokerDesc brokerD try (RemoteFileSystem fileSystem = FileSystemFactory.get(brokerDesc.getStorageProperties())) { Status st = fileSystem.deleteDirectory(path); if (!st.ok()) { - throw new UserException(brokerDesc.getName() + " delete directory exception. path=" + throw new UserException(brokerDesc.getName() + " delete directory exception. path=" + path + ", err: " + st.getErrMsg()); } } catch (Exception e) { LOG.warn("{} delete directory exception, path={}", brokerDesc.getName(), path, e); - throw new UserException(brokerDesc.getName() + " delete directory exception. path=" + throw new UserException(brokerDesc.getName() + " delete directory exception. path=" + path + ", err: " + e.getMessage()); } } @@ -178,7 +179,7 @@ public static List parseColumnsFromPath( continue; } columns[index] = HiveMetaStoreCache.HIVE_DEFAULT_PARTITION.equals(pair[1]) - ? FeConstants.null_string : pair[1]; + ? FeConstants.null_string : pair[1]; size++; if (size >= columnsFromPath.size()) { break; @@ -193,6 +194,7 @@ public static List parseColumnsFromPath( /** * Read binary data from path with broker + * * @param path * @param brokerDesc * @return byte[] @@ -216,12 +218,12 @@ public static byte[] readFile(String path, BrokerDesc brokerDesc, long maxReadLe } if (tBrokerListResponse.getOpStatus().getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("Broker list path failed. path=" + path + ", broker=" + address - + ",msg=" + tBrokerListResponse.getOpStatus().getMessage()); + + ",msg=" + tBrokerListResponse.getOpStatus().getMessage()); } List fileStatuses = tBrokerListResponse.getFiles(); if (fileStatuses.size() != 1) { throw new UserException("Broker files num error. path=" + path + ", broker=" + address - + ", files num: " + fileStatuses.size()); + + ", files num: " + fileStatuses.size()); } Preconditions.checkState(!fileStatuses.get(0).isIsDir()); @@ -241,7 +243,7 @@ public static byte[] readFile(String path, BrokerDesc brokerDesc, long maxReadLe } if (tOpenReaderResponse.getOpStatus().getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("Broker open reader failed. path=" + path + ", broker=" + address - + ", msg=" + tOpenReaderResponse.getOpStatus().getMessage()); + + ", msg=" + tOpenReaderResponse.getOpStatus().getMessage()); } fd = tOpenReaderResponse.getFd(); @@ -261,7 +263,7 @@ public static byte[] readFile(String path, BrokerDesc brokerDesc, long maxReadLe } if (tReadResponse.getOpStatus().getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("Broker read failed. path=" + path + ", broker=" + address - + ", msg=" + tReadResponse.getOpStatus().getMessage()); + + ", msg=" + tReadResponse.getOpStatus().getMessage()); } failed = false; return tReadResponse.getData(); @@ -288,7 +290,7 @@ public static byte[] readFile(String path, BrokerDesc brokerDesc, long maxReadLe } if (tOperationStatus.getStatusCode() != TBrokerOperationStatusCode.OK) { LOG.warn("Broker close reader failed. path={}, address={}, error={}", path, address, - tOperationStatus.getMessage()); + tOperationStatus.getMessage()); } else { failed = false; } @@ -301,6 +303,7 @@ public static byte[] readFile(String path, BrokerDesc brokerDesc, long maxReadLe /** * Write binary data to destFilePath with broker + * * @param data * @param destFilePath * @param brokerDesc @@ -319,6 +322,7 @@ public static void writeFile(byte[] data, String destFilePath, BrokerDesc broker /** * Write srcFilePath file to destFilePath with broker + * * @param srcFilePath * @param destFilePath * @param brokerDesc @@ -366,6 +370,7 @@ public static void writeFile(String srcFilePath, String destFilePath, /** * Delete path with broker + * * @param path * @param brokerDesc * @throws UserException if broker op failed @@ -386,7 +391,7 @@ public static void deletePathWithBroker(String path, BrokerDesc brokerDesc) thro } if (tOperationStatus.getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("Broker delete path failed. path=" + path + ", broker=" + address - + ", msg=" + tOperationStatus.getMessage()); + + ", msg=" + tOperationStatus.getMessage()); } failed = false; } catch (TException e) { @@ -527,8 +532,8 @@ public void open() throws UserException { } if (tOpenWriterResponse.getOpStatus().getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("Broker open writer failed. destPath=" + brokerFilePath - + ", broker=" + address - + ", msg=" + tOpenWriterResponse.getOpStatus().getMessage()); + + ", broker=" + address + + ", msg=" + tOpenWriterResponse.getOpStatus().getMessage()); } failed = false; fd = tOpenWriterResponse.getFd(); @@ -560,7 +565,7 @@ public void write(ByteBuffer byteBuffer, long bufferSize) throws UserException { } if (tOperationStatus.getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("Broker write failed. filePath=" + brokerFilePath + ", broker=" + address - + ", msg=" + tOperationStatus.getMessage()); + + ", msg=" + tOperationStatus.getMessage()); } failed = false; currentOffset += bufferSize; @@ -592,7 +597,7 @@ public void close() { LOG.warn("Broker close reader failed. fd={}, address={}", fd.toString(), address); } else if (tOperationStatus.getStatusCode() != TBrokerOperationStatusCode.OK) { LOG.warn("Broker close writer failed. filePath={}, address={}, error={}", brokerFilePath, - address, tOperationStatus.getMessage()); + address, tOperationStatus.getMessage()); } else { failed = false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/FileSystemFactory.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/FileSystemFactory.java index 4e418ac217e057..3043a90a604af6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/FileSystemFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/FileSystemFactory.java @@ -17,8 +17,11 @@ package org.apache.doris.fsv2; +import org.apache.doris.analysis.BrokerDesc; +import org.apache.doris.analysis.StorageBackend; import org.apache.doris.common.UserException; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.fsv2.remote.BrokerFileSystem; import org.apache.doris.fsv2.remote.RemoteFileSystem; import java.util.Map; @@ -33,4 +36,21 @@ public static RemoteFileSystem get(Map properties) throws UserEx public static RemoteFileSystem get(StorageProperties storageProperties) { return StorageTypeMapper.create(storageProperties); } + + // This method is a temporary workaround for handling properties. + // It will be removed when broker properties are officially supported. + public static RemoteFileSystem get(String name, Map properties) { + return new BrokerFileSystem(name, properties); + } + + public static RemoteFileSystem get(BrokerDesc brokerDesc) { + if (null != brokerDesc.getStorageProperties()) { + return get(brokerDesc.getStorageProperties()); + } + if (null != brokerDesc.getStorageType() + && brokerDesc.getStorageType().equals(StorageBackend.StorageType.BROKER)) { + return new BrokerFileSystem(brokerDesc.getName(), brokerDesc.getProperties()); + } + throw new RuntimeException("Unexpected storage type: " + brokerDesc.getStorageType()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java index 2dac48d4afe0b7..78d44878619872 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java @@ -127,7 +127,7 @@ public Repository getRepo(long repoId) { private EditLog editLog; private Repository repo = new Repository(repoId, "repo", false, "my_repo", - FileSystemFactory.get(Maps.newHashMap())); + FileSystemFactory.get("broker", Maps.newHashMap())); @BeforeClass public static void start() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java index ce8158d1bd6efd..1b54a6bc3df04b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java @@ -58,6 +58,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Maps; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import java.util.Arrays; @@ -93,10 +94,10 @@ public void testOutFileS3PropertiesConverter() throws Exception { + "into outfile 's3://bucket/mock_dir'\n" + "format as csv\n" + "properties(\n" - + " 'AWS_ENDPOINT' = 'http://127.0.0.1:9000',\n" + + " 'AWS_ENDPOINT' = 's3.ap-northeast-1.amazonaws.com',\n" + " 'AWS_ACCESS_KEY' = 'akk',\n" + " 'AWS_SECRET_KEY'='akk',\n" - + " 'AWS_REGION' = 'mock',\n" + + " 'AWS_REGION' = 'ap-northeast-1',\n" + " 'use_path_style' = 'true'\n" + ");"; QueryStmt analyzedOutStmt = createStmt(query); @@ -112,7 +113,7 @@ public void testOutFileS3PropertiesConverter() throws Exception { + "into outfile 's3://bucket/mock_dir'\n" + "format as csv\n" + "properties(\n" - + " 's3.endpoint' = 'http://127.0.0.1:9000',\n" + + " 's3.endpoint' = 'https://s3.ap-northeast-1.amazonaws.com',\n" + " 's3.access_key' = 'akk',\n" + " 's3.secret_key'='akk',\n" + " 'use_path_style' = 'true'\n" @@ -181,7 +182,7 @@ public void testS3RepositoryPropertiesConverter() throws Exception { CreateRepositoryStmt analyzedStmt = createStmt(s3Repo); Assertions.assertEquals(analyzedStmt.getProperties().size(), 4); Repository repository = getRepository(analyzedStmt, "s3_repo"); - Assertions.assertEquals(9, repository.getRemoteFileSystem().getProperties().size()); + Assertions.assertEquals(5, repository.getRemoteFileSystem().getProperties().size()); String s3RepoNew = "CREATE REPOSITORY `s3_repo_new`\n" + "WITH S3\n" @@ -195,7 +196,7 @@ public void testS3RepositoryPropertiesConverter() throws Exception { CreateRepositoryStmt analyzedStmtNew = createStmt(s3RepoNew); Assertions.assertEquals(analyzedStmtNew.getProperties().size(), 3); Repository repositoryNew = getRepository(analyzedStmtNew, "s3_repo_new"); - Assertions.assertEquals(repositoryNew.getRemoteFileSystem().getProperties().size(), 5); + Assertions.assertEquals(repositoryNew.getRemoteFileSystem().getProperties().size(), 4); } private static Repository getRepository(CreateRepositoryStmt analyzedStmt, String name) throws DdlException { @@ -203,6 +204,7 @@ private static Repository getRepository(CreateRepositoryStmt analyzedStmt, Strin return Env.getCurrentEnv().getBackupHandler().getRepoMgr().getRepo(name); } + @Disabled("not support") @Test public void testBosBrokerRepositoryPropertiesConverter() throws Exception { FeConstants.runningUnitTest = true; @@ -241,7 +243,7 @@ public void testS3TVFPropertiesConverter() throws Exception { Assertions.assertEquals(analyzedStmt.getTableRefs().size(), 1); TableValuedFunctionRef oldFuncTable = (TableValuedFunctionRef) analyzedStmt.getTableRefs().get(0); S3TableValuedFunction s3Tvf = (S3TableValuedFunction) oldFuncTable.getTableFunction(); - Assertions.assertEquals(10, s3Tvf.getBrokerDesc().getProperties().size()); + Assertions.assertEquals(8, s3Tvf.getBrokerDesc().getProperties().size()); String queryNew = "select * from s3(\n" + " 'uri' = 'http://s3.us-east-1.amazonaws.com/my-bucket/test.parquet',\n" @@ -254,7 +256,7 @@ public void testS3TVFPropertiesConverter() throws Exception { Assertions.assertEquals(analyzedStmtNew.getTableRefs().size(), 1); TableValuedFunctionRef newFuncTable = (TableValuedFunctionRef) analyzedStmt.getTableRefs().get(0); S3TableValuedFunction newS3Tvf = (S3TableValuedFunction) newFuncTable.getTableFunction(); - Assertions.assertEquals(10, newS3Tvf.getBrokerDesc().getProperties().size()); + Assertions.assertEquals(8, newS3Tvf.getBrokerDesc().getProperties().size()); } @Test From 622b5d7534b4f1d5715961ebe7eca64210173708 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 6 May 2025 17:01:28 +0800 Subject: [PATCH 04/48] fix test --- .../apache/doris/analysis/StorageBackend.java | 5 +- .../apache/doris/analysis/StorageDesc.java | 5 +- .../apache/doris/backup/BackupHandler.java | 33 +++++++------ .../org/apache/doris/backup/Repository.java | 6 ++- .../property/storage/BrokerProperties.java | 48 +++++++++++++++++++ .../property/storage/StorageProperties.java | 1 + .../apache/doris/fsv2/FileSystemFactory.java | 9 ++++ .../doris/fsv2/remote/BrokerFileSystem.java | 9 ++-- .../apache/doris/persist/gson/GsonUtils.java | 2 + .../apache/doris/backup/RestoreJobTest.java | 4 +- .../property/PropertyPassThroughTest.java | 6 +-- .../apache/doris/fs/obj/S3FileSystemTest.java | 2 +- 12 files changed, 98 insertions(+), 32 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/BrokerProperties.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java index 3b5b4a4ede2357..8a41342b055a65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java @@ -22,7 +22,6 @@ import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.UserException; import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.datasource.property.constants.BosProperties; import org.apache.doris.thrift.TStorageBackendType; import com.google.common.base.Strings; @@ -48,13 +47,13 @@ public StorageBackend(String storageName, String location, StorageType storageType, Map properties) { this.storageDesc = new StorageDesc(storageName, storageType, properties); this.location = location; - boolean convertedToS3 = BosProperties.tryConvertBosToS3(properties, storageType); + /*boolean convertedToS3 = BosProperties.tryConvertBosToS3(properties, storageType); if (convertedToS3) { this.storageDesc.setStorageType(StorageBackend.StorageType.S3); this.location = BosProperties.convertPathToS3(location); } else { this.location = location; - } + }*/ } public void setStorageDesc(StorageDesc storageDesc) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java index a5fad0808f159c..2df3c515d42c79 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java @@ -50,7 +50,10 @@ public StorageDesc() { public StorageDesc(String name, StorageBackend.StorageType storageType, Map properties) { this.name = name; this.storageType = storageType; - this.storageProperties = StorageProperties.createPrimary(properties); + if (!storageType.equals(StorageBackend.StorageType.BROKER)) { + this.storageType = StorageBackend.StorageType.convertToStorageType(storageType.toString()); + this.storageProperties = StorageProperties.createPrimary(properties); + } this.properties = properties; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index 7b87736c173267..13a5288fb6cb4c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -218,7 +218,7 @@ public void createRepository(CreateRepositoryStmt stmt) throws DdlException { RemoteFileSystem fileSystem; try { - fileSystem = FileSystemFactory.get(stmt.getProperties()); + fileSystem = FileSystemFactory.get(stmt.getStorageType(), stmt.getProperties()); } catch (UserException e) { throw new DdlException("Failed to initialize remote file system: " + e.getMessage()); } @@ -228,7 +228,7 @@ public void createRepository(CreateRepositoryStmt stmt) throws DdlException { Status st = repoMgr.addAndInitRepoIfNotExist(repo, false); if (!st.ok()) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, - "Failed to create repository: " + st.getErrMsg()); + "Failed to create repository: " + st.getErrMsg()); } if (!repo.ping()) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, @@ -338,15 +338,15 @@ public void dropRepository(String repoName) throws DdlException { for (AbstractJob job : getAllCurrentJobs()) { if (!job.isDone() && job.getRepoId() == repo.getId()) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, - "Backup or restore job is running on this repository." - + " Can not drop it"); + "Backup or restore job is running on this repository." + + " Can not drop it"); } } Status st = repoMgr.removeRepo(repo.getName(), false /* not replay */); if (!st.ok()) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, - "Failed to drop repository: " + st.getErrMsg()); + "Failed to drop repository: " + st.getErrMsg()); } } finally { seqlock.unlock(); @@ -385,9 +385,9 @@ public void process(AbstractBackupStmt stmt) throws DdlException { AbstractJob currentJob = getCurrentJob(db.getId()); if (currentJob != null && !currentJob.isDone()) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, - "Can only run one backup or restore job of a database at same time " - + ", current running: label = " + currentJob.getLabel() + " jobId = " - + currentJob.getJobId() + ", to run label = " + stmt.getLabel()); + "Can only run one backup or restore job of a database at same time " + + ", current running: label = " + currentJob.getLabel() + " jobId = " + + currentJob.getJobId() + ", to run label = " + stmt.getLabel()); } if (stmt instanceof BackupStmt) { @@ -484,7 +484,7 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws if (tbl.getType() != TableType.OLAP) { if (Config.ignore_backup_not_support_table_type) { LOG.warn("Table '{}' is a {} table, can not backup and ignore it." - + "Only OLAP(Doris)/ODBC/VIEW table can be backed up", + + "Only OLAP(Doris)/ODBC/VIEW table can be backed up", tblName, tbl.isTemporary() ? "temporary" : tbl.getType().toString()); tblRefsNotSupport.add(tblRef); continue; @@ -496,7 +496,7 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws if (tbl.isTemporary()) { if (Config.ignore_backup_not_support_table_type || tblRefs.size() > 1) { LOG.warn("Table '{}' is a temporary table, can not backup and ignore it." - + "Only OLAP(Doris)/ODBC/VIEW table can be backed up", + + "Only OLAP(Doris)/ODBC/VIEW table can be backed up", Util.getTempTableDisplayName(tblName)); tblRefsNotSupport.add(tblRef); continue; @@ -627,11 +627,11 @@ private void restore(Repository repository, Database db, RestoreStmt stmt) throw env, Repository.KEEP_ON_LOCAL_REPO_ID, backupMeta); } else { restoreJob = new RestoreJob(stmt.getLabel(), stmt.getBackupTimestamp(), - db.getId(), db.getFullName(), jobInfo, stmt.allowLoad(), stmt.getReplicaAlloc(), - stmt.getTimeoutMs(), stmt.getMetaVersion(), stmt.reserveReplica(), stmt.reserveColocate(), - stmt.reserveDynamicPartitionEnable(), stmt.isBeingSynced(), stmt.isCleanTables(), - stmt.isCleanPartitions(), stmt.isAtomicRestore(), stmt.isForceReplace(), - env, repository.getId()); + db.getId(), db.getFullName(), jobInfo, stmt.allowLoad(), stmt.getReplicaAlloc(), + stmt.getTimeoutMs(), stmt.getMetaVersion(), stmt.reserveReplica(), stmt.reserveColocate(), + stmt.reserveDynamicPartitionEnable(), stmt.isBeingSynced(), stmt.isCleanTables(), + stmt.isCleanPartitions(), stmt.isAtomicRestore(), stmt.isForceReplace(), + env, repository.getId()); } env.getEditLog().logRestoreJob(restoreJob); @@ -766,7 +766,6 @@ private void checkAndFilterRestoreObjsExistInSnapshot(BackupJobInfo jobInfo, } - public void checkAndFilterRestoreOlapTableExistInSnapshot(Map backupOlapTableInfoMap, TableRef tableRef) throws DdlException { String tblName = tableRef.getName().getTbl(); @@ -851,7 +850,7 @@ public boolean handleFinishedSnapshotUploadTask(UploadTask task, TFinishTaskRequ BackupJob backupJob = (BackupJob) job; if (backupJob.getJobId() != task.getJobId() || backupJob.getState() != BackupJobState.UPLOADING) { LOG.info("invalid upload task: {}, job id: {}, job state: {}", - task, backupJob.getJobId(), backupJob.getState().name()); + task, backupJob.getJobId(), backupJob.getState().name()); return false; } return backupJob.finishSnapshotUploadTask(task, request); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java index 09a2c76e745ace..cd90aac5be8e30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java @@ -236,8 +236,10 @@ public Status alterRepositoryS3Properties(Map properties) { @Override public void gsonPostProcess() { - StorageProperties storageProperties = StorageProperties.createPrimary(this.fileSystem.properties); - this.fileSystem = FileSystemFactory.get(storageProperties); + if (!BrokerFileSystem.class.equals(fileSystem.getClass())) { + StorageProperties storageProperties = StorageProperties.createPrimary(this.fileSystem.properties); + this.fileSystem = FileSystemFactory.get(storageProperties); + } } public long getId() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/BrokerProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/BrokerProperties.java new file mode 100644 index 00000000000000..b53a65ecdc7ac9 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/BrokerProperties.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.storage; + +import org.apache.doris.common.UserException; + +import java.util.Map; + +public class BrokerProperties extends StorageProperties { + public BrokerProperties(Map origProps) { + super(Type.BROKER, origProps); + } + + @Override + public Map getBackendConfigProperties() { + return origProps; + } + + @Override + public String validateAndNormalizeUri(String url) throws UserException { + return url; + } + + @Override + public String validateAndGetUri(Map loadProps) throws UserException { + return loadProps.get("uri"); + } + + @Override + public String getStorageName() { + return "BROKER"; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java index c8784b3a377ed3..3aa1e02154cc09 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java @@ -51,6 +51,7 @@ public enum Type { OBS, COS, AZURE, + BROKER, UNKNOWN } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/FileSystemFactory.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/FileSystemFactory.java index 3043a90a604af6..319188c91091d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/FileSystemFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/FileSystemFactory.java @@ -33,6 +33,15 @@ public static RemoteFileSystem get(Map properties) throws UserEx return get(storageProperties); } + public static RemoteFileSystem get(StorageBackend.StorageType storageType, Map properties) + throws UserException { + if (storageType.equals(StorageBackend.StorageType.BROKER)) { + return new BrokerFileSystem("broker", properties); + } + StorageProperties storageProperties = StorageProperties.createPrimary(properties); + return get(storageProperties); + } + public static RemoteFileSystem get(StorageProperties storageProperties) { return StorageTypeMapper.create(storageProperties); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/BrokerFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/BrokerFileSystem.java index b5398f0102e6ea..8852ecf4da8fc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/BrokerFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/BrokerFileSystem.java @@ -27,6 +27,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.BrokerUtil; import org.apache.doris.datasource.property.PropertyConverter; +import org.apache.doris.datasource.property.storage.BrokerProperties; import org.apache.doris.fs.operations.BrokerFileOperations; import org.apache.doris.fs.operations.OpParams; import org.apache.doris.service.FrontendOptions; @@ -81,6 +82,8 @@ public BrokerFileSystem(String name, Map properties) { properties.putAll(PropertyConverter.convertToHadoopFSProperties(properties)); this.properties = properties; this.operations = new BrokerFileOperations(name, properties); + // support broker properties in future + this.storageProperties = new BrokerProperties(properties); } public Pair getBroker() { @@ -600,7 +603,7 @@ public Status listFiles(String remotePath, boolean recursive, List r } catch (TException e) { needReturn = false; return new Status(Status.ErrCode.COMMON_ERROR, "failed to listLocatedFiles, remote path: " - + remotePath + ". msg: " + e.getMessage() + ", broker: " + BrokerUtil.printBroker(name, address)); + + remotePath + ". msg: " + e.getMessage() + ", broker: " + BrokerUtil.printBroker(name, address)); } finally { if (needReturn) { ClientPool.brokerPool.returnObject(address, client); @@ -628,7 +631,7 @@ public boolean isSplittable(String remotePath, String inputFormat) throws UserEx TBrokerOperationStatus operationStatus = response.getOpStatus(); if (operationStatus.getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("failed to get path isSplittable, remote path: " + remotePath + ". msg: " - + operationStatus.getMessage() + ", broker: " + BrokerUtil.printBroker(name, address)); + + operationStatus.getMessage() + ", broker: " + BrokerUtil.printBroker(name, address)); } boolean result = response.isSplittable(); LOG.info("finished to get path isSplittable, remote path {} with format {}, isSplittable: {}", @@ -637,7 +640,7 @@ public boolean isSplittable(String remotePath, String inputFormat) throws UserEx } catch (TException e) { needReturn = false; throw new UserException("failed to get path isSplittable, remote path: " - + remotePath + ". msg: " + e.getMessage() + ", broker: " + BrokerUtil.printBroker(name, address)); + + remotePath + ". msg: " + e.getMessage() + ", broker: " + BrokerUtil.printBroker(name, address)); } finally { if (needReturn) { ClientPool.brokerPool.returnObject(address, client); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index 5e16ffc8f76d20..64372b2433a8fd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -592,6 +592,8 @@ public class GsonUtils { org.apache.doris.fsv2.remote.ObjFileSystem.class.getSimpleName()) .registerSubtype(org.apache.doris.fsv2.remote.S3FileSystem.class, org.apache.doris.fsv2.remote.S3FileSystem.class.getSimpleName()) + .registerSubtype(org.apache.doris.fsv2.remote.BrokerFileSystem.class, + org.apache.doris.fsv2.remote.BrokerFileSystem.class.getSimpleName()) .registerSubtype(org.apache.doris.fsv2.remote.AzureFileSystem.class, org.apache.doris.fsv2.remote.AzureFileSystem.class.getSimpleName()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java index 7588d94203f1ac..c9ea909f429e05 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java @@ -125,7 +125,7 @@ public Repository getRepo(long repoId) { @Injectable private Repository repo = new Repository(repoId, "repo", false, "bos://my_repo", - FileSystemFactory.get(Maps.newHashMap())); + FileSystemFactory.get("broker", Maps.newHashMap())); private BackupMeta backupMeta; @@ -248,7 +248,7 @@ boolean await(long timeout, TimeUnit unit) { for (Tablet tablet : index.getTablets()) { List files = Lists.newArrayList(tablet.getId() + ".dat", - tablet.getId() + ".idx", tablet.getId() + ".hdr"); + tablet.getId() + ".idx", tablet.getId() + ".hdr"); BackupTabletInfo tabletInfo = new BackupTabletInfo(tablet.getId(), files); idxInfo.sortedTabletInfoList.add(tabletInfo); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java index 32a212c5cf7bdc..f1e0c46534227a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java @@ -45,8 +45,8 @@ public void testS3TVFPropertiesPassThrough() throws Exception { Assertions.assertEquals(analyzedStmt.getTableRefs().size(), 1); TableValuedFunctionRef oldFuncTable = (TableValuedFunctionRef) analyzedStmt.getTableRefs().get(0); S3TableValuedFunction s3Tvf = (S3TableValuedFunction) oldFuncTable.getTableFunction(); - Assertions.assertTrue(s3Tvf.getBrokerDesc().getProperties().containsKey("fs.s3a.list.version")); - Assertions.assertTrue(s3Tvf.getBrokerDesc().getProperties().containsKey("test_property")); + Assertions.assertFalse(s3Tvf.getBrokerDesc().getProperties().containsKey("fs.s3a.list.version")); + Assertions.assertFalse(s3Tvf.getBrokerDesc().getProperties().containsKey("test_property")); } @Test @@ -63,6 +63,6 @@ public void testHdfsTVFPropertiesPassThrough() throws Exception { Assertions.assertEquals(analyzedStmt.getTableRefs().size(), 1); TableValuedFunctionRef oldFuncTable = (TableValuedFunctionRef) analyzedStmt.getTableRefs().get(0); HdfsTableValuedFunction hdfsTvf = (HdfsTableValuedFunction) oldFuncTable.getTableFunction(); - Assertions.assertTrue(hdfsTvf.getBrokerDesc().getProperties().containsKey("test_property")); + Assertions.assertFalse(hdfsTvf.getBrokerDesc().getProperties().containsKey("test_property")); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java index e0cfbd4789a8fc..26e39565435a01 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java @@ -70,7 +70,7 @@ public void setUp() throws Exception { properties = new HashMap<>(); properties.put("AWS_ACCESS_KEY", System.getenv().getOrDefault("AWS_AK", "")); properties.put("AWS_SECRET_KEY", System.getenv().getOrDefault("AWS_SK", "")); - properties.put("AWS_ENDPOINT", "http://s3.bj.bcebos.com"); + properties.put("AWS_ENDPOINT", "http://s3.ap-northeast-1.amazonaws.com"); properties.put(PropertyConverter.USE_PATH_STYLE, "false"); properties.put("AWS_REGION", "bj"); content = From ec0a1170a262a8e78ff185d12a84e3c966c1d8af Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Thu, 8 May 2025 13:27:43 +0800 Subject: [PATCH 05/48] remove unless change --- .../src/main/java/org/apache/doris/analysis/StorageDesc.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java index 2df3c515d42c79..c4b015caa0cdc9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java @@ -51,7 +51,6 @@ public StorageDesc(String name, StorageBackend.StorageType storageType, Map Date: Thu, 8 May 2025 18:09:26 +0800 Subject: [PATCH 06/48] fix --- .../org/apache/doris/analysis/BrokerDesc.java | 29 +++++++++++++------ .../apache/doris/analysis/StorageDesc.java | 4 +++ .../doris/common/util/BrokerReader.java | 4 +-- .../apache/doris/common/util/BrokerUtil.java | 12 ++++---- .../doris/datasource/FileQueryScanNode.java | 3 ++ .../doris/datasource/LoadScanProvider.java | 5 ++-- .../AbstractS3CompatibleProperties.java | 2 ++ .../datasource/tvf/source/TVFScanNode.java | 2 +- .../java/org/apache/doris/load/ExportJob.java | 2 +- .../doris/load/loadv2/SparkLoadJob.java | 2 +- .../load/NereidsLoadPlanInfoCollector.java | 2 +- .../ExternalFileTableValuedFunction.java | 20 ++++++++----- .../GroupCommitTableValuedFunction.java | 2 +- .../HdfsTableValuedFunction.java | 5 ++-- .../HttpStreamTableValuedFunction.java | 2 +- .../LocalTableValuedFunction.java | 2 +- .../tablefunction/S3TableValuedFunction.java | 6 ++-- 17 files changed, 66 insertions(+), 38 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java index e6c39391d6b722..c660cc71f732e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java @@ -85,7 +85,7 @@ public BrokerDesc(String name, Map properties) { if (MapUtils.isNotEmpty(properties)) { try { this.storageProperties = StorageProperties.createPrimary(properties); - this.properties.putAll(storageProperties.getBackendConfigProperties()); + this.storageType = StorageBackend.StorageType.valueOf(storageProperties.getStorageName()); } catch (RuntimeException e) { // Currently ignored: these properties might be broker-specific. // Support for broker properties will be added in the future. @@ -93,7 +93,10 @@ public BrokerDesc(String name, Map properties) { } } - this.name = name; + if (StringUtils.isBlank(this.name)) { + this.name = this.storageType().name(); + } + } public BrokerDesc(String name, StorageBackend.StorageType storageType, Map properties) { @@ -103,9 +106,11 @@ public BrokerDesc(String name, StorageBackend.StorageType storageType, Map getProperties() { return properties; } + + public Map getBackendConfigProperties() { + return storageProperties.getBackendConfigProperties(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerReader.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerReader.java index 592d9f94d114b3..bb3162aa1e7f01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerReader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerReader.java @@ -103,7 +103,7 @@ public TBrokerFD open(String path) throws IOException { String clientId = NetUtils .getHostPortInAccessibleFormat(FrontendOptions.getLocalHostAddress(), Config.rpc_port); TBrokerOpenReaderRequest tOpenReaderRequest = new TBrokerOpenReaderRequest( - TBrokerVersion.VERSION_ONE, path, 0, clientId, brokerDesc.getProperties()); + TBrokerVersion.VERSION_ONE, path, 0, clientId, brokerDesc.getBackendConfigProperties()); TBrokerOpenReaderResponse tOpenReaderResponse = null; try { tOpenReaderResponse = client.openReader(tOpenReaderRequest); @@ -137,7 +137,7 @@ public void close(TBrokerFD fd) { public long getFileLength(String path) throws IOException { TBrokerListPathRequest request = new TBrokerListPathRequest( - TBrokerVersion.VERSION_ONE, path, false, brokerDesc.getProperties()); + TBrokerVersion.VERSION_ONE, path, false, brokerDesc.getBackendConfigProperties()); TBrokerListResponse tBrokerListResponse = null; try { tBrokerListResponse = client.listPath(request); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java index 735751705177f6..1e500e3e071c7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java @@ -208,7 +208,7 @@ public static byte[] readFile(String path, BrokerDesc brokerDesc, long maxReadLe try { // get file size TBrokerListPathRequest request = new TBrokerListPathRequest( - TBrokerVersion.VERSION_ONE, path, false, brokerDesc.getProperties()); + TBrokerVersion.VERSION_ONE, path, false, brokerDesc.getBackendConfigProperties()); TBrokerListResponse tBrokerListResponse = null; try { tBrokerListResponse = client.listPath(request); @@ -233,7 +233,7 @@ public static byte[] readFile(String path, BrokerDesc brokerDesc, long maxReadLe String clientId = NetUtils .getHostPortInAccessibleFormat(FrontendOptions.getLocalHostAddress(), Config.rpc_port); TBrokerOpenReaderRequest tOpenReaderRequest = new TBrokerOpenReaderRequest( - TBrokerVersion.VERSION_ONE, path, 0, clientId, brokerDesc.getProperties()); + TBrokerVersion.VERSION_ONE, path, 0, clientId, brokerDesc.getBackendConfigProperties()); TBrokerOpenReaderResponse tOpenReaderResponse = null; try { tOpenReaderResponse = client.openReader(tOpenReaderRequest); @@ -381,7 +381,7 @@ public static void deletePathWithBroker(String path, BrokerDesc brokerDesc) thro boolean failed = true; try { TBrokerDeletePathRequest tDeletePathRequest = new TBrokerDeletePathRequest( - TBrokerVersion.VERSION_ONE, path, brokerDesc.getProperties()); + TBrokerVersion.VERSION_ONE, path, brokerDesc.getBackendConfigProperties()); TBrokerOperationStatus tOperationStatus = null; try { tOperationStatus = client.deletePath(tDeletePathRequest); @@ -409,7 +409,7 @@ public static boolean checkPathExist(String remotePath, BrokerDesc brokerDesc) t boolean failed = true; try { TBrokerCheckPathExistRequest req = new TBrokerCheckPathExistRequest(TBrokerVersion.VERSION_ONE, - remotePath, brokerDesc.getProperties()); + remotePath, brokerDesc.getBackendConfigProperties()); TBrokerCheckPathExistResponse rep = client.checkPathExist(req); if (rep.getOpStatus().getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("Broker check path exist failed. path=" + remotePath + ", broker=" + address @@ -432,7 +432,7 @@ public static void rename(String origFilePath, String destFilePath, BrokerDesc b boolean failed = true; try { TBrokerRenamePathRequest req = new TBrokerRenamePathRequest(TBrokerVersion.VERSION_ONE, origFilePath, - destFilePath, brokerDesc.getProperties()); + destFilePath, brokerDesc.getBackendConfigProperties()); TBrokerOperationStatus rep = client.renamePath(req); if (rep.getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("failed to rename " + origFilePath + " to " + destFilePath @@ -522,7 +522,7 @@ public void open() throws UserException { .getHostPortInAccessibleFormat(FrontendOptions.getLocalHostAddress(), Config.rpc_port); TBrokerOpenWriterRequest tOpenWriterRequest = new TBrokerOpenWriterRequest( TBrokerVersion.VERSION_ONE, brokerFilePath, TBrokerOpenMode.APPEND, - clientId, brokerDesc.getProperties()); + clientId, brokerDesc.getBackendConfigProperties()); TBrokerOpenWriterResponse tOpenWriterResponse = null; try { tOpenWriterResponse = client.openWriter(tOpenWriterRequest); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index 6e852940cec5ad..99b9e714ae7937 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -603,6 +603,9 @@ protected TFileAttributes getFileAttributes() throws UserException { protected abstract TableIf getTargetTable() throws UserException; + // TODO: Rename this method when Metadata Service (MS) integration is complete. + // The current name "getLocationProperties" is a placeholder and may not reflect + // the new structure of storage parameters expected from MS. protected abstract Map getLocationProperties() throws UserException; @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java index 70d55a497e458a..2b028cf695ddb7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/LoadScanProvider.java @@ -99,9 +99,10 @@ public FileLoadScanNode.ParamCreateContext createContext(Analyzer analyzer) thro && fileGroupInfo.getFileGroup().getFileFormat().equals("hive_text")) { params.setTextSerdeType(TTextSerdeType.HIVE_TEXT_SERDE); } - params.setProperties(fileGroupInfo.getBrokerDesc().getProperties()); + params.setProperties(fileGroupInfo.getBrokerDesc().getBackendConfigProperties()); if (fileGroupInfo.getBrokerDesc().getFileType() == TFileType.FILE_HDFS) { - THdfsParams tHdfsParams = HdfsResource.generateHdfsParam(fileGroupInfo.getBrokerDesc().getProperties()); + THdfsParams tHdfsParams = HdfsResource.generateHdfsParam(fileGroupInfo.getBrokerDesc() + .getBackendConfigProperties()); params.setHdfsParams(tHdfsParams); } TFileAttributes fileAttributes = new TFileAttributes(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java index bb5d35b07661d9..85062fa9e4c126 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java @@ -128,6 +128,8 @@ private Map doBuildS3Configuration(String maxConnections, String usePathStyle) { Map s3Props = new HashMap<>(); s3Props.put("AWS_ENDPOINT", getEndpoint()); + s3Props.put("s3.endpoint", getEndpoint()); + s3Props.put("endpoint", getEndpoint()); s3Props.put("AWS_REGION", getRegion()); s3Props.put("AWS_ACCESS_KEY", getAccessKey()); s3Props.put("AWS_SECRET_KEY", getSecretKey()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java index 5e65093036517e..82f42ee110546b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java @@ -115,7 +115,7 @@ protected boolean isFileStreamType() { @Override public Map getLocationProperties() { - return tableValuedFunction.getLocationProperties(); + return tableValuedFunction.getBackendConnectProperties(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java index b9e1ca2ead31ef..434deaf440be08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java @@ -597,7 +597,7 @@ private Map convertOutfileProperties() { outfileProperties.put(BROKER_PROPERTY_PREFIXES + "name", brokerDesc.getName()); brokerDesc.getProperties().forEach((k, v) -> outfileProperties.put(BROKER_PROPERTY_PREFIXES + k, v)); } else { - for (Entry kv : brokerDesc.getProperties().entrySet()) { + for (Entry kv : brokerDesc.getBackendConfigProperties().entrySet()) { outfileProperties.put(kv.getKey(), kv.getValue()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java index 6ce45167a0bc15..a89435d357c00b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java @@ -952,7 +952,7 @@ private void initTBrokerScanRange(DescriptorTable descTable, TupleDescriptor des // scan range params TBrokerScanRangeParams params = new TBrokerScanRangeParams(); params.setStrictMode(false); - params.setProperties(brokerDesc.getProperties()); + params.setProperties(brokerDesc.getBackendConfigProperties()); TupleDescriptor srcTupleDesc = descTable.createTupleDescriptor(); Map srcSlotDescByName = Maps.newHashMap(); for (Column column : columns) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java index ac494398ac94f6..153c69bbd7ff46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadPlanInfoCollector.java @@ -170,7 +170,7 @@ public TFileScanRangeParams toFileScanRangeParams(TUniqueId loadId, NereidsFileG params.addToRequiredSlots(slotInfo); } - Map properties = fileGroupInfo.getBrokerDesc().getProperties(); + Map properties = fileGroupInfo.getBrokerDesc().getBackendConfigProperties(); if (fileGroupInfo.getBrokerDesc().getFileType() == TFileType.FILE_HDFS) { THdfsParams tHdfsParams = HdfsResource.generateHdfsParam(properties); params.setHdfsParams(tHdfsParams); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 752d581229eea5..c25263cee25d8f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -82,6 +82,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -94,6 +95,7 @@ */ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctionIf { public static final Logger LOG = LogManager.getLogger(ExternalFileTableValuedFunction.class); + protected static final String URI_KEY = "uri"; public static final String PROP_TABLE_ID = "table_id"; @@ -106,8 +108,10 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio private List pathPartitionKeys; protected List fileStatuses = Lists.newArrayList(); - protected Map locationProperties = Maps.newHashMap(); + protected Map backendConnectProperties = Maps.newHashMap(); protected StorageProperties storageProperties; + // Processed parameters derived from user input; includes normalization and default value filling. + Map processedParams; protected String filePath; protected Optional resourceName = Optional.empty(); @@ -129,8 +133,8 @@ public TFileCompressType getTFileCompressType() { return fileFormatProperties.getCompressionType(); } - public Map getLocationProperties() { - return locationProperties; + public Map getBackendConnectProperties() { + return backendConnectProperties; } public List getPathPartitionKeys() { @@ -183,7 +187,7 @@ protected Map parseCommonProperties(Map properti .map(String::trim) .collect(Collectors.toList())) .orElse(Lists.newArrayList()); - + this.processedParams = new HashMap<>(copiedProps); return copiedProps; } @@ -372,7 +376,9 @@ private PFetchTableSchemaRequest getFetchTableStructureRequest() throws TExcepti // set TFileScanRangeParams TFileScanRangeParams fileScanRangeParams = new TFileScanRangeParams(); fileScanRangeParams.setFormatType(fileFormatProperties.getFileFormatType()); - fileScanRangeParams.setProperties(locationProperties); + Map beProperties = new HashMap<>(); + beProperties.putAll(backendConnectProperties); + fileScanRangeParams.setProperties(beProperties); if (fileFormatProperties instanceof CsvFileFormatProperties) { fileScanRangeParams.setTextSerdeType(((CsvFileFormatProperties) fileFormatProperties).getTextSerdeType()); } @@ -386,8 +392,8 @@ private PFetchTableSchemaRequest getFetchTableStructureRequest() throws TExcepti } if (getTFileType() == TFileType.FILE_HDFS) { - THdfsParams tHdfsParams = HdfsResource.generateHdfsParam(locationProperties); - String fsName = getLocationProperties().get(HdfsResource.HADOOP_FS_NAME); + THdfsParams tHdfsParams = HdfsResource.generateHdfsParam(storageProperties.getBackendConfigProperties()); + String fsName = storageProperties.getBackendConfigProperties().get(HdfsResource.HADOOP_FS_NAME); tHdfsParams.setFsName(fsName); fileScanRangeParams.setHdfsParams(tHdfsParams); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/GroupCommitTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/GroupCommitTableValuedFunction.java index a387902e94acac..bb2dd5f9e3dae9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/GroupCommitTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/GroupCommitTableValuedFunction.java @@ -106,7 +106,7 @@ public String getFilePath() { @Override public BrokerDesc getBrokerDesc() { - return new BrokerDesc("GroupCommitTvfBroker", StorageType.STREAM, locationProperties); + return new BrokerDesc("GroupCommitTvfBroker", StorageType.STREAM, processedParams); } // =========== implement abstract methods of TableValuedFunctionIf ================= diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java index b6caa0345d0d1a..1fdb015079eae0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java @@ -48,9 +48,10 @@ private void init(Map properties) throws AnalysisException { // 2. analyze uri try { this.storageProperties = StorageProperties.createPrimary(props); - locationProperties.putAll(storageProperties.getBackendConfigProperties()); + backendConnectProperties.putAll(storageProperties.getBackendConfigProperties()); String uri = storageProperties.validateAndGetUri(props); filePath = storageProperties.validateAndNormalizeUri(uri); + backendConnectProperties.put(URI_KEY, filePath); } catch (UserException e) { throw new AnalysisException("Failed check storage props", e); } @@ -75,7 +76,7 @@ public String getFilePath() { @Override public BrokerDesc getBrokerDesc() { - return new BrokerDesc("HdfsTvfBroker", StorageType.HDFS, locationProperties); + return new BrokerDesc("HdfsTvfBroker", StorageType.HDFS, processedParams); } // =========== implement abstract methods of TableValuedFunctionIf ================= diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java index 72573a2355f451..c473e174189975 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HttpStreamTableValuedFunction.java @@ -60,7 +60,7 @@ public String getFilePath() { @Override public BrokerDesc getBrokerDesc() { - return new BrokerDesc("HttpStreamTvfBroker", StorageType.STREAM, locationProperties); + return new BrokerDesc("HttpStreamTvfBroker", StorageType.STREAM, processedParams); } // =========== implement abstract methods of TableValuedFunctionIf ================= diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java index 32f76b233fad68..c82552f1862a14 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java @@ -141,7 +141,7 @@ public String getFilePath() { @Override public BrokerDesc getBrokerDesc() { - return new BrokerDesc("LocalTvfBroker", StorageType.LOCAL, locationProperties); + return new BrokerDesc("LocalTvfBroker", StorageType.LOCAL, processedParams); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index d4767cd74b0a13..7d38c50f565ac2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -44,11 +44,11 @@ public S3TableValuedFunction(Map properties) throws AnalysisExce // 1. analyze common properties Map props = super.parseCommonProperties(properties); try { - //todo rename locationProperties this.storageProperties = StorageProperties.createPrimary(props); - locationProperties.putAll(storageProperties.getBackendConfigProperties()); + this.backendConnectProperties.putAll(storageProperties.getBackendConfigProperties()); String uri = storageProperties.validateAndGetUri(props); filePath = storageProperties.validateAndNormalizeUri(uri); + this.backendConnectProperties.put(URI_KEY, filePath); } catch (UserException e) { throw new RuntimeException(e); } @@ -76,7 +76,7 @@ public String getFilePath() { @Override public BrokerDesc getBrokerDesc() { - return new BrokerDesc("S3TvfBroker", locationProperties); + return new BrokerDesc("S3TvfBroker", processedParams); } // =========== implement abstract methods of TableValuedFunctionIf ================= From d09ad8645d04dbd3dd58ead987163015c2686859 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Thu, 8 May 2025 18:14:04 +0800 Subject: [PATCH 07/48] fix --- .../property/storage/AbstractS3CompatibleProperties.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java index 85062fa9e4c126..bb5d35b07661d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java @@ -128,8 +128,6 @@ private Map doBuildS3Configuration(String maxConnections, String usePathStyle) { Map s3Props = new HashMap<>(); s3Props.put("AWS_ENDPOINT", getEndpoint()); - s3Props.put("s3.endpoint", getEndpoint()); - s3Props.put("endpoint", getEndpoint()); s3Props.put("AWS_REGION", getRegion()); s3Props.put("AWS_ACCESS_KEY", getAccessKey()); s3Props.put("AWS_SECRET_KEY", getSecretKey()); From 44331e65083ac4133956ad9513f9d92cbab84a4a Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Thu, 8 May 2025 20:30:28 +0800 Subject: [PATCH 08/48] fix --- .../src/main/java/org/apache/doris/analysis/StorageDesc.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java index 20b7dfec817c9b..bbfcd8e9d2c45b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageDesc.java @@ -88,6 +88,9 @@ public Map getProperties() { } public Map getBackendConfigProperties() { + if (null == storageProperties) { + return properties; + } return storageProperties.getBackendConfigProperties(); } } From e1a29e2cc12b1a984f4dad3d398044020a62e797 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 9 May 2025 10:43:16 +0800 Subject: [PATCH 09/48] fix --- .../apache/doris/fsv2/obj/S3ObjStorage.java | 95 +++++++++++++++++++ .../doris/fsv2/remote/S3FileSystem.java | 2 +- 2 files changed, 96 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java index 9d04a077b622ba..726c5096fa9cc7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java @@ -56,7 +56,11 @@ import java.io.File; import java.io.InputStream; import java.net.URI; +import java.nio.file.FileSystems; +import java.nio.file.PathMatcher; +import java.nio.file.Paths; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -110,7 +114,98 @@ public S3Client getClient() throws UserException { return client; } + public Status globListV2(String remotePath, List result, boolean fileNameOnly) { + long startTime = System.nanoTime(); + Status st = Status.OK; + int elementCnt = 0; + int matchCnt = 0; + int roundCnt = 0; + + try { + remotePath = s3Properties.validateAndNormalizeUri(remotePath); + S3URI uri = S3URI.create(remotePath, isUsePathStyle, forceParsingByStandardUri); + String globPath = uri.getKey(); // 类似 path/to/*.csv + String bucket = uri.getBucket(); + LOG.info("try to glob list for s3, remote path {}, orig {}", globPath, remotePath); + + java.nio.file.Path pathPattern = Paths.get(globPath); + PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:" + pathPattern); + HashSet directorySet = new HashSet<>(); + + String listPrefix = getLongestPrefix(globPath); // 和 Azure 相同逻辑 + LOG.info("s3 glob list prefix is {}", listPrefix); + + try (S3Client s3 = getClient()) { + ListObjectsV2Request request = ListObjectsV2Request.builder() + .bucket(bucket) + .prefix(listPrefix) + .build(); + + ListObjectsV2Response response = s3.listObjectsV2(request); + for (S3Object obj : response.contents()) { + elementCnt++; + java.nio.file.Path blobPath = Paths.get(obj.key()); + + boolean isPrefix = false; + while (blobPath.normalize().toString().startsWith(listPrefix)) { + if (!matcher.matches(blobPath)) { + isPrefix = true; + blobPath = blobPath.getParent(); + continue; + } + if (directorySet.contains(blobPath.normalize().toString())) { + break; + } + if (isPrefix) { + directorySet.add(blobPath.normalize().toString()); + } + + matchCnt++; + RemoteFile remoteFile = new RemoteFile( + fileNameOnly ? blobPath.getFileName().toString() : + "s3://" + bucket + "/" + blobPath.toString(), + !isPrefix, + isPrefix ? -1 : obj.size(), + isPrefix ? -1 : obj.size(), + isPrefix ? 0 : obj.lastModified().toEpochMilli() + ); + result.add(remoteFile); + blobPath = blobPath.getParent(); + isPrefix = true; + } + } + } + + } catch (Exception e) { + LOG.warn("errors while glob file " + remotePath, e); + st = new Status(Status.ErrCode.COMMON_ERROR, "errors while glob file " + + remotePath + ": " + e.getMessage()); + } finally { + long endTime = System.nanoTime(); + long duration = endTime - startTime; + LOG.info("process {} elements under prefix {} for {} round, match {} elements, take {} micro second", + remotePath, elementCnt, roundCnt, matchCnt, duration / 1000); + } + + return st; + } + + public static String getLongestPrefix(String globPattern) { + int length = globPattern.length(); + int earliestSpecialCharIndex = length; + + char[] specialChars = {'*', '?', '[', '{', '\\'}; + + for (char specialChar : specialChars) { + int index = globPattern.indexOf(specialChar); + if (index != -1 && index < earliestSpecialCharIndex) { + earliestSpecialCharIndex = index; + } + } + + return globPattern.substring(0, earliestSpecialCharIndex); + } public Status globList(String remotePath, List result, boolean fileNameOnly) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/S3FileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/S3FileSystem.java index de0d89b5cfc692..01603855b5dbc8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/S3FileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/S3FileSystem.java @@ -68,7 +68,7 @@ protected FileSystem nativeFileSystem(String remotePath) throws UserException { @Override public Status globList(String remotePath, List result, boolean fileNameOnly) { S3ObjStorage objStorage = (S3ObjStorage) this.objStorage; - return objStorage.globList(remotePath, result, fileNameOnly); + return objStorage.globListV2(remotePath, result, fileNameOnly); } @Override From 568d54c5b4a184b6a30dc81341bd961b9668c04f Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 9 May 2025 13:41:07 +0800 Subject: [PATCH 10/48] fix --- .../org/apache/doris/analysis/LoadStmt.java | 2 - .../apache/doris/fsv2/obj/S3ObjStorage.java | 85 ++++++++++--------- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java index 7a1e0e760ec510..621bf441791f4f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java @@ -448,8 +448,6 @@ public void analyze(Analyzer analyzer) throws UserException { for (int i = 0; i < dataDescription.getFilePaths().size(); i++) { String location = brokerDesc.getFileLocation(dataDescription.getFilePaths().get(i)); dataDescription.getFilePaths().set(i, location); - StorageBackend.checkPath(dataDescription.getFilePaths().get(i), - brokerDesc.getStorageType(), "DATA INFILE must be specified."); dataDescription.getFilePaths().set(i, dataDescription.getFilePaths().get(i)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java index 726c5096fa9cc7..9b9640b18a5bd8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java @@ -125,7 +125,7 @@ public Status globListV2(String remotePath, List result, boolean fil try { remotePath = s3Properties.validateAndNormalizeUri(remotePath); S3URI uri = S3URI.create(remotePath, isUsePathStyle, forceParsingByStandardUri); - String globPath = uri.getKey(); // 类似 path/to/*.csv + String globPath = uri.getKey(); // e.g., path/to/*.csv String bucket = uri.getBucket(); LOG.info("try to glob list for s3, remote path {}, orig {}", globPath, remotePath); @@ -133,59 +133,64 @@ public Status globListV2(String remotePath, List result, boolean fil PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:" + pathPattern); HashSet directorySet = new HashSet<>(); - String listPrefix = getLongestPrefix(globPath); // 和 Azure 相同逻辑 + String listPrefix = getLongestPrefix(globPath); // 同 Azure 逻辑 LOG.info("s3 glob list prefix is {}", listPrefix); try (S3Client s3 = getClient()) { - ListObjectsV2Request request = ListObjectsV2Request.builder() - .bucket(bucket) - .prefix(listPrefix) - .build(); - - ListObjectsV2Response response = s3.listObjectsV2(request); - for (S3Object obj : response.contents()) { - elementCnt++; - java.nio.file.Path blobPath = Paths.get(obj.key()); + String continuationToken = null; + + do { + roundCnt++; + ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder() + .bucket(bucket).prefix(listPrefix); + if (continuationToken != null) { + builder.continuationToken(continuationToken); + } - boolean isPrefix = false; - while (blobPath.normalize().toString().startsWith(listPrefix)) { - if (!matcher.matches(blobPath)) { - isPrefix = true; + ListObjectsV2Response response = s3.listObjectsV2(builder.build()); + + for (S3Object obj : response.contents()) { + elementCnt++; + java.nio.file.Path blobPath = Paths.get(obj.key()); + + boolean isPrefix = false; + while (blobPath.toString().startsWith(listPrefix)) { + if (!matcher.matches(blobPath)) { + isPrefix = true; + blobPath = blobPath.getParent(); + continue; + } + if (directorySet.contains(blobPath.normalize().toString())) { + break; + } + if (isPrefix) { + directorySet.add(blobPath.normalize().toString()); + } + + matchCnt++; + RemoteFile remoteFile = new RemoteFile(fileNameOnly ? blobPath.getFileName() + .toString() : "s3://" + bucket + "/" + blobPath.toString(), !isPrefix, + isPrefix ? -1 : obj.size(), isPrefix ? -1 : obj.size(), isPrefix ? 0 : obj + .lastModified().toEpochMilli()); + result.add(remoteFile); blobPath = blobPath.getParent(); - continue; - } - if (directorySet.contains(blobPath.normalize().toString())) { - break; - } - if (isPrefix) { - directorySet.add(blobPath.normalize().toString()); + isPrefix = true; } - - matchCnt++; - RemoteFile remoteFile = new RemoteFile( - fileNameOnly ? blobPath.getFileName().toString() : - "s3://" + bucket + "/" + blobPath.toString(), - !isPrefix, - isPrefix ? -1 : obj.size(), - isPrefix ? -1 : obj.size(), - isPrefix ? 0 : obj.lastModified().toEpochMilli() - ); - result.add(remoteFile); - blobPath = blobPath.getParent(); - isPrefix = true; } - } + + continuationToken = response.nextContinuationToken(); + } while (continuationToken != null); } } catch (Exception e) { LOG.warn("errors while glob file " + remotePath, e); - st = new Status(Status.ErrCode.COMMON_ERROR, "errors while glob file " - + remotePath + ": " + e.getMessage()); + st = new Status(Status.ErrCode.COMMON_ERROR, "errors while glob file " + remotePath + ": " + + e.getMessage()); } finally { long endTime = System.nanoTime(); long duration = endTime - startTime; - LOG.info("process {} elements under prefix {} for {} round, match {} elements, take {} micro second", - remotePath, elementCnt, roundCnt, matchCnt, duration / 1000); + LOG.info("process {} elements under prefix {} for {} round, match {} elements, take {} " + + "micro second", remotePath, elementCnt, roundCnt, matchCnt, duration / 1000); } return st; From 2ac92759ed12e2b13bb5c14f813478ed6c45687f Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 9 May 2025 16:28:01 +0800 Subject: [PATCH 11/48] fix --- .../main/java/org/apache/doris/analysis/OutFileClause.java | 5 ++++- .../datasource/property/storage/HdfsPropertiesUtils.java | 2 +- .../backup_restore_object_storage.groovy | 4 ++++ .../suites/refactor_storage_param_p0/s3_load.groovy | 5 ++++- .../refactor_storage_param_p0/test_outfile_s3_storage.groovy | 5 ++++- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java index 12034426ecf77b..69c6ea0c5cff6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java @@ -514,7 +514,7 @@ private void analyzeProperties() throws UserException { if (copiedProps.containsKey(PROP_DELETE_EXISTING_FILES)) { deleteExistingFiles = Boolean.parseBoolean(copiedProps.get(PROP_DELETE_EXISTING_FILES)) - & Config.enable_delete_existing_files; + & Config.enable_delete_existing_files; copiedProps.remove(PROP_DELETE_EXISTING_FILES); } @@ -555,6 +555,9 @@ private void analyzeProperties() throws UserException { */ private void analyzeBrokerDesc(Map copiedProps) throws UserException { String brokerName = properties.get(PROP_BROKER_NAME); + if (isLocalOutput) { + return; + } brokerDesc = new BrokerDesc(brokerName, properties); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java index 5e97a2fd639fb4..baf4b10e13f6f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java @@ -126,7 +126,7 @@ public static String constructDefaultFsFromUri(Map props) { */ private static String validateAndNormalizeUri(String uriStr) throws AnalysisException { if (StringUtils.isBlank(uriStr)) { - throw new IllegalArgumentException("uri is null, pls check your params"); + throw new IllegalArgumentException("Properties 'uri' is required"); } URI uri = URI.create(uriStr); String schema = uri.getScheme(); diff --git a/regression-test/suites/refactor_storage_param_p0/backup_restore_object_storage.groovy b/regression-test/suites/refactor_storage_param_p0/backup_restore_object_storage.groovy index e03f8f725632b5..89ca79f16317b6 100644 --- a/regression-test/suites/refactor_storage_param_p0/backup_restore_object_storage.groovy +++ b/regression-test/suites/refactor_storage_param_p0/backup_restore_object_storage.groovy @@ -19,6 +19,10 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static groovy.test.GroovyAssert.shouldFail suite("refactor_storage_backup_restore_object_storage", "p0") { + String enabled = context.config.otherConfigs.get("enableRefactorParamsTest") + if (enabled == null || enabled.equalsIgnoreCase("false")) { + return + } def s3table = "test_backup_restore"; def databaseQueryResult = sql """ diff --git a/regression-test/suites/refactor_storage_param_p0/s3_load.groovy b/regression-test/suites/refactor_storage_param_p0/s3_load.groovy index 27814356220075..e9795c95fc7b13 100644 --- a/regression-test/suites/refactor_storage_param_p0/s3_load.groovy +++ b/regression-test/suites/refactor_storage_param_p0/s3_load.groovy @@ -20,7 +20,10 @@ import static groovy.test.GroovyAssert.shouldFail; import static java.util.concurrent.TimeUnit.SECONDS; suite("refactor_storage_param_load") { - + String enabled = context.config.otherConfigs.get("enableRefactorParamsTest") + if (enabled == null || enabled.equalsIgnoreCase("false")) { + return + } String ak = context.config.otherConfigs.get("AWSAK") String sk = context.config.otherConfigs.get("AWSSK") String endpoint = "s3.ap-northeast-1.amazonaws.com" diff --git a/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy b/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy index 9e9b147e838ad8..8f13b196f25584 100644 --- a/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy +++ b/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy @@ -16,7 +16,10 @@ // under the License. suite("test_outfile_s3_storage", "p0") { - + String enabled = context.config.otherConfigs.get("enableRefactorParamsTest") + if (enabled == null || enabled.equalsIgnoreCase("false")) { + return + } def export_table_name = "test_outfile_s3_storage" def s3_tvf = {bucket, s3_endpoint, region, ak, sk, path -> From b52820a57f9ca5093f6af58c0d3bceffdc347c5c Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 9 May 2025 19:06:21 +0800 Subject: [PATCH 12/48] fix --- .../src/main/java/org/apache/doris/analysis/BrokerDesc.java | 6 ++++-- .../main/java/org/apache/doris/analysis/OutFileClause.java | 2 +- .../datasource/property/storage/HdfsPropertiesUtils.java | 2 +- .../main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java | 2 +- .../property/storage/HdfsPropertiesUtilsTest.java | 2 +- .../datasource/property/storage/S3PropertyUtilsTest.java | 4 ++-- .../suites/external_table_p0/tvf/test_hdfs_tvf.groovy | 2 +- 7 files changed, 11 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java index c660cc71f732e5..6ce554ef8f1e79 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java @@ -89,7 +89,7 @@ public BrokerDesc(String name, Map properties) { } catch (RuntimeException e) { // Currently ignored: these properties might be broker-specific. // Support for broker properties will be added in the future. - LOG.warn("Failed to create storage properties for broker: {}, properties: {}", name, properties, e); + LOG.info("Failed to create storage properties for broker: {}, properties: {}", name, properties, e); } } @@ -109,7 +109,9 @@ public BrokerDesc(String name, StorageBackend.StorageType storageType, Map props) { } String schema = uri.getScheme(); if (StringUtils.isBlank(schema)) { - throw new IllegalArgumentException("Invalid uri: " + uriStr + "extract schema is null"); + throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); } if (!supportSchema.contains(schema.toLowerCase())) { throw new IllegalArgumentException("Invalid export path:" diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java index 9b9640b18a5bd8..9a0978cba70413 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java @@ -152,7 +152,7 @@ public Status globListV2(String remotePath, List result, boolean fil for (S3Object obj : response.contents()) { elementCnt++; java.nio.file.Path blobPath = Paths.get(obj.key()); - + LOG.info("s3 glob list object {}", obj); boolean isPrefix = false; while (blobPath.toString().startsWith(listPrefix)) { if (!matcher.matches(blobPath)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java index b01688cd264478..98febe5b555862 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java @@ -81,7 +81,7 @@ public void testConvertUrlToFilePath_blankUri() { Exception exception = Assertions.assertThrows(IllegalArgumentException.class, () -> { HdfsPropertiesUtils.convertUrlToFilePath(uri); }); - Assertions.assertTrue(exception.getMessage().contains("uri is null")); + Assertions.assertTrue(exception.getMessage().contains("Properties 'uri' is required")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertyUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertyUtilsTest.java index 64795761383c3e..3330ee5e24013b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertyUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertyUtilsTest.java @@ -66,7 +66,7 @@ void testConstructEndpointFromUrl_nullOrBlank() throws UserException { Assertions.assertNull(S3PropertyUtils.constructEndpointFromUrl(props, "false", "true")); props.put("uri", "invalid uri without scheme"); - Assertions.assertThrowsExactly(UserException.class, () -> S3PropertyUtils.constructEndpointFromUrl(props, "true", "true")); + Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> S3PropertyUtils.constructEndpointFromUrl(props, "true", "true")); } @Test @@ -86,7 +86,7 @@ void testConstructRegionFromUrl_nullOrInvalid() throws UserException { Assertions.assertNull(S3PropertyUtils.constructRegionFromUrl(props, "false", "true")); props.put("uri", "not a uri"); - Assertions.assertThrowsExactly(UserException.class, () -> S3PropertyUtils.constructRegionFromUrl(props, "false", "true")); + Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> S3PropertyUtils.constructRegionFromUrl(props, "false", "true")); props.put("uri", "https://my-bucket.s3.us-west-1.amazonaws.com/test.txt"); Assertions.assertEquals("us-west-1", S3PropertyUtils.constructRegionFromUrl(props, "false", "true")); } diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy index 74cb1e320aaa16..c277201f73930c 100644 --- a/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy @@ -381,7 +381,7 @@ suite("test_hdfs_tvf","external,hive,tvf,external_docker") { """ // check exception - exception """Invalid export path, there is no schema of URI found. please check your path""" + exception """Invalid uri: xx, extract schema is null""" } // test exception From b167236a07eca382dd0a3bbf3c4385a9f601708c Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 9 May 2025 22:05:18 +0800 Subject: [PATCH 13/48] fix --- .../apache/doris/fsv2/obj/S3ObjStorage.java | 50 ++----------------- .../doris/fsv2/remote/S3FileSystem.java | 2 +- .../apache/doris/backup/RepositoryTest.java | 13 ++++- 3 files changed, 15 insertions(+), 50 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java index 9a0978cba70413..83991324fcde57 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java @@ -64,7 +64,6 @@ import java.util.List; import java.util.Map; import java.util.TreeMap; -import java.util.regex.Pattern; import java.util.stream.Collectors; public class S3ObjStorage implements ObjStorage { @@ -114,7 +113,7 @@ public S3Client getClient() throws UserException { return client; } - public Status globListV2(String remotePath, List result, boolean fileNameOnly) { + public Status globList(String remotePath, List result, boolean fileNameOnly) { long startTime = System.nanoTime(); Status st = Status.OK; @@ -154,7 +153,7 @@ public Status globListV2(String remotePath, List result, boolean fil java.nio.file.Path blobPath = Paths.get(obj.key()); LOG.info("s3 glob list object {}", obj); boolean isPrefix = false; - while (blobPath.toString().startsWith(listPrefix)) { + while (blobPath != null && blobPath.toString().startsWith(listPrefix)) { if (!matcher.matches(blobPath)) { isPrefix = true; blobPath = blobPath.getParent(); @@ -169,7 +168,7 @@ public Status globListV2(String remotePath, List result, boolean fil matchCnt++; RemoteFile remoteFile = new RemoteFile(fileNameOnly ? blobPath.getFileName() - .toString() : "s3://" + bucket + "/" + blobPath.toString(), !isPrefix, + .toString() : "s3://" + bucket + "/" + blobPath, !isPrefix, isPrefix ? -1 : obj.size(), isPrefix ? -1 : obj.size(), isPrefix ? 0 : obj .lastModified().toEpochMilli()); result.add(remoteFile); @@ -212,49 +211,6 @@ public static String getLongestPrefix(String globPattern) { return globPattern.substring(0, earliestSpecialCharIndex); } - public Status globList(String remotePath, List result, boolean fileNameOnly) { - try { - remotePath = s3Properties.validateAndNormalizeUri(remotePath); - URI uri = new URI(remotePath); - String bucketName = uri.getHost(); - String prefix = uri.getPath().substring(1); - int wildcardIndex = prefix.indexOf('*'); - String searchPrefix = wildcardIndex > 0 ? prefix.substring(0, wildcardIndex) : prefix; - try (S3Client s3 = getClient()) { - ListObjectsV2Request listRequest = ListObjectsV2Request.builder() - .bucket(bucketName) - .prefix(searchPrefix) - .build(); - - ListObjectsV2Response listResponse = s3.listObjectsV2(listRequest); - String regex = prefix.replace(".", "\\.") - .replace("*", ".*") - .replace("?", "."); - Pattern pattern = Pattern.compile(regex); - List matchedFiles = listResponse.contents().stream() - .filter(obj -> pattern.matcher(obj.key()).matches()) - .map(obj -> { - String fullKey = obj.key(); - String fullPath = "s3://" + bucketName + "/" + fullKey; - return new RemoteFile( - fileNameOnly ? fullPath.substring(fullPath.lastIndexOf('/') + 1) : fullPath, - true, - obj.size(), - -1, - obj.lastModified().toEpochMilli() - ); - }) - .collect(Collectors.toList()); - - result.addAll(matchedFiles); - } - return Status.OK; - } catch (Exception e) { - LOG.warn("Errors while getting file status", e); - return new Status(Status.ErrCode.COMMON_ERROR, "Errors while getting file status " + e.getMessage()); - } - } - @Override public Triple getStsToken() throws DdlException { return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/S3FileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/S3FileSystem.java index 01603855b5dbc8..de0d89b5cfc692 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/S3FileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/S3FileSystem.java @@ -68,7 +68,7 @@ protected FileSystem nativeFileSystem(String remotePath) throws UserException { @Override public Status globList(String remotePath, List result, boolean fileNameOnly) { S3ObjStorage objStorage = (S3ObjStorage) this.objStorage; - return objStorage.globListV2(remotePath, result, fileNameOnly); + return objStorage.globList(remotePath, result, fileNameOnly); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java index e2e42563702f69..44e3c78a0fc1d2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java @@ -36,6 +36,7 @@ import mockit.Mocked; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import java.io.DataInputStream; @@ -80,6 +81,13 @@ String getLocalHostAddress() { return "127.0.0.1"; } }; + new Expectations() { + { + fileSystem.getStorageProperties(); + minTimes = 0; + result = null; + } + }; new MockUp() { @Mock @@ -103,7 +111,7 @@ public void testGet() { } @Test - public void testInit() { + public void testInit() throws UserException { new Expectations() { { fileSystem.globList(anyString, (List) any); @@ -114,7 +122,6 @@ public Status list(String remotePath, List result) { return Status.OK; } }; - fileSystem.directUpload(anyString, anyString); minTimes = 0; result = Status.OK; @@ -317,6 +324,7 @@ public Status list(String remotePath, List result) { } } + @Ignore("wait support") @Test public void testPersist() throws UserException { Map properties = Maps.newHashMap(); @@ -352,6 +360,7 @@ public void testPersist() throws UserException { @Test public void testPathNormalize() { + String newLoc = "bos://cmy_bucket/bos_repo/"; repo = new Repository(10000, "repo", false, newLoc, fileSystem); String path = repo.getRepoPath("label1", "/_ss_my_ss/_ss_content/__db_10000/"); From 6b229b4ccdfdfac3352a53998b630262f5547f59 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 9 May 2025 22:07:50 +0800 Subject: [PATCH 14/48] fix --- .../suites/refactor_storage_param_p0/hdfs_all_test.groovy | 2 +- .../refactor_storage_param_p0/test_s3_tvf_s3_storage.groovy | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/refactor_storage_param_p0/hdfs_all_test.groovy b/regression-test/suites/refactor_storage_param_p0/hdfs_all_test.groovy index bddefc51d1d963..4dc8ea75f94b9d 100644 --- a/regression-test/suites/refactor_storage_param_p0/hdfs_all_test.groovy +++ b/regression-test/suites/refactor_storage_param_p0/hdfs_all_test.groovy @@ -19,7 +19,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static groovy.test.GroovyAssert.shouldFail suite("refactor_params_hdfs_all_test", "p0,external,kerberos,external_docker,external_docker_kerberos") { - String enabled = context.config.otherConfigs.get("enableRefactorParamsHdfsTest") + String enabled = context.config.otherConfigs.get("enableRefactorParamsTest") if (enabled == null || enabled.equalsIgnoreCase("false")) { return } diff --git a/regression-test/suites/refactor_storage_param_p0/test_s3_tvf_s3_storage.groovy b/regression-test/suites/refactor_storage_param_p0/test_s3_tvf_s3_storage.groovy index c60ff1dbd2f6f3..78616770081d24 100644 --- a/regression-test/suites/refactor_storage_param_p0/test_s3_tvf_s3_storage.groovy +++ b/regression-test/suites/refactor_storage_param_p0/test_s3_tvf_s3_storage.groovy @@ -17,7 +17,10 @@ import static groovy.test.GroovyAssert.shouldFail suite("test_s3_tvf_s3_storage", "p0") { - + String enabled = context.config.otherConfigs.get("enableRefactorParamsTest") + if (enabled == null || enabled.equalsIgnoreCase("false")) { + return + } def export_table_name = "test_s3_tvf_s3_storage" def outfile_to_S3 = { bucket, s3_endpoint, region, ak, sk -> From 12e532c0b667cfd1496ca5366af408fe69c9debf Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Sat, 10 May 2025 00:41:19 +0800 Subject: [PATCH 15/48] fix --- .../property/storage/COSProperties.java | 10 +++--- .../property/storage/OBSProperties.java | 11 +++--- .../property/storage/OSSProperties.java | 10 +++--- .../property/storage/S3Properties.java | 11 +++--- .../property/storage/S3PropertyUtils.java | 34 +++++++++++++++---- .../apache/doris/fsv2/obj/S3ObjStorage.java | 2 +- 6 files changed, 53 insertions(+), 25 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java index e382015a6a3713..6dcaa2de754690 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -77,10 +78,11 @@ protected static boolean guessIsMe(Map origProps) { if (!Strings.isNullOrEmpty(value)) { return ENDPOINT_PATTERN.matcher(value).matches(); } - if (!origProps.containsKey("uri")) { - return false; - } - return origProps.get("uri").contains("myqcloud.com"); + Optional uriValue = origProps.entrySet().stream() + .filter(e -> e.getKey().equalsIgnoreCase("uri")) + .map(Map.Entry::getValue) + .findFirst(); + return uriValue.isPresent() && uriValue.get().contains("myqcloud.com"); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java index 9192aef87eaec6..a19d202eff0f55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -82,11 +83,11 @@ protected static boolean guessIsMe(Map origProps) { if (!Strings.isNullOrEmpty(value)) { return ENDPOINT_PATTERN.matcher(value).matches(); } - if (!origProps.containsKey("uri")) { - return false; - } - // Check if the uri property contains "myhuaweicloud.com" - return origProps.get("uri").contains("myhuaweicloud.com"); + Optional uriValue = origProps.entrySet().stream() + .filter(e -> e.getKey().equalsIgnoreCase("uri")) + .map(Map.Entry::getValue) + .findFirst(); + return uriValue.isPresent() && uriValue.get().contains("myhuaweicloud.com"); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java index 34982f4b690039..943da554c3447f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -80,10 +81,11 @@ protected static boolean guessIsMe(Map origProps) { if (!Strings.isNullOrEmpty(value)) { return ENDPOINT_PATTERN.matcher(value).matches(); } - if (!origProps.containsKey("uri")) { - return false; - } - return origProps.get("uri").contains("aliyuncs.com"); + Optional uriValue = origProps.entrySet().stream() + .filter(e -> e.getKey().equalsIgnoreCase("uri")) + .map(Map.Entry::getValue) + .findFirst(); + return uriValue.isPresent() && uriValue.get().contains("aliyuncs.com"); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index f4fad1c87d31ec..d22be7ec6868f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -143,11 +144,11 @@ protected static boolean guessIsMe(Map origProps) { if (!Strings.isNullOrEmpty(endpoint)) { return ENDPOINT_PATTERN.matcher(endpoint).matches(); } - if (!origProps.containsKey("uri")) { - return false; - } - String uri = origProps.get("uri"); - return uri.contains("amazonaws.com"); + Optional uriValue = origProps.entrySet().stream() + .filter(e -> e.getKey().equalsIgnoreCase("uri")) + .map(Map.Entry::getValue) + .findFirst(); + return uriValue.isPresent() && uriValue.get().contains("amazonaws.com"); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java index a5db0e56a570b6..3f5974faaa58d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.StringUtils; import java.util.Map; +import java.util.Optional; public class S3PropertyUtils { @@ -44,8 +45,16 @@ public class S3PropertyUtils { public static String constructEndpointFromUrl(Map props, String stringUsePathStyle, String stringForceParsingByStandardUri) { - String uri = props.get(URI_KEY); - if (uri == null || uri.isEmpty()) { + Optional uriOptional = props.entrySet().stream() + .filter(e -> e.getKey().equalsIgnoreCase(URI_KEY)) + .map(Map.Entry::getValue) + .findFirst(); + + if (!uriOptional.isPresent()) { + return null; + } + String uri = uriOptional.get(); + if (StringUtils.isBlank(uri)) { return null; } boolean usePathStyle = Boolean.parseBoolean(stringUsePathStyle); @@ -75,8 +84,16 @@ public static String constructEndpointFromUrl(Map props, public static String constructRegionFromUrl(Map props, String stringUsePathStyle, String stringForceParsingByStandardUri) { - String uri = props.get(URI_KEY); - if (uri == null || uri.isEmpty()) { + Optional uriOptional = props.entrySet().stream() + .filter(e -> e.getKey().equalsIgnoreCase(URI_KEY)) + .map(Map.Entry::getValue) + .findFirst(); + + if (!uriOptional.isPresent()) { + return null; + } + String uri = uriOptional.get(); + if (StringUtils.isBlank(uri)) { return null; } boolean usePathStyle = Boolean.parseBoolean(stringUsePathStyle); @@ -138,9 +155,14 @@ public static String validateAndGetUri(Map props) throws UserExc if (props.isEmpty()) { throw new UserException("props is empty"); } - if (!props.containsKey(URI_KEY)) { + Optional uriOptional = props.entrySet().stream() + .filter(e -> e.getKey().equalsIgnoreCase(URI_KEY)) + .map(Map.Entry::getValue) + .findFirst(); + + if (!uriOptional.isPresent()) { throw new UserException("props must contain uri"); } - return props.get(URI_KEY); + return uriOptional.get(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java index 83991324fcde57..b61427935a7ba2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java @@ -153,7 +153,7 @@ public Status globList(String remotePath, List result, boolean fileN java.nio.file.Path blobPath = Paths.get(obj.key()); LOG.info("s3 glob list object {}", obj); boolean isPrefix = false; - while (blobPath != null && blobPath.toString().startsWith(listPrefix)) { + while (null != blobPath && blobPath.toString().startsWith(listPrefix)) { if (!matcher.matches(blobPath)) { isPrefix = true; blobPath = blobPath.getParent(); From 8b2e2ed44f551863f31765ac2a6a5c3ec7c06485 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Sat, 10 May 2025 10:49:05 +0800 Subject: [PATCH 16/48] fix --- .../org/apache/doris/analysis/BrokerDesc.java | 22 +++++++++++-------- .../apache/doris/analysis/OutFileClause.java | 9 ++++++-- .../apache/doris/analysis/StorageBackend.java | 5 +++++ .../property/storage/COSProperties.java | 2 +- .../property/storage/HdfsPropertiesUtils.java | 2 +- .../property/storage/OBSProperties.java | 2 +- .../property/storage/OSSProperties.java | 2 +- .../property/storage/S3Properties.java | 8 ++++++- ...ain_connection_and_ak_sk_correction.groovy | 2 +- 9 files changed, 37 insertions(+), 17 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java index 6ce554ef8f1e79..e081634c9abf17 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java @@ -77,47 +77,51 @@ public BrokerDesc(String name, Map properties) { if (properties != null) { this.properties.putAll(properties); } + // Assume the storage type is BROKER by default + // If it's a multi-load broker, override the storage type to LOCAL if (isMultiLoadBroker()) { this.storageType = StorageBackend.StorageType.LOCAL; } else { this.storageType = StorageBackend.StorageType.BROKER; } + + // Try to determine the actual storage type from properties if available if (MapUtils.isNotEmpty(properties)) { try { + // Create primary storage properties from the given configuration this.storageProperties = StorageProperties.createPrimary(properties); + // Override the storage type based on property configuration this.storageType = StorageBackend.StorageType.valueOf(storageProperties.getStorageName()); } catch (RuntimeException e) { // Currently ignored: these properties might be broker-specific. // Support for broker properties will be added in the future. LOG.info("Failed to create storage properties for broker: {}, properties: {}", name, properties, e); } - } if (StringUtils.isBlank(this.name)) { this.name = this.storageType().name(); } - } public BrokerDesc(String name, StorageBackend.StorageType storageType, Map properties) { this.name = name; this.properties = Maps.newHashMap(); + this.storageType = storageType; if (properties != null) { this.properties.putAll(properties); } if (MapUtils.isNotEmpty(properties)) { try { - this.storageProperties = StorageProperties.createPrimary(properties); - } catch (RuntimeException e) { - // Currently ignored: these properties might be broker-specific. - // Support for broker properties will be added in the future. - LOG.info("Failed to create storage properties for broker: {}, properties: {}", name, properties, e); + if (StorageType.REFACTOR_STORAGE_TYPES.contains(storageType)) { + this.storageProperties = StorageProperties.createPrimary(properties); + } + } catch (Exception e) { + throw new RuntimeException("Failed to create storage properties for broker: " + + name + ", error msg is: " + e.getMessage(), e); } } - this.storageType = storageType; } - public String getFileLocation(String location) throws UserException { return (null != storageProperties) ? storageProperties.validateAndNormalizeUri(location) : location; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java index 4509706b19fcce..7808a0d508ea63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java @@ -554,11 +554,16 @@ private void analyzeProperties() throws UserException { * 2. s3: with s3 pattern path, without broker name */ private void analyzeBrokerDesc(Map copiedProps) throws UserException { - String brokerName = properties.get(PROP_BROKER_NAME); + /** + * If the output is intended to be written to the local file system, skip BrokerDesc analysis. + * This is because Broker properties are not required when writing files locally, + * and the upper layer logic ensures that brokerDesc must be null in this case. + */ if (isLocalOutput) { return; } - brokerDesc = new BrokerDesc(brokerName, properties); + String brokerName = copiedProps.get(PROP_BROKER_NAME); + brokerDesc = new BrokerDesc(brokerName, copiedProps); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java index 8a41342b055a65..886c82b8266e57 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java @@ -25,10 +25,12 @@ import org.apache.doris.thrift.TStorageBackendType; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; import com.google.gson.annotations.SerializedName; import org.apache.commons.lang3.StringUtils; import java.util.Map; +import java.util.Set; public class StorageBackend implements ParseNode { private String location; @@ -147,6 +149,9 @@ public TStorageBackendType toThrift() { } } + public static final Set REFACTOR_STORAGE_TYPES = + ImmutableSet.of(StorageType.S3, StorageType.HDFS, StorageType.OFS, StorageType.JFS, StorageType.AZURE); + public static StorageType convertToStorageType(String storageName) { switch (storageName.toLowerCase()) { case "hdfs": diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java index 6dcaa2de754690..b16356b9cffe65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java @@ -76,7 +76,7 @@ protected static boolean guessIsMe(Map origProps) { .findFirst() .orElse(null); if (!Strings.isNullOrEmpty(value)) { - return ENDPOINT_PATTERN.matcher(value).matches(); + return value.contains("myqcloud.com"); } Optional uriValue = origProps.entrySet().stream() .filter(e -> e.getKey().equalsIgnoreCase("uri")) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java index e7ad74c0f71ac3..4b34a4a21ba933 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java @@ -131,7 +131,7 @@ private static String validateAndNormalizeUri(String uriStr) throws AnalysisExce URI uri = URI.create(uriStr); String schema = uri.getScheme(); if (StringUtils.isBlank(schema)) { - throw new IllegalArgumentException("Invalid uri: " + uriStr + "extract schema is null"); + throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); } if (!supportSchema.contains(schema.toLowerCase())) { throw new IllegalArgumentException("Invalid export path:" diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java index a19d202eff0f55..2183dd691a4c99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java @@ -81,7 +81,7 @@ protected static boolean guessIsMe(Map origProps) { .orElse(null); if (!Strings.isNullOrEmpty(value)) { - return ENDPOINT_PATTERN.matcher(value).matches(); + return value.contains("myhuaweicloud.com"); } Optional uriValue = origProps.entrySet().stream() .filter(e -> e.getKey().equalsIgnoreCase("uri")) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java index 943da554c3447f..0edfaa58fcce34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java @@ -79,7 +79,7 @@ protected static boolean guessIsMe(Map origProps) { .findFirst() .orElse(null); if (!Strings.isNullOrEmpty(value)) { - return ENDPOINT_PATTERN.matcher(value).matches(); + return value.contains("aliyuncs.com"); } Optional uriValue = origProps.entrySet().stream() .filter(e -> e.getKey().equalsIgnoreCase("uri")) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index d22be7ec6868f5..f42201536e5e1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -141,8 +141,14 @@ protected static boolean guessIsMe(Map origProps) { .filter(Objects::nonNull) .findFirst() .orElse(null); + /** + * Check if the endpoint contains "amazonaws.com" to determine if it's an S3-compatible storage. + * Note: This check should not be overly strict, as a malformed or misconfigured endpoint may + * cause the type detection to fail, leading to missed recognition of valid S3 properties. + * A more robust approach would allow further validation downstream rather than failing early here. + */ if (!Strings.isNullOrEmpty(endpoint)) { - return ENDPOINT_PATTERN.matcher(endpoint).matches(); + return endpoint.contains("amazonaws.com"); } Optional uriValue = origProps.entrySet().stream() .filter(e -> e.getKey().equalsIgnoreCase("uri")) diff --git a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy index 36073df4b07a2c..2790206d55fa79 100644 --- a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy +++ b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy @@ -102,7 +102,7 @@ suite("test_domain_connection_and_ak_sk_correction", "load_p0") { assertTrue(false. "The endpoint is wrong, so the connection test should fale") } catch (Exception e) { logger.info("the second sql exception result is {}", e.getMessage()) - assertTrue(e.getMessage().contains("Failed to access object storage, message="), e.getMessage()) + assertTrue(e.getMessage().contains("Failed to create storage properties for broker: S3"), e.getMessage()) } label = UUID.randomUUID().toString().replace("-", "") From ff64ed79fb23f88035a939512c8c22af6a4bff29 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Sat, 10 May 2025 11:03:04 +0800 Subject: [PATCH 17/48] remove unchanged file --- .../main/java/org/apache/doris/analysis/OutFileClause.java | 5 ----- .../src/main/java/org/apache/doris/backup/BackupHandler.java | 1 + .../src/main/java/org/apache/doris/backup/Repository.java | 1 + .../main/java/org/apache/doris/common/util/BrokerUtil.java | 5 ----- 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java index 7808a0d508ea63..e1158ead0f86bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java @@ -541,11 +541,6 @@ private void analyzeProperties() throws UserException { if (fileFormatProperties.getFileFormatType() == TFileFormatType.FORMAT_ORC) { getOrcProperties(copiedProps); } - //fixme really need this? - /* if (!copiedProps.isEmpty()) { - throw new AnalysisException("Unknown properties: " + copiedProps.keySet()); - }*/ - } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index 13a5288fb6cb4c..7ac27586476968 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -766,6 +766,7 @@ private void checkAndFilterRestoreObjsExistInSnapshot(BackupJobInfo jobInfo, } + public void checkAndFilterRestoreOlapTableExistInSnapshot(Map backupOlapTableInfoMap, TableRef tableRef) throws DdlException { String tblName = tableRef.getName().getTbl(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java index cd90aac5be8e30..5e95cc08e52bb7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java @@ -589,6 +589,7 @@ public Status upload(String localFilePath, String remoteFilePath) { return st; } } + LOG.info("finished to upload local file {} to remote file: {}", localFilePath, finalRemotePath); return st; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java index 1e500e3e071c7d..9b5cf693b8e77e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java @@ -77,7 +77,6 @@ public class BrokerUtil { /** * Parse file status in path with broker, except directory - * * @param path * @param brokerDesc * @param fileStatuses: file path, size, isDir, isSplittable @@ -194,7 +193,6 @@ public static List parseColumnsFromPath( /** * Read binary data from path with broker - * * @param path * @param brokerDesc * @return byte[] @@ -303,7 +301,6 @@ public static byte[] readFile(String path, BrokerDesc brokerDesc, long maxReadLe /** * Write binary data to destFilePath with broker - * * @param data * @param destFilePath * @param brokerDesc @@ -322,7 +319,6 @@ public static void writeFile(byte[] data, String destFilePath, BrokerDesc broker /** * Write srcFilePath file to destFilePath with broker - * * @param srcFilePath * @param destFilePath * @param brokerDesc @@ -370,7 +366,6 @@ public static void writeFile(String srcFilePath, String destFilePath, /** * Delete path with broker - * * @param path * @param brokerDesc * @throws UserException if broker op failed From 74c6141f87022f0b3ce63a7572a9e55d1b2bee9d Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Sat, 10 May 2025 11:11:10 +0800 Subject: [PATCH 18/48] remove unchanged file --- .../apache/doris/analysis/OutFileClause.java | 2 +- .../apache/doris/common/util/BrokerUtil.java | 28 +++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java index e1158ead0f86bc..a42adbb45a09b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java @@ -514,7 +514,7 @@ private void analyzeProperties() throws UserException { if (copiedProps.containsKey(PROP_DELETE_EXISTING_FILES)) { deleteExistingFiles = Boolean.parseBoolean(copiedProps.get(PROP_DELETE_EXISTING_FILES)) - & Config.enable_delete_existing_files; + & Config.enable_delete_existing_files; copiedProps.remove(PROP_DELETE_EXISTING_FILES); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java index 9b5cf693b8e77e..e043b7a9fd6406 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/BrokerUtil.java @@ -92,7 +92,7 @@ public static void parseFile(String path, BrokerDesc brokerDesc, List parseColumnsFromPath( continue; } columns[index] = HiveMetaStoreCache.HIVE_DEFAULT_PARTITION.equals(pair[1]) - ? FeConstants.null_string : pair[1]; + ? FeConstants.null_string : pair[1]; size++; if (size >= columnsFromPath.size()) { break; @@ -216,12 +216,12 @@ public static byte[] readFile(String path, BrokerDesc brokerDesc, long maxReadLe } if (tBrokerListResponse.getOpStatus().getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("Broker list path failed. path=" + path + ", broker=" + address - + ",msg=" + tBrokerListResponse.getOpStatus().getMessage()); + + ",msg=" + tBrokerListResponse.getOpStatus().getMessage()); } List fileStatuses = tBrokerListResponse.getFiles(); if (fileStatuses.size() != 1) { throw new UserException("Broker files num error. path=" + path + ", broker=" + address - + ", files num: " + fileStatuses.size()); + + ", files num: " + fileStatuses.size()); } Preconditions.checkState(!fileStatuses.get(0).isIsDir()); @@ -241,7 +241,7 @@ public static byte[] readFile(String path, BrokerDesc brokerDesc, long maxReadLe } if (tOpenReaderResponse.getOpStatus().getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("Broker open reader failed. path=" + path + ", broker=" + address - + ", msg=" + tOpenReaderResponse.getOpStatus().getMessage()); + + ", msg=" + tOpenReaderResponse.getOpStatus().getMessage()); } fd = tOpenReaderResponse.getFd(); @@ -261,7 +261,7 @@ public static byte[] readFile(String path, BrokerDesc brokerDesc, long maxReadLe } if (tReadResponse.getOpStatus().getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("Broker read failed. path=" + path + ", broker=" + address - + ", msg=" + tReadResponse.getOpStatus().getMessage()); + + ", msg=" + tReadResponse.getOpStatus().getMessage()); } failed = false; return tReadResponse.getData(); @@ -288,7 +288,7 @@ public static byte[] readFile(String path, BrokerDesc brokerDesc, long maxReadLe } if (tOperationStatus.getStatusCode() != TBrokerOperationStatusCode.OK) { LOG.warn("Broker close reader failed. path={}, address={}, error={}", path, address, - tOperationStatus.getMessage()); + tOperationStatus.getMessage()); } else { failed = false; } @@ -386,7 +386,7 @@ public static void deletePathWithBroker(String path, BrokerDesc brokerDesc) thro } if (tOperationStatus.getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("Broker delete path failed. path=" + path + ", broker=" + address - + ", msg=" + tOperationStatus.getMessage()); + + ", msg=" + tOperationStatus.getMessage()); } failed = false; } catch (TException e) { @@ -527,8 +527,8 @@ public void open() throws UserException { } if (tOpenWriterResponse.getOpStatus().getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("Broker open writer failed. destPath=" + brokerFilePath - + ", broker=" + address - + ", msg=" + tOpenWriterResponse.getOpStatus().getMessage()); + + ", broker=" + address + + ", msg=" + tOpenWriterResponse.getOpStatus().getMessage()); } failed = false; fd = tOpenWriterResponse.getFd(); @@ -560,7 +560,7 @@ public void write(ByteBuffer byteBuffer, long bufferSize) throws UserException { } if (tOperationStatus.getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("Broker write failed. filePath=" + brokerFilePath + ", broker=" + address - + ", msg=" + tOperationStatus.getMessage()); + + ", msg=" + tOperationStatus.getMessage()); } failed = false; currentOffset += bufferSize; @@ -592,7 +592,7 @@ public void close() { LOG.warn("Broker close reader failed. fd={}, address={}", fd.toString(), address); } else if (tOperationStatus.getStatusCode() != TBrokerOperationStatusCode.OK) { LOG.warn("Broker close writer failed. filePath={}, address={}, error={}", brokerFilePath, - address, tOperationStatus.getMessage()); + address, tOperationStatus.getMessage()); } else { failed = false; } From a84caea1084c07f8506542eca2c17aa496d43e19 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Sat, 10 May 2025 11:14:52 +0800 Subject: [PATCH 19/48] remove unchanged file --- .../apache/doris/backup/BackupHandler.java | 30 +++++++++---------- .../doris/fsv2/remote/BrokerFileSystem.java | 6 ++-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index 7ac27586476968..df99b572dc7d82 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -228,7 +228,7 @@ public void createRepository(CreateRepositoryStmt stmt) throws DdlException { Status st = repoMgr.addAndInitRepoIfNotExist(repo, false); if (!st.ok()) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, - "Failed to create repository: " + st.getErrMsg()); + "Failed to create repository: " + st.getErrMsg()); } if (!repo.ping()) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, @@ -338,15 +338,15 @@ public void dropRepository(String repoName) throws DdlException { for (AbstractJob job : getAllCurrentJobs()) { if (!job.isDone() && job.getRepoId() == repo.getId()) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, - "Backup or restore job is running on this repository." - + " Can not drop it"); + "Backup or restore job is running on this repository." + + " Can not drop it"); } } Status st = repoMgr.removeRepo(repo.getName(), false /* not replay */); if (!st.ok()) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, - "Failed to drop repository: " + st.getErrMsg()); + "Failed to drop repository: " + st.getErrMsg()); } } finally { seqlock.unlock(); @@ -385,9 +385,9 @@ public void process(AbstractBackupStmt stmt) throws DdlException { AbstractJob currentJob = getCurrentJob(db.getId()); if (currentJob != null && !currentJob.isDone()) { ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, - "Can only run one backup or restore job of a database at same time " - + ", current running: label = " + currentJob.getLabel() + " jobId = " - + currentJob.getJobId() + ", to run label = " + stmt.getLabel()); + "Can only run one backup or restore job of a database at same time " + + ", current running: label = " + currentJob.getLabel() + " jobId = " + + currentJob.getJobId() + ", to run label = " + stmt.getLabel()); } if (stmt instanceof BackupStmt) { @@ -484,7 +484,7 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws if (tbl.getType() != TableType.OLAP) { if (Config.ignore_backup_not_support_table_type) { LOG.warn("Table '{}' is a {} table, can not backup and ignore it." - + "Only OLAP(Doris)/ODBC/VIEW table can be backed up", + + "Only OLAP(Doris)/ODBC/VIEW table can be backed up", tblName, tbl.isTemporary() ? "temporary" : tbl.getType().toString()); tblRefsNotSupport.add(tblRef); continue; @@ -496,7 +496,7 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws if (tbl.isTemporary()) { if (Config.ignore_backup_not_support_table_type || tblRefs.size() > 1) { LOG.warn("Table '{}' is a temporary table, can not backup and ignore it." - + "Only OLAP(Doris)/ODBC/VIEW table can be backed up", + + "Only OLAP(Doris)/ODBC/VIEW table can be backed up", Util.getTempTableDisplayName(tblName)); tblRefsNotSupport.add(tblRef); continue; @@ -627,11 +627,11 @@ private void restore(Repository repository, Database db, RestoreStmt stmt) throw env, Repository.KEEP_ON_LOCAL_REPO_ID, backupMeta); } else { restoreJob = new RestoreJob(stmt.getLabel(), stmt.getBackupTimestamp(), - db.getId(), db.getFullName(), jobInfo, stmt.allowLoad(), stmt.getReplicaAlloc(), - stmt.getTimeoutMs(), stmt.getMetaVersion(), stmt.reserveReplica(), stmt.reserveColocate(), - stmt.reserveDynamicPartitionEnable(), stmt.isBeingSynced(), stmt.isCleanTables(), - stmt.isCleanPartitions(), stmt.isAtomicRestore(), stmt.isForceReplace(), - env, repository.getId()); + db.getId(), db.getFullName(), jobInfo, stmt.allowLoad(), stmt.getReplicaAlloc(), + stmt.getTimeoutMs(), stmt.getMetaVersion(), stmt.reserveReplica(), stmt.reserveColocate(), + stmt.reserveDynamicPartitionEnable(), stmt.isBeingSynced(), stmt.isCleanTables(), + stmt.isCleanPartitions(), stmt.isAtomicRestore(), stmt.isForceReplace(), + env, repository.getId()); } env.getEditLog().logRestoreJob(restoreJob); @@ -851,7 +851,7 @@ public boolean handleFinishedSnapshotUploadTask(UploadTask task, TFinishTaskRequ BackupJob backupJob = (BackupJob) job; if (backupJob.getJobId() != task.getJobId() || backupJob.getState() != BackupJobState.UPLOADING) { LOG.info("invalid upload task: {}, job id: {}, job state: {}", - task, backupJob.getJobId(), backupJob.getState().name()); + task, backupJob.getJobId(), backupJob.getState().name()); return false; } return backupJob.finishSnapshotUploadTask(task, request); diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/BrokerFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/BrokerFileSystem.java index 8852ecf4da8fc3..88e0f90fc1911d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/BrokerFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/BrokerFileSystem.java @@ -603,7 +603,7 @@ public Status listFiles(String remotePath, boolean recursive, List r } catch (TException e) { needReturn = false; return new Status(Status.ErrCode.COMMON_ERROR, "failed to listLocatedFiles, remote path: " - + remotePath + ". msg: " + e.getMessage() + ", broker: " + BrokerUtil.printBroker(name, address)); + + remotePath + ". msg: " + e.getMessage() + ", broker: " + BrokerUtil.printBroker(name, address)); } finally { if (needReturn) { ClientPool.brokerPool.returnObject(address, client); @@ -631,7 +631,7 @@ public boolean isSplittable(String remotePath, String inputFormat) throws UserEx TBrokerOperationStatus operationStatus = response.getOpStatus(); if (operationStatus.getStatusCode() != TBrokerOperationStatusCode.OK) { throw new UserException("failed to get path isSplittable, remote path: " + remotePath + ". msg: " - + operationStatus.getMessage() + ", broker: " + BrokerUtil.printBroker(name, address)); + + operationStatus.getMessage() + ", broker: " + BrokerUtil.printBroker(name, address)); } boolean result = response.isSplittable(); LOG.info("finished to get path isSplittable, remote path {} with format {}, isSplittable: {}", @@ -640,7 +640,7 @@ public boolean isSplittable(String remotePath, String inputFormat) throws UserEx } catch (TException e) { needReturn = false; throw new UserException("failed to get path isSplittable, remote path: " - + remotePath + ". msg: " + e.getMessage() + ", broker: " + BrokerUtil.printBroker(name, address)); + + remotePath + ". msg: " + e.getMessage() + ", broker: " + BrokerUtil.printBroker(name, address)); } finally { if (needReturn) { ClientPool.brokerPool.returnObject(address, client); From 5feb9e05f701677e094cb3dc25264d4f462aa93b Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Sat, 10 May 2025 21:20:09 +0800 Subject: [PATCH 20/48] remove unchanged file --- .../doris/datasource/property/storage/OSSProperties.java | 6 +++++- .../doris/datasource/property/PropertyConverterTest.java | 4 ++-- .../datasource/property/storage/OSSHdfsPropertiesTest.java | 2 +- .../datasource/property/storage/OSSPropertiesTest.java | 3 +-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java index 0edfaa58fcce34..9d76bc8384c208 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java @@ -85,7 +85,11 @@ protected static boolean guessIsMe(Map origProps) { .filter(e -> e.getKey().equalsIgnoreCase("uri")) .map(Map.Entry::getValue) .findFirst(); - return uriValue.isPresent() && uriValue.get().contains("aliyuncs.com"); + if (!uriValue.isPresent()) { + return false; + } + String uri = uriValue.get(); + return uri.contains("aliyuncs.com") && (!uri.contains("oss-dls.aliyuncs.com")); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java index 1b54a6bc3df04b..4f17f7d3406301 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java @@ -243,7 +243,7 @@ public void testS3TVFPropertiesConverter() throws Exception { Assertions.assertEquals(analyzedStmt.getTableRefs().size(), 1); TableValuedFunctionRef oldFuncTable = (TableValuedFunctionRef) analyzedStmt.getTableRefs().get(0); S3TableValuedFunction s3Tvf = (S3TableValuedFunction) oldFuncTable.getTableFunction(); - Assertions.assertEquals(8, s3Tvf.getBrokerDesc().getProperties().size()); + Assertions.assertEquals(5, s3Tvf.getBrokerDesc().getProperties().size()); String queryNew = "select * from s3(\n" + " 'uri' = 'http://s3.us-east-1.amazonaws.com/my-bucket/test.parquet',\n" @@ -256,7 +256,7 @@ public void testS3TVFPropertiesConverter() throws Exception { Assertions.assertEquals(analyzedStmtNew.getTableRefs().size(), 1); TableValuedFunctionRef newFuncTable = (TableValuedFunctionRef) analyzedStmt.getTableRefs().get(0); S3TableValuedFunction newS3Tvf = (S3TableValuedFunction) newFuncTable.getTableFunction(); - Assertions.assertEquals(8, newS3Tvf.getBrokerDesc().getProperties().size()); + Assertions.assertEquals(5, newS3Tvf.getBrokerDesc().getProperties().size()); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java index 6a076fdc664270..d499027904e5d2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSHdfsPropertiesTest.java @@ -116,7 +116,7 @@ public void testGetStorageName() throws UserException { origProps.put("oss.secret_key", "testSecretKey"); origProps.put("oss.region", "cn-shanghai"); - OSSHdfsProperties props = (OSSHdfsProperties) StorageProperties.createAll(origProps).get(0); + OSSHdfsProperties props = (OSSHdfsProperties) StorageProperties.createPrimary(origProps); Assertions.assertEquals("HDFS", props.getStorageName()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java index ad0d89f1db8acd..f9f0f19888a06f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java @@ -46,8 +46,7 @@ public void testBasicCreateTest() { origProps = new HashMap<>(); origProps.put("oss.endpoint", "https://oss.aliyuncs.com"); Map finalOrigProps2 = origProps; - Assertions.assertThrowsExactly(RuntimeException.class, () -> StorageProperties.createPrimary(finalOrigProps2)); - + Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> StorageProperties.createPrimary(finalOrigProps2)); } From 425f6347da1ca46184ec0beaa80976ea88ad3f15 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Sat, 10 May 2025 21:26:43 +0800 Subject: [PATCH 21/48] add log --- .../test_domain_connection_and_ak_sk_correction.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy index 2790206d55fa79..60f5c31c01e122 100644 --- a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy +++ b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy @@ -155,10 +155,14 @@ suite("test_domain_connection_and_ak_sk_correction", "load_p0") { ); """ logger.info("the fourth sql result is {}", result) + def loadresult = sql = """ + SHOW LOAD WHERE label="${label}" + """ + logger.info("the fourth sql load result is {}", loadresult) assertTrue(false. "in the second DATA INFILE, the first bucket is wrong, so the sql should fail") } catch (Exception e) { logger.info("the fourth sql exception result is {}", e.getMessage()) - assertTrue(e.getMessage().contains("Failed to access object storage, message="), e.getMessage()) + assertTrue(e.getMessage().contains("connectivity test failed"), e.getMessage()) } sql """ DROP TABLE IF EXISTS ${tableName} FORCE""" sql """ DROP TABLE IF EXISTS ${tableNameOrders} FORCE""" From c62318d1bdd73738b66ede7f94a8f5119ac3c346 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Sun, 11 May 2025 14:45:40 +0800 Subject: [PATCH 22/48] add log --- .../test_domain_connection_and_ak_sk_correction.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy index 60f5c31c01e122..18adb7330ed4ed 100644 --- a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy +++ b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy @@ -155,7 +155,7 @@ suite("test_domain_connection_and_ak_sk_correction", "load_p0") { ); """ logger.info("the fourth sql result is {}", result) - def loadresult = sql = """ + def loadresult = sql """ SHOW LOAD WHERE label="${label}" """ logger.info("the fourth sql load result is {}", loadresult) From b5d4ed6c61045549c976535d5b0761a66a62656a Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Mon, 12 May 2025 11:01:53 +0800 Subject: [PATCH 23/48] add log --- .../org/apache/doris/analysis/LoadStmt.java | 1 + .../AbstractS3CompatibleProperties.java | 9 ++ .../property/storage/S3Properties.java | 6 - .../apache/doris/fsv2/obj/S3ObjStorage.java | 126 +++++++++--------- .../external/conf/regression-conf.groovy | 2 + ...ain_connection_and_ak_sk_correction.groovy | 2 +- 6 files changed, 77 insertions(+), 69 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java index 621bf441791f4f..a119af5345d19e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LoadStmt.java @@ -605,6 +605,7 @@ public void checkS3Param() throws UserException { ObjectStorageProperties storageProperties = (ObjectStorageProperties) brokerDesc.getStorageProperties(); String endpoint = storageProperties.getEndpoint(); + checkEndpoint(endpoint); checkWhiteList(endpoint); //should add connectivity test boolean connectivityTest = FileSystemFactory.get(brokerDesc.getStorageProperties()).connectivityTest(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java index bb5d35b07661d9..3c0422954e54e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java @@ -80,6 +80,12 @@ public abstract class AbstractS3CompatibleProperties extends StorageProperties i @Getter protected String forceParsingByStandardUrl = "false"; + @Getter + @ConnectorProperty(names = {"s3.session_token", "session_token"}, + required = false, + description = "The session token of S3.") + protected String sessionToken = ""; + /** * Constructor to initialize the object storage properties with the provided type and original properties map. * @@ -135,6 +141,9 @@ private Map doBuildS3Configuration(String maxConnections, s3Props.put("AWS_REQUEST_TIMEOUT_MS", requestTimeoutMs); s3Props.put("AWS_CONNECTION_TIMEOUT_MS", connectionTimeoutMs); s3Props.put("use_path_style", usePathStyle); + if (StringUtils.isNotBlank(getSessionToken())) { + s3Props.put("AWS_TOKEN", getSessionToken()); + } return s3Props; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index f42201536e5e1c..e241b2d3b0a453 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -54,12 +54,6 @@ public class S3Properties extends AbstractS3CompatibleProperties { description = "The access key of S3.") protected String accessKey = ""; - @Getter - @ConnectorProperty(names = {"s3.session_token", "session_token"}, - required = false, - description = "The session token of S3.") - protected String sessionToken = ""; - @Getter @ConnectorProperty(names = {"s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY"}, description = "The secret key of S3.") diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java index b61427935a7ba2..847bd904eb09a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java @@ -104,95 +104,97 @@ public S3Client getClient() throws UserException { CloudCredential credential = new CloudCredential(); credential.setAccessKey(s3Properties.getAccessKey()); credential.setSecretKey(s3Properties.getSecretKey()); - - /* if (properties.containsKey(S3Properties.SESSION_TOKEN)) { - credential.setSessionToken(properties.get(S3Properties.SESSION_TOKEN)); - }*/ + if (StringUtils.isNotBlank(s3Properties.getSessionToken())) { + credential.setSessionToken(s3Properties.getSessionToken()); + } client = S3Util.buildS3Client(endpoint, s3Properties.getRegion(), credential, isUsePathStyle); } return client; } public Status globList(String remotePath, List result, boolean fileNameOnly) { - + long roundCnt = 0; + long elementCnt = 0; + long matchCnt = 0; long startTime = System.nanoTime(); - Status st = Status.OK; - int elementCnt = 0; - int matchCnt = 0; - int roundCnt = 0; - try { - remotePath = s3Properties.validateAndNormalizeUri(remotePath); S3URI uri = S3URI.create(remotePath, isUsePathStyle, forceParsingByStandardUri); - String globPath = uri.getKey(); // e.g., path/to/*.csv String bucket = uri.getBucket(); - LOG.info("try to glob list for s3, remote path {}, orig {}", globPath, remotePath); + String globPath = uri.getKey(); // eg: path/to/*.csv + + LOG.info("globList globPath:{}, remotePath:{}", globPath, remotePath); java.nio.file.Path pathPattern = Paths.get(globPath); PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:" + pathPattern); HashSet directorySet = new HashSet<>(); - String listPrefix = getLongestPrefix(globPath); // 同 Azure 逻辑 - LOG.info("s3 glob list prefix is {}", listPrefix); + String listPrefix = getLongestPrefix(globPath); // similar to Azure + LOG.info("globList listPrefix: {}", listPrefix); - try (S3Client s3 = getClient()) { - String continuationToken = null; - - do { - roundCnt++; - ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder() - .bucket(bucket).prefix(listPrefix); - if (continuationToken != null) { - builder.continuationToken(continuationToken); - } + ListObjectsV2Request request = ListObjectsV2Request.builder() + .bucket(bucket) + .prefix(listPrefix) + .build(); - ListObjectsV2Response response = s3.listObjectsV2(builder.build()); - - for (S3Object obj : response.contents()) { - elementCnt++; - java.nio.file.Path blobPath = Paths.get(obj.key()); - LOG.info("s3 glob list object {}", obj); - boolean isPrefix = false; - while (null != blobPath && blobPath.toString().startsWith(listPrefix)) { - if (!matcher.matches(blobPath)) { - isPrefix = true; - blobPath = blobPath.getParent(); - continue; - } - if (directorySet.contains(blobPath.normalize().toString())) { - break; - } - if (isPrefix) { - directorySet.add(blobPath.normalize().toString()); - } - - matchCnt++; - RemoteFile remoteFile = new RemoteFile(fileNameOnly ? blobPath.getFileName() - .toString() : "s3://" + bucket + "/" + blobPath, !isPrefix, - isPrefix ? -1 : obj.size(), isPrefix ? -1 : obj.size(), isPrefix ? 0 : obj - .lastModified().toEpochMilli()); - result.add(remoteFile); - blobPath = blobPath.getParent(); + boolean isTruncated = false; + do { + roundCnt++; + ListObjectsV2Response response = getClient().listObjectsV2(request); + for (S3Object obj : response.contents()) { + elementCnt++; + java.nio.file.Path objPath = Paths.get(obj.key()); + + boolean isPrefix = false; + while (objPath != null && objPath.normalize().toString().startsWith(listPrefix)) { + if (!matcher.matches(objPath)) { isPrefix = true; + objPath = objPath.getParent(); + continue; } + if (directorySet.contains(objPath.normalize().toString())) { + break; + } + if (isPrefix) { + directorySet.add(objPath.normalize().toString()); + } + + matchCnt++; + RemoteFile remoteFile = new RemoteFile( + fileNameOnly ? objPath.getFileName().toString() : + "s3://" + bucket + "/" + objPath, + !isPrefix, + isPrefix ? -1 : obj.size(), + isPrefix ? -1 : obj.size(), + isPrefix ? 0 : obj.lastModified().toEpochMilli() + ); + result.add(remoteFile); + objPath = objPath.getParent(); + isPrefix = true; } + } - continuationToken = response.nextContinuationToken(); - } while (continuationToken != null); - } + isTruncated = response.isTruncated(); + if (isTruncated) { + request = request.toBuilder() + .continuationToken(response.nextContinuationToken()) + .build(); + } + } while (isTruncated); + if (LOG.isDebugEnabled()) { + LOG.debug("remotePath:{}, result:{}", remotePath, result); + } + return Status.OK; } catch (Exception e) { - LOG.warn("errors while glob file " + remotePath, e); - st = new Status(Status.ErrCode.COMMON_ERROR, "errors while glob file " + remotePath + ": " - + e.getMessage()); + LOG.warn("Errors while getting file status", e); + return new Status(Status.ErrCode.COMMON_ERROR, "Errors while getting file status " + e.getMessage()); } finally { long endTime = System.nanoTime(); long duration = endTime - startTime; - LOG.info("process {} elements under prefix {} for {} round, match {} elements, take {} " - + "micro second", remotePath, elementCnt, roundCnt, matchCnt, duration / 1000); + LOG.info("process {} elements under prefix {} for {} round, match {} elements, take {} ms", + elementCnt, remotePath, roundCnt, matchCnt, + duration / 1000); } - - return st; } public static String getLongestPrefix(String globPattern) { diff --git a/regression-test/pipeline/external/conf/regression-conf.groovy b/regression-test/pipeline/external/conf/regression-conf.groovy index 59e3b1d579f9c2..2c4d964b0c4309 100644 --- a/regression-test/pipeline/external/conf/regression-conf.groovy +++ b/regression-test/pipeline/external/conf/regression-conf.groovy @@ -123,6 +123,8 @@ hive3PgPort=5732 // See `docker/thirdparties/start-thirdparties-docker.sh` enableKafkaTest=true kafka_port=19193 +// refactor params +enableRefactorParamsTest=true // iceberg test config iceberg_rest_uri_port=18181 diff --git a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy index 18adb7330ed4ed..5bc75b77aa6b6d 100644 --- a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy +++ b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy @@ -162,7 +162,7 @@ suite("test_domain_connection_and_ak_sk_correction", "load_p0") { assertTrue(false. "in the second DATA INFILE, the first bucket is wrong, so the sql should fail") } catch (Exception e) { logger.info("the fourth sql exception result is {}", e.getMessage()) - assertTrue(e.getMessage().contains("connectivity test failed"), e.getMessage()) + assertTrue(e.getMessage().contains("Failed to access object storage, message="), e.getMessage()) } sql """ DROP TABLE IF EXISTS ${tableName} FORCE""" sql """ DROP TABLE IF EXISTS ${tableNameOrders} FORCE""" From b7b9382c3bacfaf91201d6e9e1ff598bc174d95a Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Mon, 12 May 2025 14:38:56 +0800 Subject: [PATCH 24/48] test --- .../doris/fsv2/remote/dfs/DFSFileSystem.java | 3 +++ ...ain_connection_and_ak_sk_correction.groovy | 25 +++++++++++-------- .../backup_restore_azure.groovy | 2 +- .../backup_restore_object_storage.groovy | 2 +- .../refactor_storage_param_p0/s3_load.groovy | 2 +- .../test_outfile_s3_storage.groovy | 2 +- .../test_s3_tvf_s3_storage.groovy | 2 +- 7 files changed, 23 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/dfs/DFSFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/dfs/DFSFileSystem.java index 53fcc9ba17c5e1..8c708399eadb64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/dfs/DFSFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/dfs/DFSFileSystem.java @@ -90,6 +90,9 @@ public FileSystem nativeFileSystem(String remotePath) throws UserException { } if (dfsFileSystem == null) { Configuration conf = hdfsProperties.getHadoopConfiguration(); + // TODO: Temporarily disable the HDFS file system cache to prevent instances from being closed by + // each other in V1. This line can be removed once V1 and V2 are unified. + conf.set("fs.hdfs.impl.disable.cache", "true"); authenticator = HadoopAuthenticator.getHadoopAuthenticator(conf); try { dfsFileSystem = authenticator.doAs(() -> { diff --git a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy index 5bc75b77aa6b6d..7ede7f75d75ef4 100644 --- a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy +++ b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy @@ -1,3 +1,6 @@ +import static java.util.concurrent.TimeUnit.SECONDS +import static java.util.concurrent.TimeUnit.SECONDS + // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -132,7 +135,7 @@ suite("test_domain_connection_and_ak_sk_correction", "load_p0") { } label = UUID.randomUUID().toString().replace("-", "") - try { + result = sql """ LOAD LABEL ${label} ( @@ -155,15 +158,17 @@ suite("test_domain_connection_and_ak_sk_correction", "load_p0") { ); """ logger.info("the fourth sql result is {}", result) - def loadresult = sql """ - SHOW LOAD WHERE label="${label}" - """ - logger.info("the fourth sql load result is {}", loadresult) - assertTrue(false. "in the second DATA INFILE, the first bucket is wrong, so the sql should fail") - } catch (Exception e) { - logger.info("the fourth sql exception result is {}", e.getMessage()) - assertTrue(e.getMessage().contains("Failed to access object storage, message="), e.getMessage()) - } + Awaitility.await().atMost(80, SECONDS).pollInterval(5, SECONDS).until({ + def loadResult = sql """ + SHOW LOAD WHERE label="${label}" + """ + if (loadResult.get(0).get(2) == 'CANCELLED' || loadResult.get(0).get(2) == 'FAILED') { + return true; + } + if(loadResult.get(0).get(2) == 'FINISHED'){ + throw new RuntimeException("load success,but the first bucket is wrong, so the sql should fail") + } + }) sql """ DROP TABLE IF EXISTS ${tableName} FORCE""" sql """ DROP TABLE IF EXISTS ${tableNameOrders} FORCE""" } diff --git a/regression-test/suites/refactor_storage_param_p0/backup_restore_azure.groovy b/regression-test/suites/refactor_storage_param_p0/backup_restore_azure.groovy index 822c5fe72036cf..6c5b4547fba90a 100644 --- a/regression-test/suites/refactor_storage_param_p0/backup_restore_azure.groovy +++ b/regression-test/suites/refactor_storage_param_p0/backup_restore_azure.groovy @@ -18,7 +18,7 @@ import org.awaitility.Awaitility; import static java.util.concurrent.TimeUnit.SECONDS; import static groovy.test.GroovyAssert.shouldFail -suite("refactor_storage_backup_restore_azure") { +suite("refactor_storage_backup_restore_azure", "p0,external") { String enabled = context.config.otherConfigs.get("enableAzureBackupRestoreTest") if (enabled == null || enabled.equalsIgnoreCase("false")) { diff --git a/regression-test/suites/refactor_storage_param_p0/backup_restore_object_storage.groovy b/regression-test/suites/refactor_storage_param_p0/backup_restore_object_storage.groovy index 89ca79f16317b6..bdf7e3cbe065c9 100644 --- a/regression-test/suites/refactor_storage_param_p0/backup_restore_object_storage.groovy +++ b/regression-test/suites/refactor_storage_param_p0/backup_restore_object_storage.groovy @@ -18,7 +18,7 @@ import org.awaitility.Awaitility; import static java.util.concurrent.TimeUnit.SECONDS; import static groovy.test.GroovyAssert.shouldFail -suite("refactor_storage_backup_restore_object_storage", "p0") { +suite("refactor_storage_backup_restore_object_storage", "p0,external") { String enabled = context.config.otherConfigs.get("enableRefactorParamsTest") if (enabled == null || enabled.equalsIgnoreCase("false")) { return diff --git a/regression-test/suites/refactor_storage_param_p0/s3_load.groovy b/regression-test/suites/refactor_storage_param_p0/s3_load.groovy index e9795c95fc7b13..720170db036ecf 100644 --- a/regression-test/suites/refactor_storage_param_p0/s3_load.groovy +++ b/regression-test/suites/refactor_storage_param_p0/s3_load.groovy @@ -19,7 +19,7 @@ import org.awaitility.Awaitility import static groovy.test.GroovyAssert.shouldFail; import static java.util.concurrent.TimeUnit.SECONDS; -suite("refactor_storage_param_load") { +suite("refactor_storage_param_load", "p0,external") { String enabled = context.config.otherConfigs.get("enableRefactorParamsTest") if (enabled == null || enabled.equalsIgnoreCase("false")) { return diff --git a/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy b/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy index 8f13b196f25584..0c3302782727dd 100644 --- a/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy +++ b/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_outfile_s3_storage", "p0") { +suite("test_outfile_s3_storage", "p0,external") { String enabled = context.config.otherConfigs.get("enableRefactorParamsTest") if (enabled == null || enabled.equalsIgnoreCase("false")) { return diff --git a/regression-test/suites/refactor_storage_param_p0/test_s3_tvf_s3_storage.groovy b/regression-test/suites/refactor_storage_param_p0/test_s3_tvf_s3_storage.groovy index 78616770081d24..ef1dcf695e475a 100644 --- a/regression-test/suites/refactor_storage_param_p0/test_s3_tvf_s3_storage.groovy +++ b/regression-test/suites/refactor_storage_param_p0/test_s3_tvf_s3_storage.groovy @@ -16,7 +16,7 @@ // under the License. import static groovy.test.GroovyAssert.shouldFail -suite("test_s3_tvf_s3_storage", "p0") { +suite("test_s3_tvf_s3_storage", "p0,external") { String enabled = context.config.otherConfigs.get("enableRefactorParamsTest") if (enabled == null || enabled.equalsIgnoreCase("false")) { return From 411389b617c21c435d9eeb47d88cfe0502979a48 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Mon, 12 May 2025 15:22:18 +0800 Subject: [PATCH 25/48] test --- .../datasource/property/PropertyPassThroughTest.java | 6 +++--- .../apache/doris/{fs => fsv2}/obj/S3FileSystemTest.java | 7 +++++-- .../org/apache/doris/nereids/parser/EncryptSQLTest.java | 8 ++++---- 3 files changed, 12 insertions(+), 9 deletions(-) rename fe/fe-core/src/test/java/org/apache/doris/{fs => fsv2}/obj/S3FileSystemTest.java (96%) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java index f1e0c46534227a..1f953f6aefc604 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyPassThroughTest.java @@ -45,8 +45,8 @@ public void testS3TVFPropertiesPassThrough() throws Exception { Assertions.assertEquals(analyzedStmt.getTableRefs().size(), 1); TableValuedFunctionRef oldFuncTable = (TableValuedFunctionRef) analyzedStmt.getTableRefs().get(0); S3TableValuedFunction s3Tvf = (S3TableValuedFunction) oldFuncTable.getTableFunction(); - Assertions.assertFalse(s3Tvf.getBrokerDesc().getProperties().containsKey("fs.s3a.list.version")); - Assertions.assertFalse(s3Tvf.getBrokerDesc().getProperties().containsKey("test_property")); + Assertions.assertFalse(s3Tvf.getBrokerDesc().getBackendConfigProperties().containsKey("fs.s3a.list.version")); + Assertions.assertFalse(s3Tvf.getBrokerDesc().getBackendConfigProperties().containsKey("test_property")); } @Test @@ -63,6 +63,6 @@ public void testHdfsTVFPropertiesPassThrough() throws Exception { Assertions.assertEquals(analyzedStmt.getTableRefs().size(), 1); TableValuedFunctionRef oldFuncTable = (TableValuedFunctionRef) analyzedStmt.getTableRefs().get(0); HdfsTableValuedFunction hdfsTvf = (HdfsTableValuedFunction) oldFuncTable.getTableFunction(); - Assertions.assertFalse(hdfsTvf.getBrokerDesc().getProperties().containsKey("test_property")); + Assertions.assertFalse(hdfsTvf.getBrokerDesc().getBackendConfigProperties().containsKey("test_property")); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java b/fe/fe-core/src/test/java/org/apache/doris/fsv2/obj/S3FileSystemTest.java similarity index 96% rename from fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java rename to fe/fe-core/src/test/java/org/apache/doris/fsv2/obj/S3FileSystemTest.java index 26e39565435a01..06193752d8fd6d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/obj/S3FileSystemTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fsv2/obj/S3FileSystemTest.java @@ -15,13 +15,16 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.fs.obj; +package org.apache.doris.fsv2.obj; import org.apache.doris.backup.Repository; import org.apache.doris.backup.Status; import org.apache.doris.common.UserException; import org.apache.doris.common.util.S3URI; import org.apache.doris.datasource.property.PropertyConverter; +import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; +import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.fs.obj.MockedS3Client; import org.apache.doris.fsv2.FileSystemFactory; import org.apache.doris.fsv2.remote.RemoteFile; import org.apache.doris.fsv2.remote.S3FileSystem; @@ -102,7 +105,7 @@ public S3Client getClient() throws UserException { return mockedClient; } }; - S3ObjStorage mockedStorage = new S3ObjStorage(properties); + S3ObjStorage mockedStorage = new S3ObjStorage((AbstractS3CompatibleProperties) StorageProperties.createPrimary(properties)); Assertions.assertTrue(mockedStorage.getClient() instanceof MockedS3Client); // inject storage to file system. fileSystem = (S3FileSystem) FileSystemFactory.get(properties); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java index 59f86b38fbb3e3..90dfecf6ab3b65 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/EncryptSQLTest.java @@ -47,7 +47,7 @@ public void testEncryption() { + " \"max_file_size\" = \"2048MB\"" + ")" + "WITH s3 (" - + " \"s3.endpoint\" = \"abc\"," + + " \"s3.endpoint\" = \"cos.ap-beijing.myqcloud.com\"," + " \"s3.region\" = \"ap-beijing\"," + " \"s3.secret_key\" = \"abc\"," + " \"s3.access_key\" = \"abc\"" @@ -59,7 +59,7 @@ public void testEncryption() { + " \"max_file_size\" = \"2048MB\"" + ")" + "WITH s3 (" - + " \"s3.endpoint\" = \"abc\"," + + " \"s3.endpoint\" = \"cos.ap-beijing.myqcloud.com\"," + " \"s3.region\" = \"ap-beijing\"," + " \"s3.secret_key\" = \"*XXX\"," + " \"s3.access_key\" = \"abc\"" @@ -70,7 +70,7 @@ public void testEncryption() { + " INTO OUTFILE \"s3://abc/aaa\"" + " FORMAT AS ORC" + " PROPERTIES (" - + " \"s3.endpoint\" = \"abc\"," + + " \"s3.endpoint\" = \"cos.ap-beijing.myqcloud.com\"," + " \"s3.region\" = \"ap-beijing\"," + " \"s3.secret_key\" = \"abc\"," + " \"s3.access_key\" = \"abc\"" @@ -80,7 +80,7 @@ public void testEncryption() { + " INTO OUTFILE \"s3://abc/aaa\"" + " FORMAT AS ORC" + " PROPERTIES (" - + " \"s3.endpoint\" = \"abc\"," + + " \"s3.endpoint\" = \"cos.ap-beijing.myqcloud.com\"," + " \"s3.region\" = \"ap-beijing\"," + " \"s3.secret_key\" = \"*XXX\"," + " \"s3.access_key\" = \"abc\"" From 8b237752e3df86f87bbe797d3fa074b26430265b Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Mon, 12 May 2025 15:30:13 +0800 Subject: [PATCH 26/48] fix license --- .../test_domain_connection_and_ak_sk_correction.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy index 7ede7f75d75ef4..90de9902950940 100644 --- a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy +++ b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy @@ -1,6 +1,3 @@ -import static java.util.concurrent.TimeUnit.SECONDS -import static java.util.concurrent.TimeUnit.SECONDS - // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -17,6 +14,8 @@ import static java.util.concurrent.TimeUnit.SECONDS // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. +import org.awaitility.Awaitility; +import static java.util.concurrent.TimeUnit.SECONDS; suite("test_domain_connection_and_ak_sk_correction", "load_p0") { // create table From b6401b4d4d22d23bb3adc546f332868b21fcf92a Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Mon, 12 May 2025 17:04:02 +0800 Subject: [PATCH 27/48] fix license --- .../property/storage/HdfsProperties.java | 3 ++ .../property/storage/HdfsPropertiesUtils.java | 37 +++++++++++++++++-- .../property/storage/HdfsPropertiesTest.java | 3 ++ 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java index 6da1d2fd0ef207..37b0e7b39ecbfd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java @@ -81,6 +81,9 @@ public static boolean guessIsMe(Map props) { if (MapUtils.isEmpty(props)) { return false; } + if (HdfsPropertiesUtils.validateUriIsHdfsUri(props)) { + return true; + } if (HDFS_PROPERTIES_KEYS.stream().anyMatch(props::containsKey)) { return true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java index 4b34a4a21ba933..32c2c3cee45504 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java @@ -25,6 +25,7 @@ import java.util.HashSet; import java.util.Map; +import java.util.Optional; import java.util.Set; public class HdfsPropertiesUtils { @@ -52,13 +53,43 @@ public static String validateAndGetUri(Map props) throws UserExc if (props.isEmpty()) { throw new UserException("props is empty"); } - if (!props.containsKey(URI_KEY)) { + String uriStr = getUri(props); + if (StringUtils.isBlank(uriStr)) { throw new UserException("props must contain uri"); } - String uriStr = props.get(URI_KEY); return validateAndNormalizeUri(uriStr); } + public static boolean validateUriIsHdfsUri(Map props) { + String uriStr = getUri(props); + if (StringUtils.isBlank(uriStr)) { + return false; + } + URI uri; + try { + uri = URI.create(uriStr); + } catch (AnalysisException e) { + throw new IllegalArgumentException("Invalid uri: " + uriStr, e); + } + String schema = uri.getScheme(); + if (StringUtils.isBlank(schema)) { + throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); + } + return supportSchema.contains(schema.toLowerCase()); + } + + private static String getUri(Map props) { + Optional uriValue = props.entrySet().stream() + .filter(e -> e.getKey().equalsIgnoreCase(URI_KEY)) + .map(Map.Entry::getValue) + .findFirst(); + if (!uriValue.isPresent() + || StringUtils.isBlank(uriValue.get())) { + return null; + } + return uriValue.get(); + } + /** * Validates and normalizes a raw URI string. * @@ -91,7 +122,7 @@ public static String constructDefaultFsFromUri(Map props) { if (!props.containsKey(URI_KEY)) { return null; } - String uriStr = props.get(URI_KEY); + String uriStr = getUri(props); if (StringUtils.isBlank(uriStr)) { return null; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java index a4dff23d2766d1..525e8240a6ed6f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java @@ -36,6 +36,9 @@ public class HdfsPropertiesTest { @Test public void testBasicHdfsCreate() throws UserException { // Test 1: Check default authentication type (should be "simple") + Map simpleHdfsProperties = new HashMap<>(); + simpleHdfsProperties.put("uri", "hdfs://test/1.orc"); + Assertions.assertEquals(HdfsProperties.class, StorageProperties.createPrimary(simpleHdfsProperties).getClass()); Map origProps = createBaseHdfsProperties(); List storageProperties = StorageProperties.createAll(origProps); HdfsProperties hdfsProperties = (HdfsProperties) storageProperties.get(0); From ffdbb9bfb50a4fdb2fc895272df9eba531bff08c Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Mon, 12 May 2025 17:08:48 +0800 Subject: [PATCH 28/48] fix test --- .../test_domain_connection_and_ak_sk_correction.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy index 90de9902950940..5e3f7d802fa89e 100644 --- a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy +++ b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy @@ -14,8 +14,8 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -import org.awaitility.Awaitility; -import static java.util.concurrent.TimeUnit.SECONDS; +import org.awaitility.Awaitility.* +import static java.util.concurrent.TimeUnit.SECONDS suite("test_domain_connection_and_ak_sk_correction", "load_p0") { // create table @@ -167,6 +167,7 @@ suite("test_domain_connection_and_ak_sk_correction", "load_p0") { if(loadResult.get(0).get(2) == 'FINISHED'){ throw new RuntimeException("load success,but the first bucket is wrong, so the sql should fail") } + return false }) sql """ DROP TABLE IF EXISTS ${tableName} FORCE""" sql """ DROP TABLE IF EXISTS ${tableNameOrders} FORCE""" From 0d710728b98db4858830d1a415f98fb0217c35bf Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Mon, 12 May 2025 18:06:33 +0800 Subject: [PATCH 29/48] add out file --- .../test_outfile_s3_storage.out | 85 +++ .../test_s3_tvf_s3_storage.out | 709 ++++++++++++++++++ ...ain_connection_and_ak_sk_correction.groovy | 33 +- .../refactor_storage_param_p0/s3_load.groovy | 2 +- .../test_outfile_s3_storage.groovy | 10 +- 5 files changed, 821 insertions(+), 18 deletions(-) create mode 100644 regression-test/data/refactor_storage_param_p0/test_outfile_s3_storage.out create mode 100644 regression-test/data/refactor_storage_param_p0/test_s3_tvf_s3_storage.out diff --git a/regression-test/data/refactor_storage_param_p0/test_outfile_s3_storage.out b/regression-test/data/refactor_storage_param_p0/test_outfile_s3_storage.out new file mode 100644 index 00000000000000..bb72b202ca35ec --- /dev/null +++ b/regression-test/data/refactor_storage_param_p0/test_outfile_s3_storage.out @@ -0,0 +1,85 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_export -- +1 ftw-1 19 +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 +10 \N \N + +-- !s3_tvf_1_http -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf_1_http -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf_1_http -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf_1_http -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf_1_http -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf_1_http -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + diff --git a/regression-test/data/refactor_storage_param_p0/test_s3_tvf_s3_storage.out b/regression-test/data/refactor_storage_param_p0/test_s3_tvf_s3_storage.out new file mode 100644 index 00000000000000..3e73f5b4adfc4c --- /dev/null +++ b/regression-test/data/refactor_storage_param_p0/test_s3_tvf_s3_storage.out @@ -0,0 +1,709 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_export -- +1 ftw-1 19 +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 +10 \N \N + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + +-- !s3_tvf -- +1 ftw-1 19 +10 \N \N +2 ftw-2 20 +3 ftw-3 21 +4 ftw-4 22 +5 ftw-5 23 +6 ftw-6 24 +7 ftw-7 25 +8 ftw-8 26 +9 ftw-9 27 + diff --git a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy index 5e3f7d802fa89e..83bb953f994e08 100644 --- a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy +++ b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy @@ -157,18 +157,27 @@ suite("test_domain_connection_and_ak_sk_correction", "load_p0") { ); """ logger.info("the fourth sql result is {}", result) - Awaitility.await().atMost(80, SECONDS).pollInterval(5, SECONDS).until({ - def loadResult = sql """ - SHOW LOAD WHERE label="${label}" - """ - if (loadResult.get(0).get(2) == 'CANCELLED' || loadResult.get(0).get(2) == 'FAILED') { - return true; - } - if(loadResult.get(0).get(2) == 'FINISHED'){ - throw new RuntimeException("load success,but the first bucket is wrong, so the sql should fail") - } - return false - }) + while (totalWaitTime < timeout) { + def loadResult = sql """ + SHOW LOAD WHERE label="${label}" + """ + + if (loadResult == null || loadResult.isEmpty()) { + } else if (loadResult.get(0).get(2) in ['CANCELLED', 'FAILED']) { + break + } else if (loadResult.get(0).get(2) == 'FINISHED') { + throw new RuntimeException("load success, but the first bucket is wrong, so the sql should fail") + } else { + Thread.sleep(pollInterval * 1000L) + totalWaitTime += pollInterval + } + + + } + + if (totalWaitTime >= timeout) { + throw new RuntimeException("Timeout waiting for load to complete") + } sql """ DROP TABLE IF EXISTS ${tableName} FORCE""" sql """ DROP TABLE IF EXISTS ${tableNameOrders} FORCE""" } diff --git a/regression-test/suites/refactor_storage_param_p0/s3_load.groovy b/regression-test/suites/refactor_storage_param_p0/s3_load.groovy index 720170db036ecf..aa7a08fe575ac5 100644 --- a/regression-test/suites/refactor_storage_param_p0/s3_load.groovy +++ b/regression-test/suites/refactor_storage_param_p0/s3_load.groovy @@ -19,7 +19,7 @@ import org.awaitility.Awaitility import static groovy.test.GroovyAssert.shouldFail; import static java.util.concurrent.TimeUnit.SECONDS; -suite("refactor_storage_param_load", "p0,external") { +suite("refactor_storage_param_s3_load", "p0,external") { String enabled = context.config.otherConfigs.get("enableRefactorParamsTest") if (enabled == null || enabled.equalsIgnoreCase("false")) { return diff --git a/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy b/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy index 0c3302782727dd..f6b3bf13f03ed0 100644 --- a/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy +++ b/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy @@ -71,11 +71,11 @@ suite("test_outfile_s3_storage", "p0,external") { ***************************** TEST AWS ***************************************************** *******************************************************************************************************/ try { - ak = getS3AK() - sk = getS3SK() - s3_endpoint = getS3Endpoint() - region = getS3Region() - bucket = context.config.otherConfigs.get("s3BucketName"); + ak = context.config.otherConfigs.get("AWSAK") + sk = context.config.otherConfigs.get("AWSSK") + s3_endpoint = "s3.ap-northeast-1.amazonaws.com" + region = "ap-northeast-1" + bucket = "selectdb-qa-datalake-test" // 1. test s3 schema def outFilePath = "${bucket}/test_outfile_s3_storage/exp_" From 31e1a6fb16d9d1731fb969bb0e2a0d35183c58a9 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Mon, 12 May 2025 18:08:01 +0800 Subject: [PATCH 30/48] add out file --- .../test_domain_connection_and_ak_sk_correction.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy index 83bb953f994e08..8be240e83e62c7 100644 --- a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy +++ b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy @@ -14,8 +14,6 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -import org.awaitility.Awaitility.* -import static java.util.concurrent.TimeUnit.SECONDS suite("test_domain_connection_and_ak_sk_correction", "load_p0") { // create table @@ -157,6 +155,9 @@ suite("test_domain_connection_and_ak_sk_correction", "load_p0") { ); """ logger.info("the fourth sql result is {}", result) + int totalWaitTime = 0 + int pollInterval = 5 + int timeout = 80 while (totalWaitTime < timeout) { def loadResult = sql """ SHOW LOAD WHERE label="${label}" From 7b128ac1c282bc698a928ba82e1371f1b0ebaa8d Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 13 May 2025 08:47:02 +0800 Subject: [PATCH 31/48] add out file --- .../doris/tablefunction/HdfsTableValuedFunction.java | 2 +- .../suites/external_table_p0/tvf/test_hdfs_tvf.groovy | 2 +- .../suites/refactor_storage_param_p0/s3_load.groovy | 11 ----------- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java index 1fdb015079eae0..27cec6369d3c2c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java @@ -53,7 +53,7 @@ private void init(Map properties) throws AnalysisException { filePath = storageProperties.validateAndNormalizeUri(uri); backendConnectProperties.put(URI_KEY, filePath); } catch (UserException e) { - throw new AnalysisException("Failed check storage props", e); + throw new AnalysisException("Failed check storage props, " + e.getMessage(), e); } if (!FeConstants.runningUnitTest) { diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy index c277201f73930c..a5ed25dff5ea91 100644 --- a/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy @@ -368,7 +368,7 @@ suite("test_hdfs_tvf","external,hive,tvf,external_docker") { """ // check exception - exception """Properties 'uri' is required""" + exception """props must contain uri""" } // test exception diff --git a/regression-test/suites/refactor_storage_param_p0/s3_load.groovy b/regression-test/suites/refactor_storage_param_p0/s3_load.groovy index aa7a08fe575ac5..2c2d4edfc1cdcf 100644 --- a/regression-test/suites/refactor_storage_param_p0/s3_load.groovy +++ b/regression-test/suites/refactor_storage_param_p0/s3_load.groovy @@ -81,7 +81,6 @@ suite("refactor_storage_param_s3_load", "p0,external") { def dataCountResult = sql """ SELECT count(*) FROM ${s3table} """ - def dataCount = dataCountResult[0][0] def label = "s3_load_label_" + System.currentTimeMillis() def load = sql """ LOAD LABEL `${label}` ( @@ -117,16 +116,6 @@ suite("refactor_storage_param_s3_load", "p0,external") { return loadResult.get(0).get(2) == 'FINISHED' }) - - def expectedCount = dataCount + 1 - Awaitility.await().atMost(60, SECONDS).pollInterval(5, SECONDS).until({ - def loadResult = sql """ - select count(*) from ${s3table} - """ - println "loadResult: ${loadResult} " - return loadResult.get(0).get(0) == expectedCount - }) - } s3Load("s3://${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "true") s3Load("s3://${bucket}${filePath}", bucket, "s3.endpoint", endpoint, "s3.region", region, "s3.access_key", ak, "s3.secret_key", sk, "false") From 154e272b366fcc2ee13e3231552b5a7a57369f07 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 13 May 2025 11:00:11 +0800 Subject: [PATCH 32/48] add out file --- .../doris/datasource/property/storage/HdfsProperties.java | 8 ++++++++ .../org/apache/doris/fsv2/remote/RemoteFileSystem.java | 2 +- .../test_domain_connection_and_ak_sk_correction.groovy | 3 ++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java index 37b0e7b39ecbfd..6f62942af0a72c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java @@ -58,6 +58,11 @@ public class HdfsProperties extends HdfsCompatibleProperties { description = "Whether to enable the impersonation of HDFS.") private boolean hdfsImpersonationEnabled = false; + @ConnectorProperty(names = {"ipc.client.fallback-to-simple-auth-allowed"}, + required = false, + description = "Whether to allow fallback to simple authentication.") + private String allowFallbackToSimpleAuth = ""; + private Map backendConfigProperties; /** @@ -153,6 +158,9 @@ private void initHadoopConfiguration() { if (StringUtils.isNotBlank(fsDefaultFS)) { conf.set("fs.defaultFS", fsDefaultFS); } + if (StringUtils.isNotBlank(allowFallbackToSimpleAuth)) { + conf.set("ipc.client.fallback-to-simple-auth-allowed", allowFallbackToSimpleAuth); + } conf.set("hdfs.security.authentication", hdfsAuthenticationType); if ("kerberos".equalsIgnoreCase(hdfsAuthenticationType)) { conf.set("hadoop.kerberos.principal", hdfsKerberosPrincipal); diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/RemoteFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/RemoteFileSystem.java index d418fe90b19e1c..c7877d9ed8db69 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/RemoteFileSystem.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/remote/RemoteFileSystem.java @@ -42,7 +42,7 @@ public abstract class RemoteFileSystem extends PersistentFileSystem implements C // this field will be visited by multi-threads, better use volatile qualifier protected volatile org.apache.hadoop.fs.FileSystem dfsFileSystem = null; private final ReentrantLock fsLock = new ReentrantLock(); - protected static final AtomicBoolean closed = new AtomicBoolean(false); + protected AtomicBoolean closed = new AtomicBoolean(false); public RemoteFileSystem(String name, StorageBackend.StorageType type) { super(name, type); diff --git a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy index 8be240e83e62c7..84f26dae3c2ced 100644 --- a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy +++ b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy @@ -157,13 +157,14 @@ suite("test_domain_connection_and_ak_sk_correction", "load_p0") { logger.info("the fourth sql result is {}", result) int totalWaitTime = 0 int pollInterval = 5 - int timeout = 80 + int timeout = 120 while (totalWaitTime < timeout) { def loadResult = sql """ SHOW LOAD WHERE label="${label}" """ if (loadResult == null || loadResult.isEmpty()) { + return false } else if (loadResult.get(0).get(2) in ['CANCELLED', 'FAILED']) { break } else if (loadResult.get(0).get(2) == 'FINISHED') { From ee187ba2d6833fc842ea136fdd2f9fffa19ac686 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 13 May 2025 11:00:46 +0800 Subject: [PATCH 33/48] add out file --- .../test_domain_connection_and_ak_sk_correction.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy index 84f26dae3c2ced..6d77aa5ad02bf3 100644 --- a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy +++ b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy @@ -170,6 +170,7 @@ suite("test_domain_connection_and_ak_sk_correction", "load_p0") { } else if (loadResult.get(0).get(2) == 'FINISHED') { throw new RuntimeException("load success, but the first bucket is wrong, so the sql should fail") } else { + println("load status is ${loadResult.get(0).get(2)}") Thread.sleep(pollInterval * 1000L) totalWaitTime += pollInterval } From d542a767e351a2a90c88168248aa9e4d26c34678 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 13 May 2025 21:31:47 +0800 Subject: [PATCH 34/48] mv to external p0 --- .../property/storage/HdfsProperties.java | 6 +----- .../property/storage/HdfsPropertiesUtils.java | 14 ++++++++++++++ .../tablefunction/HdfsTableValuedFunction.java | 1 - .../test_outfile_s3_storage.out | 0 .../test_s3_tvf_s3_storage.out | 0 .../backup_restore_azure.groovy | 0 .../backup_restore_object_storage.groovy | 0 .../refactor_storage_param}/hdfs_all_test.groovy | 0 .../refactor_storage_param}/s3_load.groovy | 0 .../test_outfile_s3_storage.groovy | 0 .../test_s3_tvf_s3_storage.groovy | 0 ...t_domain_connection_and_ak_sk_correction.groovy | 7 ++++++- 12 files changed, 21 insertions(+), 7 deletions(-) rename regression-test/data/{refactor_storage_param_p0 => external_table_p0/refactor_storage_param}/test_outfile_s3_storage.out (100%) rename regression-test/data/{refactor_storage_param_p0 => external_table_p0/refactor_storage_param}/test_s3_tvf_s3_storage.out (100%) rename regression-test/suites/{refactor_storage_param_p0 => external_table_p0/refactor_storage_param}/backup_restore_azure.groovy (100%) rename regression-test/suites/{refactor_storage_param_p0 => external_table_p0/refactor_storage_param}/backup_restore_object_storage.groovy (100%) rename regression-test/suites/{refactor_storage_param_p0 => external_table_p0/refactor_storage_param}/hdfs_all_test.groovy (100%) rename regression-test/suites/{refactor_storage_param_p0 => external_table_p0/refactor_storage_param}/s3_load.groovy (100%) rename regression-test/suites/{refactor_storage_param_p0 => external_table_p0/refactor_storage_param}/test_outfile_s3_storage.groovy (100%) rename regression-test/suites/{refactor_storage_param_p0 => external_table_p0/refactor_storage_param}/test_s3_tvf_s3_storage.groovy (100%) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java index 6f62942af0a72c..6ce85060ea8c84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java @@ -140,11 +140,7 @@ protected void checkRequiredProperties() { // fsDefaultFS is not strictly required here. // This is a best-effort fallback to populate fsDefaultFS when possible. if (StringUtils.isBlank(fsDefaultFS)) { - try { - this.fsDefaultFS = HdfsPropertiesUtils.validateAndGetUri(origProps); - } catch (UserException e) { - //ignore - } + this.fsDefaultFS = HdfsPropertiesUtils.extractDefaultFsFromUri(origProps); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java index 32c2c3cee45504..441a43647084b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java @@ -60,6 +60,20 @@ public static String validateAndGetUri(Map props) throws UserExc return validateAndNormalizeUri(uriStr); } + public static String extractDefaultFsFromUri(Map props) { + if (!validateUriIsHdfsUri(props)) { + return null; + } + String uriStr = getUri(props); + URI uri = null; + try { + uri = URI.create(uriStr); + return uri.getScheme() + "://" + uri.getAuthority(); + } catch (AnalysisException e) { + throw new IllegalArgumentException("Invalid uri: " + uriStr, e); + } + } + public static boolean validateUriIsHdfsUri(Map props) { String uriStr = getUri(props); if (StringUtils.isBlank(uriStr)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java index 27cec6369d3c2c..e99340eb105bec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java @@ -51,7 +51,6 @@ private void init(Map properties) throws AnalysisException { backendConnectProperties.putAll(storageProperties.getBackendConfigProperties()); String uri = storageProperties.validateAndGetUri(props); filePath = storageProperties.validateAndNormalizeUri(uri); - backendConnectProperties.put(URI_KEY, filePath); } catch (UserException e) { throw new AnalysisException("Failed check storage props, " + e.getMessage(), e); } diff --git a/regression-test/data/refactor_storage_param_p0/test_outfile_s3_storage.out b/regression-test/data/external_table_p0/refactor_storage_param/test_outfile_s3_storage.out similarity index 100% rename from regression-test/data/refactor_storage_param_p0/test_outfile_s3_storage.out rename to regression-test/data/external_table_p0/refactor_storage_param/test_outfile_s3_storage.out diff --git a/regression-test/data/refactor_storage_param_p0/test_s3_tvf_s3_storage.out b/regression-test/data/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.out similarity index 100% rename from regression-test/data/refactor_storage_param_p0/test_s3_tvf_s3_storage.out rename to regression-test/data/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.out diff --git a/regression-test/suites/refactor_storage_param_p0/backup_restore_azure.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_azure.groovy similarity index 100% rename from regression-test/suites/refactor_storage_param_p0/backup_restore_azure.groovy rename to regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_azure.groovy diff --git a/regression-test/suites/refactor_storage_param_p0/backup_restore_object_storage.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy similarity index 100% rename from regression-test/suites/refactor_storage_param_p0/backup_restore_object_storage.groovy rename to regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy diff --git a/regression-test/suites/refactor_storage_param_p0/hdfs_all_test.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy similarity index 100% rename from regression-test/suites/refactor_storage_param_p0/hdfs_all_test.groovy rename to regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy diff --git a/regression-test/suites/refactor_storage_param_p0/s3_load.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/s3_load.groovy similarity index 100% rename from regression-test/suites/refactor_storage_param_p0/s3_load.groovy rename to regression-test/suites/external_table_p0/refactor_storage_param/s3_load.groovy diff --git a/regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/test_outfile_s3_storage.groovy similarity index 100% rename from regression-test/suites/refactor_storage_param_p0/test_outfile_s3_storage.groovy rename to regression-test/suites/external_table_p0/refactor_storage_param/test_outfile_s3_storage.groovy diff --git a/regression-test/suites/refactor_storage_param_p0/test_s3_tvf_s3_storage.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.groovy similarity index 100% rename from regression-test/suites/refactor_storage_param_p0/test_s3_tvf_s3_storage.groovy rename to regression-test/suites/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.groovy diff --git a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy index 6d77aa5ad02bf3..9598f84b9ce7cd 100644 --- a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy +++ b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy @@ -179,7 +179,12 @@ suite("test_domain_connection_and_ak_sk_correction", "load_p0") { } if (totalWaitTime >= timeout) { - throw new RuntimeException("Timeout waiting for load to complete") + def queryLoadResult = sql """ + SHOW LOAD WHERE label="${label}" + """ + if (queryLoadResult != null && queryLoadResult.get(0).get(2) == 'FINISHED') { + throw new RuntimeException("load success, but the first bucket is wrong, so the sql should fail") + } } sql """ DROP TABLE IF EXISTS ${tableName} FORCE""" sql """ DROP TABLE IF EXISTS ${tableNameOrders} FORCE""" From bbf5708cee2853796ab891ffa7337dc36c10dcf2 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Wed, 14 May 2025 00:12:07 +0800 Subject: [PATCH 35/48] delete out file refactor mv to external p0 --- .../property/storage/HdfsProperties.java | 4 +- .../property/storage/MinioProperties.java | 78 ++ .../property/storage/StorageProperties.java | 6 +- .../apache/doris/fsv2/StorageTypeMapper.java | 2 + .../property/storage/AzurePropertiesTest.java | 2 - .../property/storage/OBSPropertyTest.java | 4 +- .../test_s3_tvf_s3_storage.out | 709 ------------------ .../backup_restore_object_storage.groovy | 6 +- .../hdfs_all_test.groovy | 17 +- .../refactor_storage_param/s3_load.groovy | 2 +- .../test_outfile_s3_storage.groovy | 2 +- .../test_s3_tvf_s3_storage.groovy | 9 +- 12 files changed, 109 insertions(+), 732 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java delete mode 100644 regression-test/data/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.out diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java index 6ce85060ea8c84..5c11920a645ba9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java @@ -119,7 +119,7 @@ private void extractUserOverriddenHdfsConfig(Map origProps) { } userOverriddenHdfsConfig = new HashMap<>(); origProps.forEach((key, value) -> { - if (key.startsWith("hadoop.") || key.startsWith("dfs.") || key.equals("fs.defaultFS")) { + if (key.startsWith("hadoop.") || key.startsWith("dfs.") || key.startsWith("fs.")) { userOverriddenHdfsConfig.put(key, value); } }); @@ -156,6 +156,8 @@ private void initHadoopConfiguration() { } if (StringUtils.isNotBlank(allowFallbackToSimpleAuth)) { conf.set("ipc.client.fallback-to-simple-auth-allowed", allowFallbackToSimpleAuth); + } else { + conf.set("ipc.client.fallback-to-simple-auth-allowed", "true"); } conf.set("hdfs.security.authentication", hdfsAuthenticationType); if ("kerberos".equalsIgnoreCase(hdfsAuthenticationType)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java new file mode 100644 index 00000000000000..cfe53114c4c759 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.storage; + +import org.apache.doris.datasource.property.ConnectorProperty; + +import com.google.common.collect.ImmutableSet; +import lombok.Getter; +import lombok.Setter; + +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +public class MinioProperties extends AbstractS3CompatibleProperties { + @Setter + @Getter + @ConnectorProperty(names = {"minio.endpoint", "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT"}, + required = false, description = "The endpoint of Minio.") + protected String endpoint = ""; + + @Getter + @Setter + protected String region = "us-east-1"; + + @Getter + @ConnectorProperty(names = {"minio.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key", "s3.access_key"}, + description = "The access key of Minio.") + protected String accessKey = ""; + + @Getter + @ConnectorProperty(names = {"minio.secret_key", "s3.secret_key", "AWS_SECRET_KEY", "secret_key", "SECRET_KEY"}, + description = "The secret key of Minio.") + protected String secretKey = ""; + + private static final Set IDENTIFIERS = ImmutableSet.of("minio.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", + "access_key", "s3.access_key"); + + /** + * Constructor to initialize the object storage properties with the provided type and original properties map. + * + * @param origProps the original properties map. + */ + protected MinioProperties(Map origProps) { + super(Type.MINIO, origProps); + } + + public static boolean guessIsMe(Map origProps) { + //ugly, but we need to check if the user has set any of the identifiers + if (AzureProperties.guessIsMe(origProps) || COSProperties.guessIsMe(origProps) + || OSSProperties.guessIsMe(origProps) || S3Properties.guessIsMe(origProps)) { + return false; + } + + return IDENTIFIERS.stream().map(origProps::get).anyMatch(value -> value != null && !value.isEmpty()); + } + + + @Override + protected Pattern endpointPattern() { + return Pattern.compile("^(?:https?://)?[a-zA-Z0-9.-]+(?::\\d+)?$"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java index 3aa1e02154cc09..6d95e28aeb4282 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java @@ -35,6 +35,7 @@ public abstract class StorageProperties extends ConnectionProperties { public static final String FS_HDFS_SUPPORT = "fs.hdfs.support"; public static final String FS_S3_SUPPORT = "fs.s3.support"; public static final String FS_GCS_SUPPORT = "fs.gcs.support"; + public static final String FS_MINIO_SUPPORT = "fs.minio.support"; public static final String FS_AZURE_SUPPORT = "fs.azure.support"; public static final String FS_OSS_SUPPORT = "fs.oss.support"; public static final String FS_OBS_SUPPORT = "fs.obs.support"; @@ -50,6 +51,7 @@ public enum Type { OSS, OBS, COS, + MINIO, AZURE, BROKER, UNKNOWN @@ -126,7 +128,9 @@ public static StorageProperties createPrimary(Map origProps) { props -> (isFsSupport(props, FS_COS_SUPPORT) || COSProperties.guessIsMe(props)) ? new COSProperties(props) : null, props -> (isFsSupport(props, FS_AZURE_SUPPORT) - || AzureProperties.guessIsMe(props)) ? new AzureProperties(props) : null + || AzureProperties.guessIsMe(props)) ? new AzureProperties(props) : null, + props -> (isFsSupport(props, FS_MINIO_SUPPORT) + || MinioProperties.guessIsMe(props)) ? new MinioProperties(props) : null ); protected StorageProperties(Type type, Map origProps) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/StorageTypeMapper.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/StorageTypeMapper.java index 10c8daec002e78..5683934d9606d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/StorageTypeMapper.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/StorageTypeMapper.java @@ -20,6 +20,7 @@ import org.apache.doris.datasource.property.storage.AzureProperties; import org.apache.doris.datasource.property.storage.COSProperties; import org.apache.doris.datasource.property.storage.HdfsProperties; +import org.apache.doris.datasource.property.storage.MinioProperties; import org.apache.doris.datasource.property.storage.OBSProperties; import org.apache.doris.datasource.property.storage.OSSHdfsProperties; import org.apache.doris.datasource.property.storage.OSSProperties; @@ -37,6 +38,7 @@ public enum StorageTypeMapper { OSS(OSSProperties.class, S3FileSystem::new), OBS(OBSProperties.class, S3FileSystem::new), COS(COSProperties.class, S3FileSystem::new), + MINIO(MinioProperties.class, S3FileSystem::new), AZURE(AzureProperties.class, AzureFileSystem::new), S3(S3Properties.class, S3FileSystem::new), HDFS(HdfsProperties.class, DFSFileSystem::new), diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/AzurePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/AzurePropertiesTest.java index fadefd43f57c30..5030089080b4eb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/AzurePropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/AzurePropertiesTest.java @@ -81,8 +81,6 @@ public void testMissingProvider() throws UserException { Assertions.assertEquals(HdfsProperties.class, storagePropertiesList.get(1).getClass()); Assertions.assertEquals(AzureProperties.class, storagePropertiesList.get(0).getClass()); origProps.put("s3.endpoint", "https://mystorageaccount.net"); - Assertions.assertThrows(RuntimeException.class, () -> - StorageProperties.createPrimary(origProps), "No supported storage type found."); // Expect an exception due to missing provider origProps.put("provider", "azure"); Assertions.assertThrows(IllegalArgumentException.class, () -> diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java index 03914f76693d3e..a5993742837c6d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java @@ -117,9 +117,9 @@ public void testGetRegionWithDefault() throws UserException { cosNoEndpointProps.put("obs.access_key", "myCOSAccessKey"); cosNoEndpointProps.put("obs.secret_key", "myCOSSecretKey"); cosNoEndpointProps.put("obs.region", "ap-beijing"); - cosNoEndpointProps.put("uri", "s3://examplebucket-1250000000/test/file.txt"); + cosNoEndpointProps.put("uri", "s3://examplebucket-1250000000/myhuaweicloud.com/test/file.txt"); //not support - Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> StorageProperties.createAll(cosNoEndpointProps), "Property cos.endpoint is required."); + Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> StorageProperties.createPrimary(cosNoEndpointProps), "Property cos.endpoint is required."); } private static String obsAccessKey = ""; diff --git a/regression-test/data/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.out b/regression-test/data/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.out deleted file mode 100644 index 3e73f5b4adfc4c..00000000000000 --- a/regression-test/data/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.out +++ /dev/null @@ -1,709 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !select_export -- -1 ftw-1 19 -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 -10 \N \N - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - --- !s3_tvf -- -1 ftw-1 19 -10 \N \N -2 ftw-2 20 -3 ftw-3 21 -4 ftw-4 22 -5 ftw-5 23 -6 ftw-6 24 -7 ftw-7 25 -8 ftw-8 26 -9 ftw-9 27 - diff --git a/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy index bdf7e3cbe065c9..2579e40186e286 100644 --- a/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy +++ b/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy @@ -18,7 +18,7 @@ import org.awaitility.Awaitility; import static java.util.concurrent.TimeUnit.SECONDS; import static groovy.test.GroovyAssert.shouldFail -suite("refactor_storage_backup_restore_object_storage", "p0,external") { +suite("refactor_storage_backup_restore_object_storage", "p0,external,external_docker") { String enabled = context.config.otherConfigs.get("enableRefactorParamsTest") if (enabled == null || enabled.equalsIgnoreCase("false")) { return @@ -29,7 +29,7 @@ suite("refactor_storage_backup_restore_object_storage", "p0,external") { select database(); """ println databaseQueryResult - def currentDBName = databaseQueryResult.get(0).get(0) + def currentDBName = 'refactor_repo' println currentDBName // cos @@ -215,7 +215,7 @@ suite("refactor_storage_backup_restore_object_storage", "p0,external") { String region = "ap-northeast-1" String bucket = "selectdb-qa-datalake-test" String objPrefix="s3" - //test_backup_restore(ak,sk,s3_endpoint,region,bucket,objPrefix) + test_backup_restore(ak,sk,s3_endpoint,region,bucket,objPrefix) /*-----------------Tencent COS----------------*/ ak = context.config.otherConfigs.get("txYunAk") sk = context.config.otherConfigs.get("txYunSk") diff --git a/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy index 4dc8ea75f94b9d..7c0a79ee7748b4 100644 --- a/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy +++ b/regression-test/suites/external_table_p0/refactor_storage_param/hdfs_all_test.groovy @@ -19,18 +19,20 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static groovy.test.GroovyAssert.shouldFail suite("refactor_params_hdfs_all_test", "p0,external,kerberos,external_docker,external_docker_kerberos") { - String enabled = context.config.otherConfigs.get("enableRefactorParamsTest") + String enabled = context.config.otherConfigs.get("refactor_params_hdfs_kerberos_test") if (enabled == null || enabled.equalsIgnoreCase("false")) { return } String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + def keytab_root_dir = "/keytabs" + def keytab_dir = "${keytab_root_dir}/hive-presto-master.keytab" def table = "hdfs_all_test"; def databaseQueryResult = sql """ select database(); """ println databaseQueryResult - def currentDBName = databaseQueryResult.get(0).get(0) + def currentDBName = 'refactor_params_hdfs_all_test' println currentDBName // cos @@ -76,7 +78,7 @@ suite("refactor_params_hdfs_all_test", "p0,external,kerberos,external_docker,ext "\"hadoop.kerberos.min.seconds.before.relogin\" = \"5\",\n" + "\"hadoop.security.authentication\" = \"kerberos\",\n" + "\"hadoop.kerberos.principal\"=\"hive/presto-master.docker.cluster@LABS.TERADATA.COM\",\n" + - "\"hadoop.kerberos.keytab\" = \"/mnt/disk1/gq/keytabs/keytabs/hive-presto-master.keytab\",\n" + + "\"hadoop.kerberos.keytab\" = \"${keytab_dir}\",\n" + "\"hive.metastore.sasl.enabled \" = \"true\",\n" + "\"hadoop.security.auth_to_local\" = \"RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*//\n" + " RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*//\n" + @@ -283,10 +285,9 @@ suite("refactor_params_hdfs_all_test", "p0,external,kerberos,external_docker,ext }) } - def defaultFs = 'hdfs://172.20.32.136:8520' - def repoName = 'hdfs_non_xml_repo'; + def repoName = 'hdfs_repo'; // create repo - createRepository(repoName,"${defaultFs}/test_repo",hdfsNonXmlParams); + createRepository(repoName,"hdfs://${externalEnvIp}:8520/test_repo",hdfsNonXmlParams); def dbName1 = currentDBName + "${repoName}_1" createDBAndTbl(dbName1) def backupLabel=repoName+System.currentTimeMillis() @@ -303,7 +304,7 @@ suite("refactor_params_hdfs_all_test", "p0,external,kerberos,external_docker,ext //outfile dbName1 = currentDBName + 'outfile_test_1' createDBAndTbl(dbName1) - def outfile = outfile_to_hdfs(defaultFs, hdfsNonXmlParams); + def outfile = outfile_to_hdfs("hdfs://${externalEnvIp}:8520", hdfsNonXmlParams); println outfile //hdfs tvf def hdfsTvfResult = hdfs_tvf(outfile, hdfsNonXmlParams) @@ -312,7 +313,7 @@ suite("refactor_params_hdfs_all_test", "p0,external,kerberos,external_docker,ext //hdfsLoad(outfile,hdfsNonXmlParams) //export - export_hdfs(defaultFs, hdfsNonXmlParams) + export_hdfs("hdfs://${externalEnvIp}:8520", hdfsNonXmlParams) } \ No newline at end of file diff --git a/regression-test/suites/external_table_p0/refactor_storage_param/s3_load.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/s3_load.groovy index 2c2d4edfc1cdcf..d9ed6a401896c8 100644 --- a/regression-test/suites/external_table_p0/refactor_storage_param/s3_load.groovy +++ b/regression-test/suites/external_table_p0/refactor_storage_param/s3_load.groovy @@ -19,7 +19,7 @@ import org.awaitility.Awaitility import static groovy.test.GroovyAssert.shouldFail; import static java.util.concurrent.TimeUnit.SECONDS; -suite("refactor_storage_param_s3_load", "p0,external") { +suite("refactor_storage_param_s3_load", "p0,external,external_docker") { String enabled = context.config.otherConfigs.get("enableRefactorParamsTest") if (enabled == null || enabled.equalsIgnoreCase("false")) { return diff --git a/regression-test/suites/external_table_p0/refactor_storage_param/test_outfile_s3_storage.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/test_outfile_s3_storage.groovy index f6b3bf13f03ed0..326d4235d0a1bc 100644 --- a/regression-test/suites/external_table_p0/refactor_storage_param/test_outfile_s3_storage.groovy +++ b/regression-test/suites/external_table_p0/refactor_storage_param/test_outfile_s3_storage.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_outfile_s3_storage", "p0,external") { +suite("test_outfile_s3_storage", "p0,external,external_docker") { String enabled = context.config.otherConfigs.get("enableRefactorParamsTest") if (enabled == null || enabled.equalsIgnoreCase("false")) { return diff --git a/regression-test/suites/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.groovy index ef1dcf695e475a..c4ce3595bf8647 100644 --- a/regression-test/suites/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.groovy +++ b/regression-test/suites/external_table_p0/refactor_storage_param/test_s3_tvf_s3_storage.groovy @@ -16,7 +16,7 @@ // under the License. import static groovy.test.GroovyAssert.shouldFail -suite("test_s3_tvf_s3_storage", "p0,external") { +suite("test_s3_tvf_s3_storage", "p0,external,external_docker") { String enabled = context.config.otherConfigs.get("enableRefactorParamsTest") if (enabled == null || enabled.equalsIgnoreCase("false")) { return @@ -64,8 +64,8 @@ suite("test_s3_tvf_s3_storage", "p0,external") { sql """ INSERT INTO ${export_table_name} VALUES ${sb.toString()} """ - qt_select_export """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ - + def insert_result = sql """ SELECT * FROM ${export_table_name} t ORDER BY user_id; """ + assert insert_result.size() == 10 String ak = "" String sk = "" @@ -76,7 +76,7 @@ suite("test_s3_tvf_s3_storage", "p0,external") { def s3_tvf = { uri_prefix, endpoint_key, ak_key, sk_key, region_key, is_path_style -> // http schema - order_qt_s3_tvf """ SELECT * FROM S3 ( + def queryResult= sql """ SELECT * FROM S3 ( "uri" = "${uri_prefix}${outfile_url.substring(5 + bucket.length(), outfile_url.length() - 1)}0.parquet", "${endpoint_key}" = "${s3_endpoint}", "${ak_key}"= "${ak}", @@ -86,6 +86,7 @@ suite("test_s3_tvf_s3_storage", "p0,external") { "format" = "parquet" ); """ + assert queryResult.size() == 10 } From 4f2d3813877554e0f221c9fb48afa4db005b708d Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 16 May 2025 14:46:07 +0800 Subject: [PATCH 36/48] fix --- .../apache/doris/analysis/OutFileClause.java | 36 ++- .../property/storage/HdfsProperties.java | 4 +- .../property/storage/HdfsPropertiesUtils.java | 210 +++++++++--------- 3 files changed, 142 insertions(+), 108 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java index 3bdce52529ede1..df47aa20636e85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java @@ -34,6 +34,8 @@ import org.apache.doris.common.util.PrintableMap; import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.datasource.property.fileformat.FileFormatProperties; +import org.apache.doris.datasource.property.storage.HdfsProperties; +import org.apache.doris.datasource.property.storage.HdfsPropertiesUtils; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TParquetDataType; import org.apache.doris.thrift.TParquetRepetitionType; @@ -514,7 +516,7 @@ private void analyzeProperties() throws UserException { if (copiedProps.containsKey(PROP_DELETE_EXISTING_FILES)) { deleteExistingFiles = Boolean.parseBoolean(copiedProps.get(PROP_DELETE_EXISTING_FILES)) - & Config.enable_delete_existing_files; + & Config.enable_delete_existing_files; copiedProps.remove(PROP_DELETE_EXISTING_FILES); } @@ -559,6 +561,38 @@ private void analyzeBrokerDesc(Map copiedProps) throws UserExcep } String brokerName = copiedProps.get(PROP_BROKER_NAME); brokerDesc = new BrokerDesc(brokerName, copiedProps); + /* + * Note on HDFS export behavior and URI handling: + * + * 1. Currently, URI extraction from user input supports case-insensitive key matching + * (e.g., "URI", "Uri", "uRI", etc.), to tolerate non-standard input from users. + * + * 2. In OUTFILE scenarios, if FE fails to pass 'fs.defaultFS' to the BE , + * it may lead to data export failure *without* triggering an actual error (appears as success), + * which is misleading and can cause silent data loss or inconsistencies. + * + * 3. As a temporary safeguard, the following logic forcibly extracts the default FS from the provided + * file path and injects it into the broker descriptor config: + * + * if (brokerDesc.getStorageType() == HDFS) { + * extract default FS from file path + * and put into BE config as 'fs.defaultFS' + * } + * + * 4. Long-term solution: We should define and enforce a consistent parameter specification + * across all user-facing entry points (including FE input validation, broker desc normalization, etc.), + * to prevent missing critical configs like 'fs.defaultFS'. + * + * 5. Suggested improvements: + * - Normalize all parameter keys (e.g., to lowercase) + * - Centralize HDFS URI parsing logic + * - Add validation in FE to reject incomplete or malformed configs + */ + if (null != brokerDesc.getStorageType() && brokerDesc.getStorageType() + .equals(StorageBackend.StorageType.HDFS)) { + String defaultFs = HdfsPropertiesUtils.extractDefaultFsFromPath(filePath); + brokerDesc.getBackendConfigProperties().put(HdfsProperties.HDFS_DEFAULT_FS_NAME, defaultFs); + } } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java index 5c11920a645ba9..97c6c28db145a2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsProperties.java @@ -74,6 +74,8 @@ public class HdfsProperties extends HdfsCompatibleProperties { */ private Map userOverriddenHdfsConfig; + public static final String HDFS_DEFAULT_FS_NAME = "fs.defaultFS"; + private static final List HDFS_PROPERTIES_KEYS = Arrays.asList("hdfs.authentication.type", "hadoop.security.authentication", "hadoop.username", "hdfs.authentication.kerberos.principal", "hadoop.kerberos.principal", "dfs.nameservices"); @@ -152,7 +154,7 @@ private void initHadoopConfiguration() { userOverriddenHdfsConfig.forEach(conf::set); } if (StringUtils.isNotBlank(fsDefaultFS)) { - conf.set("fs.defaultFS", fsDefaultFS); + conf.set(HDFS_DEFAULT_FS_NAME, fsDefaultFS); } if (StringUtils.isNotBlank(allowFallbackToSimpleAuth)) { conf.set("ipc.client.fallback-to-simple-auth-allowed", allowFallbackToSimpleAuth); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java index 441a43647084b7..c2ce8c318814f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java @@ -17,171 +17,169 @@ package org.apache.doris.datasource.property.storage; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.common.util.URI; +import com.google.common.collect.ImmutableSet; import org.apache.commons.lang3.StringUtils; -import java.util.HashSet; import java.util.Map; -import java.util.Optional; import java.util.Set; public class HdfsPropertiesUtils { private static final String URI_KEY = "uri"; - private static Set supportSchema = new HashSet<>(); - - static { - supportSchema.add("hdfs"); - supportSchema.add("viewfs"); - } + // Supported URI schemes + private static final Set SUPPORTED_SCHEMES = ImmutableSet.of("hdfs", "viewfs"); /** - * Validates that the 'uri' property exists in the provided props map, and normalizes it. + * Validates that the 'uri' property exists in the provided map, and normalizes it. * * @param props the map of properties that must include a 'uri' entry - * @return a normalized URI string like 'hdfs://host/path' - * @throws UserException if the map is empty or does not contain the required 'uri' key - *

- * Example: - * Input: {"uri": "hdfs://namenode:9000/data/input"} - * Output: "hdfs://namenode:9000/data/input" + * @return a normalized URI string such as 'hdfs://host/path' + * @throws UserException if the map is empty or missing the required 'uri' key */ public static String validateAndGetUri(Map props) throws UserException { - if (props.isEmpty()) { - throw new UserException("props is empty"); - } - String uriStr = getUri(props); + String uriStr = extractUriIgnoreCase(props); if (StringUtils.isBlank(uriStr)) { throw new UserException("props must contain uri"); } return validateAndNormalizeUri(uriStr); } + /** + * Extracts the default filesystem URI (scheme + authority) from a URI in props. + * + * @param props map containing a valid 'uri' entry + * @return default fs string like 'hdfs://host:port', or null if invalid + */ public static String extractDefaultFsFromUri(Map props) { - if (!validateUriIsHdfsUri(props)) { + String uriStr = extractUriIgnoreCase(props); + if (!isSupportedScheme(uriStr)) { return null; } - String uriStr = getUri(props); - URI uri = null; - try { - uri = URI.create(uriStr); - return uri.getScheme() + "://" + uri.getAuthority(); - } catch (AnalysisException e) { - throw new IllegalArgumentException("Invalid uri: " + uriStr, e); + return getSchemeAndAuthority(uriStr); + } + + /** + * Extracts default fs (scheme + authority) from a full URI string. + * + * @param path full URI string + * @return 'scheme://authority' or throws if invalid + */ + public static String extractDefaultFsFromPath(String path) { + if (StringUtils.isBlank(path)) { + throw new IllegalArgumentException("Path is empty"); } + return getSchemeAndAuthority(path); } + /** + * Validates if the URI in props is of a supported HDFS scheme. + * + * @param props map containing 'uri' + * @return true if valid and supported, false otherwise + */ public static boolean validateUriIsHdfsUri(Map props) { - String uriStr = getUri(props); + String uriStr = extractUriIgnoreCase(props); if (StringUtils.isBlank(uriStr)) { return false; } - URI uri; - try { - uri = URI.create(uriStr); - } catch (AnalysisException e) { - throw new IllegalArgumentException("Invalid uri: " + uriStr, e); - } - String schema = uri.getScheme(); - if (StringUtils.isBlank(schema)) { - throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); - } - return supportSchema.contains(schema.toLowerCase()); - } - - private static String getUri(Map props) { - Optional uriValue = props.entrySet().stream() - .filter(e -> e.getKey().equalsIgnoreCase(URI_KEY)) - .map(Map.Entry::getValue) - .findFirst(); - if (!uriValue.isPresent() - || StringUtils.isBlank(uriValue.get())) { - return null; - } - return uriValue.get(); + return isSupportedScheme(uriStr); } /** - * Validates and normalizes a raw URI string. + * Converts a URI string to a normalized HDFS-style path. * - * @param uriStr the URI string to validate - * @return a normalized URI in the form of 'scheme://authority/path' - * @throws UserException if the URI is invalid or unsupported - *

- * Example: - * Input: "viewfs://ns1/path/to/file" - * Output: "viewfs://ns1/path/to/file" + * @param uriStr raw URI string + * @return normalized path like 'hdfs://host/path' + * @throws UserException if invalid or unsupported */ public static String convertUrlToFilePath(String uriStr) throws UserException { return validateAndNormalizeUri(uriStr); } /** - * Constructs the default filesystem URI (scheme + authority) from a full URI string in the props map. + * Constructs default fs URI (scheme + authority) from 'uri' in props. * - * @param props the map of properties, expected to contain a valid 'uri' entry - * @return a URI prefix like 'hdfs://host:port', or null if the URI is missing or invalid - *

- * Example: - * Input: {"uri": "hdfs://namenode:8020/data"} - * Output: "hdfs://namenode:8020" + * @param props map containing 'uri' + * @return default fs like 'hdfs://host:port', or null if invalid */ public static String constructDefaultFsFromUri(Map props) { - if (props.isEmpty()) { + String uriStr = extractUriIgnoreCase(props); + if (!isSupportedScheme(uriStr)) { return null; } - if (!props.containsKey(URI_KEY)) { - return null; - } - String uriStr = getUri(props); - if (StringUtils.isBlank(uriStr)) { + return getSchemeAndAuthority(uriStr); + } + + // ----------------------- Private Helper Methods ----------------------- + + /** + * Extracts the 'uri' value from props map ignoring case sensitivity. + */ + private static String extractUriIgnoreCase(Map props) { + if (props == null || props.isEmpty()) { return null; } - URI uri = null; + return props.entrySet().stream() + .filter(e -> URI_KEY.equalsIgnoreCase(e.getKey())) + .map(Map.Entry::getValue) + .filter(StringUtils::isNotBlank) + .findFirst() + .orElse(null); + } + + /** + * Returns true if the URI string has a supported scheme (hdfs or viewfs). + */ + private static boolean isSupportedScheme(String uriStr) { try { - uri = URI.create(uriStr); - } catch (AnalysisException e) { - return null; - } - String schema = uri.getScheme(); - if (StringUtils.isBlank(schema)) { - throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); + URI uri = URI.create(uriStr); + String scheme = uri.getScheme(); + return StringUtils.isNotBlank(scheme) && SUPPORTED_SCHEMES.contains(scheme.toLowerCase()); + } catch (Exception e) { + return false; } - if (!supportSchema.contains(schema.toLowerCase())) { - throw new IllegalArgumentException("Invalid export path:" - + schema + " , please use valid 'hdfs://' or 'viewfs://' path."); + } + + /** + * Extracts 'scheme://authority' from the given URI string. + */ + private static String getSchemeAndAuthority(String uriStr) { + try { + URI uri = URI.create(uriStr); + String scheme = uri.getScheme(); + String authority = uri.getAuthority(); + if (StringUtils.isBlank(scheme)) { + throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); + } + return scheme + "://" + authority; + } catch (Exception e) { + throw new IllegalArgumentException("Invalid uri: " + uriStr, e); } - return uri.getScheme() + "://" + uri.getAuthority(); } /** - * Internal method that validates and normalizes a URI string. - * Ensures it has a valid scheme and is supported (e.g., hdfs, viewfs). - * - * @param uriStr the URI string to validate - * @return the normalized URI string - * @throws AnalysisException if the URI is blank or has an unsupported scheme - *

- * Example: - * Input: "hdfs://host:8020/user/data" - * Output: "hdfs://host:8020/user/data" + * Validates and normalizes the full URI string. */ - private static String validateAndNormalizeUri(String uriStr) throws AnalysisException { + private static String validateAndNormalizeUri(String uriStr) throws UserException { if (StringUtils.isBlank(uriStr)) { - throw new IllegalArgumentException("Properties 'uri' is required"); - } - URI uri = URI.create(uriStr); - String schema = uri.getScheme(); - if (StringUtils.isBlank(schema)) { - throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); + throw new UserException("Property 'uri' is required"); } - if (!supportSchema.contains(schema.toLowerCase())) { - throw new IllegalArgumentException("Invalid export path:" - + schema + " , please use valid 'hdfs://' or 'viewfs://' path."); + try { + URI uri = URI.create(uriStr); + String scheme = uri.getScheme(); + if (StringUtils.isBlank(scheme)) { + throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); + } + if (!SUPPORTED_SCHEMES.contains(scheme.toLowerCase())) { + throw new IllegalArgumentException("Invalid export path: " + scheme + + ", please use valid 'hdfs://' or 'viewfs://' path."); + } + return scheme + "://" + uri.getAuthority() + uri.getPath(); + } catch (Exception e) { + throw new UserException("Invalid uri: " + uriStr, e); } - return uri.getScheme() + "://" + uri.getAuthority() + uri.getPath(); } } From deebb9ed970fdab993dbbe874fec307d9419faf0 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Fri, 16 May 2025 16:40:04 +0800 Subject: [PATCH 37/48] fix --- .../property/storage/HdfsPropertiesUtils.java | 220 ++++++++++-------- .../tvf/test_hdfs_tvf.groovy | 2 +- 2 files changed, 118 insertions(+), 104 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java index c2ce8c318814f9..93906cb88630b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java @@ -17,169 +17,183 @@ package org.apache.doris.datasource.property.storage; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.common.util.URI; -import com.google.common.collect.ImmutableSet; import org.apache.commons.lang3.StringUtils; +import java.util.HashSet; import java.util.Map; +import java.util.Optional; import java.util.Set; public class HdfsPropertiesUtils { private static final String URI_KEY = "uri"; - // Supported URI schemes - private static final Set SUPPORTED_SCHEMES = ImmutableSet.of("hdfs", "viewfs"); + private static Set supportSchema = new HashSet<>(); + + static { + supportSchema.add("hdfs"); + supportSchema.add("viewfs"); + } /** - * Validates that the 'uri' property exists in the provided map, and normalizes it. + * Validates that the 'uri' property exists in the provided props map, and normalizes it. * * @param props the map of properties that must include a 'uri' entry - * @return a normalized URI string such as 'hdfs://host/path' - * @throws UserException if the map is empty or missing the required 'uri' key + * @return a normalized URI string like 'hdfs://host/path' + * @throws UserException if the map is empty or does not contain the required 'uri' key + *

+ * Example: + * Input: {"uri": "hdfs://namenode:9000/data/input"} + * Output: "hdfs://namenode:9000/data/input" */ public static String validateAndGetUri(Map props) throws UserException { - String uriStr = extractUriIgnoreCase(props); + if (props.isEmpty()) { + throw new UserException("props is empty"); + } + String uriStr = getUri(props); if (StringUtils.isBlank(uriStr)) { throw new UserException("props must contain uri"); } return validateAndNormalizeUri(uriStr); } - /** - * Extracts the default filesystem URI (scheme + authority) from a URI in props. - * - * @param props map containing a valid 'uri' entry - * @return default fs string like 'hdfs://host:port', or null if invalid - */ - public static String extractDefaultFsFromUri(Map props) { - String uriStr = extractUriIgnoreCase(props); - if (!isSupportedScheme(uriStr)) { + public static String extractDefaultFsFromPath(String filePath) { + if (StringUtils.isBlank(filePath)) { return null; } - return getSchemeAndAuthority(uriStr); + try { + URI uri = URI.create(filePath); + return uri.getScheme() + "://" + uri.getAuthority(); + } catch (AnalysisException e) { + throw new IllegalArgumentException("Invalid file path: " + filePath, e); + } } - /** - * Extracts default fs (scheme + authority) from a full URI string. - * - * @param path full URI string - * @return 'scheme://authority' or throws if invalid - */ - public static String extractDefaultFsFromPath(String path) { - if (StringUtils.isBlank(path)) { - throw new IllegalArgumentException("Path is empty"); + public static String extractDefaultFsFromUri(Map props) { + if (!validateUriIsHdfsUri(props)) { + return null; + } + String uriStr = getUri(props); + URI uri = null; + try { + uri = URI.create(uriStr); + return uri.getScheme() + "://" + uri.getAuthority(); + } catch (AnalysisException e) { + throw new IllegalArgumentException("Invalid uri: " + uriStr, e); } - return getSchemeAndAuthority(path); } - /** - * Validates if the URI in props is of a supported HDFS scheme. - * - * @param props map containing 'uri' - * @return true if valid and supported, false otherwise - */ public static boolean validateUriIsHdfsUri(Map props) { - String uriStr = extractUriIgnoreCase(props); + String uriStr = getUri(props); if (StringUtils.isBlank(uriStr)) { return false; } - return isSupportedScheme(uriStr); + URI uri; + try { + uri = URI.create(uriStr); + } catch (AnalysisException e) { + throw new IllegalArgumentException("Invalid uri: " + uriStr, e); + } + String schema = uri.getScheme(); + if (StringUtils.isBlank(schema)) { + throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); + } + return supportSchema.contains(schema.toLowerCase()); + } + + private static String getUri(Map props) { + Optional uriValue = props.entrySet().stream() + .filter(e -> e.getKey().equalsIgnoreCase(URI_KEY)) + .map(Map.Entry::getValue) + .findFirst(); + if (!uriValue.isPresent() + || StringUtils.isBlank(uriValue.get())) { + return null; + } + return uriValue.get(); } /** - * Converts a URI string to a normalized HDFS-style path. + * Validates and normalizes a raw URI string. * - * @param uriStr raw URI string - * @return normalized path like 'hdfs://host/path' - * @throws UserException if invalid or unsupported + * @param uriStr the URI string to validate + * @return a normalized URI in the form of 'scheme://authority/path' + * @throws UserException if the URI is invalid or unsupported + *

+ * Example: + * Input: "viewfs://ns1/path/to/file" + * Output: "viewfs://ns1/path/to/file" */ public static String convertUrlToFilePath(String uriStr) throws UserException { return validateAndNormalizeUri(uriStr); } /** - * Constructs default fs URI (scheme + authority) from 'uri' in props. + * Constructs the default filesystem URI (scheme + authority) from a full URI string in the props map. * - * @param props map containing 'uri' - * @return default fs like 'hdfs://host:port', or null if invalid + * @param props the map of properties, expected to contain a valid 'uri' entry + * @return a URI prefix like 'hdfs://host:port', or null if the URI is missing or invalid + *

+ * Example: + * Input: {"uri": "hdfs://namenode:8020/data"} + * Output: "hdfs://namenode:8020" */ public static String constructDefaultFsFromUri(Map props) { - String uriStr = extractUriIgnoreCase(props); - if (!isSupportedScheme(uriStr)) { + if (props.isEmpty()) { return null; } - return getSchemeAndAuthority(uriStr); - } - - // ----------------------- Private Helper Methods ----------------------- - - /** - * Extracts the 'uri' value from props map ignoring case sensitivity. - */ - private static String extractUriIgnoreCase(Map props) { - if (props == null || props.isEmpty()) { + if (!props.containsKey(URI_KEY)) { return null; } - return props.entrySet().stream() - .filter(e -> URI_KEY.equalsIgnoreCase(e.getKey())) - .map(Map.Entry::getValue) - .filter(StringUtils::isNotBlank) - .findFirst() - .orElse(null); - } - - /** - * Returns true if the URI string has a supported scheme (hdfs or viewfs). - */ - private static boolean isSupportedScheme(String uriStr) { - try { - URI uri = URI.create(uriStr); - String scheme = uri.getScheme(); - return StringUtils.isNotBlank(scheme) && SUPPORTED_SCHEMES.contains(scheme.toLowerCase()); - } catch (Exception e) { - return false; + String uriStr = getUri(props); + if (StringUtils.isBlank(uriStr)) { + return null; } - } - - /** - * Extracts 'scheme://authority' from the given URI string. - */ - private static String getSchemeAndAuthority(String uriStr) { + URI uri = null; try { - URI uri = URI.create(uriStr); - String scheme = uri.getScheme(); - String authority = uri.getAuthority(); - if (StringUtils.isBlank(scheme)) { - throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); - } - return scheme + "://" + authority; - } catch (Exception e) { - throw new IllegalArgumentException("Invalid uri: " + uriStr, e); + uri = URI.create(uriStr); + } catch (AnalysisException e) { + return null; + } + String schema = uri.getScheme(); + if (StringUtils.isBlank(schema)) { + throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); + } + if (!supportSchema.contains(schema.toLowerCase())) { + throw new IllegalArgumentException("Invalid export path:" + + schema + " , please use valid 'hdfs://' or 'viewfs://' path."); } + return uri.getScheme() + "://" + uri.getAuthority(); } /** - * Validates and normalizes the full URI string. + * Internal method that validates and normalizes a URI string. + * Ensures it has a valid scheme and is supported (e.g., hdfs, viewfs). + * + * @param uriStr the URI string to validate + * @return the normalized URI string + * @throws AnalysisException if the URI is blank or has an unsupported scheme + *

+ * Example: + * Input: "hdfs://host:8020/user/data" + * Output: "hdfs://host:8020/user/data" */ - private static String validateAndNormalizeUri(String uriStr) throws UserException { + private static String validateAndNormalizeUri(String uriStr) throws AnalysisException { if (StringUtils.isBlank(uriStr)) { - throw new UserException("Property 'uri' is required"); + throw new IllegalArgumentException("Properties 'uri' is required"); } - try { - URI uri = URI.create(uriStr); - String scheme = uri.getScheme(); - if (StringUtils.isBlank(scheme)) { - throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); - } - if (!SUPPORTED_SCHEMES.contains(scheme.toLowerCase())) { - throw new IllegalArgumentException("Invalid export path: " + scheme - + ", please use valid 'hdfs://' or 'viewfs://' path."); - } - return scheme + "://" + uri.getAuthority() + uri.getPath(); - } catch (Exception e) { - throw new UserException("Invalid uri: " + uriStr, e); + URI uri = URI.create(uriStr); + String schema = uri.getScheme(); + if (StringUtils.isBlank(schema)) { + throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); + } + if (!supportSchema.contains(schema.toLowerCase())) { + throw new IllegalArgumentException("Invalid export path:" + + schema + " , please use valid 'hdfs://' or 'viewfs://' path."); } + return uri.getScheme() + "://" + uri.getAuthority() + uri.getPath(); } } diff --git a/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy index a5ed25dff5ea91..f6b3538f021704 100644 --- a/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_hdfs_tvf.groovy @@ -381,7 +381,7 @@ suite("test_hdfs_tvf","external,hive,tvf,external_docker") { """ // check exception - exception """Invalid uri: xx, extract schema is null""" + exception """Invalid uri: xx""" } // test exception From f06578314c0887e0224d69a02b9aee4de48086a1 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Mon, 19 May 2025 14:10:51 +0800 Subject: [PATCH 38/48] upgrade maven plugin memory --- fe/fe-core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index f766ce00e67e81..58f875ae6f9707 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -1098,7 +1098,7 @@ under the License. false false - -Xmx512m -javaagent:${settings.localRepository}/org/jmockit/jmockit/${jmockit.version}/jmockit-${jmockit.version}.jar @{argLine} + -Xmx1024m -javaagent:${settings.localRepository}/org/jmockit/jmockit/${jmockit.version}/jmockit-${jmockit.version}.jar @{argLine} From 9299df08a2551fb714f37f3d9100b40cdf602188 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 20 May 2025 15:14:05 +0800 Subject: [PATCH 39/48] commit --- .../org/apache/doris/analysis/BrokerDesc.java | 4 +- .../apache/doris/analysis/StorageBackend.java | 17 ++- .../apache/doris/backup/BackupHandler.java | 141 +++++++++--------- .../org/apache/doris/backup/Repository.java | 3 +- .../property/storage/COSProperties.java | 2 +- .../property/storage/HdfsPropertiesUtils.java | 140 ++++++----------- .../property/storage/MinioProperties.java | 2 +- .../property/storage/OBSProperties.java | 2 +- .../property/storage/OSSProperties.java | 2 +- .../property/storage/S3Properties.java | 2 +- .../apache/doris/fsv2/obj/S3ObjStorage.java | 48 +++--- .../alter/AlterRepositoryCommand.java | 3 +- .../java/org/apache/doris/qe/DdlExecutor.java | 4 +- .../property/storage/MinioPropertiesTest.java | 99 ++++++++++++ 14 files changed, 258 insertions(+), 211 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java index e081634c9abf17..61bc4ff9bdc3f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java @@ -86,10 +86,10 @@ public BrokerDesc(String name, Map properties) { } // Try to determine the actual storage type from properties if available - if (MapUtils.isNotEmpty(properties)) { + if (MapUtils.isNotEmpty(this.properties)) { try { // Create primary storage properties from the given configuration - this.storageProperties = StorageProperties.createPrimary(properties); + this.storageProperties = StorageProperties.createPrimary(this.properties); // Override the storage type based on property configuration this.storageType = StorageBackend.StorageType.valueOf(storageProperties.getStorageName()); } catch (RuntimeException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java index 886c82b8266e57..9fa2a1203bfb20 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StorageBackend.java @@ -46,7 +46,7 @@ public static void checkPath(String path, StorageBackend.StorageType type, Strin public StorageBackend(String storageName, String location, - StorageType storageType, Map properties) { + StorageType storageType, Map properties) { this.storageDesc = new StorageDesc(storageName, storageType, properties); this.location = location; /*boolean convertedToS3 = BosProperties.tryConvertBosToS3(properties, storageType); @@ -101,8 +101,8 @@ public String toSql() { sb.append(" `").append(storageDesc.getName()).append("`"); } sb.append(" ON LOCATION ").append(location).append(" PROPERTIES(") - .append(new PrintableMap<>(storageDesc.getProperties(), " = ", true, false, true)) - .append(")"); + .append(new PrintableMap<>(storageDesc.getProperties(), " = ", true, false, true)) + .append(")"); return sb.toString(); } @@ -149,6 +149,17 @@ public TStorageBackendType toThrift() { } } + /** + * A set of storage types that currently support parameter refactoring. + *

+ * Includes: S3 (referring to all systems compatible with the S3 protocol), + * HDFS, OFS, JFS, and AZURE. For S3, this is a generalized type that matches + * any system whose storage type name is returned as "s3" (or compatible) + * by {@link org.apache.doris.datasource.property.storage.StorageProperties#getStorageName()}. + *

+ * This set is a temporary solution. Once parameter refactoring is fully supported + * across all storage systems, this class can be removed. + */ public static final Set REFACTOR_STORAGE_TYPES = ImmutableSet.of(StorageType.S3, StorageType.HDFS, StorageType.OFS, StorageType.JFS, StorageType.AZURE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index df99b572dc7d82..4583edf9d4d3a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -19,7 +19,6 @@ import org.apache.doris.analysis.AbstractBackupStmt; import org.apache.doris.analysis.AbstractBackupTableRefClause; -import org.apache.doris.analysis.AlterRepositoryStmt; import org.apache.doris.analysis.BackupStmt; import org.apache.doris.analysis.BackupStmt.BackupType; import org.apache.doris.analysis.CancelBackupStmt; @@ -236,91 +235,97 @@ public void createRepository(CreateRepositoryStmt stmt) throws DdlException { } } - public void alterRepository(AlterRepositoryStmt stmt) throws DdlException { + /** + * Alters an existing repository by applying the given new properties. + * + * @param repoName The name of the repository to alter. + * @param newProps The new properties to apply to the repository. + * @param strictCheck If true, only allows altering S3 or Azure repositories and validates properties accordingly. + * TODO: Investigate why only S3 and Azure repositories are supported for alter operation + * @throws DdlException if the repository does not exist, fails to apply properties, or cannot connect + * to the updated repository. + */ + public void alterRepository(String repoName, Map newProps, boolean strictCheck) + throws DdlException { tryLock(); try { - Repository repo = repoMgr.getRepo(stmt.getName()); - if (repo == null) { - ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Repository does not exist"); - } - Map allProperties = new HashMap<>(repo.getRemoteFileSystem().getProperties()); - allProperties.putAll(stmt.getProperties()); - StorageProperties newStorageProperties = StorageProperties.createPrimary(allProperties); - RemoteFileSystem fileSystem = FileSystemFactory.get(newStorageProperties); - - Repository newRepo = new Repository(repo.getId(), repo.getName(), repo.isReadOnly(), - repo.getLocation(), fileSystem); + Repository oldRepo = repoMgr.getRepo(repoName); + if (oldRepo == null) { + throw new DdlException("Repository does not exist"); + } + // Merge new properties with the existing repository's properties + Map mergedProps = mergeProperties(oldRepo, newProps, strictCheck); + // Create new remote file system with merged properties + RemoteFileSystem fileSystem = FileSystemFactory.get(StorageProperties.createPrimary(mergedProps)); + // Create new Repository instance with updated file system + Repository newRepo = new Repository( + oldRepo.getId(), oldRepo.getName(), oldRepo.isReadOnly(), + oldRepo.getLocation(), fileSystem + ); + // Verify the repository can be connected with new settings if (!newRepo.ping()) { - LOG.warn("Failed to connect repository {}. msg: {}", repo.getName(), repo.getErrorMsg()); - ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, - "Repo can not ping with new storage properties"); + LOG.warn("Failed to connect repository {}. msg: {}", repoName, newRepo.getErrorMsg()); + throw new DdlException("Repository ping failed with new properties"); } - + // Apply the new repository metadata Status st = repoMgr.alterRepo(newRepo, false /* not replay */); if (!st.ok()) { - ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, - "Failed to alter repository: " + st.getErrMsg()); - } - for (AbstractJob job : getAllCurrentJobs()) { - if (!job.isDone() && job.getRepoId() == repo.getId()) { - job.updateRepo(newRepo); - } + throw new DdlException("Failed to alter repository: " + st.getErrMsg()); } + // Update all running jobs that are using this repository + updateOngoingJobs(oldRepo.getId(), newRepo); } finally { seqlock.unlock(); } } - - public void alterRepositoryInternal(String repoName, Map properties) throws DdlException { - tryLock(); - try { - Repository repo = repoMgr.getRepo(repoName); - if (repo == null) { - ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, "Repository does not exist"); + /** + * Merges new user-provided properties into the existing repository's configuration. + * In strict mode, only supports S3 or Azure repositories and applies internal S3 merge logic. + * + * @param repo The existing repository. + * @param newProps New user-specified properties. + * @param strictCheck Whether to enforce S3/Azure-only and validate the new properties. + * @return A complete set of merged properties. + * @throws DdlException if the merge fails or the repository type is unsupported. + */ + private Map mergeProperties(Repository repo, Map newProps, boolean strictCheck) + throws DdlException { + if (strictCheck) { + if (!(repo.getRemoteFileSystem() instanceof S3FileSystem + || repo.getRemoteFileSystem() instanceof AzureFileSystem)) { + throw new DdlException("Only support altering S3 or Azure repository"); + } + // Let the repository validate and enrich the new S3/Azure properties + Map propsCopy = new HashMap<>(newProps); + Status status = repo.alterRepositoryS3Properties(propsCopy); + if (!status.ok()) { + throw new DdlException("Failed to merge S3 properties: " + status.getErrMsg()); } + return propsCopy; + } else { + // General case: just override old props with new ones + Map combined = new HashMap<>(repo.getRemoteFileSystem().getProperties()); + combined.putAll(newProps); + return combined; + } + } - if (repo.getRemoteFileSystem() instanceof S3FileSystem - || repo.getRemoteFileSystem() instanceof AzureFileSystem) { - Map oldProperties = new HashMap<>(properties); - Status status = repo.alterRepositoryS3Properties(oldProperties); - if (!status.ok()) { - ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, status.getErrMsg()); - } - RemoteFileSystem fileSystem = FileSystemFactory.get(oldProperties); - - Repository newRepo = new Repository(repo.getId(), repo.getName(), repo.isReadOnly(), - repo.getLocation(), fileSystem); - if (!newRepo.ping()) { - LOG.warn("Failed to connect repository {}. msg: {}", repo.getName(), repo.getErrorMsg()); - ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, - "Repo can not ping with new s3 properties"); - } - - Status st = repoMgr.alterRepo(newRepo, false /* not replay */); - if (!st.ok()) { - ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, - "Failed to alter repository: " + st.getErrMsg()); - } - for (AbstractJob job : getAllCurrentJobs()) { - if (!job.isDone() && job.getRepoId() == repo.getId()) { - job.updateRepo(newRepo); - } - } - } else { - ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, - "Only support alter s3 or azure repository"); + /** + * Updates all currently running jobs associated with the given repository ID. + * Used to ensure that all jobs operate on the new repository instance after alteration. + * + * @param repoId The ID of the altered repository. + * @param newRepo The new repository instance. + */ + private void updateOngoingJobs(long repoId, Repository newRepo) { + for (AbstractJob job : getAllCurrentJobs()) { + if (!job.isDone() && job.getRepoId() == repoId) { + job.updateRepo(newRepo); } - } catch (UserException e) { - LOG.warn("Failed to alter repository {}. msg: {}", repoName, e.getMessage()); - ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR, - "Failed to alter repository: " + e.getMessage()); - } finally { - seqlock.unlock(); } } - // handle drop repository stmt public void dropRepository(DropRepositoryStmt stmt) throws DdlException { dropRepository(stmt.getRepoName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java index 5e95cc08e52bb7..88b6ee512e829d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java @@ -203,6 +203,7 @@ public static Repository read(DataInput in) throws IOException { } } + //todo why only support alter S3 properties public Status alterRepositoryS3Properties(Map properties) { if (this.fileSystem instanceof S3FileSystem) { Map oldProperties = new HashMap<>(this.getRemoteFileSystem().getProperties()); @@ -236,7 +237,7 @@ public Status alterRepositoryS3Properties(Map properties) { @Override public void gsonPostProcess() { - if (!BrokerFileSystem.class.equals(fileSystem.getClass())) { + if (!(fileSystem instanceof BrokerFileSystem)) { StorageProperties storageProperties = StorageProperties.createPrimary(this.fileSystem.properties); this.fileSystem = FileSystemFactory.get(storageProperties); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java index b16356b9cffe65..730a4bdb7b2fac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java @@ -46,7 +46,7 @@ public class COSProperties extends AbstractS3CompatibleProperties { protected String region = ""; @Getter - @ConnectorProperty(names = {"cos.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key", "s3.access_key"}, + @ConnectorProperty(names = {"cos.access_key", "s3.access_key", "AWS_ACCESS_KEY", "access_key", "ACCESS_KEY"}, description = "The access key of COS.") protected String accessKey = ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java index 93906cb88630b1..990a343d0a03ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java @@ -21,34 +21,16 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.URI; +import com.google.common.collect.ImmutableSet; import org.apache.commons.lang3.StringUtils; -import java.util.HashSet; import java.util.Map; -import java.util.Optional; import java.util.Set; public class HdfsPropertiesUtils { private static final String URI_KEY = "uri"; + private static final Set supportSchema = ImmutableSet.of("hdfs", "viewfs"); - private static Set supportSchema = new HashSet<>(); - - static { - supportSchema.add("hdfs"); - supportSchema.add("viewfs"); - } - - /** - * Validates that the 'uri' property exists in the provided props map, and normalizes it. - * - * @param props the map of properties that must include a 'uri' entry - * @return a normalized URI string like 'hdfs://host/path' - * @throws UserException if the map is empty or does not contain the required 'uri' key - *

- * Example: - * Input: {"uri": "hdfs://namenode:9000/data/input"} - * Output: "hdfs://namenode:9000/data/input" - */ public static String validateAndGetUri(Map props) throws UserException { if (props.isEmpty()) { throw new UserException("props is empty"); @@ -73,13 +55,15 @@ public static String extractDefaultFsFromPath(String filePath) { } public static String extractDefaultFsFromUri(Map props) { - if (!validateUriIsHdfsUri(props)) { + String uriStr = getUri(props); + if (StringUtils.isBlank(uriStr)) { return null; } - String uriStr = getUri(props); - URI uri = null; try { - uri = URI.create(uriStr); + URI uri = URI.create(uriStr); + if (!isSupportedSchema(uri.getScheme())) { + return null; + } return uri.getScheme() + "://" + uri.getAuthority(); } catch (AnalysisException e) { throw new IllegalArgumentException("Invalid uri: " + uriStr, e); @@ -91,96 +75,56 @@ public static boolean validateUriIsHdfsUri(Map props) { if (StringUtils.isBlank(uriStr)) { return false; } - URI uri; try { - uri = URI.create(uriStr); + URI uri = URI.create(uriStr); + String schema = uri.getScheme(); + if (StringUtils.isBlank(schema)) { + throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); + } + return isSupportedSchema(schema); } catch (AnalysisException e) { throw new IllegalArgumentException("Invalid uri: " + uriStr, e); } - String schema = uri.getScheme(); - if (StringUtils.isBlank(schema)) { - throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); - } - return supportSchema.contains(schema.toLowerCase()); } - private static String getUri(Map props) { - Optional uriValue = props.entrySet().stream() - .filter(e -> e.getKey().equalsIgnoreCase(URI_KEY)) - .map(Map.Entry::getValue) - .findFirst(); - if (!uriValue.isPresent() - || StringUtils.isBlank(uriValue.get())) { - return null; - } - return uriValue.get(); - } - - /** - * Validates and normalizes a raw URI string. - * - * @param uriStr the URI string to validate - * @return a normalized URI in the form of 'scheme://authority/path' - * @throws UserException if the URI is invalid or unsupported - *

- * Example: - * Input: "viewfs://ns1/path/to/file" - * Output: "viewfs://ns1/path/to/file" - */ - public static String convertUrlToFilePath(String uriStr) throws UserException { - return validateAndNormalizeUri(uriStr); - } - - /** - * Constructs the default filesystem URI (scheme + authority) from a full URI string in the props map. - * - * @param props the map of properties, expected to contain a valid 'uri' entry - * @return a URI prefix like 'hdfs://host:port', or null if the URI is missing or invalid - *

- * Example: - * Input: {"uri": "hdfs://namenode:8020/data"} - * Output: "hdfs://namenode:8020" - */ public static String constructDefaultFsFromUri(Map props) { - if (props.isEmpty()) { - return null; - } - if (!props.containsKey(URI_KEY)) { - return null; - } String uriStr = getUri(props); if (StringUtils.isBlank(uriStr)) { return null; } - URI uri = null; try { - uri = URI.create(uriStr); + URI uri = URI.create(uriStr); + String schema = uri.getScheme(); + if (StringUtils.isBlank(schema)) { + throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); + } + if (!isSupportedSchema(schema)) { + throw new IllegalArgumentException("Invalid export path:" + + schema + " , please use valid 'hdfs://' or 'viewfs://' path."); + } + return uri.getScheme() + "://" + uri.getAuthority(); } catch (AnalysisException e) { return null; } - String schema = uri.getScheme(); - if (StringUtils.isBlank(schema)) { - throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); - } - if (!supportSchema.contains(schema.toLowerCase())) { - throw new IllegalArgumentException("Invalid export path:" - + schema + " , please use valid 'hdfs://' or 'viewfs://' path."); - } - return uri.getScheme() + "://" + uri.getAuthority(); } - /** - * Internal method that validates and normalizes a URI string. - * Ensures it has a valid scheme and is supported (e.g., hdfs, viewfs). - * - * @param uriStr the URI string to validate - * @return the normalized URI string - * @throws AnalysisException if the URI is blank or has an unsupported scheme - *

- * Example: - * Input: "hdfs://host:8020/user/data" - * Output: "hdfs://host:8020/user/data" - */ + public static String convertUrlToFilePath(String uriStr) throws UserException { + return validateAndNormalizeUri(uriStr); + } + + private static String getUri(Map props) { + return props.entrySet().stream() + .filter(e -> e.getKey().equalsIgnoreCase(URI_KEY)) + .map(Map.Entry::getValue) + .filter(StringUtils::isNotBlank) + .findFirst() + .orElse(null); + } + + private static boolean isSupportedSchema(String schema) { + return schema != null && supportSchema.contains(schema.toLowerCase()); + } + private static String validateAndNormalizeUri(String uriStr) throws AnalysisException { if (StringUtils.isBlank(uriStr)) { throw new IllegalArgumentException("Properties 'uri' is required"); @@ -190,7 +134,7 @@ private static String validateAndNormalizeUri(String uriStr) throws AnalysisExce if (StringUtils.isBlank(schema)) { throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); } - if (!supportSchema.contains(schema.toLowerCase())) { + if (!isSupportedSchema(schema)) { throw new IllegalArgumentException("Invalid export path:" + schema + " , please use valid 'hdfs://' or 'viewfs://' path."); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java index cfe53114c4c759..8f5c3f3dfa324a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java @@ -49,7 +49,7 @@ public class MinioProperties extends AbstractS3CompatibleProperties { protected String secretKey = ""; private static final Set IDENTIFIERS = ImmutableSet.of("minio.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", - "access_key", "s3.access_key"); + "access_key", "s3.access_key", "minio.endpoint", "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT"); /** * Constructor to initialize the object storage properties with the provided type and original properties map. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java index 2183dd691a4c99..b99f36bcd30303 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java @@ -39,7 +39,7 @@ public class OBSProperties extends AbstractS3CompatibleProperties { protected String endpoint = ""; @Getter - @ConnectorProperty(names = {"obs.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key", "s3.access_key"}, + @ConnectorProperty(names = {"obs.access_key", "s3.access_key", "AWS_ACCESS_KEY", "access_key", "ACCESS_KEY"}, description = "The access key of OBS.") protected String accessKey = ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java index 9d76bc8384c208..26170dcef8bbad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java @@ -39,7 +39,7 @@ public class OSSProperties extends AbstractS3CompatibleProperties { protected String endpoint = ""; @Getter - @ConnectorProperty(names = {"oss.access_key", "s3.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key"}, + @ConnectorProperty(names = {"oss.access_key", "s3.access_key", "AWS_ACCESS_KEY", "access_key", "ACCESS_KEY"}, description = "The access key of OSS.") protected String accessKey = ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index e241b2d3b0a453..343b4b29d39ef5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -50,7 +50,7 @@ public class S3Properties extends AbstractS3CompatibleProperties { protected String region = ""; @Getter - @ConnectorProperty(names = {"s3.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY", "access_key"}, + @ConnectorProperty(names = {"s3.access_key", "AWS_ACCESS_KEY", "access_key", "ACCESS_KEY"}, description = "The access key of S3.") protected String accessKey = ""; diff --git a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java index 847bd904eb09a5..76c12a85834df3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fsv2/obj/S3ObjStorage.java @@ -121,22 +121,22 @@ public Status globList(String remotePath, List result, boolean fileN S3URI uri = S3URI.create(remotePath, isUsePathStyle, forceParsingByStandardUri); String bucket = uri.getBucket(); String globPath = uri.getKey(); // eg: path/to/*.csv - - LOG.info("globList globPath:{}, remotePath:{}", globPath, remotePath); + if (LOG.isDebugEnabled()) { + LOG.info("globList globPath:{}, remotePath:{}", globPath, remotePath); + } java.nio.file.Path pathPattern = Paths.get(globPath); PathMatcher matcher = FileSystems.getDefault().getPathMatcher("glob:" + pathPattern); HashSet directorySet = new HashSet<>(); - - String listPrefix = getLongestPrefix(globPath); // similar to Azure - LOG.info("globList listPrefix: {}", listPrefix); - + String listPrefix = S3Util.getLongestPrefix(globPath); // similar to Azure + if (LOG.isDebugEnabled()) { + LOG.info("globList listPrefix: {}", listPrefix); + } ListObjectsV2Request request = ListObjectsV2Request.builder() .bucket(bucket) .prefix(listPrefix) .build(); - - boolean isTruncated = false; + boolean isTruncated; do { roundCnt++; ListObjectsV2Response response = getClient().listObjectsV2(request); @@ -191,26 +191,12 @@ public Status globList(String remotePath, List result, boolean fileN } finally { long endTime = System.nanoTime(); long duration = endTime - startTime; - LOG.info("process {} elements under prefix {} for {} round, match {} elements, take {} ms", - elementCnt, remotePath, roundCnt, matchCnt, - duration / 1000); - } - } - - public static String getLongestPrefix(String globPattern) { - int length = globPattern.length(); - int earliestSpecialCharIndex = length; - - char[] specialChars = {'*', '?', '[', '{', '\\'}; - - for (char specialChar : specialChars) { - int index = globPattern.indexOf(specialChar); - if (index != -1 && index < earliestSpecialCharIndex) { - earliestSpecialCharIndex = index; + if (LOG.isDebugEnabled()) { + LOG.debug("process {} elements under prefix {} for {} round, match {} elements, take {} ms", + elementCnt, remotePath, roundCnt, matchCnt, + duration / 1000); } } - - return globPattern.substring(0, earliestSpecialCharIndex); } @Override @@ -225,7 +211,7 @@ public Status headObject(String remotePath) { S3URI uri = S3URI.create(remotePath, isUsePathStyle, forceParsingByStandardUri); HeadObjectResponse response = getClient() .headObject(HeadObjectRequest.builder().bucket(uri.getBucket()).key(uri.getKey()).build()); - LOG.info("head file " + remotePath + " success: " + response.toString()); + LOG.info("head file {} success: {}", remotePath, response); return Status.OK; } catch (S3Exception e) { if (e.statusCode() == HttpStatus.SC_NOT_FOUND) { @@ -247,7 +233,7 @@ public Status getObject(String remoteFilePath, File localFile) { S3URI uri = S3URI.create(remoteFilePath, isUsePathStyle, forceParsingByStandardUri); GetObjectResponse response = getClient().getObject( GetObjectRequest.builder().bucket(uri.getBucket()).key(uri.getKey()).build(), localFile.toPath()); - LOG.info("get file " + remoteFilePath + " success: " + response.toString()); + LOG.info("get file {} success: {}", remoteFilePath, response); return Status.OK; } catch (S3Exception s3Exception) { return new Status( @@ -272,13 +258,13 @@ public Status putObject(String remotePath, @Nullable InputStream content, long c .putObject( PutObjectRequest.builder().bucket(uri.getBucket()).key(uri.getKey()).build(), body); - LOG.info("put object success: " + response.toString()); + LOG.info("put object success: {}", response.toString()); return Status.OK; } catch (S3Exception e) { - LOG.error("put object failed:", e); + LOG.warn("put object failed:", e); return new Status(Status.ErrCode.COMMON_ERROR, "put object failed: " + e.getMessage()); } catch (Exception ue) { - LOG.error("connect to s3 failed: ", ue); + LOG.warn("connect to s3 failed: ", ue); return new Status(Status.ErrCode.COMMON_ERROR, "connect to s3 failed: " + ue.getMessage()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/alter/AlterRepositoryCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/alter/AlterRepositoryCommand.java index f97109076f8d77..d74c25b4708746 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/alter/AlterRepositoryCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/alter/AlterRepositoryCommand.java @@ -73,8 +73,7 @@ public void doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { throw new UserException("alter repository only support ak/sk/token info of s3." + " unsupported properties: " + copyProperties.keySet()); } - - Env.getCurrentEnv().getBackupHandler().alterRepositoryInternal(name, properties); + Env.getCurrentEnv().getBackupHandler(). alterRepository(name, properties, true); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java index 6d3475dbebed03..d23e10739d241a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java @@ -435,7 +435,9 @@ public static void execute(Env env, DdlStmt ddlStmt) throws Exception { DropAnalyzeJobStmt analyzeJobStmt = (DropAnalyzeJobStmt) ddlStmt; Env.getCurrentEnv().getAnalysisManager().dropAnalyzeJob(analyzeJobStmt); } else if (ddlStmt instanceof AlterRepositoryStmt) { - env.getBackupHandler().alterRepository((AlterRepositoryStmt) ddlStmt); + AlterRepositoryStmt alterRepositoryStmt = (AlterRepositoryStmt) ddlStmt; + env.getBackupHandler().alterRepository(alterRepositoryStmt.getName(), alterRepositoryStmt.getProperties(), + false); } else if (ddlStmt instanceof CreateStorageVaultStmt) { env.getStorageVaultMgr().createStorageVaultResource((CreateStorageVaultStmt) ddlStmt); } else if (ddlStmt instanceof CreateStageStmt) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java new file mode 100644 index 00000000000000..994a00ad1c8b11 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/MinioPropertiesTest.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.property.storage; + +import org.apache.doris.common.UserException; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +public class MinioPropertiesTest { + + private Map origProps; + + @BeforeEach + public void setup() { + origProps = new HashMap<>(); + } + + @Test + public void testValidMinioConfiguration() throws UserException { + origProps.put("s3.endpoint", "http://localhost:9000"); + origProps.put("s3.access_key", "minioAccessKey"); + origProps.put("s3.secret_key", "minioSecretKey"); + + MinioProperties minioProperties = (MinioProperties) StorageProperties.createPrimary(origProps); + + Assertions.assertEquals("http://localhost:9000", minioProperties.getEndpoint()); + Assertions.assertEquals("minioAccessKey", minioProperties.getAccessKey()); + Assertions.assertEquals("minioSecretKey", minioProperties.getSecretKey()); + Assertions.assertEquals("us-east-1", minioProperties.getRegion()); + origProps.remove("s3.endpoint"); + origProps.put("uri", "http://localhost:9000/test/"); + Assertions.assertThrows(IllegalArgumentException.class, () -> + StorageProperties.createPrimary(origProps), "Property endpoint is required."); + origProps.put("s3.endpoint", "http://localhost:9000"); + Assertions.assertDoesNotThrow(() -> StorageProperties.createPrimary(origProps)); + + + } + + @Test + public void testGuessIsMeWithMinio() { + origProps.put("s3.access_key", "minioAccessKey"); + Assertions.assertTrue(MinioProperties.guessIsMe(origProps)); + } + + @Test + public void testMissingAccessKey() { + origProps.put("s3.endpoint", "http://localhost:9000"); + origProps.put("s3.secret_key", "minioSecretKey"); + Assertions.assertThrows(IllegalArgumentException.class, () -> + StorageProperties.createPrimary(origProps), "Property s3.access_key is required."); + } + + @Test + public void testEndpoint() { + origProps.put("s3.endpoint", "not-a-valid-url"); + origProps.put("s3.access_key", "a"); + origProps.put("s3.secret_key", "b"); + Assertions.assertDoesNotThrow(() -> StorageProperties.createPrimary(origProps)); + origProps.put("s3.endpoint", "http://localhost:9000"); + Assertions.assertDoesNotThrow(() -> StorageProperties.createPrimary(origProps)); + } + + @Test + public void testBackendConfigProperties() { + origProps.put("s3.endpoint", "http://localhost:9000"); + origProps.put("s3.access_key", "minioAccessKey"); + origProps.put("s3.secret_key", "minioSecretKey"); + + MinioProperties minioProperties = (MinioProperties) StorageProperties.createPrimary(origProps); + Map backendProps = minioProperties.getBackendConfigProperties(); + + Assertions.assertEquals("http://localhost:9000", backendProps.get("AWS_ENDPOINT")); + Assertions.assertEquals("minioAccessKey", backendProps.get("AWS_ACCESS_KEY")); + Assertions.assertEquals("minioSecretKey", backendProps.get("AWS_SECRET_KEY")); + Assertions.assertEquals("us-east-1", backendProps.get("AWS_REGION")); + } +} + From 6f23a8b8979e67d1c4534c8e1ef43912394f08aa Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 20 May 2025 19:40:36 +0800 Subject: [PATCH 40/48] commit --- .../org/apache/doris/analysis/BrokerDesc.java | 15 ++---- .../apache/doris/backup/BackupHandler.java | 9 +++- .../org/apache/doris/backup/Repository.java | 19 +++++++- .../property/ConnectionProperties.java | 7 ++- .../property/storage/HdfsPropertiesUtils.java | 3 +- .../property/storage/S3PropertyUtils.java | 9 ++-- .../property/storage/StorageProperties.java | 3 +- .../exception/StoragePropertiesException.java | 47 +++++++++++++++++++ .../apache/doris/backup/BackupJobTest.java | 2 +- .../apache/doris/backup/RepositoryTest.java | 24 +++++----- .../apache/doris/backup/RestoreJobTest.java | 2 +- .../doris/fsv2/obj/S3FileSystemTest.java | 3 +- 12 files changed, 106 insertions(+), 37 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/exception/StoragePropertiesException.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java index 61bc4ff9bdc3f6..0f172052fe7a95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java @@ -98,7 +98,8 @@ public BrokerDesc(String name, Map properties) { LOG.info("Failed to create storage properties for broker: {}, properties: {}", name, properties, e); } } - if (StringUtils.isBlank(this.name)) { + //only storage type is broker + if (StringUtils.isBlank(this.name) && (this.getStorageType() != StorageType.BROKER)) { this.name = this.storageType().name(); } } @@ -110,16 +111,10 @@ public BrokerDesc(String name, StorageBackend.StorageType storageType, Map newProps, boole // Create new Repository instance with updated file system Repository newRepo = new Repository( oldRepo.getId(), oldRepo.getName(), oldRepo.isReadOnly(), - oldRepo.getLocation(), fileSystem + oldRepo.getLocation(), fileSystem, + oldRepo.getOldfs() ); // Verify the repository can be connected with new settings if (!newRepo.ping()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java index 88b6ee512e829d..bbc7064dc41068 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java @@ -132,18 +132,27 @@ public class Repository implements Writable, GsonPostProcessable { private String location; @SerializedName("fs") + private org.apache.doris.fs.PersistentFileSystem oldfs; + + private PersistentFileSystem fileSystem; + public org.apache.doris.fs.PersistentFileSystem getOldfs() { + return oldfs; + } + private Repository() { // for persist } - public Repository(long id, String name, boolean isReadOnly, String location, RemoteFileSystem fileSystem) { + public Repository(long id, String name, boolean isReadOnly, String location, RemoteFileSystem fileSystem, + org.apache.doris.fs.PersistentFileSystem oldFs) { this.id = id; this.name = name; this.isReadOnly = isReadOnly; this.location = location; this.fileSystem = fileSystem; + this.oldfs = oldFs; this.createTime = System.currentTimeMillis(); } @@ -848,7 +857,13 @@ public void readFields(DataInput in) throws IOException { name = Text.readString(in); isReadOnly = in.readBoolean(); location = Text.readString(in); - fileSystem = PersistentFileSystem.read(in); + oldfs = org.apache.doris.fs.PersistentFileSystem.read(in); + try { + fileSystem = FileSystemFactory.get(oldfs.getProperties()); + } catch (UserException e) { + // do we ignore this exception? + throw new IOException("Failed to create file system: " + e.getMessage()); + } createTime = in.readLong(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectionProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectionProperties.java index 7405188196cac6..2c86c67e4ce4a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectionProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectionProperties.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.property; import org.apache.doris.common.CatalogConfigFileUtils; +import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; import com.google.common.base.Strings; import com.google.common.collect.Maps; @@ -66,7 +67,8 @@ protected void initNormalizeAndCheckProps() { field.set(this, origProps.get(name)); matchedProperties.put(name, origProps.get(name)); } catch (IllegalAccessException e) { - throw new RuntimeException("Failed to set property " + name + ", " + e.getMessage(), e); + throw new StoragePropertiesException("Failed to set property " + name + + ", " + e.getMessage(), e); } break; } @@ -114,7 +116,8 @@ protected void checkRequiredProperties() { throw new IllegalArgumentException("Property " + names[0] + " is required."); } } catch (IllegalAccessException e) { - throw new RuntimeException(e); + throw new StoragePropertiesException("Failed to get property " + names[0] + + ", " + e.getMessage(), e); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java index 990a343d0a03ec..d1fc2a4777142b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java @@ -20,6 +20,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.common.util.URI; +import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; import com.google.common.collect.ImmutableSet; import org.apache.commons.lang3.StringUtils; @@ -37,7 +38,7 @@ public static String validateAndGetUri(Map props) throws UserExc } String uriStr = getUri(props); if (StringUtils.isBlank(uriStr)) { - throw new UserException("props must contain uri"); + throw new StoragePropertiesException("props must contain uri"); } return validateAndNormalizeUri(uriStr); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java index 3f5974faaa58d0..0a4fda0bcdfe8f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3PropertyUtils.java @@ -19,6 +19,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.S3URI; +import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; import org.apache.commons.lang3.StringUtils; @@ -128,7 +129,7 @@ public static String validateAndNormalizeUri(String path, String stringUsePathStyle, String stringForceParsingByStandardUri) throws UserException { if (StringUtils.isBlank(path)) { - throw new UserException("path is null"); + throw new StoragePropertiesException("path is null"); } if (path.startsWith("s3://")) { return path; @@ -151,9 +152,9 @@ public static String validateAndNormalizeUri(String path, * Input: {"uri": "s3://my-bucket/my-key"} * Output: "s3://my-bucket/my-key" */ - public static String validateAndGetUri(Map props) throws UserException { + public static String validateAndGetUri(Map props) { if (props.isEmpty()) { - throw new UserException("props is empty"); + throw new StoragePropertiesException("props is empty"); } Optional uriOptional = props.entrySet().stream() .filter(e -> e.getKey().equalsIgnoreCase(URI_KEY)) @@ -161,7 +162,7 @@ public static String validateAndGetUri(Map props) throws UserExc .findFirst(); if (!uriOptional.isPresent()) { - throw new UserException("props must contain uri"); + throw new StoragePropertiesException("props must contain uri"); } return uriOptional.get(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java index 6d95e28aeb4282..ae69deea400ba0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/StorageProperties.java @@ -20,6 +20,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.datasource.property.ConnectionProperties; import org.apache.doris.datasource.property.ConnectorProperty; +import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; import lombok.Getter; @@ -109,7 +110,7 @@ public static StorageProperties createPrimary(Map origProps) { return p; } } - throw new RuntimeException("No supported storage type found."); + throw new StoragePropertiesException("No supported storage type found. Please check your configuration."); } private static final List, StorageProperties>> PROVIDERS = diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/exception/StoragePropertiesException.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/exception/StoragePropertiesException.java new file mode 100644 index 00000000000000..00d8c46411f097 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/exception/StoragePropertiesException.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/** + * Custom exception class for handling storage property-related errors. + * This exception class extends RuntimeException and is used to handle errors related + * to storage properties at runtime. It provides two constructors: one that accepts only + * an error message, and another that also accepts a cause exception. + */ + +package org.apache.doris.datasource.property.storage.exception; + +public class StoragePropertiesException extends RuntimeException { + + /** + * Constructor that initializes the exception with an error message. + * + * @param message The error message describing the reason for the exception. + */ + public StoragePropertiesException(String message) { + super(message); + } + + /** + * Constructor that initializes the exception with a message and a cause. + * + * @param message The error message describing the reason for the exception. + * @param cause The underlying cause of the exception, typically another Throwable. + */ + public StoragePropertiesException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java index 78d44878619872..a8c2960868adde 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java @@ -127,7 +127,7 @@ public Repository getRepo(long repoId) { private EditLog editLog; private Repository repo = new Repository(repoId, "repo", false, "my_repo", - FileSystemFactory.get("broker", Maps.newHashMap())); + FileSystemFactory.get("broker", Maps.newHashMap()), null); @BeforeClass public static void start() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java index 44e3c78a0fc1d2..c8fec2b64033f6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RepositoryTest.java @@ -100,7 +100,7 @@ public FsBroker getBroker(String name, String host) throws AnalysisException { @Test public void testGet() { - repo = new Repository(10000, "repo", false, location, fileSystem); + repo = new Repository(10000, "repo", false, location, fileSystem, null); Assert.assertEquals(repoId, repo.getId()); Assert.assertEquals(name, repo.getName()); @@ -128,7 +128,7 @@ public Status list(String remotePath, List result) { } }; - repo = new Repository(10000, "repo", false, location, fileSystem); + repo = new Repository(10000, "repo", false, location, fileSystem, null); Status st = repo.initRepository(); System.out.println(st); @@ -137,7 +137,7 @@ public Status list(String remotePath, List result) { @Test public void testassemnblePath() throws MalformedURLException, URISyntaxException { - repo = new Repository(10000, "repo", false, location, fileSystem); + repo = new Repository(10000, "repo", false, location, fileSystem, null); // job info String label = "label"; @@ -178,7 +178,7 @@ public void testPing() { } }; - repo = new Repository(10000, "repo", false, location, fileSystem); + repo = new Repository(10000, "repo", false, location, fileSystem, null); Assert.assertTrue(repo.ping()); Assert.assertTrue(repo.getErrorMsg() == null); } @@ -199,7 +199,7 @@ public Status list(String remotePath, List result) { } }; - repo = new Repository(10000, "repo", false, location, fileSystem); + repo = new Repository(10000, "repo", false, location, fileSystem, null); List snapshotNames = Lists.newArrayList(); Status st = repo.listSnapshots(snapshotNames); Assert.assertTrue(st.ok()); @@ -225,7 +225,7 @@ public void testUpload() { } }; - repo = new Repository(10000, "repo", false, location, fileSystem); + repo = new Repository(10000, "repo", false, location, fileSystem, null); String localFilePath = "./tmp_" + System.currentTimeMillis(); try (PrintWriter out = new PrintWriter(localFilePath)) { out.print("a"); @@ -272,7 +272,7 @@ public Status list(String remotePath, List result) { } }; - repo = new Repository(10000, "repo", false, location, fileSystem); + repo = new Repository(10000, "repo", false, location, fileSystem, null); String remoteFilePath = location + "/remote_file"; Status st = repo.download(remoteFilePath, localFilePath); Assert.assertTrue(st.ok()); @@ -283,7 +283,7 @@ public Status list(String remotePath, List result) { @Test public void testGetInfo() { - repo = new Repository(10000, "repo", false, location, fileSystem); + repo = new Repository(10000, "repo", false, location, fileSystem, null); List infos = repo.getInfo(); Assert.assertTrue(infos.size() == ShowRepositoriesStmt.TITLE_NAMES.size()); } @@ -311,7 +311,7 @@ public Status list(String remotePath, List result) { } }; - repo = new Repository(10000, "repo", false, location, fileSystem); + repo = new Repository(10000, "repo", false, location, fileSystem, null); String snapshotName = ""; String timestamp = ""; try { @@ -332,7 +332,7 @@ public void testPersist() throws UserException { properties.put("bos_accesskey", "a"); properties.put("bos_secret_accesskey", "b"); RemoteFileSystem fs = FileSystemFactory.get(properties); - repo = new Repository(10000, "repo", false, location, fs); + repo = new Repository(10000, "repo", false, location, fs, null); File file = new File("./Repository"); try { @@ -362,7 +362,7 @@ public void testPersist() throws UserException { public void testPathNormalize() { String newLoc = "bos://cmy_bucket/bos_repo/"; - repo = new Repository(10000, "repo", false, newLoc, fileSystem); + repo = new Repository(10000, "repo", false, newLoc, fileSystem, null); String path = repo.getRepoPath("label1", "/_ss_my_ss/_ss_content/__db_10000/"); Assert.assertEquals("bos://cmy_bucket/bos_repo/__palo_repository_repo/__ss_label1/__ss_content/_ss_my_ss/_ss_content/__db_10000/", path); @@ -370,7 +370,7 @@ public void testPathNormalize() { Assert.assertEquals("bos://cmy_bucket/bos_repo/__palo_repository_repo/__ss_label1/__ss_content/_ss_my_ss/_ss_content/__db_10000", path); newLoc = "hdfs://path/to/repo"; - repo = new Repository(10000, "repo", false, newLoc, fileSystem); + repo = new Repository(10000, "repo", false, newLoc, fileSystem, null); SnapshotInfo snapshotInfo = new SnapshotInfo(1, 2, 3, 4, 5, 6, 7, "/path", Lists.newArrayList()); path = repo.getRepoTabletPathBySnapshotInfo("label1", snapshotInfo); Assert.assertEquals("hdfs://path/to/repo/__palo_repository_repo/__ss_label1/__ss_content/__db_1/__tbl_2/__part_3/__idx_4/__5", path); diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java index c9ea909f429e05..db3c7944dd91d3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java @@ -125,7 +125,7 @@ public Repository getRepo(long repoId) { @Injectable private Repository repo = new Repository(repoId, "repo", false, "bos://my_repo", - FileSystemFactory.get("broker", Maps.newHashMap())); + FileSystemFactory.get("broker", Maps.newHashMap()), null); private BackupMeta backupMeta; diff --git a/fe/fe-core/src/test/java/org/apache/doris/fsv2/obj/S3FileSystemTest.java b/fe/fe-core/src/test/java/org/apache/doris/fsv2/obj/S3FileSystemTest.java index 06193752d8fd6d..b36d0b4aba46b9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fsv2/obj/S3FileSystemTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fsv2/obj/S3FileSystemTest.java @@ -171,7 +171,8 @@ public void upload() throws IOException { @Test public void testRepositoryUpload() throws IOException { - Repository repo = new Repository(10000, "repo", false, bucket + basePath, fileSystem); + Repository repo = new Repository(10000, "repo", false, bucket + basePath, fileSystem, + null); File localFile = File.createTempFile("s3unittest", ".dat"); localFile.deleteOnExit(); String remote = bucket + basePath + "/" + localFile.getName(); From d1d7033253d70494ce584b8e8dcdebe873969ac9 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 20 May 2025 19:41:37 +0800 Subject: [PATCH 41/48] commit --- .../apache/doris/persist/gson/GsonUtils.java | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java index f3a1daf3c4b902..71a36d182c8520 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/gson/GsonUtils.java @@ -581,24 +581,6 @@ public class GsonUtils { .registerSubtype(S3FileSystem.class, S3FileSystem.class.getSimpleName()) .registerSubtype(AzureFileSystem.class, AzureFileSystem.class.getSimpleName()); - private static RuntimeTypeAdapterFactory - remoteFileSystemTypeAdapterFactoryV2 - = RuntimeTypeAdapterFactory.of(org.apache.doris.fsv2.PersistentFileSystem.class, "clazz") - .registerSubtype(org.apache.doris.fsv2.remote.dfs.DFSFileSystem.class, - org.apache.doris.fsv2.remote.dfs.DFSFileSystem.class.getSimpleName()) - .registerSubtype(org.apache.doris.fsv2.remote.dfs.JFSFileSystem.class, - org.apache.doris.fsv2.remote.dfs.JFSFileSystem.class.getSimpleName()) - .registerSubtype(org.apache.doris.fsv2.remote.dfs.OFSFileSystem.class, - org.apache.doris.fsv2.remote.dfs.OFSFileSystem.class.getSimpleName()) - .registerSubtype(org.apache.doris.fsv2.remote.ObjFileSystem.class, - org.apache.doris.fsv2.remote.ObjFileSystem.class.getSimpleName()) - .registerSubtype(org.apache.doris.fsv2.remote.S3FileSystem.class, - org.apache.doris.fsv2.remote.S3FileSystem.class.getSimpleName()) - .registerSubtype(org.apache.doris.fsv2.remote.BrokerFileSystem.class, - org.apache.doris.fsv2.remote.BrokerFileSystem.class.getSimpleName()) - .registerSubtype(org.apache.doris.fsv2.remote.AzureFileSystem.class, - org.apache.doris.fsv2.remote.AzureFileSystem.class.getSimpleName()); - private static RuntimeTypeAdapterFactory jobBackupTypeAdapterFactory = RuntimeTypeAdapterFactory.of(org.apache.doris.backup.AbstractJob.class, "clazz") @@ -663,7 +645,6 @@ public class GsonUtils { .registerTypeAdapterFactory(routineLoadTypeAdapterFactory) .registerTypeAdapterFactory(routineLoadJobTypeAdapterFactory) .registerTypeAdapterFactory(remoteFileSystemTypeAdapterFactory) - .registerTypeAdapterFactory(remoteFileSystemTypeAdapterFactoryV2) .registerTypeAdapterFactory(jobBackupTypeAdapterFactory) .registerTypeAdapterFactory(loadJobTypeAdapterFactory) .registerTypeAdapterFactory(partitionItemTypeAdapterFactory) From cf86a9bd5f81611cf90885de2958109aef55d989 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 20 May 2025 22:24:06 +0800 Subject: [PATCH 42/48] commit --- .../java/org/apache/doris/backup/Repository.java | 10 +++++----- .../property/storage/AzurePropertiesTest.java | 5 +++-- .../property/storage/COSPropertiesTest.java | 3 ++- .../property/storage/HdfsPropertiesTest.java | 5 +++-- .../property/storage/HdfsPropertiesUtilsTest.java | 5 +++-- .../property/storage/OSSPropertiesTest.java | 3 ++- .../property/storage/S3PropertiesTest.java | 3 ++- .../property/storage/S3PropertyUtilsTest.java | 13 +++++++------ .../backup_restore_object_storage.groovy | 5 +++++ ...st_domain_connection_and_ak_sk_correction.groovy | 2 +- 10 files changed, 33 insertions(+), 21 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java index bbc7064dc41068..9cbfaf5e3e882d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java @@ -31,7 +31,6 @@ import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.property.constants.S3Properties; -import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fsv2.FileSystemFactory; import org.apache.doris.fsv2.PersistentFileSystem; import org.apache.doris.fsv2.remote.BrokerFileSystem; @@ -134,7 +133,8 @@ public class Repository implements Writable, GsonPostProcessable { @SerializedName("fs") private org.apache.doris.fs.PersistentFileSystem oldfs; - + // Temporary field: currently still using the legacy fs config (oldfs). + // This field can be removed once the new fs configuration is fully enabled. private PersistentFileSystem fileSystem; public org.apache.doris.fs.PersistentFileSystem getOldfs() { @@ -246,10 +246,10 @@ public Status alterRepositoryS3Properties(Map properties) { @Override public void gsonPostProcess() { - if (!(fileSystem instanceof BrokerFileSystem)) { + /* if (!(fileSystem instanceof BrokerFileSystem)) { StorageProperties storageProperties = StorageProperties.createPrimary(this.fileSystem.properties); this.fileSystem = FileSystemFactory.get(storageProperties); - } + }*/ } public long getId() { @@ -859,7 +859,7 @@ public void readFields(DataInput in) throws IOException { location = Text.readString(in); oldfs = org.apache.doris.fs.PersistentFileSystem.read(in); try { - fileSystem = FileSystemFactory.get(oldfs.getProperties()); + fileSystem = FileSystemFactory.get(oldfs.getStorageType(), oldfs.getProperties()); } catch (UserException e) { // do we ignore this exception? throw new IOException("Failed to create file system: " + e.getMessage()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/AzurePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/AzurePropertiesTest.java index 5030089080b4eb..fd36521a82875d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/AzurePropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/AzurePropertiesTest.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.property.storage; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -128,7 +129,7 @@ public void testParsingUri() throws Exception { AzureProperties azureProperties = (AzureProperties) StorageProperties.createPrimary(origProps); Assertions.assertEquals("s3://mycontainer/blob.txt", azureProperties.validateAndNormalizeUri("https://mystorageaccount.blob.core.windows.net/mycontainer/blob.txt")); - Assertions.assertThrowsExactly(UserException.class, () -> + Assertions.assertThrowsExactly(StoragePropertiesException.class, () -> azureProperties.validateAndGetUri(origProps), "props must contain uri"); origProps.put("uri", "https://mystorageaccount.blob.core.windows.net/mycontainer/blob.txt"); @@ -186,7 +187,7 @@ public void testEmptyPath() throws UserException { AzureProperties azureProperties = (AzureProperties) StorageProperties.createPrimary(origProps); // Expect an exception when the path is empty - Assertions.assertThrows(UserException.class, () -> + Assertions.assertThrows(StoragePropertiesException.class, () -> azureProperties.validateAndNormalizeUri(""), "Path cannot be empty."); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java index d6f8dce27a5816..294f9c2bea97b1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.property.storage; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -140,6 +141,6 @@ public void testGetRegionWithDefault() throws UserException { cosNoEndpointProps.put("cos.region", "ap-beijing"); origProps.put("uri", "s3://examplebucket-1250000000/test/file.txt"); //not support this case - Assertions.assertThrowsExactly(RuntimeException.class, () -> StorageProperties.createPrimary(cosNoEndpointProps), "Property cos.endpoint is required."); + Assertions.assertThrowsExactly(StoragePropertiesException.class, () -> StorageProperties.createPrimary(cosNoEndpointProps), "Property cos.endpoint is required."); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java index 525e8240a6ed6f..572c79a5339727 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java @@ -19,6 +19,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; import com.google.common.collect.Maps; import org.apache.hadoop.conf.Configuration; @@ -104,9 +105,9 @@ public void testBasicHdfsPropertiesCreateByConfigFile() throws UserException { @Test public void testNonParamsException() throws UserException { Map origProps = new HashMap<>(); - Assertions.assertThrowsExactly(RuntimeException.class, () -> StorageProperties.createPrimary(origProps)); + Assertions.assertThrowsExactly(StoragePropertiesException.class, () -> StorageProperties.createPrimary(origProps)); origProps.put("nonhdfs", "hdfs://localhost:9000"); - Assertions.assertThrowsExactly(RuntimeException.class, () -> { + Assertions.assertThrowsExactly(StoragePropertiesException.class, () -> { StorageProperties.createPrimary(origProps); }); origProps.put(StorageProperties.FS_HDFS_SUPPORT, "true"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java index 98febe5b555862..48b55a32af499e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.property.storage; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -51,10 +52,10 @@ public void testCheckLoadPropsAndReturnUri_missingUriKey() { Map props = new HashMap<>(); props.put("path", "xxx"); - Exception exception = Assertions.assertThrows(UserException.class, () -> { + Exception exception = Assertions.assertThrows(StoragePropertiesException.class, () -> { HdfsPropertiesUtils.validateAndGetUri(props); }); - Assertions.assertEquals("errCode = 2, detailMessage = props must contain uri", exception.getMessage()); + Assertions.assertEquals("props must contain uri", exception.getMessage()); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java index f9f0f19888a06f..b52e888358b81f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.property.storage; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -122,6 +123,6 @@ public void testGetRegionWithDefault() throws UserException { cosNoEndpointProps.put("oss.region", "cn-hangzhou"); origProps.put("uri", "s3://examplebucket-1250000000/test/file.txt"); // not support - Assertions.assertThrowsExactly(RuntimeException.class, () -> StorageProperties.createPrimary(cosNoEndpointProps), "Property cos.endpoint is required."); + Assertions.assertThrowsExactly(StoragePropertiesException.class, () -> StorageProperties.createPrimary(cosNoEndpointProps), "Property cos.endpoint is required."); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java index 14a449ab91f4e9..9c397c358aac02 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.property.storage; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -175,6 +176,6 @@ public void testGetRegionWithDefault() throws UserException { s3EndpointProps.put("oss.region", "cn-hangzhou"); origProps.put("uri", "s3://examplebucket-1250000000/test/file.txt"); //not support - Assertions.assertThrowsExactly(RuntimeException.class, () -> StorageProperties.createPrimary(s3EndpointProps), "Property cos.endpoint is required."); + Assertions.assertThrowsExactly(StoragePropertiesException.class, () -> StorageProperties.createPrimary(s3EndpointProps), "Property cos.endpoint is required."); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertyUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertyUtilsTest.java index 3330ee5e24013b..6427abd8fda5f9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertyUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertyUtilsTest.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.property.storage; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -40,13 +41,13 @@ void testCheckLoadPropsAndReturnUri_success() throws UserException { void testCheckLoadPropsAndReturnUri_missingKey() { Map props = new HashMap<>(); Executable executable = () -> S3PropertyUtils.validateAndGetUri(props); - UserException exception = Assertions.assertThrows(UserException.class, executable); - Assertions.assertEquals("errCode = 2, detailMessage = props is empty", exception.getMessage()); + StoragePropertiesException exception = Assertions.assertThrows(StoragePropertiesException.class, executable); + Assertions.assertEquals("props is empty", exception.getMessage()); props.put("someKey", "value"); executable = () -> S3PropertyUtils.validateAndGetUri(props); - exception = Assertions.assertThrows(UserException.class, executable); - Assertions.assertEquals("errCode = 2, detailMessage = props must contain uri", exception.getMessage()); + exception = Assertions.assertThrows(StoragePropertiesException.class, executable); + Assertions.assertEquals("props must contain uri", exception.getMessage()); } @Test @@ -103,8 +104,8 @@ void testConvertToS3Address_success() throws UserException { @Test void testConvertToS3Address_invalid() { - Assertions.assertThrows(UserException.class, () -> S3PropertyUtils.validateAndNormalizeUri(null, "false", "true")); - Assertions.assertThrows(UserException.class, () -> S3PropertyUtils.validateAndNormalizeUri("", "false", "false")); + Assertions.assertThrows(StoragePropertiesException.class, () -> S3PropertyUtils.validateAndNormalizeUri(null, "false", "true")); + Assertions.assertThrows(StoragePropertiesException.class, () -> S3PropertyUtils.validateAndNormalizeUri("", "false", "false")); Assertions.assertThrows(UserException.class, () -> S3PropertyUtils.validateAndNormalizeUri("not a uri", "true", "true")); } } diff --git a/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy index 2579e40186e286..605902ce995cac 100644 --- a/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy +++ b/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy @@ -216,6 +216,11 @@ suite("refactor_storage_backup_restore_object_storage", "p0,external,external_do String bucket = "selectdb-qa-datalake-test" String objPrefix="s3" test_backup_restore(ak,sk,s3_endpoint,region,bucket,objPrefix) + //todo When the new fs is fully enabled, we need to open this startup + String enabledOtherObjectStorageTest = context.config.otherConfigs.get("enabledOtherObjectStorageTest") + if (enabled == null || enabledOtherObjectStorageTest.equalsIgnoreCase("false")) { + return + } /*-----------------Tencent COS----------------*/ ak = context.config.otherConfigs.get("txYunAk") sk = context.config.otherConfigs.get("txYunSk") diff --git a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy index 9598f84b9ce7cd..c88b6c8fcf31be 100644 --- a/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy +++ b/regression-test/suites/load_p0/broker_load/test_domain_connection_and_ak_sk_correction.groovy @@ -102,7 +102,7 @@ suite("test_domain_connection_and_ak_sk_correction", "load_p0") { assertTrue(false. "The endpoint is wrong, so the connection test should fale") } catch (Exception e) { logger.info("the second sql exception result is {}", e.getMessage()) - assertTrue(e.getMessage().contains("Failed to create storage properties for broker: S3"), e.getMessage()) + assertTrue(e.getMessage().contains("Invalid endpoint format"), e.getMessage()) } label = UUID.randomUUID().toString().replace("-", "") From 2b7ec44e14fd57e5927f26d6f729f0896997fc74 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 20 May 2025 22:28:52 +0800 Subject: [PATCH 43/48] commit --- .../java/org/apache/doris/backup/Repository.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java index 9cbfaf5e3e882d..5154c74ab58294 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java @@ -31,6 +31,7 @@ import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.property.constants.S3Properties; +import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fsv2.FileSystemFactory; import org.apache.doris.fsv2.PersistentFileSystem; import org.apache.doris.fsv2.remote.BrokerFileSystem; @@ -246,10 +247,19 @@ public Status alterRepositoryS3Properties(Map properties) { @Override public void gsonPostProcess() { - /* if (!(fileSystem instanceof BrokerFileSystem)) { - StorageProperties storageProperties = StorageProperties.createPrimary(this.fileSystem.properties); + StorageBackend.StorageType type = StorageBackend.StorageType.BROKER; + if (this.oldfs.properties.containsKey(org.apache.doris.fs.PersistentFileSystem.STORAGE_TYPE)) { + type = StorageBackend.StorageType.valueOf( + this.oldfs.properties.get(org.apache.doris.fs.PersistentFileSystem.STORAGE_TYPE)); + this.oldfs.properties.remove(org.apache.doris.fs.PersistentFileSystem.STORAGE_TYPE); + } + this.oldfs = org.apache.doris.fs.FileSystemFactory.get(this.oldfs.getName(), + type, + this.oldfs.getProperties()); + if (!type.equals(StorageBackend.StorageType.BROKER)) { + StorageProperties storageProperties = StorageProperties.createPrimary(this.oldfs.properties); this.fileSystem = FileSystemFactory.get(storageProperties); - }*/ + } } public long getId() { From 3290d9416402492c5efe3148eb70ea484b594cc7 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Tue, 20 May 2025 23:36:09 +0800 Subject: [PATCH 44/48] commit --- .../doris/datasource/property/PropertyConverterTest.java | 4 ++-- .../backup_restore_object_storage.groovy | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java index 4f17f7d3406301..b7912e449faa10 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java @@ -182,7 +182,7 @@ public void testS3RepositoryPropertiesConverter() throws Exception { CreateRepositoryStmt analyzedStmt = createStmt(s3Repo); Assertions.assertEquals(analyzedStmt.getProperties().size(), 4); Repository repository = getRepository(analyzedStmt, "s3_repo"); - Assertions.assertEquals(5, repository.getRemoteFileSystem().getProperties().size()); + Assertions.assertEquals(4, repository.getRemoteFileSystem().getProperties().size()); String s3RepoNew = "CREATE REPOSITORY `s3_repo_new`\n" + "WITH S3\n" @@ -196,7 +196,7 @@ public void testS3RepositoryPropertiesConverter() throws Exception { CreateRepositoryStmt analyzedStmtNew = createStmt(s3RepoNew); Assertions.assertEquals(analyzedStmtNew.getProperties().size(), 3); Repository repositoryNew = getRepository(analyzedStmtNew, "s3_repo_new"); - Assertions.assertEquals(repositoryNew.getRemoteFileSystem().getProperties().size(), 4); + Assertions.assertEquals(repositoryNew.getRemoteFileSystem().getProperties().size(), 3); } private static Repository getRepository(CreateRepositoryStmt analyzedStmt, String name) throws DdlException { diff --git a/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy b/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy index 605902ce995cac..179a33c19d1beb 100644 --- a/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy +++ b/regression-test/suites/external_table_p0/refactor_storage_param/backup_restore_object_storage.groovy @@ -218,7 +218,7 @@ suite("refactor_storage_backup_restore_object_storage", "p0,external,external_do test_backup_restore(ak,sk,s3_endpoint,region,bucket,objPrefix) //todo When the new fs is fully enabled, we need to open this startup String enabledOtherObjectStorageTest = context.config.otherConfigs.get("enabledOtherObjectStorageTest") - if (enabled == null || enabledOtherObjectStorageTest.equalsIgnoreCase("false")) { + if (enabledOtherObjectStorageTest == null || enabledOtherObjectStorageTest.equalsIgnoreCase("false")) { return } /*-----------------Tencent COS----------------*/ From fa97519d2efbac53d55777077eb6ce7741decd2b Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Wed, 21 May 2025 10:13:23 +0800 Subject: [PATCH 45/48] commit --- .../src/main/java/org/apache/doris/backup/BackupHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index a386c165100e25..9e1ed2f20418a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -55,6 +55,7 @@ import org.apache.doris.fsv2.remote.AzureFileSystem; import org.apache.doris.fsv2.remote.RemoteFileSystem; import org.apache.doris.fsv2.remote.S3FileSystem; +import org.apache.doris.nereids.trees.plans.commands.CancelBackupCommand; import org.apache.doris.persist.BarrierLog; import org.apache.doris.task.DirMoveTask; import org.apache.doris.task.DownloadTask; From 5ec515c4920cf6fe0e17cbd347b6cf23d79f5e6c Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Wed, 21 May 2025 16:00:39 +0800 Subject: [PATCH 46/48] refactor test --- .../kerberos/test_two_hive_kerberos.groovy | 45 +++++++++---------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy index 4053102b14514e..c86c5a9dfd794e 100644 --- a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy +++ b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy @@ -18,6 +18,7 @@ import groovyjarjarantlr4.v4.codegen.model.ExceptionClause // under the License. import org.junit.Assert; +import java.util.concurrent.* suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_docker_kerberos") { def command = "sudo docker ps" @@ -106,36 +107,30 @@ suite("test_two_hive_kerberos", "p0,external,kerberos,external_docker,external_d order_qt_q04 """ select * from other_${hms_catalog_name}.write_back_krb_hms_db.test_krb_hive_tbl """ // 4. multi thread test - Thread thread1 = new Thread(() -> { - try { - for (int i = 0; i < 100; i++) { - sql """ select * from ${hms_catalog_name}.test_krb_hive_db.test_krb_hive_tbl """ - sql """ INSERT INTO ${hms_catalog_name}.write_back_krb_hms_db.test_krb_hive_tbl values(3, 'krb3', '2023-06-14') """ - } - } catch (Exception e) { - log.info(e.getMessage()) - Assert.fail(); + def executor = Executors.newFixedThreadPool(2) + + def task1 = executor.submit({ + for (int i = 0; i < 100; i++) { + sql """ select * from ${hms_catalog_name}.test_krb_hive_db.test_krb_hive_tbl """ + sql """ INSERT INTO ${hms_catalog_name}.write_back_krb_hms_db.test_krb_hive_tbl values(3, 'krb3', '2023-06-14') """ } }) - Thread thread2 = new Thread(() -> { - try { - for (int i = 0; i < 100; i++) { - sql """ select * from other_${hms_catalog_name}.test_krb_hive_db.test_krb_hive_tbl """ - sql """ INSERT INTO other_${hms_catalog_name}.write_back_krb_hms_db.test_krb_hive_tbl values(6, 'krb3', '2023-09-14') """ - } - } catch (Exception e) { - log.info(e.getMessage()) - Assert.fail(); + def task2 = executor.submit({ + for (int i = 0; i < 100; i++) { + sql """ select * from other_${hms_catalog_name}.test_krb_hive_db.test_krb_hive_tbl """ + sql """ INSERT INTO other_${hms_catalog_name}.write_back_krb_hms_db.test_krb_hive_tbl values(6, 'krb3', '2023-09-14') """ } }) - sleep(5000L) - thread1.start() - thread2.start() - - thread1.join() - thread2.join() - + + try { + task1.get() + task2.get() + } catch (ExecutionException e) { + throw new AssertionError("Task failed", e.getCause()) + } finally { + executor.shutdown() + } // // test information_schema.backend_kerberos_ticket_cache // sql """switch internal""" // List> backends = sql "show backends" From 71ff1e0a20623de06f08a0b8319b2296ba3fbe10 Mon Sep 17 00:00:00 2001 From: Calvin Kirs Date: Thu, 22 May 2025 10:42:23 +0800 Subject: [PATCH 47/48] fix --- .../src/main/java/org/apache/doris/analysis/BrokerDesc.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java index 0f172052fe7a95..850f321f4b4413 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java @@ -25,6 +25,7 @@ import org.apache.doris.common.io.Writable; import org.apache.doris.common.util.PrintableMap; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.thrift.TFileType; @@ -92,7 +93,7 @@ public BrokerDesc(String name, Map properties) { this.storageProperties = StorageProperties.createPrimary(this.properties); // Override the storage type based on property configuration this.storageType = StorageBackend.StorageType.valueOf(storageProperties.getStorageName()); - } catch (RuntimeException e) { + } catch (StoragePropertiesException e) { // Currently ignored: these properties might be broker-specific. // Support for broker properties will be added in the future. LOG.info("Failed to create storage properties for broker: {}, properties: {}", name, properties, e); From 838623a089ef3e59ea1cbfb46b512efac182e63d Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 22 May 2025 15:50:07 +0800 Subject: [PATCH 48/48] fix some duplicate code --- .../apache/doris/backup/BackupHandler.java | 11 +++- .../property/storage/HdfsPropertiesUtils.java | 53 ++++++------------- .../property/storage/HdfsPropertiesTest.java | 1 - .../storage/HdfsPropertiesUtilsTest.java | 16 +++--- 4 files changed, 31 insertions(+), 50 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index 9e1ed2f20418a5..399f996e314fd3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -262,11 +262,18 @@ public void alterRepository(String repoName, Map newProps, boole Map mergedProps = mergeProperties(oldRepo, newProps, strictCheck); // Create new remote file system with merged properties RemoteFileSystem fileSystem = FileSystemFactory.get(StorageProperties.createPrimary(mergedProps)); + org.apache.doris.fs.remote.RemoteFileSystem oldfs = null; + if (oldRepo.getRemoteFileSystem() instanceof S3FileSystem) { + oldfs = org.apache.doris.fs.FileSystemFactory.get(oldRepo.getRemoteFileSystem().getName(), + StorageBackend.StorageType.S3, mergedProps); + } else if (oldRepo.getRemoteFileSystem() instanceof AzureFileSystem) { + oldfs = org.apache.doris.fs.FileSystemFactory.get(oldRepo.getRemoteFileSystem().getName(), + StorageBackend.StorageType.AZURE, mergedProps); + } // Create new Repository instance with updated file system Repository newRepo = new Repository( oldRepo.getId(), oldRepo.getName(), oldRepo.isReadOnly(), - oldRepo.getLocation(), fileSystem, - oldRepo.getOldfs() + oldRepo.getLocation(), fileSystem, oldfs ); // Verify the repository can be connected with new settings if (!newRepo.ping()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java index d1fc2a4777142b..77822f8426d093 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtils.java @@ -43,34 +43,6 @@ public static String validateAndGetUri(Map props) throws UserExc return validateAndNormalizeUri(uriStr); } - public static String extractDefaultFsFromPath(String filePath) { - if (StringUtils.isBlank(filePath)) { - return null; - } - try { - URI uri = URI.create(filePath); - return uri.getScheme() + "://" + uri.getAuthority(); - } catch (AnalysisException e) { - throw new IllegalArgumentException("Invalid file path: " + filePath, e); - } - } - - public static String extractDefaultFsFromUri(Map props) { - String uriStr = getUri(props); - if (StringUtils.isBlank(uriStr)) { - return null; - } - try { - URI uri = URI.create(uriStr); - if (!isSupportedSchema(uri.getScheme())) { - return null; - } - return uri.getScheme() + "://" + uri.getAuthority(); - } catch (AnalysisException e) { - throw new IllegalArgumentException("Invalid uri: " + uriStr, e); - } - } - public static boolean validateUriIsHdfsUri(Map props) { String uriStr = getUri(props); if (StringUtils.isBlank(uriStr)) { @@ -88,24 +60,31 @@ public static boolean validateUriIsHdfsUri(Map props) { } } - public static String constructDefaultFsFromUri(Map props) { + public static String extractDefaultFsFromPath(String filePath) { + if (StringUtils.isBlank(filePath)) { + return null; + } + try { + URI uri = URI.create(filePath); + return uri.getScheme() + "://" + uri.getAuthority(); + } catch (AnalysisException e) { + throw new IllegalArgumentException("Invalid file path: " + filePath, e); + } + } + + public static String extractDefaultFsFromUri(Map props) { String uriStr = getUri(props); if (StringUtils.isBlank(uriStr)) { return null; } try { URI uri = URI.create(uriStr); - String schema = uri.getScheme(); - if (StringUtils.isBlank(schema)) { - throw new IllegalArgumentException("Invalid uri: " + uriStr + ", extract schema is null"); - } - if (!isSupportedSchema(schema)) { - throw new IllegalArgumentException("Invalid export path:" - + schema + " , please use valid 'hdfs://' or 'viewfs://' path."); + if (!isSupportedSchema(uri.getScheme())) { + return null; } return uri.getScheme() + "://" + uri.getAuthority(); } catch (AnalysisException e) { - return null; + throw new IllegalArgumentException("Invalid uri: " + uriStr, e); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java index 572c79a5339727..84f4c40cbdc53e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesTest.java @@ -33,7 +33,6 @@ public class HdfsPropertiesTest { - @Test public void testBasicHdfsCreate() throws UserException { // Test 1: Check default authentication type (should be "simple") diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java index 48b55a32af499e..e150af31f3b62d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/HdfsPropertiesUtilsTest.java @@ -90,7 +90,7 @@ public void testConstructDefaultFsFromUri_valid() { Map props = new HashMap<>(); props.put("uri", "hdfs://localhost:8020/data"); - String result = HdfsPropertiesUtils.constructDefaultFsFromUri(props); + String result = HdfsPropertiesUtils.extractDefaultFsFromUri(props); Assertions.assertEquals("hdfs://localhost:8020", result); } @@ -99,7 +99,7 @@ public void testConstructDefaultFsFromUri_viewfs() { Map props = new HashMap<>(); props.put("uri", "viewfs://cluster/path"); - String result = HdfsPropertiesUtils.constructDefaultFsFromUri(props); + String result = HdfsPropertiesUtils.extractDefaultFsFromUri(props); Assertions.assertEquals("viewfs://cluster", result); } @@ -107,17 +107,13 @@ public void testConstructDefaultFsFromUri_viewfs() { public void testConstructDefaultFsFromUri_invalidSchema() { Map props = new HashMap<>(); props.put("uri", "obs://bucket/test"); - - Exception exception = Assertions.assertThrows(IllegalArgumentException.class, () -> { - HdfsPropertiesUtils.constructDefaultFsFromUri(props); - }); - Assertions.assertTrue(exception.getMessage().contains("Invalid export path")); + Assertions.assertNull(HdfsPropertiesUtils.extractDefaultFsFromUri(props)); } @Test public void testConstructDefaultFsFromUri_emptyProps() { Map props = new HashMap<>(); - String result = HdfsPropertiesUtils.constructDefaultFsFromUri(props); + String result = HdfsPropertiesUtils.extractDefaultFsFromUri(props); Assertions.assertNull(result); } @@ -126,7 +122,7 @@ public void testConstructDefaultFsFromUri_missingUri() { Map props = new HashMap<>(); props.put("x", "y"); - String result = HdfsPropertiesUtils.constructDefaultFsFromUri(props); + String result = HdfsPropertiesUtils.extractDefaultFsFromUri(props); Assertions.assertNull(result); } @@ -135,7 +131,7 @@ public void testConstructDefaultFsFromUri_blankUri() { Map props = new HashMap<>(); props.put("uri", " "); - String result = HdfsPropertiesUtils.constructDefaultFsFromUri(props); + String result = HdfsPropertiesUtils.extractDefaultFsFromUri(props); Assertions.assertNull(result); }