From a9eafa7145f72071b24d2d432b0e22bd91adef3e Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Mon, 23 Dec 2024 15:25:34 +0800 Subject: [PATCH 1/4] [improvement](jdbc catalog) Optimize JdbcCatalog case mapping stability --- .../apache/doris/datasource/CatalogIf.java | 10 + .../apache/doris/datasource/CatalogMgr.java | 12 +- .../doris/datasource/ExternalCatalog.java | 196 ++++++++-- .../doris/datasource/ExternalDatabase.java | 227 +++++++++-- .../datasource/ExternalMetaCacheMgr.java | 3 +- .../doris/datasource/ExternalTable.java | 26 +- .../doris/datasource/InitCatalogLog.java | 7 +- .../doris/datasource/InitDatabaseLog.java | 7 +- .../datasource/es/EsExternalDatabase.java | 12 +- .../doris/datasource/es/EsExternalTable.java | 9 +- .../datasource/hive/HMSExternalCatalog.java | 2 +- .../datasource/hive/HMSExternalDatabase.java | 12 +- .../datasource/hive/HMSExternalTable.java | 10 +- .../iceberg/IcebergExternalDatabase.java | 11 +- .../iceberg/IcebergExternalTable.java | 5 +- .../ExternalInfoSchemaDatabase.java | 8 +- .../infoschema/ExternalInfoSchemaTable.java | 6 +- .../infoschema/ExternalMysqlDatabase.java | 8 +- .../infoschema/ExternalMysqlTable.java | 6 +- .../datasource/jdbc/JdbcExternalCatalog.java | 74 +++- .../datasource/jdbc/JdbcExternalDatabase.java | 11 +- .../datasource/jdbc/JdbcExternalTable.java | 68 +++- ...Mapping.java => JdbcSchemaCacheValue.java} | 30 +- .../datasource/jdbc/client/JdbcClient.java | 47 +-- .../jdbc/client/JdbcGbaseClient.java | 4 +- .../jdbc/client/JdbcMySQLClient.java | 4 +- .../jdbc/client/JdbcOracleClient.java | 4 +- .../jdbc/client/JdbcPostgreSQLClient.java | 4 +- .../lakesoul/LakeSoulExternalDatabase.java | 11 +- .../lakesoul/LakeSoulExternalTable.java | 19 +- .../datasource/mapping/IdentifierMapping.java | 311 +-------------- .../mapping/JdbcIdentifierMapping.java | 345 +++++++++++++++++ .../MaxComputeExternalDatabase.java | 12 +- .../maxcompute/MaxComputeExternalTable.java | 7 +- .../doris/datasource/metacache/MetaCache.java | 22 +- .../paimon/PaimonExternalDatabase.java | 11 +- .../paimon/PaimonExternalTable.java | 5 +- .../datasource/test/TestExternalDatabase.java | 11 +- .../datasource/test/TestExternalTable.java | 4 +- .../TrinoConnectorExternalDatabase.java | 12 +- .../TrinoConnectorExternalTable.java | 5 +- .../constraint/ConstraintPersistTest.java | 8 +- .../doris/datasource/CatalogMgrTest.java | 8 +- .../hive/HiveDDLAndDMLPlanTest.java | 4 +- .../datasource/hive/HiveMetadataOpsTest.java | 2 +- .../iceberg/CreateIcebergTableTest.java | 2 +- ...meComparedLowercaseMetaCacheFalseTest.java | 136 +++++++ ...meComparedLowercaseMetaCacheTrueTest.java} | 21 +- ...NameStoredLowercaseMetaCacheFalseTest.java | 144 +++++++ ...NameStoredLowercaseMetaCacheTrueTest.java} | 21 +- .../mapping/JdbcIdentifierMappingTest.java | 277 +++++++++++++ .../doris/external/hms/HmsCatalogTest.java | 2 +- .../apache/doris/qe/HmsQueryCacheTest.java | 2 +- .../StatisticsAutoCollectorTest.java | 6 +- .../statistics/util/StatisticsUtilTest.java | 20 +- .../lower_case/test_conflict_name.out | 5 + .../test_lower_case_meta_show_and_select.out | 25 ++ ..._with_lower_table_conf_show_and_select.out | 97 +++++ .../lower_case/test_lower_case_mtmv.out | 3 + ...test_meta_cache_select_without_refresh.out | 10 + .../lower_case/test_meta_names_mapping.out | 13 + .../lower_case/upgrade/load.out | 7 + .../test_upgrade_lower_case_catalog.out | 7 + .../lower_case/test_conflict_name.groovy | 94 +++++ .../test_lower_case_meta_include.groovy | 158 ++++++++ ...est_lower_case_meta_show_and_select.groovy | 229 +++++++++++ ...th_lower_table_conf_show_and_select.groovy | 363 ++++++++++++++++++ .../lower_case/test_lower_case_mtmv.groovy | 64 +++ ...t_meta_cache_select_without_refresh.groovy | 92 +++++ .../lower_case/test_meta_names_mapping.groovy | 287 ++++++++++++++ .../test_timing_refresh_catalog.groovy | 161 ++++++++ .../lower_case/upgrade/load.groovy | 89 +++++ .../test_upgrade_lower_case_catalog.groovy | 47 +++ 73 files changed, 3403 insertions(+), 599 deletions(-) rename fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/{JdbcIdentifierMapping.java => JdbcSchemaCacheValue.java} (51%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/JdbcIdentifierMapping.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheFalseTest.java rename fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/{ExternalTableNameComparedLowercaseTest.java => ExternalTableNameComparedLowercaseMetaCacheTrueTest.java} (82%) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheFalseTest.java rename fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/{ExternalTableNameStoredLowercaseTest.java => ExternalTableNameStoredLowercaseMetaCacheTrueTest.java} (83%) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/mapping/JdbcIdentifierMappingTest.java create mode 100644 regression-test/data/external_table_p0/lower_case/test_conflict_name.out create mode 100644 regression-test/data/external_table_p0/lower_case/test_lower_case_meta_show_and_select.out create mode 100644 regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out create mode 100644 regression-test/data/external_table_p0/lower_case/test_lower_case_mtmv.out create mode 100644 regression-test/data/external_table_p0/lower_case/test_meta_cache_select_without_refresh.out create mode 100644 regression-test/data/external_table_p0/lower_case/test_meta_names_mapping.out create mode 100644 regression-test/data/external_table_p0/lower_case/upgrade/load.out create mode 100644 regression-test/data/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.out create mode 100644 regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index 41cb44ef0b5fa3..a11061afa38e38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -220,4 +220,14 @@ default Optional getMetaTableFunction(String dbName, String default Optional getMetaTableFunctionRef(String dbName, String sourceNameWithMetaName) { return Optional.empty(); } + + // Convert from remote database name to local database name, overridden by subclass if necessary + default String fromRemoteDatabaseName(String remoteDatabaseName) { + return remoteDatabaseName; + } + + // Convert from remote table name to local table name, overridden by subclass if necessary + default String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) { + return remoteTableName; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index f90a2a32fdc3ec..96bce5a48f04e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -48,6 +48,7 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; @@ -690,8 +691,8 @@ public boolean externalTableExistInLocal(String dbName, String tableName, String } public void registerExternalTableFromEvent(String dbName, String tableName, - String catalogName, long updateTime, - boolean ignoreIfExists) throws DdlException { + String catalogName, long updateTime, + boolean ignoreIfExists) throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { throw new DdlException("No catalog found with name: " + catalogName); @@ -721,7 +722,8 @@ public void registerExternalTableFromEvent(String dbName, String tableName, db.writeLock(); try { - HMSExternalTable namedTable = new HMSExternalTable(tblId, tableName, dbName, (HMSExternalCatalog) catalog); + HMSExternalTable namedTable = ((HMSExternalDatabase) db) + .buildTableForInit(tableName, tableName, tblId, hmsCatalog, (HMSExternalDatabase) db, false); namedTable.setUpdateTime(updateTime); db.registerTable(namedTable); } finally { @@ -767,7 +769,7 @@ public void registerExternalDatabaseFromEvent(String dbName, String catalogName) } public void addExternalPartitions(String catalogName, String dbName, String tableName, - List partitionNames, long updateTime, boolean ignoreIfNotExists) + List partitionNames, long updateTime, boolean ignoreIfNotExists) throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { @@ -802,7 +804,7 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl } public void dropExternalPartitions(String catalogName, String dbName, String tableName, - List partitionNames, long updateTime, boolean ignoreIfNotExists) + List partitionNames, long updateTime, boolean ignoreIfNotExists) throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index d1df51177fd496..840baa3ac6b9bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -106,9 +106,13 @@ public abstract class ExternalCatalog public static final String DORIS_VERSION = "doris.version"; public static final String DORIS_VERSION_VALUE = Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH; public static final String USE_META_CACHE = "use_meta_cache"; + public static final String CREATE_TIME = "create_time"; public static final boolean DEFAULT_USE_META_CACHE = true; + public static final String FOUND_CONFLICTING = "Found conflicting"; + public static final String ONLY_TEST_LOWER_CASE_TABLE_NAMES = "only_test_lower_case_table_names"; + // Properties that should not be shown in the `show create catalog` result public static final Set HIDDEN_PROPERTIES = Sets.newHashSet( CREATE_TIME, @@ -191,6 +195,7 @@ private Configuration buildConf() { /** * set some default properties when creating catalog + * * @return list of database names in this catalog */ protected List listDatabaseNames() { @@ -272,8 +277,9 @@ public final synchronized void makeSureInitialized() { OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_meta_object_cache_num, ignored -> getFilteredDatabaseNames(), - dbName -> Optional.ofNullable( - buildDbForInit(dbName, Util.genIdByName(name, dbName), logType, true)), + localDbName -> Optional.ofNullable( + buildDbForInit(null, localDbName, Util.genIdByName(name, localDbName), logType, + true)), (key, value, cause) -> value.ifPresent(v -> v.setUnInitialized(invalidCacheInInit))); } setLastUpdateTime(System.currentTimeMillis()); @@ -370,21 +376,28 @@ private void init() { InitCatalogLog initCatalogLog = new InitCatalogLog(); initCatalogLog.setCatalogId(id); initCatalogLog.setType(logType); - List filteredDatabases = getFilteredDatabaseNames(); - for (String dbName : filteredDatabases) { + List> remoteToLocalPairs = getFilteredDatabaseNames(); + for (Pair pair : remoteToLocalPairs) { + String remoteDbName = pair.key(); + String localDbName = pair.value(); long dbId; - if (dbNameToId != null && dbNameToId.containsKey(dbName)) { - dbId = dbNameToId.get(dbName); - tmpDbNameToId.put(dbName, dbId); + if (dbNameToId != null && dbNameToId.containsKey(localDbName)) { + dbId = dbNameToId.get(localDbName); + tmpDbNameToId.put(localDbName, dbId); ExternalDatabase db = idToDb.get(dbId); + // If the remote name is missing during upgrade, all databases in the Map will be reinitialized. + if (Strings.isNullOrEmpty(db.getRemoteName())) { + db.setRemoteName(remoteDbName); + } tmpIdToDb.put(dbId, db); initCatalogLog.addRefreshDb(dbId); } else { dbId = Env.getCurrentEnv().getNextId(); - tmpDbNameToId.put(dbName, dbId); - ExternalDatabase db = buildDbForInit(dbName, dbId, logType, false); + tmpDbNameToId.put(localDbName, dbId); + ExternalDatabase db = + buildDbForInit(remoteDbName, localDbName, dbId, logType, false); tmpIdToDb.put(dbId, db); - initCatalogLog.addCreateDb(dbId, dbName); + initCatalogLog.addCreateDb(dbId, localDbName, remoteDbName); } } @@ -395,15 +408,41 @@ private void init() { Env.getCurrentEnv().getEditLog().logInitCatalog(initCatalogLog); } + /** + * Retrieves a filtered list of database names and their corresponding local database names. + * The method applies to include and exclude filters based on the database properties, ensuring + * only the relevant databases are included for further operations. + *

+ * The method also handles conflicts in database names under case-insensitive conditions + * and throws an exception if such conflicts are detected. + *

+ * Steps: + * 1. Fetch all database names from the remote source. + * 2. Apply to include and exclude database filters: + * - Exclude filters take precedence over include filters. + * - If a database is in the exclude list, it is ignored. + * - If a database is not in the include list and the include list is not empty, it is ignored. + * 3. Map the filtered remote database names to local database names. + * 4. Handle conflicts when `lower_case_meta_names` is enabled: + * - Detect cases where multiple remote database names map to the same lower-cased local name. + * - Throw an exception if conflicts are found. + * + * @return A list of pairs where each pair contains the remote database name and local database name. + * @throws RuntimeException if there are conflicting database names under case-insensitive conditions. + */ @NotNull - private List getFilteredDatabaseNames() { + private List> getFilteredDatabaseNames() { List allDatabases = Lists.newArrayList(listDatabaseNames()); allDatabases.remove(InfoSchemaDb.DATABASE_NAME); allDatabases.add(InfoSchemaDb.DATABASE_NAME); allDatabases.remove(MysqlDb.DATABASE_NAME); allDatabases.add(MysqlDb.DATABASE_NAME); + Map includeDatabaseMap = getIncludeDatabaseMap(); Map excludeDatabaseMap = getExcludeDatabaseMap(); + + List> remoteToLocalPairs = Lists.newArrayList(); + allDatabases = allDatabases.stream().filter(dbName -> { if (!dbName.equals(InfoSchemaDb.DATABASE_NAME) && !dbName.equals(MysqlDb.DATABASE_NAME)) { // Exclude database map take effect with higher priority over include database map @@ -416,7 +455,40 @@ private List getFilteredDatabaseNames() { } return true; }).collect(Collectors.toList()); - return allDatabases; + + for (String remoteDbName : allDatabases) { + String localDbName = fromRemoteDatabaseName(remoteDbName); + remoteToLocalPairs.add(Pair.of(remoteDbName, localDbName)); + } + + // Check for conflicts when lower_case_meta_names = true + if (Boolean.parseBoolean(getLowerCaseMetaNames())) { + // Map to track lowercase local names and their corresponding remote names + Map> lowerCaseToRemoteNames = Maps.newHashMap(); + + // Collect lowercased local names and their remote counterparts + for (Pair pair : remoteToLocalPairs) { + String lowerCaseLocalName = pair.second.toLowerCase(); + lowerCaseToRemoteNames.computeIfAbsent(lowerCaseLocalName, k -> Lists.newArrayList()).add(pair.first); + } + + // Identify conflicts: multiple remote names mapping to the same lowercase local name + List conflicts = lowerCaseToRemoteNames.values().stream() + .filter(remoteNames -> remoteNames.size() > 1) // Conflict: more than one remote name + .flatMap(List::stream) // Collect all conflicting remote names + .collect(Collectors.toList()); + + // Throw exception if conflicts are found + if (!conflicts.isEmpty()) { + throw new RuntimeException(String.format( + FOUND_CONFLICTING + " database names under case-insensitive conditions. " + + "Conflicting remote database names: %s in catalog %s. " + + "Please use meta_names_mapping to handle name mapping.", + String.join(", ", conflicts), name)); + } + } + + return remoteToLocalPairs; } public void onRefresh(boolean invalidCache) { @@ -493,6 +565,7 @@ public void setComment(String comment) { /** * Different from 'listDatabases()', this method will return dbnames from cache. * while 'listDatabases()' will return dbnames from remote datasource. + * * @return names of database in this catalog. */ @Override @@ -649,8 +722,17 @@ private void removeAccessController() { } public void replayInitCatalog(InitCatalogLog log) { + // If the remote name is missing during upgrade, all databases in the Map will be reinitialized. + if (log.getRemoteDbNames() == null || log.getRemoteDbNames().isEmpty()) { + dbNameToId = Maps.newConcurrentMap(); + idToDb = Maps.newConcurrentMap(); + lastUpdateTime = log.getLastUpdateTime(); + initialized = false; + return; + } + Map tmpDbNameToId = Maps.newConcurrentMap(); - Map> tmpIdToDb = Maps.newConcurrentMap(); + Map> tmpIdToDb = Maps.newConcurrentMap(); for (int i = 0; i < log.getRefreshCount(); i++) { Optional> db = getDbForReplay(log.getRefreshDbIds().get(i)); // Should not return null. @@ -667,7 +749,8 @@ public void replayInitCatalog(InitCatalogLog log) { } for (int i = 0; i < log.getCreateCount(); i++) { ExternalDatabase db = - buildDbForInit(log.getCreateDbNames().get(i), log.getCreateDbIds().get(i), log.getType(), false); + buildDbForInit(log.getRemoteDbNames().get(i), log.getCreateDbNames().get(i), + log.getCreateDbIds().get(i), log.getType(), false); if (db != null) { tmpDbNameToId.put(db.getFullName(), db.getId()); tmpIdToDb.put(db.getId(), db); @@ -694,58 +777,92 @@ public Optional> getDbForReplay(long d * Build a database instance. * If checkExists is true, it will check if the database exists in the remote system. * - * @param dbName + * @param remoteDbName * @param dbId * @param logType * @param checkExists * @return */ - protected ExternalDatabase buildDbForInit(String dbName, long dbId, - InitCatalogLog.Type logType, boolean checkExists) { + protected ExternalDatabase buildDbForInit(String remoteDbName, String localDbName, + long dbId, InitCatalogLog.Type logType, boolean checkExists) { + // Step 1: Map local database name if not already provided + if (localDbName == null && remoteDbName != null) { + localDbName = fromRemoteDatabaseName(remoteDbName); + } + + // Step 2: // When running ut, disable this check to make ut pass. // Because in ut, the database is not created in remote system. if (checkExists && (!FeConstants.runningUnitTest || this instanceof TestExternalCatalog)) { try { List dbNames = getDbNames(); - if (!dbNames.contains(dbName)) { - dbNames = getFilteredDatabaseNames(); - if (!dbNames.contains(dbName)) { + if (!dbNames.contains(localDbName)) { + dbNames = getFilteredDatabaseNames().stream() + .map(Pair::value) + .collect(Collectors.toList()); + if (!dbNames.contains(localDbName)) { + LOG.warn("Database {} does not exist in the remote system. Skipping initialization.", + localDbName); return null; } } - } catch (Throwable t) { + } catch (RuntimeException e) { + // Handle "Found conflicting" exception explicitly + if (e.getMessage().contains(FOUND_CONFLICTING)) { + LOG.error(e.getMessage()); + throw e; // Rethrow to let the caller handle this critical issue + } else { + // Any errors other than name conflicts, we default to not finding the database + LOG.warn("Failed to check db {} exist in remote system, ignore it.", localDbName, e); + return null; + } + } catch (Exception e) { // If connection failed, it will throw exception. // ignore it and treat it as not exist. - LOG.warn("Failed to check db {} exist in remote system, ignore it.", dbName, t); + LOG.warn("Failed to check db {} exist in remote system, ignore it.", localDbName, e); return null; } } - if (dbName.equals(InfoSchemaDb.DATABASE_NAME)) { + // Step 3: Resolve remote database name if using meta cache + if (remoteDbName == null && useMetaCache.orElse(false)) { + if (Boolean.parseBoolean(getLowerCaseMetaNames()) || !Strings.isNullOrEmpty(getMetaNamesMapping())) { + remoteDbName = metaCache.getRemoteName(localDbName); + if (remoteDbName == null) { + LOG.warn("Could not resolve remote database name for local database: {}", localDbName); + return null; + } + } else { + remoteDbName = localDbName; + } + } + + // Step 4: Instantiate the appropriate ExternalDatabase based on logType + if (localDbName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) { return new ExternalInfoSchemaDatabase(this, dbId); } - if (dbName.equals(MysqlDb.DATABASE_NAME)) { + if (localDbName.equalsIgnoreCase(MysqlDb.DATABASE_NAME)) { return new ExternalMysqlDatabase(this, dbId); } switch (logType) { case HMS: - return new HMSExternalDatabase(this, dbId, dbName); + return new HMSExternalDatabase(this, dbId, localDbName, remoteDbName); case ES: - return new EsExternalDatabase(this, dbId, dbName); + return new EsExternalDatabase(this, dbId, localDbName, remoteDbName); case JDBC: - return new JdbcExternalDatabase(this, dbId, dbName); + return new JdbcExternalDatabase(this, dbId, localDbName, remoteDbName); case ICEBERG: - return new IcebergExternalDatabase(this, dbId, dbName); + return new IcebergExternalDatabase(this, dbId, localDbName, remoteDbName); case MAX_COMPUTE: - return new MaxComputeExternalDatabase(this, dbId, dbName); + return new MaxComputeExternalDatabase(this, dbId, localDbName, remoteDbName); case LAKESOUL: - return new LakeSoulExternalDatabase(this, dbId, dbName); + return new LakeSoulExternalDatabase(this, dbId, localDbName, remoteDbName); case TEST: - return new TestExternalDatabase(this, dbId, dbName); + return new TestExternalDatabase(this, dbId, localDbName, remoteDbName); case PAIMON: - return new PaimonExternalDatabase(this, dbId, dbName); + return new PaimonExternalDatabase(this, dbId, localDbName, remoteDbName); case TRINO_CONNECTOR: - return new TrinoConnectorExternalDatabase(this, dbId, dbName); + return new TrinoConnectorExternalDatabase(this, dbId, localDbName, remoteDbName); default: break; } @@ -890,6 +1007,19 @@ private Map getSpecifiedDatabaseMap(String catalogPropertyKey) return specifiedDatabaseMap; } + + public String getLowerCaseMetaNames() { + return catalogProperty.getOrDefault(Resource.LOWER_CASE_META_NAMES, "false"); + } + + public int getOnlyTestLowerCaseTableNames() { + return Integer.parseInt(catalogProperty.getOrDefault(ONLY_TEST_LOWER_CASE_TABLE_NAMES, "0")); + } + + public String getMetaNamesMapping() { + return catalogProperty.getOrDefault(Resource.META_NAMES_MAPPING, ""); + } + public String bindBrokerName() { return catalogProperty.getProperties().get(HMSExternalCatalog.BIND_BROKER_NAME); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index eda98efb9b6a03..13f5af7b46d9de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -25,7 +25,9 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; @@ -35,12 +37,14 @@ import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase; import org.apache.doris.datasource.infoschema.ExternalMysqlTable; import org.apache.doris.datasource.metacache.MetaCache; +import org.apache.doris.datasource.test.TestExternalDatabase; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.MasterCatalogExecutor; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -76,6 +80,8 @@ public abstract class ExternalDatabase protected long id; @SerializedName(value = "name") protected String name; + @SerializedName(value = "remoteName") + protected String remoteName; @SerializedName(value = "dbProperties") protected DatabaseProperty dbProperties = new DatabaseProperty(); @SerializedName(value = "initialized") @@ -100,11 +106,14 @@ public abstract class ExternalDatabase * @param extCatalog The catalog this database belongs to. * @param id Database id. * @param name Database name. + * @param remoteName Remote database name. */ - public ExternalDatabase(ExternalCatalog extCatalog, long id, String name, InitDatabaseLog.Type dbLogType) { + public ExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName, + InitDatabaseLog.Type dbLogType) { this.extCatalog = extCatalog; this.id = id; this.name = name; + this.remoteName = remoteName; this.dbLogType = dbLogType; } @@ -112,6 +121,10 @@ public void setExtCatalog(ExternalCatalog extCatalog) { this.extCatalog = extCatalog; } + public void setRemoteName(String remoteName) { + this.remoteName = remoteName; + } + public void setTableExtCatalog(ExternalCatalog extCatalog) { for (T table : idToTbl.values()) { table.setCatalog(extCatalog); @@ -150,9 +163,10 @@ public final synchronized void makeSureInitialized() { OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_meta_object_cache_num, ignored -> listTableNames(), - tableName -> Optional.ofNullable( - buildTableForInit(tableName, - Util.genIdByName(extCatalog.getName(), name, tableName), extCatalog)), + localTableName -> Optional.ofNullable( + buildTableForInit(null, localTableName, + Util.genIdByName(extCatalog.getName(), name, localTableName), extCatalog, + this, true)), (key, value, cause) -> value.ifPresent(ExternalTable::unsetObjectCreated)); } setLastUpdateTime(System.currentTimeMillis()); @@ -176,6 +190,15 @@ public final synchronized void makeSureInitialized() { } public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) { + // If the remote name is missing during upgrade, all tables in the Map will be reinitialized. + if (log.getRemoteTableNames() == null || log.getRemoteTableNames().isEmpty()) { + tableNameToId = Maps.newConcurrentMap(); + idToTbl = Maps.newConcurrentMap(); + lastUpdateTime = log.getLastUpdateTime(); + initialized = false; + return; + } + Map tmpTableNameToId = Maps.newConcurrentMap(); Map tmpIdToTbl = Maps.newConcurrentMap(); for (int i = 0; i < log.getRefreshCount(); i++) { @@ -191,7 +214,9 @@ public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) { } } for (int i = 0; i < log.getCreateCount(); i++) { - T table = buildTableForInit(log.getCreateTableNames().get(i), log.getCreateTableIds().get(i), catalog); + T table = + buildTableForInit(log.getRemoteTableNames().get(i), log.getCreateTableNames().get(i), + log.getCreateTableIds().get(i), catalog, this, false); tmpTableNameToId.put(table.getName(), table.getId()); tmpIdToTbl.put(table.getId(), table); } @@ -206,24 +231,34 @@ private void init() { initDatabaseLog.setType(dbLogType); initDatabaseLog.setCatalogId(extCatalog.getId()); initDatabaseLog.setDbId(id); - List tableNames = listTableNames(); - if (tableNames != null) { + List> tableNamePairs = listTableNames(); + if (tableNamePairs != null) { Map tmpTableNameToId = Maps.newConcurrentMap(); Map tmpIdToTbl = Maps.newHashMap(); - for (String tableName : tableNames) { + for (Pair pair : tableNamePairs) { + String remoteTableName = pair.first; + String localTableName = pair.second; long tblId; - if (tableNameToId != null && tableNameToId.containsKey(tableName)) { - tblId = tableNameToId.get(tableName); - tmpTableNameToId.put(tableName, tblId); + if (tableNameToId != null && tableNameToId.containsKey(localTableName)) { + tblId = tableNameToId.get(localTableName); + tmpTableNameToId.put(localTableName, tblId); T table = idToTbl.get(tblId); + // If the remote name is missing during upgrade, all tables in the Map will be reinitialized. + if (Strings.isNullOrEmpty(table.getRemoteName())) { + table.setRemoteName(remoteTableName); + } + // If the db is missing, set it. + if (table.getDb() == null) { + table.setDb(this); + } tmpIdToTbl.put(tblId, table); initDatabaseLog.addRefreshTable(tblId); } else { tblId = Env.getCurrentEnv().getNextId(); - tmpTableNameToId.put(tableName, tblId); - T table = buildTableForInit(tableName, tblId, extCatalog); + tmpTableNameToId.put(localTableName, tblId); + T table = buildTableForInit(remoteTableName, localTableName, tblId, extCatalog, this, false); tmpIdToTbl.put(tblId, table); - initDatabaseLog.addCreateTable(tblId, tableName); + initDatabaseLog.addCreateTable(tblId, localTableName, remoteTableName); } } tableNameToId = tmpTableNameToId; @@ -235,26 +270,121 @@ private void init() { Env.getCurrentEnv().getEditLog().logInitExternalDb(initDatabaseLog); } - private List listTableNames() { - List tableNames; + private List> listTableNames() { + List> tableNames; if (name.equals(InfoSchemaDb.DATABASE_NAME)) { - tableNames = ExternalInfoSchemaDatabase.listTableNames(); + tableNames = ExternalInfoSchemaDatabase.listTableNames().stream() + .map(tableName -> Pair.of(tableName, tableName)) + .collect(Collectors.toList()); } else if (name.equals(MysqlDb.DATABASE_NAME)) { - tableNames = ExternalMysqlDatabase.listTableNames(); + tableNames = ExternalMysqlDatabase.listTableNames().stream() + .map(tableName -> Pair.of(tableName, tableName)) + .collect(Collectors.toList()); } else { - tableNames = extCatalog.listTableNames(null, name).stream().map(tableName -> { - lowerCaseToTableName.put(tableName.toLowerCase(), tableName); - if (Env.isStoredTableNamesLowerCase()) { - return tableName.toLowerCase(); - } else { - return tableName; + tableNames = extCatalog.listTableNames(null, remoteName).stream().map(tableName -> { + String localTableName = extCatalog.fromRemoteTableName(remoteName, tableName); + if (Env.isStoredTableNamesLowerCase() + || extCatalog.getOnlyTestLowerCaseTableNames() == 1) { + localTableName = localTableName.toLowerCase(); } + lowerCaseToTableName.put(tableName.toLowerCase(), tableName); + return Pair.of(tableName, localTableName); }).collect(Collectors.toList()); } + // Check for conflicts when stored table names or meta names are case-insensitive + if ((Env.isStoredTableNamesLowerCase() || Env.isTableNamesCaseInsensitive()) + || Boolean.parseBoolean(extCatalog.getLowerCaseMetaNames()) + || (extCatalog.getOnlyTestLowerCaseTableNames() == 1 + || extCatalog.getOnlyTestLowerCaseTableNames() == 2)) { + // Map to track lowercased local names and their corresponding remote names + Map> lowerCaseToRemoteNames = Maps.newHashMap(); + + // Collect lowercased local names and their remote counterparts + for (Pair pair : tableNames) { + String lowerCaseLocalName = pair.value().toLowerCase(); + lowerCaseToRemoteNames.computeIfAbsent(lowerCaseLocalName, k -> Lists.newArrayList()).add(pair.key()); + } + + // Identify conflicts: multiple remote names mapping to the same lowercased local name + List conflicts = lowerCaseToRemoteNames.values().stream() + .filter(remoteNames -> remoteNames.size() > 1) // Conflict: more than one remote name + .flatMap(List::stream) // Collect all conflicting remote names + .collect(Collectors.toList()); + + // Throw exception if conflicts are found + if (!conflicts.isEmpty()) { + throw new RuntimeException(String.format( + ExternalCatalog.FOUND_CONFLICTING + " table names under case-insensitive conditions. " + + "Conflicting remote table names: %s in remote database '%s' under catalog '%s'. " + + "Please use meta_names_mapping to handle name mapping.", + String.join(", ", conflicts), remoteName, extCatalog.getName())); + } + } return tableNames; } - protected abstract T buildTableForInit(String tableName, long tblId, ExternalCatalog catalog); + public T buildTableForInit(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, ExternalDatabase db, boolean checkExists) { + + // Step 1: Resolve local table name if not provided + if (localTableName == null && remoteTableName != null) { + localTableName = extCatalog.fromRemoteTableName(remoteName, remoteTableName); + } + + // Step 2: Check if the table exists in the system, if the `checkExists` flag is enabled + if (checkExists && (!FeConstants.runningUnitTest || this instanceof TestExternalDatabase)) { + try { + List tblNames = Lists.newArrayList(getTableNamesWithLock()); + if (!tblNames.contains(localTableName)) { + tblNames = listTableNames().stream() + .map(Pair::value) + .collect(Collectors.toList()); + if (!tblNames.contains(localTableName)) { + LOG.warn("Table {} does not exist in the remote system. Skipping initialization.", + localTableName); + return null; + } + } + } catch (RuntimeException e) { + // Handle "Found conflicting" exception explicitly + if (e.getMessage().contains(ExternalCatalog.FOUND_CONFLICTING)) { + LOG.error(e.getMessage()); + throw e; // Rethrow to let the caller handle this critical issue + } else { + // Any errors other than name conflicts, we default to not finding the table + LOG.warn("Failed to check existence of table {} in the remote system. Ignoring this table.", + localTableName, e); + return null; + } + } catch (Exception e) { + // If connection fails, treat the table as non-existent + LOG.warn("Failed to check existence of table {} in the remote system. Ignoring this table.", + localTableName, e); + return null; + } + } + + // Step 3: Resolve remote table name if using meta cache and it is not provided + if (remoteTableName == null && extCatalog.useMetaCache.get()) { + if (Boolean.parseBoolean(extCatalog.getLowerCaseMetaNames()) + || !Strings.isNullOrEmpty(extCatalog.getMetaNamesMapping()) + || extCatalog.getOnlyTestLowerCaseTableNames() == 1) { + remoteTableName = metaCache.getRemoteName(localTableName); + if (remoteTableName == null) { + LOG.warn("Could not resolve remote table name for local table: {}", localTableName); + return null; + } + } else { + remoteTableName = localTableName; + } + } + + // Step 4: Build and return the table instance using the resolved names and other parameters + return buildTableInternal(remoteTableName, localTableName, tblId, catalog, db); + } + + protected abstract T buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, ExternalDatabase db); public Optional getTableForReplay(long tableId) { if (extCatalog.getUseMetaCache().get()) { @@ -328,6 +458,10 @@ public String getFullName() { return name; } + public String getRemoteName() { + return remoteName; + } + @Override public DatabaseProperty getDbProperties() { return dbProperties; @@ -335,10 +469,18 @@ public DatabaseProperty getDbProperties() { @Override public boolean isTableExist(String tableName) { - if (Env.isTableNamesCaseInsensitive()) { - tableName = lowerCaseToTableName.get(tableName.toLowerCase()); - if (tableName == null) { - return false; + if (Env.isTableNamesCaseInsensitive() || extCatalog.getOnlyTestLowerCaseTableNames() == 2) { + String realTableName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (realTableName == null) { + // Here we need to execute listTableNames() once to fill in lowerCaseToTableName + // to prevent lowerCaseToTableName from being empty in some cases + listTableNames(); + tableName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (tableName == null) { + return false; + } + } else { + tableName = realTableName; } } return extCatalog.tableExist(ConnectContext.get().getSessionContext(), name, tableName); @@ -391,15 +533,26 @@ public Set getTableNamesWithLock() { @Override public T getTableNullable(String tableName) { makeSureInitialized(); - if (Env.isStoredTableNamesLowerCase()) { + if (Env.isStoredTableNamesLowerCase() || extCatalog.getOnlyTestLowerCaseTableNames() == 1) { tableName = tableName.toLowerCase(); } - if (Env.isTableNamesCaseInsensitive()) { - tableName = lowerCaseToTableName.get(tableName.toLowerCase()); - if (tableName == null) { - return null; + if (Env.isTableNamesCaseInsensitive() || extCatalog.getOnlyTestLowerCaseTableNames() == 2) { + String realTableName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (realTableName == null) { + // Here we need to execute listTableNames() once to fill in lowerCaseToTableName + // to prevent lowerCaseToTableName from being empty in some cases + listTableNames(); + tableName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (tableName == null) { + return null; + } + } else { + tableName = realTableName; } } + if (extCatalog.getLowerCaseMetaNames().equalsIgnoreCase("true")) { + tableName = tableName.toLowerCase(); + } if (extCatalog.getUseMetaCache().get()) { // must use full qualified name to generate id. // otherwise, if 2 databases have the same table name, the id will be the same. @@ -480,7 +633,7 @@ public void gsonPostProcess() throws IOException { @Override public void unregisterTable(String tableName) { makeSureInitialized(); - if (Env.isStoredTableNamesLowerCase()) { + if (Env.isStoredTableNamesLowerCase() || extCatalog.getOnlyTestLowerCaseTableNames() == 1) { tableName = tableName.toLowerCase(); } if (LOG.isDebugEnabled()) { @@ -528,7 +681,9 @@ public boolean registerTable(TableIf tableIf) { } else { if (!tableNameToId.containsKey(tableName)) { tableNameToId.put(tableName, tableId); - idToTbl.put(tableId, buildTableForInit(tableName, tableId, extCatalog)); + idToTbl.put(tableId, + buildTableForInit(tableName, extCatalog.fromRemoteTableName(this.remoteName, tableName), + tableId, extCatalog, this, false)); lowerCaseToTableName.put(tableName.toLowerCase(), tableName); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 24f55e74266863..48d170b59166b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.Config; +import org.apache.doris.common.Pair; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; @@ -302,7 +303,7 @@ public void invalidatePartitionsCache(long catalogId, String dbName, String tabl public MetaCache buildMetaCache(String name, OptionalLong expireAfterWriteSec, OptionalLong refreshAfterWriteSec, long maxSize, - CacheLoader> namesCacheLoader, + CacheLoader>> namesCacheLoader, CacheLoader> metaObjCacheLoader, RemovalListener> removalListener) { MetaCache metaCache = new MetaCache<>(name, commonRefreshExecutor, expireAfterWriteSec, refreshAfterWriteSec, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java index 91df061678f154..dcad165afa22ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java @@ -71,10 +71,13 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable { protected long id; @SerializedName(value = "name") protected String name; + @SerializedName(value = "remoteName") + protected String remoteName; @SerializedName(value = "type") protected TableType type = null; @SerializedName(value = "timestamp") protected long timestamp; + // dbName is temporarily retained and will be deleted later. To use dbName, please use db.getFullName() @SerializedName(value = "dbName") protected String dbName; @SerializedName(value = "ta") @@ -86,6 +89,7 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable { protected long dbId; protected boolean objectCreated; protected ExternalCatalog catalog; + protected ExternalDatabase db; /** * No args constructor for persist. @@ -99,15 +103,19 @@ public ExternalTable() { * * @param id Table id. * @param name Table name. + * @param remoteName Remote table name. * @param catalog ExternalCatalog this table belongs to. - * @param dbName Name of the db the this table belongs to. + * @param db ExternalDatabase this table belongs to. * @param type Table type. */ - public ExternalTable(long id, String name, ExternalCatalog catalog, String dbName, TableType type) { + public ExternalTable(long id, String name, String remoteName, ExternalCatalog catalog, ExternalDatabase db, + TableType type) { this.id = id; this.name = name; + this.remoteName = remoteName; this.catalog = catalog; - this.dbName = dbName; + this.db = db; + this.dbName = db.getFullName(); this.type = type; this.objectCreated = false; } @@ -116,6 +124,14 @@ public void setCatalog(ExternalCatalog catalog) { this.catalog = catalog; } + public void setDb(ExternalDatabase db) { + this.db = db; + } + + public void setRemoteName(String remoteName) { + this.remoteName = remoteName; + } + public boolean isView() { return false; } @@ -141,6 +157,10 @@ public String getName() { return name; } + public String getRemoteName() { + return remoteName; + } + @Override public TableType getType() { return type; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java index 7834c0c8826daf..023eecc2fa4c00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java @@ -64,6 +64,9 @@ public enum Type { @SerializedName(value = "createDbNames") private List createDbNames; + @SerializedName(value = "remoteDbNames") + private List remoteDbNames; + @SerializedName(value = "type") private Type type; @@ -77,6 +80,7 @@ public InitCatalogLog() { refreshDbIds = Lists.newArrayList(); createDbIds = Lists.newArrayList(); createDbNames = Lists.newArrayList(); + remoteDbNames = Lists.newArrayList(); type = Type.UNKNOWN; } @@ -85,10 +89,11 @@ public void addRefreshDb(long id) { refreshDbIds.add(id); } - public void addCreateDb(long id, String name) { + public void addCreateDb(long id, String name, String remoteName) { createCount += 1; createDbIds.add(id); createDbNames.add(name); + remoteDbNames.add(remoteName); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java index c652113cf0de88..44ee0a39c56943 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java @@ -68,6 +68,9 @@ public enum Type { @SerializedName(value = "createTableNames") private List createTableNames; + @SerializedName(value = "remoteTableNames") + private List remoteTableNames; + @SerializedName(value = "type") private Type type; @@ -82,6 +85,7 @@ public InitDatabaseLog() { refreshTableIds = Lists.newArrayList(); createTableIds = Lists.newArrayList(); createTableNames = Lists.newArrayList(); + remoteTableNames = Lists.newArrayList(); type = Type.UNKNOWN; } @@ -90,10 +94,11 @@ public void addRefreshTable(long id) { refreshTableIds.add(id); } - public void addCreateTable(long id, String name) { + public void addCreateTable(long id, String name, String remoteName) { createCount += 1; createTableIds.add(id); createTableNames.add(name); + remoteTableNames.add(remoteName); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java index 3c77b112d60160..39db1452833eac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java @@ -32,14 +32,18 @@ public class EsExternalDatabase extends ExternalDatabase { * @param extCatalog External data source this database belongs to. * @param id database id. * @param name database name. + * @param remoteName remote database name. */ - public EsExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.ES); + public EsExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.ES); } @Override - protected EsExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new EsExternalTable(tblId, tableName, name, (EsExternalCatalog) extCatalog); + public EsExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new EsExternalTable(tblId, localTableName, remoteTableName, (EsExternalCatalog) extCatalog, + (EsExternalDatabase) db); } public void addTableForTest(EsExternalTable tbl) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java index 4f05d6d29cd89e..6e9e5731f41532 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java @@ -42,11 +42,12 @@ public class EsExternalTable extends ExternalTable { * * @param id Table id. * @param name Table name. - * @param dbName Database name. - * @param catalog HMSExternalDataSource. + * @param remoteName Remote table name. + * @param catalog EsExternalDataSource. + * @param db Database. */ - public EsExternalTable(long id, String name, String dbName, EsExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.ES_EXTERNAL_TABLE); + public EsExternalTable(long id, String name, String remoteName, EsExternalCatalog catalog, EsExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.ES_EXTERNAL_TABLE); } protected synchronized void makeSureInitialized() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 85b999f1111047..7498e6edd93c76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -263,7 +263,7 @@ public void registerDatabase(long dbId, String dbName) { LOG.debug("create database [{}]", dbName); } - ExternalDatabase db = buildDbForInit(dbName, dbId, logType, false); + ExternalDatabase db = buildDbForInit(dbName, null, dbId, logType, false); if (useMetaCache.get()) { if (isInitialized()) { metaCache.updateCache(dbName, db, Util.genIdByName(getQualifiedName(dbName))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java index 3ae9fbcd6e75a9..86f99527fe421c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java @@ -32,14 +32,18 @@ public class HMSExternalDatabase extends ExternalDatabase { * @param extCatalog External catalog this database belongs to. * @param id database id. * @param name database name. + * @param remoteName remote database name. */ - public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.HMS); + public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.HMS); } @Override - protected HMSExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new HMSExternalTable(tblId, tableName, name, (HMSExternalCatalog) extCatalog); + public HMSExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new HMSExternalTable(tblId, localTableName, remoteTableName, (HMSExternalCatalog) extCatalog, + (HMSExternalDatabase) db); } public void addTableForTest(HMSExternalTable tbl) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index cbea5e94dd1e44..8f6626baead4e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -168,11 +168,13 @@ public enum DLAType { * * @param id Table id. * @param name Table name. - * @param dbName Database name. - * @param catalog HMSExternalCatalog. + * @param remoteName Remote table name. + * @param catalog HMSExternalDataSource. + * @param db Database. */ - public HMSExternalTable(long id, String name, String dbName, HMSExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.HMS_EXTERNAL_TABLE); + public HMSExternalTable(long id, String name, String remoteName, HMSExternalCatalog catalog, + HMSExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.HMS_EXTERNAL_TABLE); } // Will throw NotSupportedException if not supported hms table. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java index f56183972e36d2..7a1a53825a15d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java @@ -28,13 +28,16 @@ public class IcebergExternalDatabase extends ExternalDatabase { - public IcebergExternalDatabase(ExternalCatalog extCatalog, Long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.ICEBERG); + public IcebergExternalDatabase(ExternalCatalog extCatalog, Long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.ICEBERG); } @Override - protected IcebergExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new IcebergExternalTable(tblId, tableName, name, (IcebergExternalCatalog) extCatalog); + public IcebergExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new IcebergExternalTable(tblId, localTableName, remoteTableName, (IcebergExternalCatalog) extCatalog, + (IcebergExternalDatabase) db); } public String getLocation() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index 7f7d2fdf578292..713ec94fd1bdd1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -96,8 +96,9 @@ public class IcebergExternalTable extends ExternalTable implements MTMVRelatedTa private boolean isValidRelatedTableCached = false; private boolean isValidRelatedTable = false; - public IcebergExternalTable(long id, String name, String dbName, IcebergExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.ICEBERG_EXTERNAL_TABLE); + public IcebergExternalTable(long id, String name, String remoteName, IcebergExternalCatalog catalog, + IcebergExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.ICEBERG_EXTERNAL_TABLE); } public String getIcebergCatalogType() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java index 837f3691962e91..e8ab0690e17c7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java @@ -36,7 +36,7 @@ public class ExternalInfoSchemaDatabase extends ExternalDatabase { * @param dbId The id of this database. */ public ExternalInfoSchemaDatabase(ExternalCatalog extCatalog, long dbId) { - super(extCatalog, dbId, InfoSchemaDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); + super(extCatalog, dbId, InfoSchemaDb.DATABASE_NAME, InfoSchemaDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); } public static List listTableNames() { @@ -44,8 +44,10 @@ public static List listTableNames() { } @Override - protected ExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new ExternalInfoSchemaTable(tblId, tableName, catalog); + public ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new ExternalInfoSchemaTable(tblId, localTableName, catalog, db); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java index 9d1336396128fd..b3e0ead9a118a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java @@ -18,9 +18,9 @@ package org.apache.doris.datasource.infoschema; import org.apache.doris.analysis.SchemaTableType; -import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.SchemaTable; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.thrift.TSchemaTable; @@ -31,8 +31,8 @@ public class ExternalInfoSchemaTable extends ExternalTable { - public ExternalInfoSchemaTable(long id, String name, ExternalCatalog catalog) { - super(id, name, catalog, InfoSchemaDb.DATABASE_NAME, TableType.SCHEMA); + public ExternalInfoSchemaTable(long id, String name, ExternalCatalog catalog, ExternalDatabase db) { + super(id, name, name, catalog, db, TableType.SCHEMA); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java index 5e0653f5278109..da40dc34c604ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java @@ -36,7 +36,7 @@ public class ExternalMysqlDatabase extends ExternalDatabase { * @param dbId The id of this database. */ public ExternalMysqlDatabase(ExternalCatalog extCatalog, long dbId) { - super(extCatalog, dbId, MysqlDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); + super(extCatalog, dbId, MysqlDb.DATABASE_NAME, MysqlDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); } public static List listTableNames() { @@ -44,8 +44,10 @@ public static List listTableNames() { } @Override - protected ExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new ExternalMysqlTable(tblId, tableName, catalog); + public ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new ExternalMysqlTable(tblId, localTableName, catalog, db); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java index 6f277a5690619b..1077abf81d8614 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java @@ -19,8 +19,8 @@ import org.apache.doris.analysis.SchemaTableType; import org.apache.doris.catalog.MysqlDBTable; -import org.apache.doris.catalog.MysqlDb; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.thrift.TSchemaTable; @@ -30,8 +30,8 @@ import java.util.Optional; public class ExternalMysqlTable extends ExternalTable { - public ExternalMysqlTable(long id, String name, ExternalCatalog catalog) { - super(id, name, catalog, MysqlDb.DATABASE_NAME, TableType.SCHEMA); + public ExternalMysqlTable(long id, String name, ExternalCatalog catalog, ExternalDatabase db) { + super(id, name, name, catalog, db, TableType.SCHEMA); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index fb26265d19fe93..fac322d21eb4da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -27,11 +27,15 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; import org.apache.doris.datasource.jdbc.client.JdbcClient; import org.apache.doris.datasource.jdbc.client.JdbcClientConfig; import org.apache.doris.datasource.jdbc.client.JdbcClientException; +import org.apache.doris.datasource.mapping.IdentifierMapping; +import org.apache.doris.datasource.mapping.JdbcIdentifierMapping; import org.apache.doris.proto.InternalService; import org.apache.doris.proto.InternalService.PJdbcTestConnectionRequest; import org.apache.doris.proto.InternalService.PJdbcTestConnectionResult; @@ -53,6 +57,7 @@ import org.apache.thrift.TException; import org.apache.thrift.TSerializer; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -71,12 +76,17 @@ public class JdbcExternalCatalog extends ExternalCatalog { // Must add "transient" for Gson to ignore this field, // or Gson will throw exception with HikariCP private transient JdbcClient jdbcClient; + private IdentifierMapping identifierMapping; public JdbcExternalCatalog(long catalogId, String name, String resource, Map props, String comment) throws DdlException { super(catalogId, name, InitCatalogLog.Type.JDBC, comment); this.catalogProperty = new CatalogProperty(resource, processCompatibleProperties(props)); + this.identifierMapping = new JdbcIdentifierMapping( + (Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()), + Boolean.parseBoolean(getLowerCaseMetaNames()), + getMetaNamesMapping()); } @Override @@ -119,12 +129,12 @@ public void onRefresh(boolean invalidCache) { super.onRefresh(invalidCache); if (jdbcClient != null) { jdbcClient.closeClient(); + jdbcClient = null; } - } - - @Override - public void onRefreshCache(boolean invalidCache) { - onRefresh(invalidCache); + this.identifierMapping = new JdbcIdentifierMapping( + (Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()), + Boolean.parseBoolean(getLowerCaseMetaNames()), + getMetaNamesMapping()); } @Override @@ -132,6 +142,7 @@ public void onClose() { super.onClose(); if (jdbcClient != null) { jdbcClient.closeClient(); + jdbcClient = null; } } @@ -182,16 +193,6 @@ public String getOnlySpecifiedDatabase() { JdbcResource.ONLY_SPECIFIED_DATABASE)); } - public String getLowerCaseMetaNames() { - return catalogProperty.getOrDefault(JdbcResource.LOWER_CASE_META_NAMES, JdbcResource.getDefaultPropertyValue( - JdbcResource.LOWER_CASE_META_NAMES)); - } - - public String getMetaNamesMapping() { - return catalogProperty.getOrDefault(JdbcResource.META_NAMES_MAPPING, JdbcResource.getDefaultPropertyValue( - JdbcResource.META_NAMES_MAPPING)); - } - public int getConnectionPoolMinSize() { return Integer.parseInt(catalogProperty.getOrDefault(JdbcResource.CONNECTION_POOL_MIN_SIZE, JdbcResource .getDefaultPropertyValue(JdbcResource.CONNECTION_POOL_MIN_SIZE))); @@ -232,8 +233,6 @@ protected void initLocalObjectsImpl() { .setDriverUrl(getDriverUrl()) .setDriverClass(getDriverClass()) .setOnlySpecifiedDatabase(getOnlySpecifiedDatabase()) - .setIsLowerCaseMetaNames(getLowerCaseMetaNames()) - .setMetaNamesMapping(getMetaNamesMapping()) .setIncludeDatabaseMap(getIncludeDatabaseMap()) .setExcludeDatabaseMap(getExcludeDatabaseMap()) .setConnectionPoolMinSize(getConnectionPoolMinSize()) @@ -245,20 +244,57 @@ protected void initLocalObjectsImpl() { jdbcClient = JdbcClient.createJdbcClient(jdbcClientConfig); } - protected List listDatabaseNames() { + @Override + public void gsonPostProcess() throws IOException { + super.gsonPostProcess(); + if (this.identifierMapping == null) { + identifierMapping = new JdbcIdentifierMapping( + (Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()), + Boolean.parseBoolean(getLowerCaseMetaNames()), + getMetaNamesMapping()); + } + } + + @Override + public List listDatabaseNames() { return jdbcClient.getDatabaseNameList(); } + @Override + public String fromRemoteDatabaseName(String remoteDatabaseName) { + return identifierMapping.fromRemoteDatabaseName(remoteDatabaseName); + } + @Override public List listTableNames(SessionContext ctx, String dbName) { makeSureInitialized(); return jdbcClient.getTablesNameList(dbName); } + @Override + public String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) { + return identifierMapping.fromRemoteTableName(remoteDatabaseName, remoteTableName); + } + @Override public boolean tableExist(SessionContext ctx, String dbName, String tblName) { makeSureInitialized(); - return jdbcClient.isTableExist(dbName, tblName); + ExternalDatabase database = this.getDbNullable(dbName); + if (database == null) { + return false; + } + ExternalTable tbl = database.getTableNullable(tblName); + if (tbl == null) { + return false; + } + String remoteDbName = ((ExternalDatabase) tbl.getDatabase()).getRemoteName(); + String remoteTblName = tbl.getRemoteName(); + return jdbcClient.isTableExist(remoteDbName, remoteTblName); + } + + public List listColumns(String remoteDbName, String remoteTblName) { + makeSureInitialized(); + return jdbcClient.getColumnsFromJdbc(remoteDbName, remoteTblName); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java index d078a3e238adbc..1737ec614a6418 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java @@ -30,13 +30,16 @@ public class JdbcExternalDatabase extends ExternalDatabase { * @param id database id. * @param name database name. */ - public JdbcExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.JDBC); + public JdbcExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.JDBC); } @Override - protected JdbcExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new JdbcExternalTable(tblId, tableName, name, (JdbcExternalCatalog) extCatalog); + public JdbcExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new JdbcExternalTable(tblId, localTableName, remoteTableName, (JdbcExternalCatalog) extCatalog, + (JdbcExternalDatabase) db); } public void addTableForTest(JdbcExternalTable tbl) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java index 772c366850b9a4..cd24cc72302f28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.JdbcResource; import org.apache.doris.catalog.JdbcTable; +import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.qe.AutoCloseConnectContext; @@ -32,6 +33,7 @@ import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.thrift.TTableDescriptor; +import com.google.common.collect.Maps; import org.apache.commons.text.StringSubstitutor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -40,6 +42,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; /** * Jdbc external table. @@ -76,11 +79,13 @@ public class JdbcExternalTable extends ExternalTable { * * @param id Table id. * @param name Table name. - * @param dbName Database name. - * @param catalog HMSExternalDataSource. + * @param remoteName Remote table name. + * @param catalog JdbcExternalCatalog. + * @param db JdbcExternalDatabase. */ - public JdbcExternalTable(long id, String name, String dbName, JdbcExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.JDBC_EXTERNAL_TABLE); + public JdbcExternalTable(long id, String name, String remoteName, JdbcExternalCatalog catalog, + JdbcExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.JDBC_EXTERNAL_TABLE); } @Override @@ -105,21 +110,60 @@ public TTableDescriptor toThrift() { @Override public Optional initSchema() { - return Optional.of(new SchemaCacheValue(((JdbcExternalCatalog) catalog).getJdbcClient() - .getColumnsFromJdbc(dbName, name))); + String remoteDbName = ((ExternalDatabase) this.getDatabase()).getRemoteName(); + List columns = ((JdbcExternalCatalog) catalog).listColumns(remoteDbName, remoteName); + List remoteColumnNames = columns.stream() + .map(Column::getName) + .collect(Collectors.toList()); + + // Checks whether remoteColumnNames contains duplicate column names that differ only in case + Map lowerCaseColumnNameMap = Maps.newHashMap(); + for (String remoteColumnName : remoteColumnNames) { + String lowerCaseColumnName = remoteColumnName.toLowerCase(); + if (lowerCaseColumnNameMap.containsKey(lowerCaseColumnName)) { + throw new RuntimeException(String.format( + "Found conflicting column names under case-insensitive conditions. " + + "Conflicting remote column names: '%s' and '%s' in remote table '%s.%s' " + + "under catalog '%s'. " + + "Please use meta_names_mapping to handle name mapping.", + lowerCaseColumnNameMap.get(lowerCaseColumnName), remoteColumnName, remoteDbName, remoteName, + catalog.getName())); + } + lowerCaseColumnNameMap.put(lowerCaseColumnName, remoteColumnName); + } + + List localColumnNames = remoteColumnNames.stream() + .map(remoteColumnName -> ((JdbcExternalCatalog) catalog).getIdentifierMapping() + .fromRemoteColumnName(remoteDbName, remoteName, remoteColumnName)) + .collect(Collectors.toList()); + for (int i = 0; i < columns.size(); i++) { + columns.get(i).setName(localColumnNames.get(i)); + } + Map remoteColumnNamesMap = Maps.newHashMap(); + for (int i = 0; i < remoteColumnNames.size(); i++) { + remoteColumnNamesMap.put(localColumnNames.get(i), remoteColumnNames.get(i)); + } + return Optional.of(new JdbcSchemaCacheValue(columns, remoteColumnNamesMap)); } private JdbcTable toJdbcTable() { List schema = getFullSchema(); JdbcExternalCatalog jdbcCatalog = (JdbcExternalCatalog) catalog; - String fullDbName = this.dbName + "." + this.name; - JdbcTable jdbcTable = new JdbcTable(this.id, fullDbName, schema, TableType.JDBC_EXTERNAL_TABLE); - jdbcCatalog.configureJdbcTable(jdbcTable, fullDbName); + String fullTableName = this.dbName + "." + this.name; + JdbcTable jdbcTable = new JdbcTable(this.id, fullTableName, schema, TableType.JDBC_EXTERNAL_TABLE); + jdbcCatalog.configureJdbcTable(jdbcTable, fullTableName); // Set remote properties - jdbcTable.setRemoteDatabaseName(jdbcCatalog.getJdbcClient().getRemoteDatabaseName(this.dbName)); - jdbcTable.setRemoteTableName(jdbcCatalog.getJdbcClient().getRemoteTableName(this.dbName, this.name)); - jdbcTable.setRemoteColumnNames(jdbcCatalog.getJdbcClient().getRemoteColumnNames(this.dbName, this.name)); + jdbcTable.setRemoteDatabaseName(((ExternalDatabase) this.getDatabase()).getRemoteName()); + jdbcTable.setRemoteTableName(this.getRemoteName()); + Map remoteColumnNames = Maps.newHashMap(); + Optional schemaCacheValue = getSchemaCacheValue(); + for (Column column : schema) { + String remoteColumnName = schemaCacheValue.map(value -> ((JdbcSchemaCacheValue) value) + .getremoteColumnName(column.getName())).orElse(column.getName()); + remoteColumnNames.put(column.getName(), remoteColumnName); + } + jdbcTable.setRemoteColumnNames(remoteColumnNames); return jdbcTable; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcIdentifierMapping.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcSchemaCacheValue.java similarity index 51% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcIdentifierMapping.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcSchemaCacheValue.java index 20a74724b3e496..4e21af4fabd99e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcIdentifierMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcSchemaCacheValue.java @@ -17,29 +17,21 @@ package org.apache.doris.datasource.jdbc; -import org.apache.doris.datasource.jdbc.client.JdbcClient; -import org.apache.doris.datasource.mapping.IdentifierMapping; +import org.apache.doris.catalog.Column; +import org.apache.doris.datasource.SchemaCacheValue; -public class JdbcIdentifierMapping extends IdentifierMapping { - private final JdbcClient jdbcClient; +import java.util.List; +import java.util.Map; - public JdbcIdentifierMapping(boolean isLowerCaseMetaNames, String metaNamesMapping, JdbcClient jdbcClient) { - super(isLowerCaseMetaNames, metaNamesMapping); - this.jdbcClient = jdbcClient; - } - - @Override - protected void loadDatabaseNames() { - jdbcClient.getDatabaseNameList(); - } +public class JdbcSchemaCacheValue extends SchemaCacheValue { + private Map remoteColumnNamesMap; - @Override - protected void loadTableNames(String localDbName) { - jdbcClient.getTablesNameList(localDbName); + public JdbcSchemaCacheValue(List schema, Map remoteColumnNamesMap) { + super(schema); + this.remoteColumnNamesMap = remoteColumnNamesMap; } - @Override - protected void loadColumnNames(String localDbName, String localTableName) { - jdbcClient.getColumnsFromJdbc(localDbName, localTableName); + public String getremoteColumnName(String columnName) { + return remoteColumnNamesMap.get(columnName); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java index 1ab32efddcc1ba..51f55ea24ba226 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java @@ -24,7 +24,6 @@ import org.apache.doris.cloud.security.SecurityChecker; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.Util; -import org.apache.doris.datasource.jdbc.JdbcIdentifierMapping; import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; import com.google.common.collect.ImmutableSet; @@ -63,11 +62,8 @@ public abstract class JdbcClient { protected ClassLoader classLoader = null; protected HikariDataSource dataSource = null; protected boolean isOnlySpecifiedDatabase; - protected boolean isLowerCaseMetaNames; - protected String metaNamesMapping; protected Map includeDatabaseMap; protected Map excludeDatabaseMap; - protected JdbcIdentifierMapping jdbcLowerCaseMetaMatching; public static JdbcClient createJdbcClient(JdbcClientConfig jdbcClientConfig) { String dbType = parseDbType(jdbcClientConfig.getJdbcUrl()); @@ -104,8 +100,6 @@ protected JdbcClient(JdbcClientConfig jdbcClientConfig) { this.catalogName = jdbcClientConfig.getCatalog(); this.jdbcUser = jdbcClientConfig.getUser(); this.isOnlySpecifiedDatabase = Boolean.parseBoolean(jdbcClientConfig.getOnlySpecifiedDatabase()); - this.isLowerCaseMetaNames = Boolean.parseBoolean(jdbcClientConfig.getIsLowerCaseMetaNames()); - this.metaNamesMapping = jdbcClientConfig.getMetaNamesMapping(); this.includeDatabaseMap = Optional.ofNullable(jdbcClientConfig.getIncludeDatabaseMap()).orElse(Collections.emptyMap()); this.excludeDatabaseMap = @@ -114,7 +108,6 @@ protected JdbcClient(JdbcClientConfig jdbcClientConfig) { this.dbType = parseDbType(jdbcUrl); initializeClassLoader(jdbcClientConfig); initializeDataSource(jdbcClientConfig); - this.jdbcLowerCaseMetaMatching = new JdbcIdentifierMapping(isLowerCaseMetaNames, metaNamesMapping, this); } protected void setJdbcDriverSystemProperties() { @@ -174,6 +167,7 @@ public static String parseDbType(String jdbcUrl) { public void closeClient() { dataSource.close(); + dataSource = null; } public Connection getConnection() throws JdbcClientException { @@ -312,10 +306,9 @@ public List getDatabaseNameList() { /** * get all tables of one database */ - public List getTablesNameList(String localDbName) { + public List getTablesNameList(String remoteDbName) { List remoteTablesNames = Lists.newArrayList(); String[] tableTypes = getTableTypes(); - String remoteDbName = getRemoteDatabaseName(localDbName); processTable(remoteDbName, null, tableTypes, (rs) -> { try { while (rs.next()) { @@ -325,14 +318,12 @@ public List getTablesNameList(String localDbName) { throw new JdbcClientException("failed to get all tables for remote database: `%s`", e, remoteDbName); } }); - return filterTableNames(remoteDbName, remoteTablesNames); + return remoteTablesNames; } - public boolean isTableExist(String localDbName, String localTableName) { + public boolean isTableExist(String remoteDbName, String remoteTableName) { final boolean[] isExist = {false}; String[] tableTypes = getTableTypes(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); processTable(remoteDbName, remoteTableName, tableTypes, (rs) -> { try { if (rs.next()) { @@ -349,12 +340,10 @@ public boolean isTableExist(String localDbName, String localTableName) { /** * get all columns of one table */ - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); @@ -381,21 +370,7 @@ public List getColumnsFromJdbc(String localDbName, String localTableName field.isAllowNull(), field.getRemarks(), true, -1)); } - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); - return filterColumnName(remoteDbName, remoteTableName, dorisTableSchema); - } - - public String getRemoteDatabaseName(String localDbname) { - return jdbcLowerCaseMetaMatching.getRemoteDatabaseName(localDbname); - } - - public String getRemoteTableName(String localDbName, String localTableName) { - return jdbcLowerCaseMetaMatching.getRemoteTableName(localDbName, localTableName); - } - - public Map getRemoteColumnNames(String localDbName, String localTableName) { - return jdbcLowerCaseMetaMatching.getRemoteColumnNames(localDbName, localTableName); + return dorisTableSchema; } // protected methods, for subclass to override @@ -454,7 +429,7 @@ protected List filterDatabaseNames(List remoteDbNames) { } filteredDatabaseNames.add(databaseName); } - return jdbcLowerCaseMetaMatching.setDatabaseNameMapping(filteredDatabaseNames); + return filteredDatabaseNames; } protected Set getFilterInternalDatabases() { @@ -465,14 +440,6 @@ protected Set getFilterInternalDatabases() { .build(); } - protected List filterTableNames(String remoteDbName, List remoteTableNames) { - return jdbcLowerCaseMetaMatching.setTableNameMapping(remoteDbName, remoteTableNames); - } - - protected List filterColumnName(String remoteDbName, String remoteTableName, List remoteColumns) { - return jdbcLowerCaseMetaMatching.setColumnNameMapping(remoteDbName, remoteTableName, remoteColumns); - } - protected abstract Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema); protected Type createDecimalOrStringType(int precision, int scale) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java index 7ba393e0d0aae6..086a8a5f393614 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java @@ -87,12 +87,10 @@ protected ResultSet getRemoteColumns(DatabaseMetaData databaseMetaData, String c } @Override - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java index b78589faa77380..f4c331cf35779c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java @@ -134,12 +134,10 @@ protected ResultSet getRemoteColumns(DatabaseMetaData databaseMetaData, String c * get all columns of one table */ @Override - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java index 9968de79ab3a7d..adffd06c244e54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java @@ -49,12 +49,10 @@ public String getTestQuery() { } @Override - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java index 481e5ea5e490c6..a8e62fc2893f8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java @@ -49,12 +49,10 @@ protected JdbcPostgreSQLClient(JdbcClientConfig jdbcClientConfig) { } @Override - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalDatabase.java index 59a7ace0dca355..32e389f52d3069 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalDatabase.java @@ -23,13 +23,16 @@ public class LakeSoulExternalDatabase extends ExternalDatabase { - public LakeSoulExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.LAKESOUL); + public LakeSoulExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.LAKESOUL); } @Override - protected LakeSoulExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new LakeSoulExternalTable(tblId, tableName, name, (LakeSoulExternalCatalog) catalog); + public LakeSoulExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new LakeSoulExternalTable(tblId, localTableName, remoteTableName, (LakeSoulExternalCatalog) extCatalog, + (LakeSoulExternalDatabase) db); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java index 9dd2f4811e98f0..e5e0447f1cb647 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java @@ -56,8 +56,9 @@ public class LakeSoulExternalTable extends ExternalTable { public final String tableId; - public LakeSoulExternalTable(long id, String name, String dbName, LakeSoulExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.LAKESOUl_EXTERNAL_TABLE); + public LakeSoulExternalTable(long id, String name, String remoteName, LakeSoulExternalCatalog catalog, + LakeSoulExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.LAKESOUl_EXTERNAL_TABLE); TableInfo tableInfo = getLakeSoulTableInfo(); if (tableInfo == null) { throw new RuntimeException(String.format("LakeSoul table %s.%s does not exist", dbName, name)); @@ -88,9 +89,9 @@ private Type arrowFiledToDorisType(Field field) { return Type.BIGINT; default: throw new IllegalArgumentException("Invalid integer bit width: " - + type.getBitWidth() - + " for LakeSoul table: " - + getTableIdentifier()); + + type.getBitWidth() + + " for LakeSoul table: " + + getTableIdentifier()); } } else if (dt instanceof ArrowType.FloatingPoint) { ArrowType.FloatingPoint type = (ArrowType.FloatingPoint) dt; @@ -101,16 +102,16 @@ private Type arrowFiledToDorisType(Field field) { return Type.DOUBLE; default: throw new IllegalArgumentException("Invalid floating point precision: " - + type.getPrecision() - + " for LakeSoul table: " - + getTableIdentifier()); + + type.getPrecision() + + " for LakeSoul table: " + + getTableIdentifier()); } } else if (dt instanceof ArrowType.Utf8) { return Type.STRING; } else if (dt instanceof ArrowType.Decimal) { ArrowType.Decimal decimalType = (ArrowType.Decimal) dt; return ScalarType.createDecimalType(PrimitiveType.DECIMAL64, decimalType.getPrecision(), - decimalType.getScale()); + decimalType.getScale()); } else if (dt instanceof ArrowType.Date) { return ScalarType.createDateV2Type(); } else if (dt instanceof ArrowType.Timestamp) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java index 363ef351152a39..9199ff985116c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java @@ -17,314 +17,11 @@ package org.apache.doris.datasource.mapping; -import org.apache.doris.catalog.Column; -import org.apache.doris.qe.GlobalVariable; +public interface IdentifierMapping { -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; + String fromRemoteDatabaseName(String remoteDatabaseName); -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; + String fromRemoteTableName(String remoteDatabaseName, String remoteTableName); -public abstract class IdentifierMapping { - private static final Logger LOG = LogManager.getLogger(IdentifierMapping.class); - - private final ObjectMapper mapper = new ObjectMapper(); - private final ConcurrentHashMap localDBToRemoteDB = new ConcurrentHashMap<>(); - private final ConcurrentHashMap> localTableToRemoteTable - = new ConcurrentHashMap<>(); - private final ConcurrentHashMap>> - localColumnToRemoteColumn = new ConcurrentHashMap<>(); - - private final AtomicBoolean dbNamesLoaded = new AtomicBoolean(false); - private final ConcurrentHashMap tableNamesLoadedMap = new ConcurrentHashMap<>(); - private final ConcurrentHashMap> columnNamesLoadedMap - = new ConcurrentHashMap<>(); - - private final boolean isLowerCaseMetaNames; - private final String metaNamesMapping; - - public IdentifierMapping(boolean isLowerCaseMetaNames, String metaNamesMapping) { - this.isLowerCaseMetaNames = isLowerCaseMetaNames; - this.metaNamesMapping = metaNamesMapping; - } - - public List setDatabaseNameMapping(List remoteDatabaseNames) { - JsonNode databasesNode = readAndParseJson(metaNamesMapping, "databases"); - - Map databaseNameMapping = Maps.newTreeMap(); - if (databasesNode.isArray()) { - for (JsonNode node : databasesNode) { - String remoteDatabase = node.path("remoteDatabase").asText(); - String mapping = node.path("mapping").asText(); - databaseNameMapping.put(remoteDatabase, mapping); - } - } - - Map> result = nameListToMapping(remoteDatabaseNames, localDBToRemoteDB, - databaseNameMapping, isLowerCaseMetaNames); - List localDatabaseNames = result.get("localNames"); - List conflictNames = result.get("conflictNames"); - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict database/schema names found when lower_case_meta_names is true: " + conflictNames - + ". Please set lower_case_meta_names to false or" - + " use meta_name_mapping to specify the names."); - } - return localDatabaseNames; - } - - public List setTableNameMapping(String remoteDbName, List remoteTableNames) { - JsonNode tablesNode = readAndParseJson(metaNamesMapping, "tables"); - - Map tableNameMapping = Maps.newTreeMap(); - if (tablesNode.isArray()) { - for (JsonNode node : tablesNode) { - String remoteDatabase = node.path("remoteDatabase").asText(); - if (remoteDbName.equals(remoteDatabase)) { - String remoteTable = node.path("remoteTable").asText(); - String mapping = node.path("mapping").asText(); - tableNameMapping.put(remoteTable, mapping); - } - } - } - - localTableToRemoteTable.putIfAbsent(remoteDbName, new ConcurrentHashMap<>()); - - List localTableNames; - List conflictNames; - - if (GlobalVariable.lowerCaseTableNames == 1) { - Map> result = nameListToMapping(remoteTableNames, - localTableToRemoteTable.get(remoteDbName), - tableNameMapping, true); - localTableNames = result.get("localNames"); - conflictNames = result.get("conflictNames"); - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict table names found in remote database/schema: " + remoteDbName - + " when lower_case_table_names is 1: " + conflictNames - + ". Please use meta_name_mapping to specify the names."); - } - } else { - Map> result = nameListToMapping(remoteTableNames, - localTableToRemoteTable.get(remoteDbName), - tableNameMapping, isLowerCaseMetaNames); - localTableNames = result.get("localNames"); - conflictNames = result.get("conflictNames"); - - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict table names found in remote database/schema: " + remoteDbName - + "when lower_case_meta_names is true: " + conflictNames - + ". Please set lower_case_meta_names to false or" - + " use meta_name_mapping to specify the table names."); - } - } - return localTableNames; - } - - public List setColumnNameMapping(String remoteDbName, String remoteTableName, List remoteColumns) { - JsonNode tablesNode = readAndParseJson(metaNamesMapping, "columns"); - - Map columnNameMapping = Maps.newTreeMap(); - if (tablesNode.isArray()) { - for (JsonNode node : tablesNode) { - String remoteDatabase = node.path("remoteDatabase").asText(); - String remoteTable = node.path("remoteTable").asText(); - if (remoteDbName.equals(remoteDatabase) && remoteTable.equals(remoteTableName)) { - String remoteColumn = node.path("remoteColumn").asText(); - String mapping = node.path("mapping").asText(); - columnNameMapping.put(remoteColumn, mapping); - } - } - } - localColumnToRemoteColumn.putIfAbsent(remoteDbName, new ConcurrentHashMap<>()); - localColumnToRemoteColumn.get(remoteDbName).putIfAbsent(remoteTableName, new ConcurrentHashMap<>()); - - List localColumnNames; - List conflictNames; - - // Get the name from localColumns and save it to List - List remoteColumnNames = Lists.newArrayList(); - for (Column remoteColumn : remoteColumns) { - remoteColumnNames.add(remoteColumn.getName()); - } - - Map> result = nameListToMapping(remoteColumnNames, - localColumnToRemoteColumn.get(remoteDbName).get(remoteTableName), - columnNameMapping, isLowerCaseMetaNames); - localColumnNames = result.get("localNames"); - conflictNames = result.get("conflictNames"); - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict column names found in remote database/schema: " + remoteDbName - + " in remote table: " + remoteTableName - + " when lower_case_meta_names is true: " + conflictNames - + ". Please set lower_case_meta_names to false or" - + " use meta_name_mapping to specify the column names."); - } - // Replace the name in remoteColumns with localColumnNames - for (int i = 0; i < remoteColumns.size(); i++) { - remoteColumns.get(i).setName(localColumnNames.get(i)); - } - return remoteColumns; - } - - public String getRemoteDatabaseName(String localDbName) { - return getRequiredMapping(localDBToRemoteDB, localDbName, "database", this::loadDatabaseNamesIfNeeded, - localDbName); - } - - public String getRemoteTableName(String localDbName, String localTableName) { - String remoteDbName = getRemoteDatabaseName(localDbName); - Map tableMap = localTableToRemoteTable.computeIfAbsent(remoteDbName, - k -> new ConcurrentHashMap<>()); - return getRequiredMapping(tableMap, localTableName, "table", () -> loadTableNamesIfNeeded(localDbName), - localTableName); - } - - public Map getRemoteColumnNames(String localDbName, String localTableName) { - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); - ConcurrentHashMap> tableColumnMap - = localColumnToRemoteColumn.computeIfAbsent(remoteDbName, k -> new ConcurrentHashMap<>()); - Map columnMap = tableColumnMap.computeIfAbsent(remoteTableName, k -> new ConcurrentHashMap<>()); - if (columnMap.isEmpty()) { - LOG.info("Column name mapping missing, loading column names for localDbName: {}, localTableName: {}", - localDbName, localTableName); - loadColumnNamesIfNeeded(localDbName, localTableName); - columnMap = tableColumnMap.get(remoteTableName); - } - if (columnMap.isEmpty()) { - LOG.warn("No remote column found for localTableName: {}. Please refresh this catalog.", localTableName); - throw new RuntimeException( - "No remote column found for localTableName: " + localTableName + ". Please refresh this catalog."); - } - return columnMap; - } - - - private void loadDatabaseNamesIfNeeded() { - if (dbNamesLoaded.compareAndSet(false, true)) { - try { - loadDatabaseNames(); - } catch (Exception e) { - dbNamesLoaded.set(false); // Reset on failure - LOG.warn("Error loading database names", e); - } - } - } - - private void loadTableNamesIfNeeded(String localDbName) { - AtomicBoolean isLoaded = tableNamesLoadedMap.computeIfAbsent(localDbName, k -> new AtomicBoolean(false)); - if (isLoaded.compareAndSet(false, true)) { - try { - loadTableNames(localDbName); - } catch (Exception e) { - tableNamesLoadedMap.get(localDbName).set(false); // Reset on failure - LOG.warn("Error loading table names for localDbName: {}", localDbName, e); - } - } - } - - private void loadColumnNamesIfNeeded(String localDbName, String localTableName) { - columnNamesLoadedMap.putIfAbsent(localDbName, new ConcurrentHashMap<>()); - AtomicBoolean isLoaded = columnNamesLoadedMap.get(localDbName) - .computeIfAbsent(localTableName, k -> new AtomicBoolean(false)); - if (isLoaded.compareAndSet(false, true)) { - try { - loadColumnNames(localDbName, localTableName); - } catch (Exception e) { - columnNamesLoadedMap.get(localDbName).get(localTableName).set(false); // Reset on failure - LOG.warn("Error loading column names for localDbName: {}, localTableName: {}", localDbName, - localTableName, e); - } - } - } - - private V getRequiredMapping(Map map, K key, String typeName, Runnable loadIfNeeded, - String entityName) { - if (map.isEmpty() || !map.containsKey(key) || map.get(key) == null) { - LOG.info("{} mapping missing, loading for {}: {}", typeName, typeName, entityName); - loadIfNeeded.run(); - } - V value = map.get(key); - if (value == null) { - LOG.warn("No remote {} found for {}: {}. Please refresh this catalog.", typeName, typeName, entityName); - throw new RuntimeException("No remote " + typeName + " found for " + typeName + ": " + entityName - + ". Please refresh this catalog."); - } - return value; - } - - // Load the database name from the data source. - // In the corresponding getDatabaseNameList(), setDatabaseNameMapping() must be used to update the mapping. - protected abstract void loadDatabaseNames(); - - // Load the table names for the specified database from the data source. - // In the corresponding getTableNameList(), setTableNameMapping() must be used to update the mapping. - protected abstract void loadTableNames(String localDbName); - - // Load the column names for a specified table in a database from the data source. - // In the corresponding getColumnNameList(), setColumnNameMapping() must be used to update the mapping. - protected abstract void loadColumnNames(String localDbName, String localTableName); - - private JsonNode readAndParseJson(String jsonPath, String nodeName) { - JsonNode rootNode; - try { - rootNode = mapper.readTree(jsonPath); - return rootNode.path(nodeName); - } catch (JsonProcessingException e) { - throw new RuntimeException("parse meta_names_mapping property error", e); - } - } - - private Map> nameListToMapping(List remoteNames, - ConcurrentHashMap localNameToRemoteName, - Map nameMapping, boolean isLowerCaseMetaNames) { - List filteredDatabaseNames = Lists.newArrayList(); - Set lowerCaseNames = Sets.newHashSet(); - Map> nameMap = Maps.newHashMap(); - List conflictNames = Lists.newArrayList(); - - for (String name : remoteNames) { - String mappedName = nameMapping.getOrDefault(name, name); - String localName = isLowerCaseMetaNames ? mappedName.toLowerCase() : mappedName; - - // Use computeIfAbsent to ensure atomicity - localNameToRemoteName.computeIfAbsent(localName, k -> name); - - if (isLowerCaseMetaNames && !lowerCaseNames.add(localName)) { - if (nameMap.containsKey(localName)) { - nameMap.get(localName).add(mappedName); - } - } else { - nameMap.putIfAbsent(localName, Lists.newArrayList(Collections.singletonList(mappedName))); - } - - filteredDatabaseNames.add(localName); - } - - for (List conflictNameList : nameMap.values()) { - if (conflictNameList.size() > 1) { - conflictNames.addAll(conflictNameList); - } - } - - Map> result = Maps.newConcurrentMap(); - result.put("localNames", filteredDatabaseNames); - result.put("conflictNames", conflictNames); - return result; - } + String fromRemoteColumnName(String remoteDatabaseName, String remoteTableName, String remoteColumnNames); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/JdbcIdentifierMapping.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/JdbcIdentifierMapping.java new file mode 100644 index 00000000000000..f584c9acee34e4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/JdbcIdentifierMapping.java @@ -0,0 +1,345 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.mapping; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; +import java.util.Set; + +public class JdbcIdentifierMapping implements IdentifierMapping { + private static final Logger LOG = LogManager.getLogger(JdbcIdentifierMapping.class); + + private final ObjectMapper mapper = new ObjectMapper(); + private final boolean isLowerCaseTableNames; + private final boolean isLowerCaseMetaNames; + private final String metaNamesMapping; + + public JdbcIdentifierMapping(boolean isLowerCaseTableNames, boolean isLowerCaseMetaNames, String metaNamesMapping) { + this.isLowerCaseTableNames = isLowerCaseTableNames; + this.isLowerCaseMetaNames = isLowerCaseMetaNames; + this.metaNamesMapping = metaNamesMapping; + validateMappings(); + } + + private boolean isMappingInvalid() { + return metaNamesMapping == null || metaNamesMapping.isEmpty(); + } + + @Override + public String fromRemoteDatabaseName(String remoteDatabaseName) { + if (!isLowerCaseMetaNames && isMappingInvalid()) { + return remoteDatabaseName; + } + JsonNode databasesNode = readAndParseJson(metaNamesMapping, "databases"); + + Map databaseNameMapping = Maps.newHashMap(); + if (databasesNode.isArray()) { + for (JsonNode node : databasesNode) { + String remoteDatabase = node.path("remoteDatabase").asText(); + String mapping = applyLowerCaseIfNeeded(node.path("mapping").asText()); + databaseNameMapping.put(remoteDatabase, mapping); + } + } + return getMappedName(remoteDatabaseName, databaseNameMapping); + } + + @Override + public String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) { + if (!isLowerCaseMetaNames && isMappingInvalid()) { + return remoteTableName; + } + JsonNode tablesNode = readAndParseJson(metaNamesMapping, "tables"); + + Map tableNameMapping = Maps.newHashMap(); + if (tablesNode.isArray()) { + for (JsonNode node : tablesNode) { + String remoteDatabase = node.path("remoteDatabase").asText(); + if (remoteDatabaseName.equals(remoteDatabase)) { + String remoteTable = node.path("remoteTable").asText(); + String mapping = applyLowerCaseIfNeeded(node.path("mapping").asText()); + tableNameMapping.put(remoteTable, mapping); + } + } + } + return getMappedName(remoteTableName, tableNameMapping); + } + + @Override + public String fromRemoteColumnName(String remoteDatabaseName, String remoteTableName, String remoteColumnName) { + if (!isLowerCaseMetaNames && isMappingInvalid()) { + return remoteColumnName; + } + JsonNode columnsNode = readAndParseJson(metaNamesMapping, "columns"); + + Map columnNameMapping = Maps.newHashMap(); + if (columnsNode.isArray()) { + for (JsonNode node : columnsNode) { + String remoteDatabase = node.path("remoteDatabase").asText(); + String remoteTable = node.path("remoteTable").asText(); + if (remoteDatabaseName.equals(remoteDatabase) && remoteTableName.equals(remoteTable)) { + String remoteColumn = node.path("remoteColumn").asText(); + String mapping = applyLowerCaseIfNeeded(node.path("mapping").asText()); + columnNameMapping.put(remoteColumn, mapping); + } + } + } + return getMappedName(remoteColumnName, columnNameMapping); + } + + private String getMappedName(String name, Map nameMapping) { + String mappedName = nameMapping.getOrDefault(name, name); + return isLowerCaseMetaNames ? mappedName.toLowerCase() : mappedName; + } + + private JsonNode readAndParseJson(String jsonPath, String nodeName) { + try { + JsonNode rootNode = mapper.readTree(jsonPath); + return rootNode.path(nodeName); + } catch (JsonProcessingException e) { + throw new RuntimeException("JSON format is incorrect, please check the metaNamesMapping property", e); + } + } + + private void validateMappings() { + Map> duplicateErrors = Maps.newLinkedHashMap(); + try { + JsonNode rootNode = mapper.readTree(metaNamesMapping); + + Map> dbMappingCheck = Maps.newHashMap(); + Map>> tableMappingCheck = Maps.newHashMap(); + Map>>> columnMappingCheck = Maps.newHashMap(); + + Set dbKeySet = Sets.newHashSet(); + Map> tableKeySet = Maps.newHashMap(); + Map>> columnKeySet = Maps.newHashMap(); + + validateNode(rootNode.path("databases"), "databases", duplicateErrors, dbMappingCheck, tableMappingCheck, + columnMappingCheck, dbKeySet, tableKeySet, columnKeySet); + validateNode(rootNode.path("tables"), "tables", duplicateErrors, dbMappingCheck, tableMappingCheck, + columnMappingCheck, dbKeySet, tableKeySet, columnKeySet); + validateNode(rootNode.path("columns"), "columns", duplicateErrors, dbMappingCheck, tableMappingCheck, + columnMappingCheck, dbKeySet, tableKeySet, columnKeySet); + + if (!duplicateErrors.isEmpty()) { + StringBuilder errorBuilder = new StringBuilder("Duplicate mapping found:\n"); + duplicateErrors.forEach((key, value) -> { + errorBuilder.append(key).append(":\n"); + value.forEach(error -> errorBuilder.append(" - ").append(error).append("\n")); + }); + throw new RuntimeException(errorBuilder.toString()); + } + } catch (JsonProcessingException e) { + throw new RuntimeException("The JSON format is incorrect, please check the metaNamesMapping property", e); + } + } + + private void validateNode(JsonNode nodes, + String nodeType, + Map> duplicateErrors, + Map> dbMappingCheck, + Map>> tableMappingCheck, + Map>>> columnMappingCheck, + Set dbKeySet, + Map> tableKeySet, + Map>> columnKeySet) { + Map mappingSet = Maps.newHashMap(); + if (nodes.isArray()) { + for (JsonNode node : nodes) { + String remoteKey; + String remoteDb = null; + String remoteTbl = null; + switch (nodeType) { + case "databases": + remoteKey = node.path("remoteDatabase").asText(); + checkDuplicateRemoteDatabaseKey(remoteKey, dbKeySet, duplicateErrors); + break; + case "tables": + remoteDb = node.path("remoteDatabase").asText(); + remoteKey = node.path("remoteTable").asText(); + checkDuplicateRemoteTableKey(remoteDb, remoteKey, tableKeySet, duplicateErrors); + break; + case "columns": + remoteDb = node.path("remoteDatabase").asText(); + remoteTbl = node.path("remoteTable").asText(); + remoteKey = node.path("remoteColumn").asText(); + checkDuplicateRemoteColumnKey(remoteDb, remoteTbl, remoteKey, columnKeySet, duplicateErrors); + break; + default: + throw new IllegalArgumentException("Unknown type: " + nodeType); + } + + String mapping = node.path("mapping").asText(); + + String existed = mappingSet.get(mapping); + if (existed != null) { + duplicateErrors + .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) + .add(String.format("Remote name: %s, duplicate mapping: %s (original: %s)", + remoteKey, mapping, existed)); + } else { + mappingSet.put(mapping, remoteKey); + } + + switch (nodeType) { + case "databases": + if (isLowerCaseMetaNames) { + checkCaseConflictForDatabase(mapping, dbMappingCheck, duplicateErrors, nodeType, remoteKey); + } + break; + case "tables": + if (isLowerCaseMetaNames || isLowerCaseTableNames) { + checkCaseConflictForTable(remoteDb, mapping, tableMappingCheck, duplicateErrors, + nodeType, remoteKey); + } + break; + case "columns": + checkCaseConflictForColumn(remoteDb, remoteTbl, mapping, columnMappingCheck, duplicateErrors, + nodeType, remoteKey); + break; + default: + break; + } + } + } + } + + private void checkDuplicateRemoteDatabaseKey(String remoteDatabase, + Set dbKeySet, + Map> duplicateErrors) { + if (dbKeySet == null) { + return; + } + if (!dbKeySet.add(remoteDatabase)) { + duplicateErrors + .computeIfAbsent("databases", k -> Sets.newLinkedHashSet()) + .add(String.format("Duplicate remoteDatabase found: %s", remoteDatabase)); + } + } + + private void checkDuplicateRemoteTableKey(String remoteDb, + String remoteTable, + Map> tableKeySet, + Map> duplicateErrors) { + if (tableKeySet == null) { + return; + } + Set tables = tableKeySet.computeIfAbsent(remoteDb, k -> Sets.newHashSet()); + if (!tables.add(remoteTable)) { + duplicateErrors + .computeIfAbsent("tables", k -> Sets.newLinkedHashSet()) + .add(String.format("Duplicate remoteTable found in database %s: %s", remoteDb, remoteTable)); + } + } + + private void checkDuplicateRemoteColumnKey(String remoteDb, + String remoteTbl, + String remoteColumn, + Map>> columnKeySet, + Map> duplicateErrors) { + if (columnKeySet == null) { + return; + } + Map> tblMap = columnKeySet.computeIfAbsent(remoteDb, k -> Maps.newHashMap()); + Set columns = tblMap.computeIfAbsent(remoteTbl, k -> Sets.newHashSet()); + if (!columns.add(remoteColumn)) { + duplicateErrors + .computeIfAbsent("columns", k -> Sets.newLinkedHashSet()) + .add(String.format("Duplicate remoteColumn found in database %s, table %s: %s", + remoteDb, remoteTbl, remoteColumn)); + } + } + + private void checkCaseConflictForDatabase(String mapping, + Map> dbMappingCheck, + Map> duplicateErrors, + String nodeType, + String remoteKey) { + if (dbMappingCheck == null) { + return; + } + String lower = mapping.toLowerCase(); + Set variants = dbMappingCheck.computeIfAbsent(lower, k -> Sets.newLinkedHashSet()); + if (!variants.isEmpty() && variants.stream().noneMatch(v -> v.equals(mapping))) { + duplicateErrors + .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) + .add(String.format("Remote name: %s, case-only different mapping found: %s (existing variants: %s)", + remoteKey, mapping, variants)); + } + variants.add(mapping); + } + + private void checkCaseConflictForTable(String remoteDb, + String mapping, + Map>> tableMappingCheck, + Map> duplicateErrors, + String nodeType, + String remoteKey) { + if (tableMappingCheck == null || remoteDb == null) { + return; + } + Map> dbMap = tableMappingCheck.computeIfAbsent(remoteDb, k -> Maps.newHashMap()); + String lower = mapping.toLowerCase(); + Set variants = dbMap.computeIfAbsent(lower, k -> Sets.newLinkedHashSet()); + if (!variants.isEmpty() && variants.stream().noneMatch(v -> v.equals(mapping))) { + duplicateErrors + .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) + .add(String.format("Remote name: %s (database: %s), " + + "case-only different mapping found: %s (existing variants: %s)", + remoteKey, remoteDb, mapping, variants)); + } + variants.add(mapping); + } + + private void checkCaseConflictForColumn(String remoteDb, + String remoteTbl, + String mapping, + Map>>> columnMappingCheck, + Map> duplicateErrors, + String nodeType, + String remoteKey) { + if (columnMappingCheck == null || remoteDb == null || remoteTbl == null) { + return; + } + Map>> dbMap = columnMappingCheck.computeIfAbsent(remoteDb, + k -> Maps.newHashMap()); + Map> tblMap = dbMap.computeIfAbsent(remoteTbl, k -> Maps.newHashMap()); + String lower = mapping.toLowerCase(); + Set variants = tblMap.computeIfAbsent(lower, k -> Sets.newLinkedHashSet()); + + if (!variants.isEmpty() && variants.stream().noneMatch(v -> v.equals(mapping))) { + duplicateErrors + .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) + .add(String.format( + "Remote name: %s (database: %s, table: %s), " + + "case-only different mapping found: %s (existing variants: %s)", + remoteKey, remoteDb, remoteTbl, mapping, variants)); + } + variants.add(mapping); + } + + private String applyLowerCaseIfNeeded(String value) { + return isLowerCaseMetaNames ? value.toLowerCase() : value; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java index 0a100e3495ff3d..7cd38b9d13a007 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java @@ -32,12 +32,16 @@ public class MaxComputeExternalDatabase extends ExternalDatabase columnNameToOdpsColumn = new HashMap(); + private Map columnNameToOdpsColumn = new HashMap(); @Override protected synchronized void makeSureInitialized() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java index 6e4198186e82e3..fffa0a04e42850 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.metacache; import org.apache.doris.common.CacheFactory; +import org.apache.doris.common.Pair; import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.LoadingCache; @@ -27,12 +28,14 @@ import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; public class MetaCache { - private LoadingCache> namesCache; + private LoadingCache>> namesCache; private Map idToName = Maps.newConcurrentMap(); private LoadingCache> metaObjCache; @@ -43,7 +46,7 @@ public MetaCache(String name, OptionalLong expireAfterWriteSec, OptionalLong refreshAfterWriteSec, long maxSize, - CacheLoader> namesCacheLoader, + CacheLoader>> namesCacheLoader, CacheLoader> metaObjCacheLoader, RemovalListener> removalListener) { this.name = name; @@ -71,7 +74,15 @@ public MetaCache(String name, } public List listNames() { - return namesCache.get(""); + return Objects.requireNonNull(namesCache.get("")).stream().map(Pair::value).collect(Collectors.toList()); + } + + public String getRemoteName(String localName) { + return Objects.requireNonNull(namesCache.getIfPresent("")).stream() + .filter(pair -> pair.value().equals(localName)) + .map(Pair::key) + .findFirst() + .orElse(null); } public Optional getMetaObj(String name, long id) { @@ -94,9 +105,9 @@ public void updateCache(String objName, T obj, long id) { metaObjCache.put(objName, Optional.of(obj)); namesCache.asMap().compute("", (k, v) -> { if (v == null) { - return Lists.newArrayList(objName); + return Lists.newArrayList(Pair.of(objName, objName)); } else { - v.add(objName); + v.add(Pair.of(objName, objName)); return v; } }); @@ -121,5 +132,4 @@ public void invalidateAll() { metaObjCache.invalidateAll(); idToName.clear(); } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java index 50265f77463428..fdbad45c5d0af9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java @@ -23,12 +23,15 @@ public class PaimonExternalDatabase extends ExternalDatabase { - public PaimonExternalDatabase(ExternalCatalog extCatalog, Long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.PAIMON); + public PaimonExternalDatabase(ExternalCatalog extCatalog, Long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.PAIMON); } @Override - protected PaimonExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new PaimonExternalTable(tblId, tableName, name, (PaimonExternalCatalog) extCatalog); + public PaimonExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new PaimonExternalTable(tblId, localTableName, remoteTableName, (PaimonExternalCatalog) extCatalog, + (PaimonExternalDatabase) db); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 8ddfca886d9fe9..7264d83621d8be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -75,8 +75,9 @@ public class PaimonExternalTable extends ExternalTable implements MTMVRelatedTab private final Table paimonTable; - public PaimonExternalTable(long id, String name, String dbName, PaimonExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.PAIMON_EXTERNAL_TABLE); + public PaimonExternalTable(long id, String name, String remoteName, PaimonExternalCatalog catalog, + PaimonExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.PAIMON_EXTERNAL_TABLE); this.paimonTable = catalog.getPaimonTable(dbName, name); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java index 2cf1f57d0e4672..0c11dc8f9400a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java @@ -23,12 +23,15 @@ public class TestExternalDatabase extends ExternalDatabase { - public TestExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.TEST); + public TestExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.TEST); } @Override - protected TestExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new TestExternalTable(tblId, tableName, name, (TestExternalCatalog) extCatalog); + public TestExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new TestExternalTable(tblId, localTableName, remoteTableName, (TestExternalCatalog) extCatalog, + (TestExternalDatabase) db); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java index 6da0981b97ef54..6d08b10403bb76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java @@ -33,8 +33,8 @@ public class TestExternalTable extends ExternalTable { private static final Logger LOG = LogManager.getLogger(TestExternalTable.class); - public TestExternalTable(long id, String name, String dbName, TestExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.TEST_EXTERNAL_TABLE); + public TestExternalTable(long id, String name, String remoteName, TestExternalCatalog catalog, TestExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.TEST_EXTERNAL_TABLE); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalDatabase.java index 33652e7e945f41..31ada04eeb68e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalDatabase.java @@ -22,12 +22,16 @@ import org.apache.doris.datasource.InitDatabaseLog.Type; public class TrinoConnectorExternalDatabase extends ExternalDatabase { - public TrinoConnectorExternalDatabase(ExternalCatalog extCatalog, Long id, String name) { - super(extCatalog, id, name, Type.TRINO_CONNECTOR); + public TrinoConnectorExternalDatabase(ExternalCatalog extCatalog, Long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, Type.TRINO_CONNECTOR); } @Override - protected TrinoConnectorExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new TrinoConnectorExternalTable(tblId, tableName, name, (TrinoConnectorExternalCatalog) extCatalog); + public TrinoConnectorExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new TrinoConnectorExternalTable(tblId, localTableName, remoteTableName, + (TrinoConnectorExternalCatalog) extCatalog, + (TrinoConnectorExternalDatabase) db); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalTable.java index 007ad864da3af8..a664d3889241a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalTable.java @@ -71,8 +71,9 @@ public class TrinoConnectorExternalTable extends ExternalTable { - public TrinoConnectorExternalTable(long id, String name, String dbName, TrinoConnectorExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.TRINO_CONNECTOR_EXTERNAL_TABLE); + public TrinoConnectorExternalTable(long id, String name, String remoteName, TrinoConnectorExternalCatalog catalog, + TrinoConnectorExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.TRINO_CONNECTOR_EXTERNAL_TABLE); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java index 07c7abf7ce0c26..a38b5f49fc070f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java @@ -239,8 +239,8 @@ void addConstraintLogPersistForExternalTableTest() throws Exception { Env.getCurrentEnv().changeCatalog(connectContext, "es"); EsExternalCatalog esCatalog = (EsExternalCatalog) getCatalog("es"); - EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1"); - EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog); + EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1", "es_db1"); + EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog, db); ImmutableList schema = ImmutableList.of(new Column("k1", PrimitiveType.INT)); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); @@ -302,8 +302,8 @@ void dropConstraintLogPersistForExternalTest() throws Exception { Env.getCurrentEnv().changeCatalog(connectContext, "es2"); EsExternalCatalog esCatalog = (EsExternalCatalog) getCatalog("es2"); - EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1"); - EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog); + EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1", "es_db1"); + EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog, db); ImmutableList schema = ImmutableList.of(new Column("k1", PrimitiveType.INT)); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index aa5fa313be3c91..9d36d843e9fb7c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -163,15 +163,15 @@ private void createDbAndTableForCatalog(CatalogIf catalog) { schema.add(new Column("k1", PrimitiveType.INT)); if (catalog instanceof HMSExternalCatalog) { HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; - HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hive_db1"); - HMSExternalTable tbl = new HMSExternalTable(10001, "hive_tbl1", "hive_db1", hmsCatalog); + HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hive_db1", "hive_db1"); + HMSExternalTable tbl = new HMSExternalTable(10001, "hive_tbl1", "hive_db1", hmsCatalog, db); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); hmsCatalog.addDatabaseForTest(db); } else if (catalog instanceof EsExternalCatalog) { EsExternalCatalog esCatalog = (EsExternalCatalog) catalog; - EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1"); - EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_tbl1", esCatalog); + EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1", "es_db1"); + EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_tbl1", esCatalog, db); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); esCatalog.addDatabaseForTest(db); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java index 2a6342a57affa3..b83b69df26f838 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java @@ -217,7 +217,7 @@ public Table getTable(String dbName, String tblName) { @Mock public ExternalDatabase getDbNullable(String dbName) { if (createdDbs.contains(dbName)) { - return new HMSExternalDatabase(hmsExternalCatalog, RandomUtils.nextLong(), dbName); + return new HMSExternalDatabase(hmsExternalCatalog, RandomUtils.nextLong(), dbName, dbName); } return null; } @@ -228,7 +228,7 @@ public ExternalDatabase getDbNullable(String dbName) { HMSExternalTable getTableNullable(String tableName) { for (Table table : createdTables) { if (table.getTableName().equals(tableName)) { - return new HMSExternalTable(0, tableName, mockedDbName, hmsExternalCatalog); + return new HMSExternalTable(0, tableName, tableName, hmsExternalCatalog, (HMSExternalDatabase) hmsExternalCatalog.getDbNullable(mockedDbName)); } } return null; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java index 46d3e1b897d111..8f268f19426c3e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java @@ -68,7 +68,7 @@ public void init() { new MockUp(HMSExternalCatalog.class) { @Mock public ExternalDatabase getDbNullable(String dbName) { - return new HMSExternalDatabase(mockedCatalog, 0L, "mockedDb"); + return new HMSExternalDatabase(mockedCatalog, 0L, "mockedDb", "mockedDb"); } @Mock diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java index 439f6f2aa7de21..2300ece6253b5f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java @@ -84,7 +84,7 @@ public static void beforeClass() throws Throwable { } else { icebergCatalog.setInitialized(true); } - IcebergExternalDatabase db = new IcebergExternalDatabase(icebergCatalog, 1L, dbName); + IcebergExternalDatabase db = new IcebergExternalDatabase(icebergCatalog, 1L, dbName, dbName); icebergCatalog.addDatabaseForTest(db); // context diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheFalseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheFalseTest.java new file mode 100644 index 00000000000000..acc357d85d684b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheFalseTest.java @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.analysis.CreateCatalogStmt; +import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshCatalogStmt; +import org.apache.doris.analysis.SwitchStmt; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; +import org.apache.doris.qe.GlobalVariable; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ExternalTableNameComparedLowercaseMetaCacheFalseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog + CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"use_meta_cache\" = \"false\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameComparedLowercaseMetaCacheFalseTest$ExternalTableNameComparedLowercaseProvider\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + } + + @Override + protected void beforeCluster() { + Config.lower_case_table_names = 2; + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + DropCatalogStmt stmt = (DropCatalogStmt) parseAndAnalyzeStmt("drop catalog test1"); + env.getCatalogMgr().dropCatalog(stmt); + } + + + @Test + public void testGlobalVariable() { + Assertions.assertEquals(2, GlobalVariable.lowerCaseTableNames); + } + + @Test + public void testGetTableWithOutList() { + RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + String tblName = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table1").getName(); + Assertions.assertEquals("TABLE1", tblName); + String tblName2 = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table2").getName(); + Assertions.assertEquals("TABLE2", tblName2); + } + + @Test + public void testTableNameLowerCase() { + Set tableNames = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNamesWithLock(); + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("TABLE1")); + Assertions.assertTrue(tableNames.contains("TABLE2")); + } + + private void switchTest() throws Exception { + SwitchStmt switchTest = (SwitchStmt) parseAndAnalyzeStmt("switch test1;"); + Env.getCurrentEnv().changeCatalog(connectContext, switchTest.getCatalogName()); + } + + public static class ExternalTableNameComparedLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tblSchemaMap1 = Maps.newHashMap(); + // db1 + tblSchemaMap1.put("TABLE1", Lists.newArrayList( + new Column("siteid", PrimitiveType.INT), + new Column("citycode", PrimitiveType.SMALLINT), + new Column("username", PrimitiveType.VARCHAR), + new Column("pv", PrimitiveType.BIGINT))); + tblSchemaMap1.put("TABLE2", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR), + new Column("k3", PrimitiveType.VARCHAR), + new Column("k4", PrimitiveType.INT), + new Column("k5", PrimitiveType.LARGEINT))); + MOCKED_META.put("db1", tblSchemaMap1); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheTrueTest.java similarity index 82% rename from fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseTest.java rename to fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheTrueTest.java index c48e18cb2711df..ade73f3fef2175 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheTrueTest.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.CreateCatalogStmt; import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshCatalogStmt; import org.apache.doris.analysis.SwitchStmt; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; @@ -27,6 +28,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; import org.apache.doris.qe.GlobalVariable; import org.apache.doris.utframe.TestWithFeService; @@ -39,7 +41,7 @@ import java.util.Map; import java.util.Set; -public class ExternalTableNameComparedLowercaseTest extends TestWithFeService { +public class ExternalTableNameComparedLowercaseMetaCacheTrueTest extends TestWithFeService { private static Env env; private ConnectContext rootCtx; @@ -50,8 +52,9 @@ protected void runBeforeAll() throws Exception { // 1. create test catalog CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + " \"type\" = \"test\",\n" + + " \"use_meta_cache\" = \"true\",\n" + " \"catalog_provider.class\" " - + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameComparedLowercaseTest$ExternalTableNameComparedLowercaseProvider\"\n" + + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameComparedLowercaseMetaCacheTrueTest$ExternalTableNameComparedLowercaseProvider\"\n" + ");", rootCtx); env.getCatalogMgr().createCatalog(testCatalog); @@ -77,6 +80,20 @@ public void testGlobalVariable() { Assertions.assertEquals(2, GlobalVariable.lowerCaseTableNames); } + @Test + public void testGetTableWithOutList() { + RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + String tblName = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table1").getName(); + Assertions.assertEquals("TABLE1", tblName); + String tblName2 = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table2").getName(); + Assertions.assertEquals("TABLE2", tblName2); + } + @Test public void testTableNameLowerCase() { Set tableNames = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNamesWithLock(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheFalseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheFalseTest.java new file mode 100644 index 00000000000000..c4e1fee1768cb8 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheFalseTest.java @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.analysis.CreateCatalogStmt; +import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshCatalogStmt; +import org.apache.doris.analysis.SwitchStmt; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; +import org.apache.doris.qe.GlobalVariable; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ExternalTableNameStoredLowercaseMetaCacheFalseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog + CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"use_meta_cache\" = \"false\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameStoredLowercaseMetaCacheFalseTest$ExternalTableNameStoredLowercaseProvider\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + } + + @Override + protected void beforeCluster() { + Config.lower_case_table_names = 1; + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + DropCatalogStmt stmt = (DropCatalogStmt) parseAndAnalyzeStmt("drop catalog test1"); + env.getCatalogMgr().dropCatalog(stmt); + } + + + @Test + public void testGlobalVariable() { + Assertions.assertEquals(1, GlobalVariable.lowerCaseTableNames); + } + + @Test + public void testGetTableWithOutList() { + RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + String tblName = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("TABLE1").getName(); + Assertions.assertEquals("table1", tblName); + String tblName3 = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table3").getName(); + Assertions.assertEquals("table3", tblName3); + } + + @Test + public void testTableNameLowerCase() { + Set tableNames = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNamesWithLock(); + Assertions.assertEquals(3, tableNames.size()); + Assertions.assertTrue(tableNames.contains("table1")); + Assertions.assertTrue(tableNames.contains("table2")); + Assertions.assertTrue(tableNames.contains("table3")); + Assertions.assertFalse(tableNames.contains("TABLE1")); + } + + private void switchTest() throws Exception { + SwitchStmt switchTest = (SwitchStmt) parseAndAnalyzeStmt("switch test1;"); + Env.getCurrentEnv().changeCatalog(connectContext, switchTest.getCatalogName()); + } + + public static class ExternalTableNameStoredLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tblSchemaMap1 = Maps.newHashMap(); + // db1 + tblSchemaMap1.put("table1", Lists.newArrayList( + new Column("siteid", PrimitiveType.INT), + new Column("citycode", PrimitiveType.SMALLINT), + new Column("username", PrimitiveType.VARCHAR), + new Column("pv", PrimitiveType.BIGINT))); + tblSchemaMap1.put("table2", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR), + new Column("k3", PrimitiveType.VARCHAR), + new Column("k4", PrimitiveType.INT), + new Column("k5", PrimitiveType.LARGEINT))); + tblSchemaMap1.put("TABLE3", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR), + new Column("k3", PrimitiveType.VARCHAR), + new Column("k4", PrimitiveType.INT), + new Column("k5", PrimitiveType.LARGEINT))); + MOCKED_META.put("db1", tblSchemaMap1); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheTrueTest.java similarity index 83% rename from fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseTest.java rename to fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheTrueTest.java index d55e209fa63d9f..d3eca35debb0b1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheTrueTest.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.CreateCatalogStmt; import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshCatalogStmt; import org.apache.doris.analysis.SwitchStmt; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; @@ -27,6 +28,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; import org.apache.doris.qe.GlobalVariable; import org.apache.doris.utframe.TestWithFeService; @@ -39,7 +41,7 @@ import java.util.Map; import java.util.Set; -public class ExternalTableNameStoredLowercaseTest extends TestWithFeService { +public class ExternalTableNameStoredLowercaseMetaCacheTrueTest extends TestWithFeService { private static Env env; private ConnectContext rootCtx; @@ -50,8 +52,9 @@ protected void runBeforeAll() throws Exception { // 1. create test catalog CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + " \"type\" = \"test\",\n" + + " \"use_meta_cache\" = \"true\",\n" + " \"catalog_provider.class\" " - + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameStoredLowercaseTest$ExternalTableNameStoredLowercaseProvider\"\n" + + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameStoredLowercaseMetaCacheTrueTest$ExternalTableNameStoredLowercaseProvider\"\n" + ");", rootCtx); env.getCatalogMgr().createCatalog(testCatalog); @@ -77,6 +80,20 @@ public void testGlobalVariable() { Assertions.assertEquals(1, GlobalVariable.lowerCaseTableNames); } + @Test + public void testGetTableWithOutList() { + RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + String tblName = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("TABLE1").getName(); + Assertions.assertEquals("table1", tblName); + String tblName3 = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table3").getName(); + Assertions.assertEquals("table3", tblName3); + } + @Test public void testTableNameLowerCase() { Set tableNames = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNamesWithLock(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/mapping/JdbcIdentifierMappingTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/mapping/JdbcIdentifierMappingTest.java new file mode 100644 index 00000000000000..edf7ab4e29b12a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/mapping/JdbcIdentifierMappingTest.java @@ -0,0 +1,277 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.mapping; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class JdbcIdentifierMappingTest { + private String validJson; + private String invalidJson; + private String duplicateMappingJson; + private String columnConflictJson; + private String tableConflictJson; + + @Before + public void setUp() { + validJson = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"mapping\": \"doris_local\"}\n" + + " ],\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"remoteTable\": \"TABLE_A\", \"mapping\": \"table_a_local\"}\n" + + " ],\n" + + " \"columns\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"remoteTable\": \"TABLE_A\", \"remoteColumn\": \"COLUMN_X\", " + + "\"mapping\": \"column_x_local\"}\n" + + " ]\n" + + "}"; + + invalidJson = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"mapping\": \"doris_local\"}\n" + + " ],\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"remoteTable\": \"TABLE_A\", \"mapping\": \"table_a_local\"}\n" + + " ], // Invalid JSON due to trailing comma\n" + + "}"; + + duplicateMappingJson = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"mapping\": \"conflict\"},\n" + + " {\"remoteDatabase\": \"DORIS_DUP\", \"mapping\": \"CONFLICT\"}\n" + + " ]\n" + + "}"; + + columnConflictJson = "{\n" + + " \"columns\": [\n" + + " {\"remoteDatabase\": \"D1\", \"remoteTable\": \"T1\", \"remoteColumn\": \"C1\", \"mapping\": \"custom_col\"},\n" + + " {\"remoteDatabase\": \"D1\", \"remoteTable\": \"T1\", \"remoteColumn\": \"C2\", \"mapping\": \"CUSTOM_COL\"}\n" + + " ]\n" + + "}"; + + tableConflictJson = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"D2\", \"remoteTable\": \"T1\", \"mapping\": \"custom_table\"},\n" + + " {\"remoteDatabase\": \"D2\", \"remoteTable\": \"T2\", \"mapping\": \"CUSTOM_TABLE\"}\n" + + " ]\n" + + "}"; + } + + @Test + public void testIsLowerCaseMetaNamesTrue() { + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(true, false, validJson); + + String databaseName = mapping.fromRemoteDatabaseName("DORIS"); + String tableName = mapping.fromRemoteTableName("DORIS", "TABLE_A"); + String columnName = mapping.fromRemoteColumnName("DORIS", "TABLE_A", "COLUMN_X"); + + Assert.assertEquals("doris_local", databaseName); + Assert.assertEquals("table_a_local", tableName); + Assert.assertEquals("column_x_local", columnName); + } + + @Test + public void testIsLowerCaseMetaNamesFalse() { + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, validJson); + + String databaseName = mapping.fromRemoteDatabaseName("DORIS"); + String tableName = mapping.fromRemoteTableName("DORIS", "TABLE_A"); + String columnName = mapping.fromRemoteColumnName("DORIS", "TABLE_A", "COLUMN_X"); + + Assert.assertEquals("doris_local", databaseName); + Assert.assertEquals("table_a_local", tableName); + Assert.assertEquals("column_x_local", columnName); + } + + @Test(expected = RuntimeException.class) + public void testInvalidJson() { + new JdbcIdentifierMapping(true, false, invalidJson); + } + + @Test + public void testDuplicateMappingWhenLowerCaseMetaNamesTrue() { + try { + new JdbcIdentifierMapping(false, true, duplicateMappingJson); + Assert.fail("Expected RuntimeException due to duplicate mappings"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("Duplicate mapping found")); + } + } + + @Test + public void testDuplicateMappingWhenLowerCaseMetaNamesFalse() { + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, duplicateMappingJson); + + String databaseName1 = mapping.fromRemoteDatabaseName("DORIS"); + String databaseName2 = mapping.fromRemoteDatabaseName("DORIS_DUP"); + + Assert.assertEquals("conflict", databaseName1); + Assert.assertEquals("CONFLICT", databaseName2); + } + + @Test + public void testColumnCaseConflictAlwaysChecked() { + try { + new JdbcIdentifierMapping(false, false, columnConflictJson); + Assert.fail("Expected RuntimeException due to column case-only conflict"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("case-only different mapping")); + } + } + + @Test + public void testTableCaseConflictWhenLowerCaseMetaNamesFalseAndLowerCaseTableNamesFalse() { + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, tableConflictJson); + String tableName1 = mapping.fromRemoteTableName("D2", "T1"); + String tableName2 = mapping.fromRemoteTableName("D2", "T2"); + Assert.assertEquals("custom_table", tableName1); + Assert.assertEquals("CUSTOM_TABLE", tableName2); + } + + @Test + public void testTableCaseConflictWhenLowerCaseMetaNamesTrue() { + try { + new JdbcIdentifierMapping(true, false, tableConflictJson); + Assert.fail("Expected RuntimeException due to table case-only conflict"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("case-only different mapping")); + } + } + + @Test + public void testTableCaseConflictWhenLowerCaseMetaNamesFalseButLowerCaseTableNamesTrue() { + try { + new JdbcIdentifierMapping(false, true, tableConflictJson); + Assert.fail("Expected RuntimeException due to table case-only conflict"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("case-only different mapping")); + } + } + + @Test + public void testUppercaseMappingForDBWhenLowerCaseMetaNamesTrue() { + String json = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"UPPER_DB\", \"mapping\": \"UPPER_LOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, true, json); + Assert.assertEquals("upper_local", mapping.fromRemoteDatabaseName("UPPER_DB")); + } + + @Test + public void testUppercaseMappingForDBWhenLowerCaseMetaNamesFalse() { + String json = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"UPPER_DB\", \"mapping\": \"UPPER_LOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_LOCAL", mapping.fromRemoteDatabaseName("UPPER_DB")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseTableNamesTrue() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(true, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseTableNamesFalse() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseMetaNamesTrue() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, true, json); + Assert.assertEquals("upper_tlocal", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseMetaNamesFalse() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenAllCaseFalse() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenAllCaseTrue() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(true, true, json); + Assert.assertEquals("upper_tlocal", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForColumnWithoutLowerCaseMetaNames() { + String json = "{\n" + + " \"columns\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"TAB\", \"remoteColumn\": \"UPPER_COL\", \"mapping\": \"UPPER_CLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_CLOCAL", mapping.fromRemoteColumnName("DB", "TAB", "UPPER_COL")); + } + + @Test + public void testUppercaseMappingForColumnWithLowerCaseMetaNamesTrue() { + String json = "{\n" + + " \"columns\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"TAB\", \"remoteColumn\": \"UPPER_COL\", \"mapping\": \"UPPER_CLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, true, json); + Assert.assertEquals("upper_clocal", mapping.fromRemoteColumnName("DB", "TAB", "UPPER_COL")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java index c875ef6bc2f141..30a233dd1a999e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java @@ -98,7 +98,7 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { List schema = Lists.newArrayList(); schema.add(new Column("k1", PrimitiveType.INT)); - HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db"); + HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db", "hms_db"); Deencapsulation.setField(db, "initialized", true); Deencapsulation.setField(tbl, "objectCreated", true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java index bddb3c8185ae72..0a981dab8a9dcc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java @@ -114,7 +114,7 @@ private void init(HMSExternalCatalog hmsCatalog) { List schema = Lists.newArrayList(); schema.add(new Column("k1", PrimitiveType.INT)); - HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db"); + HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db", "hms_db"); Deencapsulation.setField(db, "initialized", true); Deencapsulation.setField(tbl, "objectCreated", true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java index 026b7fb65b069a..b8067e7e6bf75b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java @@ -117,7 +117,7 @@ public void testSupportAutoAnalyze() { OlapTable table1 = new OlapTable(200, "testTable", schema, null, null, null); Assertions.assertTrue(collector.supportAutoAnalyze(table1)); - ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbcdb", null); + ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbcdb", null, null); Assertions.assertFalse(collector.supportAutoAnalyze(externalTable)); new MockUp() { @@ -126,7 +126,7 @@ public DLAType getDlaType() { return DLAType.ICEBERG; } }; - ExternalTable icebergExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null); + ExternalTable icebergExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null, null); Assertions.assertFalse(collector.supportAutoAnalyze(icebergExternalTable)); new MockUp() { @@ -135,7 +135,7 @@ public DLAType getDlaType() { return DLAType.HIVE; } }; - ExternalTable hiveExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null); + ExternalTable hiveExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null, null); Assertions.assertTrue(collector.supportAutoAnalyze(hiveExternalTable)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java index ef1e9ca02970b1..e1ecff04e7a806 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java @@ -30,10 +30,12 @@ import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.jdbc.JdbcExternalCatalog; +import org.apache.doris.datasource.jdbc.JdbcExternalDatabase; import org.apache.doris.datasource.jdbc.JdbcExternalTable; import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.AnalysisManager; @@ -181,6 +183,11 @@ void testNeedAnalyzeColumn() throws DdlException { OlapTable table = new OlapTable(200, "testTable", schema, null, null, null); HMSExternalCatalog externalCatalog = new HMSExternalCatalog(); + long dbId = 100; + String dbName = "testDb"; + HMSExternalDatabase externalDatabase = new HMSExternalDatabase(externalCatalog, dbId, dbName, dbName); + + // Test olap table auto analyze disabled. Map properties = new HashMap<>(); properties.put(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY, "disable"); @@ -195,7 +202,7 @@ protected synchronized void makeSureInitialized() { }; // Test auto analyze catalog disabled. - HMSExternalTable hmsTable = new HMSExternalTable(1, "name", "dbName", externalCatalog); + HMSExternalTable hmsTable = new HMSExternalTable(1, "name", "name", externalCatalog, externalDatabase); Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(hmsTable, Pair.of("index", column.getName()))); // Test catalog auto analyze enabled. @@ -210,7 +217,7 @@ public TableStatsMeta findTableStatsStatus(long tblId) { // Test external table auto analyze enabled. externalCatalog.getCatalogProperty().addProperty(ExternalCatalog.ENABLE_AUTO_ANALYZE, "false"); - HMSExternalTable hmsTable1 = new HMSExternalTable(1, "name", "dbName", externalCatalog); + HMSExternalTable hmsTable1 = new HMSExternalTable(1, "name", "name", externalCatalog, externalDatabase); externalCatalog.setAutoAnalyzePolicy("dbName", "name", "enable"); Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(hmsTable1, Pair.of("index", column.getName()))); @@ -246,8 +253,9 @@ protected synchronized void makeSureInitialized() { } }; // Test not supported external table type. - ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbcdb", - new JdbcExternalCatalog(1, "name", "resource", new HashMap<>(), "")); + JdbcExternalCatalog jdbcExternalCatalog = new JdbcExternalCatalog(1, "name", "resource", new HashMap<>(), ""); + JdbcExternalDatabase jdbcExternalDatabase = new JdbcExternalDatabase(jdbcExternalCatalog, 1, "jdbcdb", "jdbcdb"); + ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbctable", jdbcExternalCatalog, jdbcExternalDatabase); Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(externalTable, Pair.of("index", column.getName()))); // Test hms external table not hive type. @@ -257,7 +265,7 @@ public DLAType getDlaType() { return DLAType.ICEBERG; } }; - ExternalTable hmsExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", externalCatalog); + ExternalTable hmsExternalTable = new HMSExternalTable(1, "hmsTable", "hmsTable", externalCatalog, externalDatabase); Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(hmsExternalTable, Pair.of("index", column.getName()))); // Test partition first load. @@ -394,7 +402,7 @@ public boolean autoAnalyzeEnabled() { return true; } }; - IcebergExternalTable icebergTable = new IcebergExternalTable(0, "", "", null); + IcebergExternalTable icebergTable = new IcebergExternalTable(0, "", "", null, null); Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(icebergTable, Pair.of("index", column.getName()))); // Test table stats meta is null. diff --git a/regression-test/data/external_table_p0/lower_case/test_conflict_name.out b/regression-test/data/external_table_p0/lower_case/test_conflict_name.out new file mode 100644 index 00000000000000..be86d4d3b3ed31 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_conflict_name.out @@ -0,0 +1,5 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_external_conflict_name_db -- + +-- !select_external_conflict_name_tbl -- + diff --git a/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_show_and_select.out b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_show_and_select.out new file mode 100644 index 00000000000000..61fe3b39c984e7 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_show_and_select.out @@ -0,0 +1,25 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_test_cache_false_lower_false1 -- +1 lower + +-- !sql_test_cache_false_lower_false2 -- +1 UPPER + +-- !sql_test_cache_false_lower_false1 -- +1 lower + +-- !sql_test_cache_false_lower_false2 -- +1 UPPER + +-- !sql_test_cache_false_lower_true1 -- +1 lower + +-- !sql_test_cache_false_lower_true2 -- +1 UPPER + +-- !sql_test_cache_true_lower_true1 -- +1 lower + +-- !sql_test_cache_true_lower_true2 -- +1 UPPER + diff --git a/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out new file mode 100644 index 00000000000000..101ca6838796a7 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out @@ -0,0 +1,97 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_test_cache_false_lower_false_with_conf1_1 -- +1 lower + +-- !sql_test_cache_false_lower_false_with_conf1_2 -- +1 lower + +-- !sql_test_cache_false_lower_false_with_conf1_3 -- +1 UPPER + +-- !sql_test_cache_false_lower_false_with_conf1_4 -- +1 UPPER + +-- !sql_test_cache_false_lower_false_with_conf2_1 -- +1 lower + +-- !sql_test_cache_false_lower_false_with_conf2_2 -- +1 lower + +-- !sql_test_cache_false_lower_false_with_conf2_3 -- +1 UPPER + +-- !sql_test_cache_false_lower_false_with_conf2_4 -- +1 UPPER + +-- !sql_test_cache_true_lower_false_with_conf1_1 -- +1 lower + +-- !sql_test_cache_true_lower_false_with_conf1_2 -- +1 lower + +-- !sql_test_cache_true_lower_false_with_conf1_3 -- +1 UPPER + +-- !sql_test_cache_true_lower_false_with_conf1_4 -- +1 UPPER + +-- !sql_test_cache_true_lower_false_with_conf2_1 -- +1 lower + +-- !sql_test_cache_true_lower_false_with_conf2_2 -- +1 lower + +-- !sql_test_cache_true_lower_false_with_conf2_3 -- +1 UPPER + +-- !sql_test_cache_true_lower_false_with_conf2_4 -- +1 UPPER + +-- !sql_test_cache_false_lower_true_with_conf1_1 -- +1 lower + +-- !sql_test_cache_false_lower_true_with_conf1_2 -- +1 lower + +-- !sql_test_cache_false_lower_true_with_conf1_3 -- +1 UPPER + +-- !sql_test_cache_false_lower_true_with_conf1_4 -- +1 UPPER + +-- !sql_test_cache_false_lower_true_with_conf2_1 -- +1 lower + +-- !sql_test_cache_false_lower_true_with_conf2_2 -- +1 lower + +-- !sql_test_cache_false_lower_true_with_conf2_3 -- +1 UPPER + +-- !sql_test_cache_false_lower_true_with_conf2_4 -- +1 UPPER + +-- !sql_test_cache_true_lower_true_with_conf1_1 -- +1 lower + +-- !sql_test_cache_true_lower_true_with_conf1_2 -- +1 lower + +-- !sql_test_cache_true_lower_true_with_conf1_3 -- +1 UPPER + +-- !sql_test_cache_true_lower_true_with_conf1_4 -- +1 UPPER + +-- !sql_test_cache_true_lower_true_with_conf2_1 -- +1 lower + +-- !sql_test_cache_true_lower_true_with_conf2_2 -- +1 lower + +-- !sql_test_cache_true_lower_true_with_conf2_3 -- +1 UPPER + +-- !sql_test_cache_true_lower_true_with_conf2_4 -- +1 UPPER + diff --git a/regression-test/data/external_table_p0/lower_case/test_lower_case_mtmv.out b/regression-test/data/external_table_p0/lower_case/test_lower_case_mtmv.out new file mode 100644 index 00000000000000..f958424e65c626 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_lower_case_mtmv.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- + diff --git a/regression-test/data/external_table_p0/lower_case/test_meta_cache_select_without_refresh.out b/regression-test/data/external_table_p0/lower_case/test_meta_cache_select_without_refresh.out new file mode 100644 index 00000000000000..7669a50d30cd21 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_meta_cache_select_without_refresh.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test_meta_cache_lower_true_select_without_refresh_select_table1 -- +1 table1 + +-- !test_meta_cache_lower_false_select_without_refresh_select_table1 -- +1 table1 + +-- !test_meta_cache_lower_false_select_without_refresh_select_table2 -- +1 TABLE2 + diff --git a/regression-test/data/external_table_p0/lower_case/test_meta_names_mapping.out b/regression-test/data/external_table_p0/lower_case/test_meta_names_mapping.out new file mode 100644 index 00000000000000..71551d16891296 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_meta_names_mapping.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_meta_mapping_select_lower -- +1 lowercase 100 + +-- !sql_meta_mapping_select_upper -- +2 UPPERCASE 200 + +-- !sql_meta_mapping_select_lower_lower_case_true -- +1 lowercase 100 + +-- !sql_meta_mapping_select_upper_lower_case_true -- +2 UPPERCASE 200 + diff --git a/regression-test/data/external_table_p0/lower_case/upgrade/load.out b/regression-test/data/external_table_p0/lower_case/upgrade/load.out new file mode 100644 index 00000000000000..bdd9b17d0c6e3d --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/upgrade/load.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_test_upgrade_lower_case_catalog_1 -- +1 lower + +-- !sql_test_upgrade_lower_case_catalog_2 -- +1 UPPER + diff --git a/regression-test/data/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.out b/regression-test/data/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.out new file mode 100644 index 00000000000000..bdd9b17d0c6e3d --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_test_upgrade_lower_case_catalog_1 -- +1 lower + +-- !sql_test_upgrade_lower_case_catalog_2 -- +1 UPPER + diff --git a/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy new file mode 100644 index 00000000000000..6187d47b645564 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_conflict_name", "p0,external,doris,meta_names_mapping") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """drop database if exists internal.external_conflict_name; """ + sql """drop database if exists internal.EXTERNAL_CONFLICT_NAME; """ + sql """create database if not exists internal.external_conflict_name; """ + sql """create database if not exists internal.EXTERNAL_CONFLICT_NAME; """ + sql """create table if not exists internal.external_conflict_name.table_test + (id int, name varchar(20), column_test int) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """drop catalog if exists test_conflict_name """ + sql """ CREATE CATALOG `test_conflict_name` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_conflict_name,EXTERNAL_CONFLICT_NAME" + )""" + + test { + sql """show databases from test_conflict_name""" + exception """Found conflicting database names under case-insensitive conditions. Conflicting remote database names: EXTERNAL_CONFLICT_NAME, external_conflict_name in catalog test_conflict_name. Please use meta_names_mapping to handle name mapping.""" + } + + sql """refresh catalog test_conflict_name""" + + test { + sql """select * from test_conflict_name.external_conflict_name.table_test""" + exception """Found conflicting database names under case-insensitive conditions. Conflicting remote database names: EXTERNAL_CONFLICT_NAME, external_conflict_name in catalog test_conflict_name. Please use meta_names_mapping to handle name mapping.""" + } + + sql """drop database if exists internal.EXTERNAL_CONFLICT_NAME; """ + + sql """refresh catalog test_conflict_name""" + + qt_select_external_conflict_name_db "select * from test_conflict_name.external_conflict_name.table_test" + + sql """create table if not exists internal.external_conflict_name.TABLE_TEST + (id int, name varchar(20), column_test int) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """refresh catalog test_conflict_name""" + + test { + sql """show tables from test_conflict_name.external_conflict_name""" + exception """Found conflicting table names under case-insensitive conditions. Conflicting remote table names: TABLE_TEST, table_test in remote database 'external_conflict_name' under catalog 'test_conflict_name'. Please use meta_names_mapping to handle name mapping.""" + } + + sql """refresh catalog test_conflict_name""" + + test { + sql """select * from test_conflict_name.external_conflict_name.TABLE_TEST""" + exception """Found conflicting table names under case-insensitive conditions. Conflicting remote table names: TABLE_TEST, table_test in remote database 'external_conflict_name' under catalog 'test_conflict_name'. Please use meta_names_mapping to handle name mapping.""" + } + + sql """drop table if exists internal.external_conflict_name.TABLE_TEST; """ + + qt_select_external_conflict_name_tbl "select * from test_conflict_name.external_conflict_name.table_test" + + sql """drop database if exists internal.external_conflict_name; """ + sql """drop database if exists internal.EXTERNAL_CONFLICT_NAME; """ +} \ No newline at end of file diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy new file mode 100644 index 00000000000000..854eb06a8e2755 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lower_case_meta_include", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + String mapping_db = """ + { + "databases": [ + {"remoteDatabase": "external_INCLUDE", "mapping": "external_include_test"}, + {"remoteDatabase": "external_EXCLUDE", "mapping": "external_exclude_test"} + ] + } + """ + + sql """drop database if exists internal.external_INCLUDE; """ + sql """drop database if exists internal.external_EXCLUDE; """ + sql """create database if not exists internal.external_INCLUDE; """ + sql """create database if not exists internal.external_EXCLUDE; """ + + // Test include + sql """drop catalog if exists test_lower_case_include """ + + sql """ CREATE CATALOG `test_lower_case_include` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_INCLUDE" + )""" + + test { + sql """show databases from test_lower_case_include""" + + rowNum 3 + + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_include"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + sql """drop catalog if exists test_lower_case_include """ + + // Test exclude + sql """drop catalog if exists test_lower_case_exclude """ + + sql """ CREATE CATALOG `test_lower_case_exclude` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "exclude_database_list" = "external_EXCLUDE" + )""" + + test { + sql """show databases from test_lower_case_exclude""" + + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_exclude"] + expectedDatabases.each { dbName -> + assertFalse(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + // Test include with mapping + sql """drop catalog if exists test_lower_case_include """ + + sql """ CREATE CATALOG `test_lower_case_include` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + 'meta_names_mapping' = '${mapping_db}', + "include_database_list" = "external_INCLUDE" + )""" + + test { + sql """show databases from test_lower_case_include""" + + rowNum 3 + + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_include_test"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + sql """drop catalog if exists test_lower_case_include """ + + // Test exclude with mapping + sql """drop catalog if exists test_lower_case_exclude """ + + sql """ CREATE CATALOG `test_lower_case_exclude` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + 'meta_names_mapping' = '${mapping_db}', + "exclude_database_list" = "external_EXCLUDE" + )""" + + test { + sql """show databases from test_lower_case_exclude""" + + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_exclude_test"] + expectedDatabases.each { dbName -> + assertFalse(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + sql """drop catalog if exists test_lower_case_exclude """ + sql """drop database if exists internal.external_INCLUDE; """ + sql """drop database if exists internal.external_EXCLUDE; """ +} diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy new file mode 100644 index 00000000000000..845a78be0b62c8 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy @@ -0,0 +1,229 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lower_case_meta_show_and_select", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """drop database if exists internal.external_test_lower; """ + sql """drop database if exists internal.external_test_UPPER; """ + sql """create database if not exists internal.external_test_lower; """ + sql """create database if not exists internal.external_test_UPPER; """ + sql """create table if not exists internal.external_test_lower.lower + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """create table if not exists internal.external_test_lower.UPPER + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_test_lower.lower values(1, 'lower')""" + sql """insert into internal.external_test_lower.UPPER values(1, 'UPPER')""" + + // Test for cache false and lower false + sql """drop catalog if exists test_cache_false_lower_false """ + + sql """ CREATE CATALOG `test_cache_false_lower_false` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower,external_test_UPPER" + )""" + + test { + sql """show databases from test_cache_false_lower_false""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_test_lower", "external_test_UPPER"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_cache_false_lower_false.external_test_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "UPPER"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_cache_false_lower_false1 "select * from test_cache_false_lower_false.external_test_lower.lower" + qt_sql_test_cache_false_lower_false2 "select * from test_cache_false_lower_false.external_test_lower.UPPER" + + sql """drop catalog if exists test_cache_false_lower_false """ + + // Test for cache true and lower false + sql """drop catalog if exists test_cache_true_lower_false """ + + sql """ CREATE CATALOG `test_cache_true_lower_false` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower,external_test_UPPER" + )""" + + test { + sql """show databases from test_cache_true_lower_false""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_test_lower", "external_test_UPPER"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_cache_true_lower_false.external_test_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "UPPER"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_cache_false_lower_false1 "select * from test_cache_true_lower_false.external_test_lower.lower" + qt_sql_test_cache_false_lower_false2 "select * from test_cache_true_lower_false.external_test_lower.UPPER" + + sql """drop catalog if exists test_cache_true_lower_false """ + + // Test for cache false and lower true + sql """drop catalog if exists test_cache_false_lower_true """ + + sql """ CREATE CATALOG `test_cache_false_lower_true` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower,external_test_UPPER" + )""" + + test { + sql """show databases from test_cache_false_lower_true""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_test_lower", "external_test_upper"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_cache_false_lower_true.external_test_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_cache_false_lower_true1 "select * from test_cache_false_lower_true.external_test_lower.lower" + qt_sql_test_cache_false_lower_true2 "select * from test_cache_false_lower_true.external_test_lower.upper" + + sql """drop catalog if exists test_cache_false_lower_true """ + + // Test for cache true and lower true + sql """drop catalog if exists test_cache_true_lower_true """ + + sql """ CREATE CATALOG `test_cache_true_lower_true` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower,external_test_UPPER" + )""" + + test { + sql """show databases from test_cache_true_lower_true""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_test_lower", "external_test_upper"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_cache_true_lower_true.external_test_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_cache_true_lower_true1 "select * from test_cache_true_lower_true.external_test_lower.lower" + qt_sql_test_cache_true_lower_true2 "select * from test_cache_true_lower_true.external_test_lower.upper" + + sql """drop catalog if exists test_cache_true_lower_true """ + + + sql """drop database if exists internal.external_test_lower; """ + sql """drop database if exists internal.external_test_UPPER; """ +} diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy new file mode 100644 index 00000000000000..d18df396d56bc4 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy @@ -0,0 +1,363 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """drop database if exists internal.external_test_lower_with_conf; """ + sql """drop database if exists internal.external_test_UPPER_with_conf; """ + sql """create database if not exists internal.external_test_lower_with_conf; """ + sql """create table if not exists internal.external_test_lower_with_conf.lower_with_conf + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """create table if not exists internal.external_test_lower_with_conf.UPPER_with_conf + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_test_lower_with_conf.lower_with_conf values(1, 'lower')""" + sql """insert into internal.external_test_lower_with_conf.UPPER_with_conf values(1, 'UPPER')""" + + // Test for cache false and lower false and lower conf 1 + sql """drop catalog if exists test_cache_false_lower_false_with_conf1 """ + + sql """ CREATE CATALOG `test_cache_false_lower_false_with_conf1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "1" + )""" + + qt_sql_test_cache_false_lower_false_with_conf1_1 "select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_2 "select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_3 "select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_4 "select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_false_with_conf1 """ + + // Test for cache false and lower false and lower conf 2 + sql """drop catalog if exists test_cache_false_lower_false_with_conf2 """ + + sql """ CREATE CATALOG `test_cache_false_lower_false_with_conf2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "2" + )""" + + qt_sql_test_cache_false_lower_false_with_conf2_1 "select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_2 "select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_3 "select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_4 "select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_false_with_conf2 """ + + + + // Test for cache true and lower false and lower conf 1 + sql """drop catalog if exists test_cache_true_lower_false_with_conf1 """ + + sql """ CREATE CATALOG `test_cache_true_lower_false_with_conf1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "1" + )""" + + qt_sql_test_cache_true_lower_false_with_conf1_1 "select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_2 "select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_3 "select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_4 "select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_false_with_conf1 """ + + // Test for cache true and lower false and lower conf 2 + sql """drop catalog if exists test_cache_true_lower_false_with_conf2 """ + + sql """ CREATE CATALOG `test_cache_true_lower_false_with_conf2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "2" + )""" + + qt_sql_test_cache_true_lower_false_with_conf2_1 "select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_2 "select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_3 "select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_4 "select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_false_with_conf2 """ + + // Test for cache false and lower true and lower conf 1 + sql """drop catalog if exists test_cache_false_lower_true_with_conf1 """ + + sql """ CREATE CATALOG `test_cache_false_lower_true_with_conf1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "1" + )""" + + qt_sql_test_cache_false_lower_true_with_conf1_1 "select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_2 "select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_3 "select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_4 "select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_true_with_conf1 """ + + // Test for cache false and lower true and lower conf 2 + sql """drop catalog if exists test_cache_false_lower_true_with_conf2 """ + + sql """ CREATE CATALOG `test_cache_false_lower_true_with_conf2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "2" + )""" + + qt_sql_test_cache_false_lower_true_with_conf2_1 "select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_2 "select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_3 "select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_4 "select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_true_with_conf2 """ + + + // Test for cache true and lower true and lower conf 1 + sql """drop catalog if exists test_cache_true_lower_true_with_conf1 """ + + sql """ CREATE CATALOG `test_cache_true_lower_true_with_conf1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "1" + )""" + + qt_sql_test_cache_true_lower_true_with_conf1_1 "select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_2 "select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_3 "select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_4 "select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_true_with_conf1 """ + + // Test for cache true and lower true and lower conf 2 + sql """drop catalog if exists test_cache_true_lower_true_with_conf2 """ + + sql """ CREATE CATALOG `test_cache_true_lower_true_with_conf2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "truee", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "2" + )""" + + qt_sql_test_cache_true_lower_true_with_conf2_1 "select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_2 "select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_3 "select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_4 "select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_true_with_conf2 """ + + + + sql """drop database if exists internal.external_test_lower_with_conf; """ +} diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy new file mode 100644 index 00000000000000..40322a22afe737 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lower_case_mtmv", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """drop database if exists internal.EXTERNAL_LOWER_MTMV; """ + sql """create database if not exists internal.EXTERNAL_LOWER_MTMV;""" + sql """create table if not exists internal.EXTERNAL_LOWER_MTMV.TABLE_TEST + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.EXTERNAL_LOWER_MTMV.TABLE_TEST values(1, 'lower')""" + + sql """drop catalog if exists test_lower_case_mtmv """ + + sql """ CREATE CATALOG `test_lower_case_mtmv` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "EXTERNAL_LOWER_MTMV" + )""" + + + sql """CREATE MATERIALIZED VIEW internal.EXTERNAL_LOWER_MTMV.MTMV_TEST + REFRESH COMPLETE ON SCHEDULE EVERY 1 minute + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + 'replication_num' = '1' + ) + AS SELECT * FROM test_lower_case_mtmv.external_lower_mtmv.table_test;""" + + qt_select """select * from internal.EXTERNAL_LOWER_MTMV.MTMV_TEST""" + + sql """drop catalog if exists test_lower_case_mtmv """ + sql """drop database if exists internal.EXTERNAL_LOWER_MTMV """ +} diff --git a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy new file mode 100644 index 00000000000000..20e6b0f0032e96 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_meta_cache_select_without_refresh", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """ drop database if exists internal.external_lower_select_without_refresh; """ + sql """create database if not exists internal.external_lower_select_without_refresh;""" + + // Test include + sql """drop catalog if exists test_meta_cache_lower_true_select_without_refresh """ + + sql """ CREATE CATALOG `test_meta_cache_lower_true_select_without_refresh` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_lower_select_without_refresh" + )""" + + sql """drop catalog if exists test_meta_cache_lower_false_select_without_refresh """ + + sql """ CREATE CATALOG `test_meta_cache_lower_false_select_without_refresh` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_lower_select_without_refresh" + )""" + + sql """create table if not exists internal.external_lower_select_without_refresh.table1 + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_lower_select_without_refresh.table1 values(1, 'table1')""" + + qt_test_meta_cache_lower_true_select_without_refresh_select_table1 "select * from test_meta_cache_lower_true_select_without_refresh.external_lower_select_without_refresh.table1;" + + qt_test_meta_cache_lower_false_select_without_refresh_select_table1 "select * from test_meta_cache_lower_false_select_without_refresh.external_lower_select_without_refresh.table1;" + + sql """create table if not exists internal.external_lower_select_without_refresh.TABLE2 + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_lower_select_without_refresh.TABLE2 values(1, 'TABLE2')""" + + test { + sql """select * from test_meta_cache_lower_true_select_without_refresh.external_lower_select_without_refresh.table2;""" + + exception "Table [table2] does not exist in database [external_lower_select_without_refresh]." + } + + qt_test_meta_cache_lower_false_select_without_refresh_select_table2 "select * from test_meta_cache_lower_false_select_without_refresh.external_lower_select_without_refresh.TABLE2;" + + sql """drop catalog if exists test_meta_cache_lower_true_select_without_refresh """ + sql """drop catalog if exists test_meta_cache_lower_false_select_without_refresh """ + sql """drop database if exists internal.external_lower_select_without_refresh; """ +} diff --git a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy new file mode 100644 index 00000000000000..170a81955b1a0f --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy @@ -0,0 +1,287 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_meta_names_mapping", "p0,external,doris,meta_names_mapping") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + String validMetaNamesMapping = """ + { + "databases": [ + {"remoteDatabase": "EXTERNAL_META_NAMES_MAPPING", "mapping": "external_meta_names_mapping_upper"}, + {"remoteDatabase": "external_meta_names_mapping", "mapping": "external_meta_names_mapping_lower"} + ], + "tables": [ + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "table_test", "mapping": "table_test_lower"}, + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "TABLE_TEST", "mapping": "table_test_upper"} + ], + "columns": [ + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "TABLE_TEST", "remoteColumn": "column_test", "mapping": "column_test_local"} + ] + } + """ + + sql """drop database if exists internal.external_meta_names_mapping; """ + sql """drop database if exists internal.EXTERNAL_META_NAMES_MAPPING; """ + sql """create database if not exists internal.external_meta_names_mapping; """ + sql """create database if not exists internal.EXTERNAL_META_NAMES_MAPPING; """ + + sql """create table if not exists internal.external_meta_names_mapping.table_test + (id int, name varchar(20), column_test int) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + sql """create table if not exists internal.external_meta_names_mapping.TABLE_TEST + (id int, name varchar(20), column_test int) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + sql """create table if not exists internal.EXTERNAL_META_NAMES_MAPPING.table_test + (id int, name varchar(20), column_test int) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_meta_names_mapping.table_test values(1, 'lowercase', 100);""" + sql """insert into internal.external_meta_names_mapping.TABLE_TEST values(2, 'UPPERCASE', 200);""" + sql """insert into internal.EXTERNAL_META_NAMES_MAPPING.table_test values(3, 'MIXEDCASE', 300);""" + + sql """drop catalog if exists test_valid_meta_names_mapping """ + sql """ CREATE CATALOG `test_valid_meta_names_mapping` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "only_specified_database" = "true", + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING", + "meta_names_mapping" = '${validMetaNamesMapping}' + )""" + + test { + sql """show databases from test_valid_meta_names_mapping""" + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_meta_names_mapping_upper", "external_meta_names_mapping_lower"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_valid_meta_names_mapping.external_meta_names_mapping_lower""" + check { result, ex, startTime, endTime -> + def expectedTables = ["table_test_lower", "table_test_upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + test { + sql """describe test_valid_meta_names_mapping.external_meta_names_mapping_lower.table_test_upper""" + check { result, ex, startTime, endTime -> + def expectedColumns = ["column_test_local"] + expectedColumns.each { columnName -> + assertTrue(result.collect { it[0] }.contains(columnName), "Expected column '${columnName}' not found in result") + } + } + } + + qt_sql_meta_mapping_select_lower "select * from test_valid_meta_names_mapping.external_meta_names_mapping_lower.table_test_lower" + qt_sql_meta_mapping_select_upper "select * from test_valid_meta_names_mapping.external_meta_names_mapping_lower.table_test_upper" + + sql """drop catalog if exists test_valid_meta_names_mapping """ + + sql """ drop catalog if exists test_conflict_meta_names """ + sql """ CREATE CATALOG `test_conflict_meta_names` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING" + )""" + + test { + sql """show databases from test_conflict_meta_names""" + exception """Found conflicting database names under case-insensitive conditions. Conflicting remote database names: EXTERNAL_META_NAMES_MAPPING, external_meta_names_mapping in catalog test_conflict_meta_names. Please use meta_names_mapping to handle name mapping.""" + } + + sql """refresh catalog test_conflict_meta_names""" + + test { + sql """select * from test_conflict_meta_names.external_meta_names_mapping.table_test""" + exception """Found conflicting database names under case-insensitive conditions. Conflicting remote database names: EXTERNAL_META_NAMES_MAPPING, external_meta_names_mapping in catalog test_conflict_meta_names. Please use meta_names_mapping to handle name mapping.""" + } + + String validMetaNamesMapping2 = """ + { + "databases": [ + {"remoteDatabase": "EXTERNAL_META_NAMES_MAPPING", "mapping": "external_meta_names_mapping_upper"}, + {"remoteDatabase": "external_meta_names_mapping", "mapping": "external_meta_names_mapping_lower"} + ] + } + """ + + sql """alter catalog test_conflict_meta_names set properties('meta_names_mapping' = '${validMetaNamesMapping2}')""" + + test { + sql """show tables from test_conflict_meta_names.external_meta_names_mapping_lower""" + exception """Found conflicting table names under case-insensitive conditions. Conflicting remote table names: TABLE_TEST, table_test in remote database 'external_meta_names_mapping' under catalog 'test_conflict_meta_names'. Please use meta_names_mapping to handle name mapping.""" + } + + sql """refresh catalog test_conflict_meta_names""" + + test { + sql """select * from test_conflict_meta_names.external_meta_names_mapping_lower.table_test""" + exception """Found conflicting table names under case-insensitive conditions. Conflicting remote table names: TABLE_TEST, table_test in remote database 'external_meta_names_mapping' under catalog 'test_conflict_meta_names'. Please use meta_names_mapping to handle name mapping.""" + } + + String validMetaNamesMapping3 = """ + { + "databases": [ + {"remoteDatabase": "EXTERNAL_META_NAMES_MAPPING", "mapping": "external_meta_names_mapping_upper"}, + {"remoteDatabase": "external_meta_names_mapping", "mapping": "external_meta_names_mapping_lower"} + ], + "tables": [ + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "TABLE_TEST", "mapping": "table_test_upper"}, + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "table_test", "mapping": "table_test_lower"} + ] + } + """ + + sql """alter catalog test_conflict_meta_names set properties('meta_names_mapping' = '${validMetaNamesMapping3}')""" + + test { + sql """show databases from test_conflict_meta_names""" + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_meta_names_mapping_upper", "external_meta_names_mapping_lower"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_conflict_meta_names.external_meta_names_mapping_lower""" + check { result, ex, startTime, endTime -> + def expectedTables = ["table_test_lower", "table_test_upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_meta_mapping_select_lower_lower_case_true "select * from test_conflict_meta_names.external_meta_names_mapping_lower.table_test_lower" + qt_sql_meta_mapping_select_upper_lower_case_true "select * from test_conflict_meta_names.external_meta_names_mapping_lower.table_test_upper" + + String error_mapping_db = """ + { + "databases": [ + {"remoteDatabase": "EXTERNAL_META_NAMES_MAPPING", "mapping": "external_meta_names_mapping_upper"}, + {"remoteDatabase": "EXTERNAL_META_NAMES_MAPPING", "mapping": "external_meta_names_mapping_lower"} + ] + } + """ + + sql """drop catalog if exists test_error_mapping_db """ + + test { + sql """ CREATE CATALOG `test_error_mapping_db` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "only_specified_database" = "true", + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING", + "meta_names_mapping" = '${error_mapping_db}' + )""" + + exception "Duplicate remoteDatabase found: EXTERNAL_META_NAMES_MAPPING" + } + + sql """drop catalog if exists test_error_mapping_db """ + + String error_mapping_tbl = """ + { + "tables": [ + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "TABLE_TEST", "mapping": "table_test_upper"}, + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "TABLE_TEST", "mapping": "table_test_lower"} + ] + } + """ + + sql """drop catalog if exists test_error_mapping_tbl """ + + test { + sql """ CREATE CATALOG `test_error_mapping_tbl` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "only_specified_database" = "true", + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING", + "meta_names_mapping" = '${error_mapping_tbl}' + )""" + + exception "Duplicate remoteTable found in database external_meta_names_mapping: TABLE_TEST" + } + + sql """drop catalog if exists test_error_mapping_tbl """ + + sql """drop catalog if exists test_alter_error_mapping """ + + sql """ CREATE CATALOG `test_alter_error_mapping` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "only_specified_database" = "true", + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING" + )""" + + test { + sql """alter catalog test_alter_error_mapping set properties('meta_names_mapping' = '${error_mapping_db}')""" + exception "Duplicate remoteDatabase found: EXTERNAL_META_NAMES_MAPPING" + } + + test { + sql """alter catalog test_alter_error_mapping set properties('meta_names_mapping' = '${error_mapping_tbl}')""" + exception "Duplicate remoteTable found in database external_meta_names_mapping: TABLE_TEST" + } + + sql """drop catalog if exists test_alter_error_mapping """ + + sql """drop database if exists internal.external_meta_names_mapping; """ + sql """drop database if exists internal.EXTERNAL_META_NAMES_MAPPING; """ +} \ No newline at end of file diff --git a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy new file mode 100644 index 00000000000000..9317928c7318fb --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_timing_refresh_catalog", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + String mapping = """ + { + "databases": [ + {"remoteDatabase": "external_timing_refresh_catalog", "mapping": "db"} + ], + "tables": [ + {"remoteDatabase": "external_timing_refresh_catalog", "remoteTable": "tbl", "mapping": "table_t"} + ], + "columns": [ + {"remoteDatabase": "external_timing_refresh_catalog", "remoteTable": "tbl", "remoteColumn": "id", "mapping": "id_c"} + ] + } + """ + + sql """drop database if exists internal.external_timing_refresh_catalog; """ + sql """create database if not exists internal.external_timing_refresh_catalog;""" + sql """create table if not exists internal.external_timing_refresh_catalog.tbl + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_timing_refresh_catalog.tbl values(1, 'lower')""" + + sql """drop catalog if exists test_timing_refresh_catalog1 """ + + sql """ CREATE CATALOG `test_timing_refresh_catalog1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "metadata_refresh_interval_seconds" = "1", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_timing_refresh_catalog" + )""" + + sql """drop catalog if exists test_timing_refresh_catalog2 """ + + sql """ CREATE CATALOG `test_timing_refresh_catalog2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "metadata_refresh_interval_seconds" = "1", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_timing_refresh_catalog" + )""" + + test { + def catalogName = "test_timing_refresh_catalog1" + for (int i = 0; i < 30; i++) { + sql """ + select * from ${catalogName}.external_timing_refresh_catalog.tbl + """ + Thread.sleep(1000) + } + } + + test { + def catalogName = "test_timing_refresh_catalog2" + for (int i = 0; i < 30; i++) { + sql """ + select * from ${catalogName}.external_timing_refresh_catalog.tbl + """ + Thread.sleep(1000) + } + } + + // with mapping + sql """drop catalog if exists test_timing_refresh_catalog1 """ + + sql """ CREATE CATALOG `test_timing_refresh_catalog1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "metadata_refresh_interval_seconds" = "1", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_timing_refresh_catalog", + 'meta_names_mapping' = '${mapping}' + )""" + + sql """drop catalog if exists test_timing_refresh_catalog2 """ + + sql """ CREATE CATALOG `test_timing_refresh_catalog2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "metadata_refresh_interval_seconds" = "1", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_timing_refresh_catalog", + 'meta_names_mapping' = '${mapping}' + )""" + + test { + def catalogName = "test_timing_refresh_catalog1" + for (int i = 0; i < 30; i++) { + sql """ + select id_c from ${catalogName}.db.table_t + """ + Thread.sleep(1000) + } + } + + test { + def catalogName = "test_timing_refresh_catalog2" + for (int i = 0; i < 30; i++) { + sql """ + select id_c from ${catalogName}.db.table_t + """ + Thread.sleep(1000) + } + } + + sql """drop catalog if exists test_timing_refresh_catalog1 """ + sql """drop catalog if exists test_timing_refresh_catalog2 """ + sql """drop database if exists internal.external_timing_refresh_catalog """ +} \ No newline at end of file diff --git a/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy new file mode 100644 index 00000000000000..0ea89d4012d1cb --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_upgrade_lower_case_catalog_prepare", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """drop database if exists internal.upgrade_lower_case_catalog_lower; """ + sql """drop database if exists internal.upgrade_lower_case_catalog_UPPER; """ + sql """create database if not exists internal.upgrade_lower_case_catalog_lower; """ + sql """create database if not exists internal.upgrade_lower_case_catalog_UPPER; """ + sql """create table if not exists internal.upgrade_lower_case_catalog_lower.lower + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """create table if not exists internal.upgrade_lower_case_catalog_lower.UPPER + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.upgrade_lower_case_catalog_lower.lower values(1, 'lower')""" + sql """insert into internal.upgrade_lower_case_catalog_lower.UPPER values(1, 'UPPER')""" + + // Test for cache false and lower false + sql """drop catalog if exists test_upgrade_lower_case_catalog """ + + sql """ CREATE CATALOG `test_upgrade_lower_case_catalog` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "upgrade_lower_case_catalog_lower,upgrade_lower_case_catalog_UPPER" + )""" + + test { + sql """show databases from test_upgrade_lower_case_catalog""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["upgrade_lower_case_catalog_lower", "upgrade_lower_case_catalog_upper"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_upgrade_lower_case_catalog_1 "select * from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower.lower" + qt_sql_test_upgrade_lower_case_catalog_2 "select * from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower.upper" + +} diff --git a/regression-test/suites/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.groovy b/regression-test/suites/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.groovy new file mode 100644 index 00000000000000..634e5e5737ef6d --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.groovy @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_upgrade_lower_case_catalog", "p0,external,doris,external_docker,external_docker_doris") { + + test { + sql """show databases from test_upgrade_lower_case_catalog""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["upgrade_lower_case_catalog_lower", "upgrade_lower_case_catalog_upper"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_upgrade_lower_case_catalog_1 "select * from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower.lower" + qt_sql_test_upgrade_lower_case_catalog_2 "select * from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower.upper" + +} \ No newline at end of file From c96964db0f0188196f3548005369989e2d51a4ba Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Mon, 23 Dec 2024 17:50:11 +0800 Subject: [PATCH 2/4] fix build --- .../datasource/iceberg/IcebergExternalTableTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java index 3ba4804e52279c..c5f53a3cb8ba10 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableTest.java @@ -54,7 +54,8 @@ public void testIsSupportedPartitionTable(@Mocked org.apache.iceberg.Table icebe @Mocked PartitionSpec spec, @Mocked PartitionField field, @Mocked Schema schema) { - IcebergExternalTable table = new IcebergExternalTable(1, "1", "2", null); + IcebergExternalDatabase database = new IcebergExternalDatabase(null, 1L, "2", "2"); + IcebergExternalTable table = new IcebergExternalTable(1, "1", "1", null, database); Map specs = Maps.newHashMap(); new MockUp() { @Mock @@ -147,7 +148,8 @@ public Table getIcebergTable() { @Test public void testGetPartitionRange() throws AnalysisException { - IcebergExternalTable table = new IcebergExternalTable(1, "1", "2", null); + IcebergExternalDatabase database = new IcebergExternalDatabase(null, 1L, "2", "2"); + IcebergExternalTable table = new IcebergExternalTable(1, "1", "1", null, database); Column c = new Column("ts", PrimitiveType.DATETIMEV2); List partitionColumns = Lists.newArrayList(c); table.setPartitionColumns(partitionColumns); @@ -196,7 +198,8 @@ public void testGetPartitionRange() throws AnalysisException { @Test public void testSortRange() throws AnalysisException { - IcebergExternalTable table = new IcebergExternalTable(1, "1", "2", null); + IcebergExternalDatabase database = new IcebergExternalDatabase(null, 1L, "2", "2"); + IcebergExternalTable table = new IcebergExternalTable(1, "1", "1", null, database); Column c = new Column("c", PrimitiveType.DATETIMEV2); ArrayList columns = Lists.newArrayList(c); table.setPartitionColumns(Lists.newArrayList(c)); From 1019638f22e026094bd3d47ff035c5205f95015f Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Tue, 24 Dec 2024 17:03:32 +0800 Subject: [PATCH 3/4] fix ut and regression test --- .../doris/datasource/ExternalDatabase.java | 35 ++-- .../datasource/jdbc/JdbcExternalTable.java | 59 ++++-- .../StatisticsAutoCollectorTest.java | 10 +- .../statistics/util/StatisticsUtilTest.java | 11 +- ..._with_lower_table_conf_show_and_select.out | 24 +++ ...th_lower_table_conf_show_and_select.groovy | 190 ++++++++++++++++++ .../lower_case/test_meta_names_mapping.groovy | 2 + 7 files changed, 291 insertions(+), 40 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 13f5af7b46d9de..099d76e98e1b1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -283,8 +283,7 @@ private List> listTableNames() { } else { tableNames = extCatalog.listTableNames(null, remoteName).stream().map(tableName -> { String localTableName = extCatalog.fromRemoteTableName(remoteName, tableName); - if (Env.isStoredTableNamesLowerCase() - || extCatalog.getOnlyTestLowerCaseTableNames() == 1) { + if (this.isStoredTableNamesLowerCase()) { localTableName = localTableName.toLowerCase(); } lowerCaseToTableName.put(tableName.toLowerCase(), tableName); @@ -292,10 +291,9 @@ private List> listTableNames() { }).collect(Collectors.toList()); } // Check for conflicts when stored table names or meta names are case-insensitive - if ((Env.isStoredTableNamesLowerCase() || Env.isTableNamesCaseInsensitive()) - || Boolean.parseBoolean(extCatalog.getLowerCaseMetaNames()) - || (extCatalog.getOnlyTestLowerCaseTableNames() == 1 - || extCatalog.getOnlyTestLowerCaseTableNames() == 2)) { + if (Boolean.parseBoolean(extCatalog.getLowerCaseMetaNames()) + || this.isStoredTableNamesLowerCase() + || this.isTableNamesCaseInsensitive()) { // Map to track lowercased local names and their corresponding remote names Map> lowerCaseToRemoteNames = Maps.newHashMap(); @@ -368,7 +366,7 @@ public T buildTableForInit(String remoteTableName, String localTableName, long t if (remoteTableName == null && extCatalog.useMetaCache.get()) { if (Boolean.parseBoolean(extCatalog.getLowerCaseMetaNames()) || !Strings.isNullOrEmpty(extCatalog.getMetaNamesMapping()) - || extCatalog.getOnlyTestLowerCaseTableNames() == 1) { + || this.isStoredTableNamesLowerCase()) { remoteTableName = metaCache.getRemoteName(localTableName); if (remoteTableName == null) { LOG.warn("Could not resolve remote table name for local table: {}", localTableName); @@ -469,7 +467,7 @@ public DatabaseProperty getDbProperties() { @Override public boolean isTableExist(String tableName) { - if (Env.isTableNamesCaseInsensitive() || extCatalog.getOnlyTestLowerCaseTableNames() == 2) { + if (this.isTableNamesCaseInsensitive()) { String realTableName = lowerCaseToTableName.get(tableName.toLowerCase()); if (realTableName == null) { // Here we need to execute listTableNames() once to fill in lowerCaseToTableName @@ -533,10 +531,10 @@ public Set getTableNamesWithLock() { @Override public T getTableNullable(String tableName) { makeSureInitialized(); - if (Env.isStoredTableNamesLowerCase() || extCatalog.getOnlyTestLowerCaseTableNames() == 1) { + if (this.isStoredTableNamesLowerCase()) { tableName = tableName.toLowerCase(); } - if (Env.isTableNamesCaseInsensitive() || extCatalog.getOnlyTestLowerCaseTableNames() == 2) { + if (this.isTableNamesCaseInsensitive()) { String realTableName = lowerCaseToTableName.get(tableName.toLowerCase()); if (realTableName == null) { // Here we need to execute listTableNames() once to fill in lowerCaseToTableName @@ -550,7 +548,8 @@ public T getTableNullable(String tableName) { tableName = realTableName; } } - if (extCatalog.getLowerCaseMetaNames().equalsIgnoreCase("true")) { + if (extCatalog.getLowerCaseMetaNames().equalsIgnoreCase("true") + && (this.isTableNamesCaseInsensitive())) { tableName = tableName.toLowerCase(); } if (extCatalog.getUseMetaCache().get()) { @@ -633,7 +632,7 @@ public void gsonPostProcess() throws IOException { @Override public void unregisterTable(String tableName) { makeSureInitialized(); - if (Env.isStoredTableNamesLowerCase() || extCatalog.getOnlyTestLowerCaseTableNames() == 1) { + if (this.isStoredTableNamesLowerCase()) { tableName = tableName.toLowerCase(); } if (LOG.isDebugEnabled()) { @@ -694,4 +693,16 @@ public boolean registerTable(TableIf tableIf) { public String getQualifiedName(String tblName) { return String.join(".", extCatalog.getName(), name, tblName); } + + private boolean isStoredTableNamesLowerCase() { + // Because we have added a test configuration item, + // it needs to be judged together with Env.isStoredTableNamesLowerCase() + return Env.isStoredTableNamesLowerCase() || extCatalog.getOnlyTestLowerCaseTableNames() == 1; + } + + private boolean isTableNamesCaseInsensitive() { + // Because we have added a test configuration item, + // it needs to be judged together with Env.isTableNamesCaseInsensitive() + return Env.isTableNamesCaseInsensitive() || extCatalog.getOnlyTestLowerCaseTableNames() == 2; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java index cd24cc72302f28..4bad19b8a6949d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java @@ -33,6 +33,7 @@ import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.thrift.TTableDescriptor; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.commons.text.StringSubstitutor; import org.apache.logging.log4j.LogManager; @@ -111,38 +112,60 @@ public TTableDescriptor toThrift() { @Override public Optional initSchema() { String remoteDbName = ((ExternalDatabase) this.getDatabase()).getRemoteName(); + + // 1. Retrieve remote column information List columns = ((JdbcExternalCatalog) catalog).listColumns(remoteDbName, remoteName); + if (columns == null || columns.isEmpty()) { + return Optional.empty(); + } + + // 2. Generate local column names from remote names List remoteColumnNames = columns.stream() .map(Column::getName) .collect(Collectors.toList()); + List localColumnNames = Lists.newArrayListWithCapacity(remoteColumnNames.size()); + for (String remoteColName : remoteColumnNames) { + String localName = ((JdbcExternalCatalog) catalog).getIdentifierMapping() + .fromRemoteColumnName(remoteDbName, remoteName, remoteColName); + localColumnNames.add(localName); + } - // Checks whether remoteColumnNames contains duplicate column names that differ only in case - Map lowerCaseColumnNameMap = Maps.newHashMap(); - for (String remoteColumnName : remoteColumnNames) { - String lowerCaseColumnName = remoteColumnName.toLowerCase(); - if (lowerCaseColumnNameMap.containsKey(lowerCaseColumnName)) { - throw new RuntimeException(String.format( - "Found conflicting column names under case-insensitive conditions. " - + "Conflicting remote column names: '%s' and '%s' in remote table '%s.%s' " - + "under catalog '%s'. " - + "Please use meta_names_mapping to handle name mapping.", - lowerCaseColumnNameMap.get(lowerCaseColumnName), remoteColumnName, remoteDbName, remoteName, - catalog.getName())); - } - lowerCaseColumnNameMap.put(lowerCaseColumnName, remoteColumnName); + // 3. Collect potential conflicts in a case-insensitive scenario + Map> lowerCaseToLocalNames = Maps.newHashMap(); + for (String localColName : localColumnNames) { + String lowerName = localColName.toLowerCase(); + lowerCaseToLocalNames + .computeIfAbsent(lowerName, k -> Lists.newArrayList()) + .add(localColName); } - List localColumnNames = remoteColumnNames.stream() - .map(remoteColumnName -> ((JdbcExternalCatalog) catalog).getIdentifierMapping() - .fromRemoteColumnName(remoteDbName, remoteName, remoteColumnName)) + // 4. Check for conflicts + List conflicts = lowerCaseToLocalNames.values().stream() + .filter(names -> names.size() > 1) + .flatMap(List::stream) + .distinct() .collect(Collectors.toList()); + + if (!conflicts.isEmpty()) { + throw new RuntimeException(String.format( + "Found conflicting column names under case-insensitive conditions. " + + "Conflicting column names: %s in remote table '%s.%s' under catalog '%s'. " + + "Please use meta_names_mapping to handle name mapping.", + String.join(", ", conflicts), remoteDbName, remoteName, catalog.getName())); + } + + // 5. Update column objects with local names for (int i = 0; i < columns.size(); i++) { columns.get(i).setName(localColumnNames.get(i)); } + + // 6. Build remote->local mapping Map remoteColumnNamesMap = Maps.newHashMap(); - for (int i = 0; i < remoteColumnNames.size(); i++) { + for (int i = 0; i < columns.size(); i++) { remoteColumnNamesMap.put(localColumnNames.get(i), remoteColumnNames.get(i)); } + + // 7. Return the SchemaCacheValue return Optional.of(new JdbcSchemaCacheValue(columns, remoteColumnNamesMap)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java index b8067e7e6bf75b..1d9ea4bd4eb399 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java @@ -24,8 +24,10 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.Pair; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; +import org.apache.doris.datasource.jdbc.JdbcExternalDatabase; import org.apache.doris.datasource.jdbc.JdbcExternalTable; import mockit.Mock; @@ -117,7 +119,8 @@ public void testSupportAutoAnalyze() { OlapTable table1 = new OlapTable(200, "testTable", schema, null, null, null); Assertions.assertTrue(collector.supportAutoAnalyze(table1)); - ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbcdb", null, null); + JdbcExternalDatabase jdbcExternalDatabase = new JdbcExternalDatabase(null, 1L, "jdbcdb", "jdbcdb"); + ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbctable", null, jdbcExternalDatabase); Assertions.assertFalse(collector.supportAutoAnalyze(externalTable)); new MockUp() { @@ -126,7 +129,8 @@ public DLAType getDlaType() { return DLAType.ICEBERG; } }; - ExternalTable icebergExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null, null); + HMSExternalDatabase hmsExternalDatabase = new HMSExternalDatabase(null, 1L, "hmsDb", "hmsDb"); + ExternalTable icebergExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null, hmsExternalDatabase); Assertions.assertFalse(collector.supportAutoAnalyze(icebergExternalTable)); new MockUp() { @@ -135,7 +139,7 @@ public DLAType getDlaType() { return DLAType.HIVE; } }; - ExternalTable hiveExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null, null); + ExternalTable hiveExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null, hmsExternalDatabase); Assertions.assertTrue(collector.supportAutoAnalyze(hiveExternalTable)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java index e1ecff04e7a806..7514d96ddb5e1d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java @@ -33,6 +33,7 @@ import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; +import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.jdbc.JdbcExternalCatalog; import org.apache.doris.datasource.jdbc.JdbcExternalDatabase; @@ -182,12 +183,7 @@ void testNeedAnalyzeColumn() throws DdlException { schema.add(column); OlapTable table = new OlapTable(200, "testTable", schema, null, null, null); HMSExternalCatalog externalCatalog = new HMSExternalCatalog(); - - long dbId = 100; - String dbName = "testDb"; - HMSExternalDatabase externalDatabase = new HMSExternalDatabase(externalCatalog, dbId, dbName, dbName); - - + HMSExternalDatabase externalDatabase = new HMSExternalDatabase(externalCatalog, 1L, "dbName", "dbName"); // Test olap table auto analyze disabled. Map properties = new HashMap<>(); properties.put(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY, "disable"); @@ -402,7 +398,8 @@ public boolean autoAnalyzeEnabled() { return true; } }; - IcebergExternalTable icebergTable = new IcebergExternalTable(0, "", "", null, null); + IcebergExternalDatabase icebergDatabase = new IcebergExternalDatabase(null, 1L, "", ""); + IcebergExternalTable icebergTable = new IcebergExternalTable(0, "", "", null, icebergDatabase); Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(icebergTable, Pair.of("index", column.getName()))); // Test table stats meta is null. diff --git a/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out index 101ca6838796a7..1dab3b825eaf28 100644 --- a/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out +++ b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out @@ -95,3 +95,27 @@ -- !sql_test_cache_true_lower_true_with_conf2_4 -- 1 UPPER +-- !sql_test_cache_false_lower_false_with_conf0_1 -- +1 lower + +-- !sql_test_cache_false_lower_false_with_conf0_2 -- +1 UPPER + +-- !sql_test_cache_true_lower_false_with_conf0_1 -- +1 lower + +-- !sql_test_cache_true_lower_false_with_conf0_2 -- +1 UPPER + +-- !sql_test_cache_false_lower_true_with_conf0_1 -- +1 lower + +-- !sql_test_cache_false_lower_true_with_conf0_2 -- +1 UPPER + +-- !sql_test_cache_true_lower_true_with_conf0_1 -- +1 lower + +-- !sql_test_cache_true_lower_true_with_conf0_2 -- +1 UPPER + diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy index d18df396d56bc4..a04ab37587445f 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy @@ -358,6 +358,196 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external sql """drop catalog if exists test_cache_true_lower_true_with_conf2 """ + // Test for cache false and lower false and lower conf 0 + sql """drop catalog if exists test_cache_false_lower_false_with_conf0 """ + + sql """ CREATE CATALOG `test_cache_false_lower_false_with_conf0` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "0" + )""" + + qt_sql_test_cache_false_lower_false_with_conf0_1 "select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf0" + qt_sql_test_cache_false_lower_false_with_conf0_2 "select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + + sql "refresh catalog test_cache_false_lower_false_with_conf0" + test { + sql "select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_false_with_conf0" + test { + sql "select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.upper_with_conf" + exception "Table [upper_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + test { + sql """show tables from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_false_with_conf0 """ + + + // Test for cache true and lower false and lower conf 0 + sql """drop catalog if exists test_cache_true_lower_false_with_conf0 """ + + sql """ CREATE CATALOG `test_cache_true_lower_false_with_conf0` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "0" + )""" + + qt_sql_test_cache_true_lower_false_with_conf0_1 "select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf0" + qt_sql_test_cache_true_lower_false_with_conf0_2 "select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + + sql "refresh catalog test_cache_true_lower_false_with_conf0" + test { + sql "select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_false_with_conf0" + test { + sql "select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.upper_with_conf" + exception "Table [upper_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + test { + sql """show tables from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_false_with_conf0 """ + + + // Test for cache false and lower true and lower conf 0 + sql """drop catalog if exists test_cache_false_lower_true_with_conf0 """ + + sql """ CREATE CATALOG `test_cache_false_lower_true_with_conf0` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "0" + )""" + + qt_sql_test_cache_false_lower_true_with_conf0_1 "select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf0" + qt_sql_test_cache_false_lower_true_with_conf0_2 "select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_false_lower_true_with_conf0" + test { + sql "select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_true_with_conf0" + test { + sql "select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + exception "Table [UPPER_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + test { + sql """show tables from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_true_with_conf0 """ + + + // Test for cache true and lower true and lower conf 0 + sql """drop catalog if exists test_cache_true_lower_true_with_conf0 """ + + sql """ CREATE CATALOG `test_cache_true_lower_true_with_conf0` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "0" + )""" + + qt_sql_test_cache_true_lower_true_with_conf0_1 "select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf0" + qt_sql_test_cache_true_lower_true_with_conf0_2 "select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_true_lower_true_with_conf0" + test { + sql "select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_true_with_conf0" + test { + sql "select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + exception "Table [UPPER_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + test { + sql """show tables from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_true_with_conf0 """ sql """drop database if exists internal.external_test_lower_with_conf; """ } diff --git a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy index 170a81955b1a0f..1cf48b17c87ab9 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy @@ -199,6 +199,8 @@ suite("test_meta_names_mapping", "p0,external,doris,meta_names_mapping") { qt_sql_meta_mapping_select_lower_lower_case_true "select * from test_conflict_meta_names.external_meta_names_mapping_lower.table_test_lower" qt_sql_meta_mapping_select_upper_lower_case_true "select * from test_conflict_meta_names.external_meta_names_mapping_lower.table_test_upper" + sql """drop catalog if exists test_conflict_meta_names """ + String error_mapping_db = """ { "databases": [ From 11adf388c567839c7c0f66284c3fe352acdc3041 Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Wed, 25 Dec 2024 11:54:42 +0800 Subject: [PATCH 4/4] add case --- .../test_lower_case_meta_show_and_select.out | 24 +++ ..._with_lower_table_conf_show_and_select.out | 120 ++++++++++++++ ...est_lower_case_meta_show_and_select.groovy | 25 +++ ...th_lower_table_conf_show_and_select.groovy | 149 ++++++++++++++++++ .../test_timing_refresh_catalog.groovy | 8 +- 5 files changed, 322 insertions(+), 4 deletions(-) diff --git a/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_show_and_select.out b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_show_and_select.out index 61fe3b39c984e7..1332a98aac4f9d 100644 --- a/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_show_and_select.out +++ b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_show_and_select.out @@ -5,21 +5,45 @@ -- !sql_test_cache_false_lower_false2 -- 1 UPPER +-- !sql_test_cache_false_lower_false1_insert -- +1 + +-- !sql_test_cache_false_lower_false2_insert -- +1 + -- !sql_test_cache_false_lower_false1 -- 1 lower -- !sql_test_cache_false_lower_false2 -- 1 UPPER +-- !sql_test_cache_false_lower_false1_insert -- +1 + +-- !sql_test_cache_false_lower_false2_insert -- +1 + -- !sql_test_cache_false_lower_true1 -- 1 lower -- !sql_test_cache_false_lower_true2 -- 1 UPPER +-- !sql_test_cache_false_lower_true1_insert -- +1 + +-- !sql_test_cache_false_lower_true2_insert -- +1 + -- !sql_test_cache_true_lower_true1 -- 1 lower -- !sql_test_cache_true_lower_true2 -- 1 UPPER +-- !sql_test_cache_true_lower_true1_insert -- +1 + +-- !sql_test_cache_true_lower_true2_insert -- +1 + diff --git a/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out index 1dab3b825eaf28..6be380ed0d0161 100644 --- a/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out +++ b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out @@ -11,6 +11,18 @@ -- !sql_test_cache_false_lower_false_with_conf1_4 -- 1 UPPER +-- !sql_test_cache_false_lower_false_with_conf1_1_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf1_2_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf1_3_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf1_4_insert -- +1 + -- !sql_test_cache_false_lower_false_with_conf2_1 -- 1 lower @@ -23,6 +35,18 @@ -- !sql_test_cache_false_lower_false_with_conf2_4 -- 1 UPPER +-- !sql_test_cache_false_lower_false_with_conf2_1_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf2_2_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf2_3_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf2_4_insert -- +1 + -- !sql_test_cache_true_lower_false_with_conf1_1 -- 1 lower @@ -35,6 +59,18 @@ -- !sql_test_cache_true_lower_false_with_conf1_4 -- 1 UPPER +-- !sql_test_cache_true_lower_false_with_conf1_1_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf1_2_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf1_3_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf1_4_insert -- +1 + -- !sql_test_cache_true_lower_false_with_conf2_1 -- 1 lower @@ -47,6 +83,18 @@ -- !sql_test_cache_true_lower_false_with_conf2_4 -- 1 UPPER +-- !sql_test_cache_true_lower_false_with_conf2_1_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf2_2_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf2_3_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf2_4_insert -- +1 + -- !sql_test_cache_false_lower_true_with_conf1_1 -- 1 lower @@ -59,6 +107,18 @@ -- !sql_test_cache_false_lower_true_with_conf1_4 -- 1 UPPER +-- !sql_test_cache_false_lower_true_with_conf1_1_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf1_2_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf1_3_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf1_4_insert -- +1 + -- !sql_test_cache_false_lower_true_with_conf2_1 -- 1 lower @@ -71,6 +131,18 @@ -- !sql_test_cache_false_lower_true_with_conf2_4 -- 1 UPPER +-- !sql_test_cache_false_lower_true_with_conf2_1_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf2_2_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf2_3_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf2_4_insert -- +1 + -- !sql_test_cache_true_lower_true_with_conf1_1 -- 1 lower @@ -83,6 +155,18 @@ -- !sql_test_cache_true_lower_true_with_conf1_4 -- 1 UPPER +-- !sql_test_cache_true_lower_true_with_conf1_1_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf1_2_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf1_3_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf1_4_insert -- +1 + -- !sql_test_cache_true_lower_true_with_conf2_1 -- 1 lower @@ -95,27 +179,63 @@ -- !sql_test_cache_true_lower_true_with_conf2_4 -- 1 UPPER +-- !sql_test_cache_true_lower_true_with_conf2_1_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf2_2_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf2_3_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf2_4_insert -- +1 + -- !sql_test_cache_false_lower_false_with_conf0_1 -- 1 lower -- !sql_test_cache_false_lower_false_with_conf0_2 -- 1 UPPER +-- !sql_test_cache_false_lower_false_with_conf0_1_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf0_2_insert -- +1 + -- !sql_test_cache_true_lower_false_with_conf0_1 -- 1 lower -- !sql_test_cache_true_lower_false_with_conf0_2 -- 1 UPPER +-- !sql_test_cache_true_lower_false_with_conf0_1_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf0_2_insert -- +1 + -- !sql_test_cache_false_lower_true_with_conf0_1 -- 1 lower -- !sql_test_cache_false_lower_true_with_conf0_2 -- 1 UPPER +-- !sql_test_cache_false_lower_true_with_conf0_1_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf0_2_insert -- +1 + -- !sql_test_cache_true_lower_true_with_conf0_1 -- 1 lower -- !sql_test_cache_true_lower_true_with_conf0_2 -- 1 UPPER +-- !sql_test_cache_true_lower_true_with_conf0_1_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf0_2_insert -- +1 + diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy index 845a78be0b62c8..d4efd141c2d468 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy @@ -43,6 +43,18 @@ suite("test_lower_case_meta_show_and_select", "p0,external,doris,external_docker sql """insert into internal.external_test_lower.lower values(1, 'lower')""" sql """insert into internal.external_test_lower.UPPER values(1, 'UPPER')""" + sql """create table if not exists internal.external_test_lower.lower_insert + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """create table if not exists internal.external_test_lower.UPPER_insert + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + // Test for cache false and lower false sql """drop catalog if exists test_cache_false_lower_false """ @@ -86,6 +98,9 @@ suite("test_lower_case_meta_show_and_select", "p0,external,doris,external_docker qt_sql_test_cache_false_lower_false1 "select * from test_cache_false_lower_false.external_test_lower.lower" qt_sql_test_cache_false_lower_false2 "select * from test_cache_false_lower_false.external_test_lower.UPPER" + qt_sql_test_cache_false_lower_false1_insert "insert into internal.external_test_lower.lower_insert select * from test_cache_false_lower_false.external_test_lower.lower" + qt_sql_test_cache_false_lower_false2_insert "insert into internal.external_test_lower.UPPER_insert select * from test_cache_false_lower_false.external_test_lower.UPPER" + sql """drop catalog if exists test_cache_false_lower_false """ // Test for cache true and lower false @@ -131,6 +146,10 @@ suite("test_lower_case_meta_show_and_select", "p0,external,doris,external_docker qt_sql_test_cache_false_lower_false1 "select * from test_cache_true_lower_false.external_test_lower.lower" qt_sql_test_cache_false_lower_false2 "select * from test_cache_true_lower_false.external_test_lower.UPPER" + qt_sql_test_cache_false_lower_false1_insert "insert into internal.external_test_lower.lower_insert select * from test_cache_true_lower_false.external_test_lower.lower" + qt_sql_test_cache_false_lower_false2_insert "insert into internal.external_test_lower.UPPER_insert select * from test_cache_true_lower_false.external_test_lower.UPPER" + + sql """drop catalog if exists test_cache_true_lower_false """ // Test for cache false and lower true @@ -176,6 +195,9 @@ suite("test_lower_case_meta_show_and_select", "p0,external,doris,external_docker qt_sql_test_cache_false_lower_true1 "select * from test_cache_false_lower_true.external_test_lower.lower" qt_sql_test_cache_false_lower_true2 "select * from test_cache_false_lower_true.external_test_lower.upper" + qt_sql_test_cache_false_lower_true1_insert "insert into internal.external_test_lower.lower_insert select * from test_cache_false_lower_true.external_test_lower.lower" + qt_sql_test_cache_false_lower_true2_insert "insert into internal.external_test_lower.UPPER_insert select * from test_cache_false_lower_true.external_test_lower.upper" + sql """drop catalog if exists test_cache_false_lower_true """ // Test for cache true and lower true @@ -221,6 +243,9 @@ suite("test_lower_case_meta_show_and_select", "p0,external,doris,external_docker qt_sql_test_cache_true_lower_true1 "select * from test_cache_true_lower_true.external_test_lower.lower" qt_sql_test_cache_true_lower_true2 "select * from test_cache_true_lower_true.external_test_lower.upper" + qt_sql_test_cache_true_lower_true1_insert "insert into internal.external_test_lower.lower_insert select * from test_cache_true_lower_true.external_test_lower.lower" + qt_sql_test_cache_true_lower_true2_insert "insert into internal.external_test_lower.UPPER_insert select * from test_cache_true_lower_true.external_test_lower.upper" + sql """drop catalog if exists test_cache_true_lower_true """ diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy index a04ab37587445f..c3da24e1f55c1d 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy @@ -42,6 +42,12 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external sql """insert into internal.external_test_lower_with_conf.lower_with_conf values(1, 'lower')""" sql """insert into internal.external_test_lower_with_conf.UPPER_with_conf values(1, 'UPPER')""" + sql """create table if not exists internal.external_test_lower_with_conf.with_conf_insert + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + // Test for cache false and lower false and lower conf 1 sql """drop catalog if exists test_cache_false_lower_false_with_conf1 """ @@ -67,6 +73,16 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external sql "refresh catalog test_cache_false_lower_false_with_conf1" qt_sql_test_cache_false_lower_false_with_conf1_4 "select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf" + + test { sql """show tables from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf""" @@ -106,6 +122,15 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external sql "refresh catalog test_cache_false_lower_false_with_conf2" qt_sql_test_cache_false_lower_false_with_conf2_4 "select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf" + test { sql """show tables from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf""" @@ -147,6 +172,15 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external sql "refresh catalog test_cache_true_lower_false_with_conf1" qt_sql_test_cache_true_lower_false_with_conf1_4 "select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf" + test { sql """show tables from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf""" @@ -186,6 +220,15 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external sql "refresh catalog test_cache_true_lower_false_with_conf2" qt_sql_test_cache_true_lower_false_with_conf2_4 "select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf" + test { sql """show tables from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf""" @@ -225,6 +268,15 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external sql "refresh catalog test_cache_false_lower_true_with_conf1" qt_sql_test_cache_false_lower_true_with_conf1_4 "select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf" + test { sql """show tables from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf""" @@ -264,6 +316,15 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external sql "refresh catalog test_cache_false_lower_true_with_conf2" qt_sql_test_cache_false_lower_true_with_conf2_4 "select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf" + test { sql """show tables from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf""" @@ -304,6 +365,15 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external sql "refresh catalog test_cache_true_lower_true_with_conf1" qt_sql_test_cache_true_lower_true_with_conf1_4 "select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf" + test { sql """show tables from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf""" @@ -343,6 +413,15 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external sql "refresh catalog test_cache_true_lower_true_with_conf2" qt_sql_test_cache_true_lower_true_with_conf2_4 "select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf" + test { sql """show tables from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf""" @@ -379,6 +458,11 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external sql "refresh catalog test_cache_false_lower_false_with_conf0" qt_sql_test_cache_false_lower_false_with_conf0_2 "select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf0" + qt_sql_test_cache_false_lower_false_with_conf0_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf0" + qt_sql_test_cache_false_lower_false_with_conf0_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf0" test { sql "select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf" @@ -391,6 +475,18 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external exception "Table [upper_with_conf] does not exist in database [external_test_lower_with_conf]." } + sql "refresh catalog test_cache_false_lower_false_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_false_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.upper_with_conf" + exception "Table [upper_with_conf] does not exist in database [external_test_lower_with_conf]." + } + test { sql """show tables from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf""" @@ -427,6 +523,12 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external sql "refresh catalog test_cache_true_lower_false_with_conf0" qt_sql_test_cache_true_lower_false_with_conf0_2 "select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf0" + qt_sql_test_cache_true_lower_false_with_conf0_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf0" + qt_sql_test_cache_true_lower_false_with_conf0_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + + sql "refresh catalog test_cache_true_lower_false_with_conf0" test { sql "select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf" @@ -439,6 +541,18 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external exception "Table [upper_with_conf] does not exist in database [external_test_lower_with_conf]." } + sql "refresh catalog test_cache_true_lower_false_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_false_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.upper_with_conf" + exception "Table [upper_with_conf] does not exist in database [external_test_lower_with_conf]." + } + test { sql """show tables from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf""" @@ -475,6 +589,11 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external sql "refresh catalog test_cache_false_lower_true_with_conf0" qt_sql_test_cache_false_lower_true_with_conf0_2 "select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf0" + qt_sql_test_cache_false_lower_true_with_conf0_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf0" + qt_sql_test_cache_false_lower_true_with_conf0_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf0" test { sql "select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf" @@ -487,6 +606,18 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external exception "Table [UPPER_with_conf] does not exist in database [external_test_lower_with_conf]." } + sql "refresh catalog test_cache_false_lower_true_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_true_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + exception "Table [UPPER_with_conf] does not exist in database [external_test_lower_with_conf]." + } + test { sql """show tables from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf""" @@ -523,6 +654,12 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external sql "refresh catalog test_cache_true_lower_true_with_conf0" qt_sql_test_cache_true_lower_true_with_conf0_2 "select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf0" + qt_sql_test_cache_true_lower_true_with_conf0_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf0" + qt_sql_test_cache_true_lower_true_with_conf0_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_true_lower_true_with_conf0" test { sql "select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf" @@ -535,6 +672,18 @@ suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external exception "Table [UPPER_with_conf] does not exist in database [external_test_lower_with_conf]." } + sql "refresh catalog test_cache_true_lower_true_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_true_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + exception "Table [UPPER_with_conf] does not exist in database [external_test_lower_with_conf]." + } + test { sql """show tables from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf""" diff --git a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy index 9317928c7318fb..19d7e2db18089a 100644 --- a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy +++ b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy @@ -82,7 +82,7 @@ suite("test_timing_refresh_catalog", "p0,external,doris,external_docker,external test { def catalogName = "test_timing_refresh_catalog1" - for (int i = 0; i < 30; i++) { + for (int i = 0; i < 10; i++) { sql """ select * from ${catalogName}.external_timing_refresh_catalog.tbl """ @@ -92,7 +92,7 @@ suite("test_timing_refresh_catalog", "p0,external,doris,external_docker,external test { def catalogName = "test_timing_refresh_catalog2" - for (int i = 0; i < 30; i++) { + for (int i = 0; i < 10; i++) { sql """ select * from ${catalogName}.external_timing_refresh_catalog.tbl """ @@ -137,7 +137,7 @@ suite("test_timing_refresh_catalog", "p0,external,doris,external_docker,external test { def catalogName = "test_timing_refresh_catalog1" - for (int i = 0; i < 30; i++) { + for (int i = 0; i < 10; i++) { sql """ select id_c from ${catalogName}.db.table_t """ @@ -147,7 +147,7 @@ suite("test_timing_refresh_catalog", "p0,external,doris,external_docker,external test { def catalogName = "test_timing_refresh_catalog2" - for (int i = 0; i < 30; i++) { + for (int i = 0; i < 10; i++) { sql """ select id_c from ${catalogName}.db.table_t """