From b62e5b5fcfc2c0f6d2c78f8d94018febb6de4652 Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Wed, 25 Dec 2024 20:15:04 +0800 Subject: [PATCH] [improvement](jdbc catalog) Optimize JdbcCatalog case mapping stability (#41510) 1. Cancel the preservation of name mapping by IdentifierMapping, and only use it as the logic of case processing 2. Save the remote name in ExternalDatabase/ExternalTable object for get 3. Use fromRemoteXXName to process the remote name and map it to the local. Currently, only JdbcExternalCatalog uses IdentifierMapping for processing in this method, and other Catalogs are not processed. If processing is required, it should be extended by yourself 4. When lower_case_meta_name is true, the database, table, and column names mapped remotely to local will be checked to see if they are only case-sensitive. 5. Since Doris's column names are case-insensitive, column names will be checked at any time 6. When Fe config's lower_case_table_names is 1 or 2, the table names mapped from remote to local will be checked for only case-sensitive. 7. include/exclude_database_lists The remote name must be used 8. meta_names_mapping is used to resolve the conflicts mentioned above 9. If the same remoteName is in include_database_lists and meta_names_mapping, the filtered remoteName will be displayed according to the name mapped by meta_names_mapping 10. Since IdentifierMapping is no longer used to store the uppercase and lowercase name mapping, ExternalObject itself is used instead, which solves the problem of frequent refresh and small mapping loss 11. Remove the logic of rebuilding JdbcClient every time refreshCatalog --- .../apache/doris/datasource/CatalogIf.java | 10 + .../apache/doris/datasource/CatalogMgr.java | 12 +- .../doris/datasource/ExternalCatalog.java | 196 ++++- .../doris/datasource/ExternalDatabase.java | 238 +++++- .../datasource/ExternalMetaCacheMgr.java | 3 +- .../doris/datasource/ExternalTable.java | 26 +- .../doris/datasource/InitCatalogLog.java | 7 +- .../doris/datasource/InitDatabaseLog.java | 7 +- .../datasource/es/EsExternalDatabase.java | 12 +- .../doris/datasource/es/EsExternalTable.java | 9 +- .../datasource/hive/HMSExternalCatalog.java | 2 +- .../datasource/hive/HMSExternalDatabase.java | 12 +- .../datasource/hive/HMSExternalTable.java | 10 +- .../iceberg/IcebergExternalDatabase.java | 11 +- .../iceberg/IcebergExternalTable.java | 5 +- .../ExternalInfoSchemaDatabase.java | 8 +- .../infoschema/ExternalInfoSchemaTable.java | 6 +- .../infoschema/ExternalMysqlDatabase.java | 8 +- .../infoschema/ExternalMysqlTable.java | 6 +- .../datasource/jdbc/JdbcExternalCatalog.java | 74 +- .../datasource/jdbc/JdbcExternalDatabase.java | 11 +- .../datasource/jdbc/JdbcExternalTable.java | 91 ++- ...Mapping.java => JdbcSchemaCacheValue.java} | 30 +- .../datasource/jdbc/client/JdbcClient.java | 47 +- .../jdbc/client/JdbcGbaseClient.java | 4 +- .../jdbc/client/JdbcMySQLClient.java | 4 +- .../jdbc/client/JdbcOracleClient.java | 4 +- .../jdbc/client/JdbcPostgreSQLClient.java | 4 +- .../lakesoul/LakeSoulExternalDatabase.java | 11 +- .../lakesoul/LakeSoulExternalTable.java | 19 +- .../datasource/mapping/IdentifierMapping.java | 311 +------- .../mapping/JdbcIdentifierMapping.java | 345 +++++++++ .../MaxComputeExternalDatabase.java | 12 +- .../maxcompute/MaxComputeExternalTable.java | 7 +- .../doris/datasource/metacache/MetaCache.java | 22 +- .../paimon/PaimonExternalDatabase.java | 11 +- .../paimon/PaimonExternalTable.java | 5 +- .../datasource/test/TestExternalDatabase.java | 11 +- .../datasource/test/TestExternalTable.java | 4 +- .../TrinoConnectorExternalDatabase.java | 12 +- .../TrinoConnectorExternalTable.java | 5 +- .../constraint/ConstraintPersistTest.java | 8 +- .../doris/datasource/CatalogMgrTest.java | 8 +- .../hive/HiveDDLAndDMLPlanTest.java | 4 +- .../datasource/hive/HiveMetadataOpsTest.java | 2 +- .../iceberg/CreateIcebergTableTest.java | 2 +- ...meComparedLowercaseMetaCacheFalseTest.java | 136 ++++ ...meComparedLowercaseMetaCacheTrueTest.java} | 21 +- ...NameStoredLowercaseMetaCacheFalseTest.java | 144 ++++ ...NameStoredLowercaseMetaCacheTrueTest.java} | 21 +- .../mapping/JdbcIdentifierMappingTest.java | 277 +++++++ .../doris/external/hms/HmsCatalogTest.java | 2 +- .../apache/doris/qe/HmsQueryCacheTest.java | 2 +- .../StatisticsAutoCollectorTest.java | 10 +- .../statistics/util/StatisticsUtilTest.java | 19 +- .../lower_case/test_conflict_name.out | 5 + .../test_lower_case_meta_show_and_select.out | 49 ++ ..._with_lower_table_conf_show_and_select.out | 241 ++++++ .../lower_case/test_lower_case_mtmv.out | 3 + ...test_meta_cache_select_without_refresh.out | 10 + .../lower_case/test_meta_names_mapping.out | 13 + .../lower_case/upgrade/load.out | 7 + .../test_upgrade_lower_case_catalog.out | 7 + .../lower_case/test_conflict_name.groovy | 94 +++ .../test_lower_case_meta_include.groovy | 158 ++++ ...est_lower_case_meta_show_and_select.groovy | 254 +++++++ ...th_lower_table_conf_show_and_select.groovy | 702 ++++++++++++++++++ .../lower_case/test_lower_case_mtmv.groovy | 64 ++ ...t_meta_cache_select_without_refresh.groovy | 92 +++ .../lower_case/test_meta_names_mapping.groovy | 289 +++++++ .../test_timing_refresh_catalog.groovy | 161 ++++ .../lower_case/upgrade/load.groovy | 89 +++ .../test_upgrade_lower_case_catalog.groovy | 47 ++ 73 files changed, 3973 insertions(+), 600 deletions(-) rename fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/{JdbcIdentifierMapping.java => JdbcSchemaCacheValue.java} (51%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/JdbcIdentifierMapping.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheFalseTest.java rename fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/{ExternalTableNameComparedLowercaseTest.java => ExternalTableNameComparedLowercaseMetaCacheTrueTest.java} (82%) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheFalseTest.java rename fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/{ExternalTableNameStoredLowercaseTest.java => ExternalTableNameStoredLowercaseMetaCacheTrueTest.java} (83%) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/mapping/JdbcIdentifierMappingTest.java create mode 100644 regression-test/data/external_table_p0/lower_case/test_conflict_name.out create mode 100644 regression-test/data/external_table_p0/lower_case/test_lower_case_meta_show_and_select.out create mode 100644 regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out create mode 100644 regression-test/data/external_table_p0/lower_case/test_lower_case_mtmv.out create mode 100644 regression-test/data/external_table_p0/lower_case/test_meta_cache_select_without_refresh.out create mode 100644 regression-test/data/external_table_p0/lower_case/test_meta_names_mapping.out create mode 100644 regression-test/data/external_table_p0/lower_case/upgrade/load.out create mode 100644 regression-test/data/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.out create mode 100644 regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy create mode 100644 regression-test/suites/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index 41cb44ef0b5fa3..a11061afa38e38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -220,4 +220,14 @@ default Optional getMetaTableFunction(String dbName, String default Optional getMetaTableFunctionRef(String dbName, String sourceNameWithMetaName) { return Optional.empty(); } + + // Convert from remote database name to local database name, overridden by subclass if necessary + default String fromRemoteDatabaseName(String remoteDatabaseName) { + return remoteDatabaseName; + } + + // Convert from remote table name to local table name, overridden by subclass if necessary + default String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) { + return remoteTableName; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 91273736ebfecc..19594f339b1546 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -48,6 +48,7 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.persist.OperationType; @@ -656,8 +657,8 @@ public boolean externalTableExistInLocal(String dbName, String tableName, String } public void registerExternalTableFromEvent(String dbName, String tableName, - String catalogName, long updateTime, - boolean ignoreIfExists) throws DdlException { + String catalogName, long updateTime, + boolean ignoreIfExists) throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { throw new DdlException("No catalog found with name: " + catalogName); @@ -687,7 +688,8 @@ public void registerExternalTableFromEvent(String dbName, String tableName, db.writeLock(); try { - HMSExternalTable namedTable = new HMSExternalTable(tblId, tableName, dbName, (HMSExternalCatalog) catalog); + HMSExternalTable namedTable = ((HMSExternalDatabase) db) + .buildTableForInit(tableName, tableName, tblId, hmsCatalog, (HMSExternalDatabase) db, false); namedTable.setUpdateTime(updateTime); db.registerTable(namedTable); } finally { @@ -733,7 +735,7 @@ public void registerExternalDatabaseFromEvent(String dbName, String catalogName) } public void addExternalPartitions(String catalogName, String dbName, String tableName, - List partitionNames, long updateTime, boolean ignoreIfNotExists) + List partitionNames, long updateTime, boolean ignoreIfNotExists) throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { @@ -768,7 +770,7 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl } public void dropExternalPartitions(String catalogName, String dbName, String tableName, - List partitionNames, long updateTime, boolean ignoreIfNotExists) + List partitionNames, long updateTime, boolean ignoreIfNotExists) throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); if (catalog == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index d1df51177fd496..840baa3ac6b9bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -106,9 +106,13 @@ public abstract class ExternalCatalog public static final String DORIS_VERSION = "doris.version"; public static final String DORIS_VERSION_VALUE = Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH; public static final String USE_META_CACHE = "use_meta_cache"; + public static final String CREATE_TIME = "create_time"; public static final boolean DEFAULT_USE_META_CACHE = true; + public static final String FOUND_CONFLICTING = "Found conflicting"; + public static final String ONLY_TEST_LOWER_CASE_TABLE_NAMES = "only_test_lower_case_table_names"; + // Properties that should not be shown in the `show create catalog` result public static final Set HIDDEN_PROPERTIES = Sets.newHashSet( CREATE_TIME, @@ -191,6 +195,7 @@ private Configuration buildConf() { /** * set some default properties when creating catalog + * * @return list of database names in this catalog */ protected List listDatabaseNames() { @@ -272,8 +277,9 @@ public final synchronized void makeSureInitialized() { OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_meta_object_cache_num, ignored -> getFilteredDatabaseNames(), - dbName -> Optional.ofNullable( - buildDbForInit(dbName, Util.genIdByName(name, dbName), logType, true)), + localDbName -> Optional.ofNullable( + buildDbForInit(null, localDbName, Util.genIdByName(name, localDbName), logType, + true)), (key, value, cause) -> value.ifPresent(v -> v.setUnInitialized(invalidCacheInInit))); } setLastUpdateTime(System.currentTimeMillis()); @@ -370,21 +376,28 @@ private void init() { InitCatalogLog initCatalogLog = new InitCatalogLog(); initCatalogLog.setCatalogId(id); initCatalogLog.setType(logType); - List filteredDatabases = getFilteredDatabaseNames(); - for (String dbName : filteredDatabases) { + List> remoteToLocalPairs = getFilteredDatabaseNames(); + for (Pair pair : remoteToLocalPairs) { + String remoteDbName = pair.key(); + String localDbName = pair.value(); long dbId; - if (dbNameToId != null && dbNameToId.containsKey(dbName)) { - dbId = dbNameToId.get(dbName); - tmpDbNameToId.put(dbName, dbId); + if (dbNameToId != null && dbNameToId.containsKey(localDbName)) { + dbId = dbNameToId.get(localDbName); + tmpDbNameToId.put(localDbName, dbId); ExternalDatabase db = idToDb.get(dbId); + // If the remote name is missing during upgrade, all databases in the Map will be reinitialized. + if (Strings.isNullOrEmpty(db.getRemoteName())) { + db.setRemoteName(remoteDbName); + } tmpIdToDb.put(dbId, db); initCatalogLog.addRefreshDb(dbId); } else { dbId = Env.getCurrentEnv().getNextId(); - tmpDbNameToId.put(dbName, dbId); - ExternalDatabase db = buildDbForInit(dbName, dbId, logType, false); + tmpDbNameToId.put(localDbName, dbId); + ExternalDatabase db = + buildDbForInit(remoteDbName, localDbName, dbId, logType, false); tmpIdToDb.put(dbId, db); - initCatalogLog.addCreateDb(dbId, dbName); + initCatalogLog.addCreateDb(dbId, localDbName, remoteDbName); } } @@ -395,15 +408,41 @@ private void init() { Env.getCurrentEnv().getEditLog().logInitCatalog(initCatalogLog); } + /** + * Retrieves a filtered list of database names and their corresponding local database names. + * The method applies to include and exclude filters based on the database properties, ensuring + * only the relevant databases are included for further operations. + *

+ * The method also handles conflicts in database names under case-insensitive conditions + * and throws an exception if such conflicts are detected. + *

+ * Steps: + * 1. Fetch all database names from the remote source. + * 2. Apply to include and exclude database filters: + * - Exclude filters take precedence over include filters. + * - If a database is in the exclude list, it is ignored. + * - If a database is not in the include list and the include list is not empty, it is ignored. + * 3. Map the filtered remote database names to local database names. + * 4. Handle conflicts when `lower_case_meta_names` is enabled: + * - Detect cases where multiple remote database names map to the same lower-cased local name. + * - Throw an exception if conflicts are found. + * + * @return A list of pairs where each pair contains the remote database name and local database name. + * @throws RuntimeException if there are conflicting database names under case-insensitive conditions. + */ @NotNull - private List getFilteredDatabaseNames() { + private List> getFilteredDatabaseNames() { List allDatabases = Lists.newArrayList(listDatabaseNames()); allDatabases.remove(InfoSchemaDb.DATABASE_NAME); allDatabases.add(InfoSchemaDb.DATABASE_NAME); allDatabases.remove(MysqlDb.DATABASE_NAME); allDatabases.add(MysqlDb.DATABASE_NAME); + Map includeDatabaseMap = getIncludeDatabaseMap(); Map excludeDatabaseMap = getExcludeDatabaseMap(); + + List> remoteToLocalPairs = Lists.newArrayList(); + allDatabases = allDatabases.stream().filter(dbName -> { if (!dbName.equals(InfoSchemaDb.DATABASE_NAME) && !dbName.equals(MysqlDb.DATABASE_NAME)) { // Exclude database map take effect with higher priority over include database map @@ -416,7 +455,40 @@ private List getFilteredDatabaseNames() { } return true; }).collect(Collectors.toList()); - return allDatabases; + + for (String remoteDbName : allDatabases) { + String localDbName = fromRemoteDatabaseName(remoteDbName); + remoteToLocalPairs.add(Pair.of(remoteDbName, localDbName)); + } + + // Check for conflicts when lower_case_meta_names = true + if (Boolean.parseBoolean(getLowerCaseMetaNames())) { + // Map to track lowercase local names and their corresponding remote names + Map> lowerCaseToRemoteNames = Maps.newHashMap(); + + // Collect lowercased local names and their remote counterparts + for (Pair pair : remoteToLocalPairs) { + String lowerCaseLocalName = pair.second.toLowerCase(); + lowerCaseToRemoteNames.computeIfAbsent(lowerCaseLocalName, k -> Lists.newArrayList()).add(pair.first); + } + + // Identify conflicts: multiple remote names mapping to the same lowercase local name + List conflicts = lowerCaseToRemoteNames.values().stream() + .filter(remoteNames -> remoteNames.size() > 1) // Conflict: more than one remote name + .flatMap(List::stream) // Collect all conflicting remote names + .collect(Collectors.toList()); + + // Throw exception if conflicts are found + if (!conflicts.isEmpty()) { + throw new RuntimeException(String.format( + FOUND_CONFLICTING + " database names under case-insensitive conditions. " + + "Conflicting remote database names: %s in catalog %s. " + + "Please use meta_names_mapping to handle name mapping.", + String.join(", ", conflicts), name)); + } + } + + return remoteToLocalPairs; } public void onRefresh(boolean invalidCache) { @@ -493,6 +565,7 @@ public void setComment(String comment) { /** * Different from 'listDatabases()', this method will return dbnames from cache. * while 'listDatabases()' will return dbnames from remote datasource. + * * @return names of database in this catalog. */ @Override @@ -649,8 +722,17 @@ private void removeAccessController() { } public void replayInitCatalog(InitCatalogLog log) { + // If the remote name is missing during upgrade, all databases in the Map will be reinitialized. + if (log.getRemoteDbNames() == null || log.getRemoteDbNames().isEmpty()) { + dbNameToId = Maps.newConcurrentMap(); + idToDb = Maps.newConcurrentMap(); + lastUpdateTime = log.getLastUpdateTime(); + initialized = false; + return; + } + Map tmpDbNameToId = Maps.newConcurrentMap(); - Map> tmpIdToDb = Maps.newConcurrentMap(); + Map> tmpIdToDb = Maps.newConcurrentMap(); for (int i = 0; i < log.getRefreshCount(); i++) { Optional> db = getDbForReplay(log.getRefreshDbIds().get(i)); // Should not return null. @@ -667,7 +749,8 @@ public void replayInitCatalog(InitCatalogLog log) { } for (int i = 0; i < log.getCreateCount(); i++) { ExternalDatabase db = - buildDbForInit(log.getCreateDbNames().get(i), log.getCreateDbIds().get(i), log.getType(), false); + buildDbForInit(log.getRemoteDbNames().get(i), log.getCreateDbNames().get(i), + log.getCreateDbIds().get(i), log.getType(), false); if (db != null) { tmpDbNameToId.put(db.getFullName(), db.getId()); tmpIdToDb.put(db.getId(), db); @@ -694,58 +777,92 @@ public Optional> getDbForReplay(long d * Build a database instance. * If checkExists is true, it will check if the database exists in the remote system. * - * @param dbName + * @param remoteDbName * @param dbId * @param logType * @param checkExists * @return */ - protected ExternalDatabase buildDbForInit(String dbName, long dbId, - InitCatalogLog.Type logType, boolean checkExists) { + protected ExternalDatabase buildDbForInit(String remoteDbName, String localDbName, + long dbId, InitCatalogLog.Type logType, boolean checkExists) { + // Step 1: Map local database name if not already provided + if (localDbName == null && remoteDbName != null) { + localDbName = fromRemoteDatabaseName(remoteDbName); + } + + // Step 2: // When running ut, disable this check to make ut pass. // Because in ut, the database is not created in remote system. if (checkExists && (!FeConstants.runningUnitTest || this instanceof TestExternalCatalog)) { try { List dbNames = getDbNames(); - if (!dbNames.contains(dbName)) { - dbNames = getFilteredDatabaseNames(); - if (!dbNames.contains(dbName)) { + if (!dbNames.contains(localDbName)) { + dbNames = getFilteredDatabaseNames().stream() + .map(Pair::value) + .collect(Collectors.toList()); + if (!dbNames.contains(localDbName)) { + LOG.warn("Database {} does not exist in the remote system. Skipping initialization.", + localDbName); return null; } } - } catch (Throwable t) { + } catch (RuntimeException e) { + // Handle "Found conflicting" exception explicitly + if (e.getMessage().contains(FOUND_CONFLICTING)) { + LOG.error(e.getMessage()); + throw e; // Rethrow to let the caller handle this critical issue + } else { + // Any errors other than name conflicts, we default to not finding the database + LOG.warn("Failed to check db {} exist in remote system, ignore it.", localDbName, e); + return null; + } + } catch (Exception e) { // If connection failed, it will throw exception. // ignore it and treat it as not exist. - LOG.warn("Failed to check db {} exist in remote system, ignore it.", dbName, t); + LOG.warn("Failed to check db {} exist in remote system, ignore it.", localDbName, e); return null; } } - if (dbName.equals(InfoSchemaDb.DATABASE_NAME)) { + // Step 3: Resolve remote database name if using meta cache + if (remoteDbName == null && useMetaCache.orElse(false)) { + if (Boolean.parseBoolean(getLowerCaseMetaNames()) || !Strings.isNullOrEmpty(getMetaNamesMapping())) { + remoteDbName = metaCache.getRemoteName(localDbName); + if (remoteDbName == null) { + LOG.warn("Could not resolve remote database name for local database: {}", localDbName); + return null; + } + } else { + remoteDbName = localDbName; + } + } + + // Step 4: Instantiate the appropriate ExternalDatabase based on logType + if (localDbName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) { return new ExternalInfoSchemaDatabase(this, dbId); } - if (dbName.equals(MysqlDb.DATABASE_NAME)) { + if (localDbName.equalsIgnoreCase(MysqlDb.DATABASE_NAME)) { return new ExternalMysqlDatabase(this, dbId); } switch (logType) { case HMS: - return new HMSExternalDatabase(this, dbId, dbName); + return new HMSExternalDatabase(this, dbId, localDbName, remoteDbName); case ES: - return new EsExternalDatabase(this, dbId, dbName); + return new EsExternalDatabase(this, dbId, localDbName, remoteDbName); case JDBC: - return new JdbcExternalDatabase(this, dbId, dbName); + return new JdbcExternalDatabase(this, dbId, localDbName, remoteDbName); case ICEBERG: - return new IcebergExternalDatabase(this, dbId, dbName); + return new IcebergExternalDatabase(this, dbId, localDbName, remoteDbName); case MAX_COMPUTE: - return new MaxComputeExternalDatabase(this, dbId, dbName); + return new MaxComputeExternalDatabase(this, dbId, localDbName, remoteDbName); case LAKESOUL: - return new LakeSoulExternalDatabase(this, dbId, dbName); + return new LakeSoulExternalDatabase(this, dbId, localDbName, remoteDbName); case TEST: - return new TestExternalDatabase(this, dbId, dbName); + return new TestExternalDatabase(this, dbId, localDbName, remoteDbName); case PAIMON: - return new PaimonExternalDatabase(this, dbId, dbName); + return new PaimonExternalDatabase(this, dbId, localDbName, remoteDbName); case TRINO_CONNECTOR: - return new TrinoConnectorExternalDatabase(this, dbId, dbName); + return new TrinoConnectorExternalDatabase(this, dbId, localDbName, remoteDbName); default: break; } @@ -890,6 +1007,19 @@ private Map getSpecifiedDatabaseMap(String catalogPropertyKey) return specifiedDatabaseMap; } + + public String getLowerCaseMetaNames() { + return catalogProperty.getOrDefault(Resource.LOWER_CASE_META_NAMES, "false"); + } + + public int getOnlyTestLowerCaseTableNames() { + return Integer.parseInt(catalogProperty.getOrDefault(ONLY_TEST_LOWER_CASE_TABLE_NAMES, "0")); + } + + public String getMetaNamesMapping() { + return catalogProperty.getOrDefault(Resource.META_NAMES_MAPPING, ""); + } + public String bindBrokerName() { return catalogProperty.getProperties().get(HMSExternalCatalog.BIND_BROKER_NAME); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index eda98efb9b6a03..099d76e98e1b1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -25,7 +25,9 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.MetaNotFoundException; +import org.apache.doris.common.Pair; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; @@ -35,12 +37,14 @@ import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase; import org.apache.doris.datasource.infoschema.ExternalMysqlTable; import org.apache.doris.datasource.metacache.MetaCache; +import org.apache.doris.datasource.test.TestExternalDatabase; import org.apache.doris.persist.gson.GsonPostProcessable; import org.apache.doris.persist.gson.GsonUtils; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.MasterCatalogExecutor; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -76,6 +80,8 @@ public abstract class ExternalDatabase protected long id; @SerializedName(value = "name") protected String name; + @SerializedName(value = "remoteName") + protected String remoteName; @SerializedName(value = "dbProperties") protected DatabaseProperty dbProperties = new DatabaseProperty(); @SerializedName(value = "initialized") @@ -100,11 +106,14 @@ public abstract class ExternalDatabase * @param extCatalog The catalog this database belongs to. * @param id Database id. * @param name Database name. + * @param remoteName Remote database name. */ - public ExternalDatabase(ExternalCatalog extCatalog, long id, String name, InitDatabaseLog.Type dbLogType) { + public ExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName, + InitDatabaseLog.Type dbLogType) { this.extCatalog = extCatalog; this.id = id; this.name = name; + this.remoteName = remoteName; this.dbLogType = dbLogType; } @@ -112,6 +121,10 @@ public void setExtCatalog(ExternalCatalog extCatalog) { this.extCatalog = extCatalog; } + public void setRemoteName(String remoteName) { + this.remoteName = remoteName; + } + public void setTableExtCatalog(ExternalCatalog extCatalog) { for (T table : idToTbl.values()) { table.setCatalog(extCatalog); @@ -150,9 +163,10 @@ public final synchronized void makeSureInitialized() { OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_meta_object_cache_num, ignored -> listTableNames(), - tableName -> Optional.ofNullable( - buildTableForInit(tableName, - Util.genIdByName(extCatalog.getName(), name, tableName), extCatalog)), + localTableName -> Optional.ofNullable( + buildTableForInit(null, localTableName, + Util.genIdByName(extCatalog.getName(), name, localTableName), extCatalog, + this, true)), (key, value, cause) -> value.ifPresent(ExternalTable::unsetObjectCreated)); } setLastUpdateTime(System.currentTimeMillis()); @@ -176,6 +190,15 @@ public final synchronized void makeSureInitialized() { } public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) { + // If the remote name is missing during upgrade, all tables in the Map will be reinitialized. + if (log.getRemoteTableNames() == null || log.getRemoteTableNames().isEmpty()) { + tableNameToId = Maps.newConcurrentMap(); + idToTbl = Maps.newConcurrentMap(); + lastUpdateTime = log.getLastUpdateTime(); + initialized = false; + return; + } + Map tmpTableNameToId = Maps.newConcurrentMap(); Map tmpIdToTbl = Maps.newConcurrentMap(); for (int i = 0; i < log.getRefreshCount(); i++) { @@ -191,7 +214,9 @@ public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) { } } for (int i = 0; i < log.getCreateCount(); i++) { - T table = buildTableForInit(log.getCreateTableNames().get(i), log.getCreateTableIds().get(i), catalog); + T table = + buildTableForInit(log.getRemoteTableNames().get(i), log.getCreateTableNames().get(i), + log.getCreateTableIds().get(i), catalog, this, false); tmpTableNameToId.put(table.getName(), table.getId()); tmpIdToTbl.put(table.getId(), table); } @@ -206,24 +231,34 @@ private void init() { initDatabaseLog.setType(dbLogType); initDatabaseLog.setCatalogId(extCatalog.getId()); initDatabaseLog.setDbId(id); - List tableNames = listTableNames(); - if (tableNames != null) { + List> tableNamePairs = listTableNames(); + if (tableNamePairs != null) { Map tmpTableNameToId = Maps.newConcurrentMap(); Map tmpIdToTbl = Maps.newHashMap(); - for (String tableName : tableNames) { + for (Pair pair : tableNamePairs) { + String remoteTableName = pair.first; + String localTableName = pair.second; long tblId; - if (tableNameToId != null && tableNameToId.containsKey(tableName)) { - tblId = tableNameToId.get(tableName); - tmpTableNameToId.put(tableName, tblId); + if (tableNameToId != null && tableNameToId.containsKey(localTableName)) { + tblId = tableNameToId.get(localTableName); + tmpTableNameToId.put(localTableName, tblId); T table = idToTbl.get(tblId); + // If the remote name is missing during upgrade, all tables in the Map will be reinitialized. + if (Strings.isNullOrEmpty(table.getRemoteName())) { + table.setRemoteName(remoteTableName); + } + // If the db is missing, set it. + if (table.getDb() == null) { + table.setDb(this); + } tmpIdToTbl.put(tblId, table); initDatabaseLog.addRefreshTable(tblId); } else { tblId = Env.getCurrentEnv().getNextId(); - tmpTableNameToId.put(tableName, tblId); - T table = buildTableForInit(tableName, tblId, extCatalog); + tmpTableNameToId.put(localTableName, tblId); + T table = buildTableForInit(remoteTableName, localTableName, tblId, extCatalog, this, false); tmpIdToTbl.put(tblId, table); - initDatabaseLog.addCreateTable(tblId, tableName); + initDatabaseLog.addCreateTable(tblId, localTableName, remoteTableName); } } tableNameToId = tmpTableNameToId; @@ -235,26 +270,119 @@ private void init() { Env.getCurrentEnv().getEditLog().logInitExternalDb(initDatabaseLog); } - private List listTableNames() { - List tableNames; + private List> listTableNames() { + List> tableNames; if (name.equals(InfoSchemaDb.DATABASE_NAME)) { - tableNames = ExternalInfoSchemaDatabase.listTableNames(); + tableNames = ExternalInfoSchemaDatabase.listTableNames().stream() + .map(tableName -> Pair.of(tableName, tableName)) + .collect(Collectors.toList()); } else if (name.equals(MysqlDb.DATABASE_NAME)) { - tableNames = ExternalMysqlDatabase.listTableNames(); + tableNames = ExternalMysqlDatabase.listTableNames().stream() + .map(tableName -> Pair.of(tableName, tableName)) + .collect(Collectors.toList()); } else { - tableNames = extCatalog.listTableNames(null, name).stream().map(tableName -> { - lowerCaseToTableName.put(tableName.toLowerCase(), tableName); - if (Env.isStoredTableNamesLowerCase()) { - return tableName.toLowerCase(); - } else { - return tableName; + tableNames = extCatalog.listTableNames(null, remoteName).stream().map(tableName -> { + String localTableName = extCatalog.fromRemoteTableName(remoteName, tableName); + if (this.isStoredTableNamesLowerCase()) { + localTableName = localTableName.toLowerCase(); } + lowerCaseToTableName.put(tableName.toLowerCase(), tableName); + return Pair.of(tableName, localTableName); }).collect(Collectors.toList()); } + // Check for conflicts when stored table names or meta names are case-insensitive + if (Boolean.parseBoolean(extCatalog.getLowerCaseMetaNames()) + || this.isStoredTableNamesLowerCase() + || this.isTableNamesCaseInsensitive()) { + // Map to track lowercased local names and their corresponding remote names + Map> lowerCaseToRemoteNames = Maps.newHashMap(); + + // Collect lowercased local names and their remote counterparts + for (Pair pair : tableNames) { + String lowerCaseLocalName = pair.value().toLowerCase(); + lowerCaseToRemoteNames.computeIfAbsent(lowerCaseLocalName, k -> Lists.newArrayList()).add(pair.key()); + } + + // Identify conflicts: multiple remote names mapping to the same lowercased local name + List conflicts = lowerCaseToRemoteNames.values().stream() + .filter(remoteNames -> remoteNames.size() > 1) // Conflict: more than one remote name + .flatMap(List::stream) // Collect all conflicting remote names + .collect(Collectors.toList()); + + // Throw exception if conflicts are found + if (!conflicts.isEmpty()) { + throw new RuntimeException(String.format( + ExternalCatalog.FOUND_CONFLICTING + " table names under case-insensitive conditions. " + + "Conflicting remote table names: %s in remote database '%s' under catalog '%s'. " + + "Please use meta_names_mapping to handle name mapping.", + String.join(", ", conflicts), remoteName, extCatalog.getName())); + } + } return tableNames; } - protected abstract T buildTableForInit(String tableName, long tblId, ExternalCatalog catalog); + public T buildTableForInit(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, ExternalDatabase db, boolean checkExists) { + + // Step 1: Resolve local table name if not provided + if (localTableName == null && remoteTableName != null) { + localTableName = extCatalog.fromRemoteTableName(remoteName, remoteTableName); + } + + // Step 2: Check if the table exists in the system, if the `checkExists` flag is enabled + if (checkExists && (!FeConstants.runningUnitTest || this instanceof TestExternalDatabase)) { + try { + List tblNames = Lists.newArrayList(getTableNamesWithLock()); + if (!tblNames.contains(localTableName)) { + tblNames = listTableNames().stream() + .map(Pair::value) + .collect(Collectors.toList()); + if (!tblNames.contains(localTableName)) { + LOG.warn("Table {} does not exist in the remote system. Skipping initialization.", + localTableName); + return null; + } + } + } catch (RuntimeException e) { + // Handle "Found conflicting" exception explicitly + if (e.getMessage().contains(ExternalCatalog.FOUND_CONFLICTING)) { + LOG.error(e.getMessage()); + throw e; // Rethrow to let the caller handle this critical issue + } else { + // Any errors other than name conflicts, we default to not finding the table + LOG.warn("Failed to check existence of table {} in the remote system. Ignoring this table.", + localTableName, e); + return null; + } + } catch (Exception e) { + // If connection fails, treat the table as non-existent + LOG.warn("Failed to check existence of table {} in the remote system. Ignoring this table.", + localTableName, e); + return null; + } + } + + // Step 3: Resolve remote table name if using meta cache and it is not provided + if (remoteTableName == null && extCatalog.useMetaCache.get()) { + if (Boolean.parseBoolean(extCatalog.getLowerCaseMetaNames()) + || !Strings.isNullOrEmpty(extCatalog.getMetaNamesMapping()) + || this.isStoredTableNamesLowerCase()) { + remoteTableName = metaCache.getRemoteName(localTableName); + if (remoteTableName == null) { + LOG.warn("Could not resolve remote table name for local table: {}", localTableName); + return null; + } + } else { + remoteTableName = localTableName; + } + } + + // Step 4: Build and return the table instance using the resolved names and other parameters + return buildTableInternal(remoteTableName, localTableName, tblId, catalog, db); + } + + protected abstract T buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, ExternalDatabase db); public Optional getTableForReplay(long tableId) { if (extCatalog.getUseMetaCache().get()) { @@ -328,6 +456,10 @@ public String getFullName() { return name; } + public String getRemoteName() { + return remoteName; + } + @Override public DatabaseProperty getDbProperties() { return dbProperties; @@ -335,10 +467,18 @@ public DatabaseProperty getDbProperties() { @Override public boolean isTableExist(String tableName) { - if (Env.isTableNamesCaseInsensitive()) { - tableName = lowerCaseToTableName.get(tableName.toLowerCase()); - if (tableName == null) { - return false; + if (this.isTableNamesCaseInsensitive()) { + String realTableName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (realTableName == null) { + // Here we need to execute listTableNames() once to fill in lowerCaseToTableName + // to prevent lowerCaseToTableName from being empty in some cases + listTableNames(); + tableName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (tableName == null) { + return false; + } + } else { + tableName = realTableName; } } return extCatalog.tableExist(ConnectContext.get().getSessionContext(), name, tableName); @@ -391,15 +531,27 @@ public Set getTableNamesWithLock() { @Override public T getTableNullable(String tableName) { makeSureInitialized(); - if (Env.isStoredTableNamesLowerCase()) { + if (this.isStoredTableNamesLowerCase()) { tableName = tableName.toLowerCase(); } - if (Env.isTableNamesCaseInsensitive()) { - tableName = lowerCaseToTableName.get(tableName.toLowerCase()); - if (tableName == null) { - return null; + if (this.isTableNamesCaseInsensitive()) { + String realTableName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (realTableName == null) { + // Here we need to execute listTableNames() once to fill in lowerCaseToTableName + // to prevent lowerCaseToTableName from being empty in some cases + listTableNames(); + tableName = lowerCaseToTableName.get(tableName.toLowerCase()); + if (tableName == null) { + return null; + } + } else { + tableName = realTableName; } } + if (extCatalog.getLowerCaseMetaNames().equalsIgnoreCase("true") + && (this.isTableNamesCaseInsensitive())) { + tableName = tableName.toLowerCase(); + } if (extCatalog.getUseMetaCache().get()) { // must use full qualified name to generate id. // otherwise, if 2 databases have the same table name, the id will be the same. @@ -480,7 +632,7 @@ public void gsonPostProcess() throws IOException { @Override public void unregisterTable(String tableName) { makeSureInitialized(); - if (Env.isStoredTableNamesLowerCase()) { + if (this.isStoredTableNamesLowerCase()) { tableName = tableName.toLowerCase(); } if (LOG.isDebugEnabled()) { @@ -528,7 +680,9 @@ public boolean registerTable(TableIf tableIf) { } else { if (!tableNameToId.containsKey(tableName)) { tableNameToId.put(tableName, tableId); - idToTbl.put(tableId, buildTableForInit(tableName, tableId, extCatalog)); + idToTbl.put(tableId, + buildTableForInit(tableName, extCatalog.fromRemoteTableName(this.remoteName, tableName), + tableId, extCatalog, this, false)); lowerCaseToTableName.put(tableName.toLowerCase(), tableName); } } @@ -539,4 +693,16 @@ public boolean registerTable(TableIf tableIf) { public String getQualifiedName(String tblName) { return String.join(".", extCatalog.getName(), name, tblName); } + + private boolean isStoredTableNamesLowerCase() { + // Because we have added a test configuration item, + // it needs to be judged together with Env.isStoredTableNamesLowerCase() + return Env.isStoredTableNamesLowerCase() || extCatalog.getOnlyTestLowerCaseTableNames() == 1; + } + + private boolean isTableNamesCaseInsensitive() { + // Because we have added a test configuration item, + // it needs to be judged together with Env.isTableNamesCaseInsensitive() + return Env.isTableNamesCaseInsensitive() || extCatalog.getOnlyTestLowerCaseTableNames() == 2; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 24f55e74266863..48d170b59166b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.Config; +import org.apache.doris.common.Pair; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; @@ -302,7 +303,7 @@ public void invalidatePartitionsCache(long catalogId, String dbName, String tabl public MetaCache buildMetaCache(String name, OptionalLong expireAfterWriteSec, OptionalLong refreshAfterWriteSec, long maxSize, - CacheLoader> namesCacheLoader, + CacheLoader>> namesCacheLoader, CacheLoader> metaObjCacheLoader, RemovalListener> removalListener) { MetaCache metaCache = new MetaCache<>(name, commonRefreshExecutor, expireAfterWriteSec, refreshAfterWriteSec, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java index 91df061678f154..dcad165afa22ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java @@ -71,10 +71,13 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable { protected long id; @SerializedName(value = "name") protected String name; + @SerializedName(value = "remoteName") + protected String remoteName; @SerializedName(value = "type") protected TableType type = null; @SerializedName(value = "timestamp") protected long timestamp; + // dbName is temporarily retained and will be deleted later. To use dbName, please use db.getFullName() @SerializedName(value = "dbName") protected String dbName; @SerializedName(value = "ta") @@ -86,6 +89,7 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable { protected long dbId; protected boolean objectCreated; protected ExternalCatalog catalog; + protected ExternalDatabase db; /** * No args constructor for persist. @@ -99,15 +103,19 @@ public ExternalTable() { * * @param id Table id. * @param name Table name. + * @param remoteName Remote table name. * @param catalog ExternalCatalog this table belongs to. - * @param dbName Name of the db the this table belongs to. + * @param db ExternalDatabase this table belongs to. * @param type Table type. */ - public ExternalTable(long id, String name, ExternalCatalog catalog, String dbName, TableType type) { + public ExternalTable(long id, String name, String remoteName, ExternalCatalog catalog, ExternalDatabase db, + TableType type) { this.id = id; this.name = name; + this.remoteName = remoteName; this.catalog = catalog; - this.dbName = dbName; + this.db = db; + this.dbName = db.getFullName(); this.type = type; this.objectCreated = false; } @@ -116,6 +124,14 @@ public void setCatalog(ExternalCatalog catalog) { this.catalog = catalog; } + public void setDb(ExternalDatabase db) { + this.db = db; + } + + public void setRemoteName(String remoteName) { + this.remoteName = remoteName; + } + public boolean isView() { return false; } @@ -141,6 +157,10 @@ public String getName() { return name; } + public String getRemoteName() { + return remoteName; + } + @Override public TableType getType() { return type; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java index 7834c0c8826daf..023eecc2fa4c00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java @@ -64,6 +64,9 @@ public enum Type { @SerializedName(value = "createDbNames") private List createDbNames; + @SerializedName(value = "remoteDbNames") + private List remoteDbNames; + @SerializedName(value = "type") private Type type; @@ -77,6 +80,7 @@ public InitCatalogLog() { refreshDbIds = Lists.newArrayList(); createDbIds = Lists.newArrayList(); createDbNames = Lists.newArrayList(); + remoteDbNames = Lists.newArrayList(); type = Type.UNKNOWN; } @@ -85,10 +89,11 @@ public void addRefreshDb(long id) { refreshDbIds.add(id); } - public void addCreateDb(long id, String name) { + public void addCreateDb(long id, String name, String remoteName) { createCount += 1; createDbIds.add(id); createDbNames.add(name); + remoteDbNames.add(remoteName); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java index c652113cf0de88..44ee0a39c56943 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java @@ -68,6 +68,9 @@ public enum Type { @SerializedName(value = "createTableNames") private List createTableNames; + @SerializedName(value = "remoteTableNames") + private List remoteTableNames; + @SerializedName(value = "type") private Type type; @@ -82,6 +85,7 @@ public InitDatabaseLog() { refreshTableIds = Lists.newArrayList(); createTableIds = Lists.newArrayList(); createTableNames = Lists.newArrayList(); + remoteTableNames = Lists.newArrayList(); type = Type.UNKNOWN; } @@ -90,10 +94,11 @@ public void addRefreshTable(long id) { refreshTableIds.add(id); } - public void addCreateTable(long id, String name) { + public void addCreateTable(long id, String name, String remoteName) { createCount += 1; createTableIds.add(id); createTableNames.add(name); + remoteTableNames.add(remoteName); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java index 3c77b112d60160..39db1452833eac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java @@ -32,14 +32,18 @@ public class EsExternalDatabase extends ExternalDatabase { * @param extCatalog External data source this database belongs to. * @param id database id. * @param name database name. + * @param remoteName remote database name. */ - public EsExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.ES); + public EsExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.ES); } @Override - protected EsExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new EsExternalTable(tblId, tableName, name, (EsExternalCatalog) extCatalog); + public EsExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new EsExternalTable(tblId, localTableName, remoteTableName, (EsExternalCatalog) extCatalog, + (EsExternalDatabase) db); } public void addTableForTest(EsExternalTable tbl) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java index 4f05d6d29cd89e..6e9e5731f41532 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java @@ -42,11 +42,12 @@ public class EsExternalTable extends ExternalTable { * * @param id Table id. * @param name Table name. - * @param dbName Database name. - * @param catalog HMSExternalDataSource. + * @param remoteName Remote table name. + * @param catalog EsExternalDataSource. + * @param db Database. */ - public EsExternalTable(long id, String name, String dbName, EsExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.ES_EXTERNAL_TABLE); + public EsExternalTable(long id, String name, String remoteName, EsExternalCatalog catalog, EsExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.ES_EXTERNAL_TABLE); } protected synchronized void makeSureInitialized() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 85b999f1111047..7498e6edd93c76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -263,7 +263,7 @@ public void registerDatabase(long dbId, String dbName) { LOG.debug("create database [{}]", dbName); } - ExternalDatabase db = buildDbForInit(dbName, dbId, logType, false); + ExternalDatabase db = buildDbForInit(dbName, null, dbId, logType, false); if (useMetaCache.get()) { if (isInitialized()) { metaCache.updateCache(dbName, db, Util.genIdByName(getQualifiedName(dbName))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java index 3ae9fbcd6e75a9..86f99527fe421c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java @@ -32,14 +32,18 @@ public class HMSExternalDatabase extends ExternalDatabase { * @param extCatalog External catalog this database belongs to. * @param id database id. * @param name database name. + * @param remoteName remote database name. */ - public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.HMS); + public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.HMS); } @Override - protected HMSExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new HMSExternalTable(tblId, tableName, name, (HMSExternalCatalog) extCatalog); + public HMSExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new HMSExternalTable(tblId, localTableName, remoteTableName, (HMSExternalCatalog) extCatalog, + (HMSExternalDatabase) db); } public void addTableForTest(HMSExternalTable tbl) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index aed965369753e5..26019350fd072d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -168,11 +168,13 @@ public enum DLAType { * * @param id Table id. * @param name Table name. - * @param dbName Database name. - * @param catalog HMSExternalCatalog. + * @param remoteName Remote table name. + * @param catalog HMSExternalDataSource. + * @param db Database. */ - public HMSExternalTable(long id, String name, String dbName, HMSExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.HMS_EXTERNAL_TABLE); + public HMSExternalTable(long id, String name, String remoteName, HMSExternalCatalog catalog, + HMSExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.HMS_EXTERNAL_TABLE); } // Will throw NotSupportedException if not supported hms table. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java index f56183972e36d2..7a1a53825a15d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java @@ -28,13 +28,16 @@ public class IcebergExternalDatabase extends ExternalDatabase { - public IcebergExternalDatabase(ExternalCatalog extCatalog, Long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.ICEBERG); + public IcebergExternalDatabase(ExternalCatalog extCatalog, Long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.ICEBERG); } @Override - protected IcebergExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new IcebergExternalTable(tblId, tableName, name, (IcebergExternalCatalog) extCatalog); + public IcebergExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new IcebergExternalTable(tblId, localTableName, remoteTableName, (IcebergExternalCatalog) extCatalog, + (IcebergExternalDatabase) db); } public String getLocation() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index feded88ea326f0..a357532f2bf1f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -36,8 +36,9 @@ public class IcebergExternalTable extends ExternalTable { - public IcebergExternalTable(long id, String name, String dbName, IcebergExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.ICEBERG_EXTERNAL_TABLE); + public IcebergExternalTable(long id, String name, String remoteName, IcebergExternalCatalog catalog, + IcebergExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.ICEBERG_EXTERNAL_TABLE); } public String getIcebergCatalogType() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java index 837f3691962e91..e8ab0690e17c7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java @@ -36,7 +36,7 @@ public class ExternalInfoSchemaDatabase extends ExternalDatabase { * @param dbId The id of this database. */ public ExternalInfoSchemaDatabase(ExternalCatalog extCatalog, long dbId) { - super(extCatalog, dbId, InfoSchemaDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); + super(extCatalog, dbId, InfoSchemaDb.DATABASE_NAME, InfoSchemaDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); } public static List listTableNames() { @@ -44,8 +44,10 @@ public static List listTableNames() { } @Override - protected ExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new ExternalInfoSchemaTable(tblId, tableName, catalog); + public ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new ExternalInfoSchemaTable(tblId, localTableName, catalog, db); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java index 9d1336396128fd..b3e0ead9a118a6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java @@ -18,9 +18,9 @@ package org.apache.doris.datasource.infoschema; import org.apache.doris.analysis.SchemaTableType; -import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.SchemaTable; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.thrift.TSchemaTable; @@ -31,8 +31,8 @@ public class ExternalInfoSchemaTable extends ExternalTable { - public ExternalInfoSchemaTable(long id, String name, ExternalCatalog catalog) { - super(id, name, catalog, InfoSchemaDb.DATABASE_NAME, TableType.SCHEMA); + public ExternalInfoSchemaTable(long id, String name, ExternalCatalog catalog, ExternalDatabase db) { + super(id, name, name, catalog, db, TableType.SCHEMA); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java index 5e0653f5278109..da40dc34c604ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlDatabase.java @@ -36,7 +36,7 @@ public class ExternalMysqlDatabase extends ExternalDatabase { * @param dbId The id of this database. */ public ExternalMysqlDatabase(ExternalCatalog extCatalog, long dbId) { - super(extCatalog, dbId, MysqlDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); + super(extCatalog, dbId, MysqlDb.DATABASE_NAME, MysqlDb.DATABASE_NAME, Type.INFO_SCHEMA_DB); } public static List listTableNames() { @@ -44,8 +44,10 @@ public static List listTableNames() { } @Override - protected ExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new ExternalMysqlTable(tblId, tableName, catalog); + public ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new ExternalMysqlTable(tblId, localTableName, catalog, db); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java index 6f277a5690619b..1077abf81d8614 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalMysqlTable.java @@ -19,8 +19,8 @@ import org.apache.doris.analysis.SchemaTableType; import org.apache.doris.catalog.MysqlDBTable; -import org.apache.doris.catalog.MysqlDb; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.thrift.TSchemaTable; @@ -30,8 +30,8 @@ import java.util.Optional; public class ExternalMysqlTable extends ExternalTable { - public ExternalMysqlTable(long id, String name, ExternalCatalog catalog) { - super(id, name, catalog, MysqlDb.DATABASE_NAME, TableType.SCHEMA); + public ExternalMysqlTable(long id, String name, ExternalCatalog catalog, ExternalDatabase db) { + super(id, name, name, catalog, db, TableType.SCHEMA); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index fb26265d19fe93..fac322d21eb4da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -27,11 +27,15 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; import org.apache.doris.datasource.jdbc.client.JdbcClient; import org.apache.doris.datasource.jdbc.client.JdbcClientConfig; import org.apache.doris.datasource.jdbc.client.JdbcClientException; +import org.apache.doris.datasource.mapping.IdentifierMapping; +import org.apache.doris.datasource.mapping.JdbcIdentifierMapping; import org.apache.doris.proto.InternalService; import org.apache.doris.proto.InternalService.PJdbcTestConnectionRequest; import org.apache.doris.proto.InternalService.PJdbcTestConnectionResult; @@ -53,6 +57,7 @@ import org.apache.thrift.TException; import org.apache.thrift.TSerializer; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -71,12 +76,17 @@ public class JdbcExternalCatalog extends ExternalCatalog { // Must add "transient" for Gson to ignore this field, // or Gson will throw exception with HikariCP private transient JdbcClient jdbcClient; + private IdentifierMapping identifierMapping; public JdbcExternalCatalog(long catalogId, String name, String resource, Map props, String comment) throws DdlException { super(catalogId, name, InitCatalogLog.Type.JDBC, comment); this.catalogProperty = new CatalogProperty(resource, processCompatibleProperties(props)); + this.identifierMapping = new JdbcIdentifierMapping( + (Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()), + Boolean.parseBoolean(getLowerCaseMetaNames()), + getMetaNamesMapping()); } @Override @@ -119,12 +129,12 @@ public void onRefresh(boolean invalidCache) { super.onRefresh(invalidCache); if (jdbcClient != null) { jdbcClient.closeClient(); + jdbcClient = null; } - } - - @Override - public void onRefreshCache(boolean invalidCache) { - onRefresh(invalidCache); + this.identifierMapping = new JdbcIdentifierMapping( + (Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()), + Boolean.parseBoolean(getLowerCaseMetaNames()), + getMetaNamesMapping()); } @Override @@ -132,6 +142,7 @@ public void onClose() { super.onClose(); if (jdbcClient != null) { jdbcClient.closeClient(); + jdbcClient = null; } } @@ -182,16 +193,6 @@ public String getOnlySpecifiedDatabase() { JdbcResource.ONLY_SPECIFIED_DATABASE)); } - public String getLowerCaseMetaNames() { - return catalogProperty.getOrDefault(JdbcResource.LOWER_CASE_META_NAMES, JdbcResource.getDefaultPropertyValue( - JdbcResource.LOWER_CASE_META_NAMES)); - } - - public String getMetaNamesMapping() { - return catalogProperty.getOrDefault(JdbcResource.META_NAMES_MAPPING, JdbcResource.getDefaultPropertyValue( - JdbcResource.META_NAMES_MAPPING)); - } - public int getConnectionPoolMinSize() { return Integer.parseInt(catalogProperty.getOrDefault(JdbcResource.CONNECTION_POOL_MIN_SIZE, JdbcResource .getDefaultPropertyValue(JdbcResource.CONNECTION_POOL_MIN_SIZE))); @@ -232,8 +233,6 @@ protected void initLocalObjectsImpl() { .setDriverUrl(getDriverUrl()) .setDriverClass(getDriverClass()) .setOnlySpecifiedDatabase(getOnlySpecifiedDatabase()) - .setIsLowerCaseMetaNames(getLowerCaseMetaNames()) - .setMetaNamesMapping(getMetaNamesMapping()) .setIncludeDatabaseMap(getIncludeDatabaseMap()) .setExcludeDatabaseMap(getExcludeDatabaseMap()) .setConnectionPoolMinSize(getConnectionPoolMinSize()) @@ -245,20 +244,57 @@ protected void initLocalObjectsImpl() { jdbcClient = JdbcClient.createJdbcClient(jdbcClientConfig); } - protected List listDatabaseNames() { + @Override + public void gsonPostProcess() throws IOException { + super.gsonPostProcess(); + if (this.identifierMapping == null) { + identifierMapping = new JdbcIdentifierMapping( + (Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()), + Boolean.parseBoolean(getLowerCaseMetaNames()), + getMetaNamesMapping()); + } + } + + @Override + public List listDatabaseNames() { return jdbcClient.getDatabaseNameList(); } + @Override + public String fromRemoteDatabaseName(String remoteDatabaseName) { + return identifierMapping.fromRemoteDatabaseName(remoteDatabaseName); + } + @Override public List listTableNames(SessionContext ctx, String dbName) { makeSureInitialized(); return jdbcClient.getTablesNameList(dbName); } + @Override + public String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) { + return identifierMapping.fromRemoteTableName(remoteDatabaseName, remoteTableName); + } + @Override public boolean tableExist(SessionContext ctx, String dbName, String tblName) { makeSureInitialized(); - return jdbcClient.isTableExist(dbName, tblName); + ExternalDatabase database = this.getDbNullable(dbName); + if (database == null) { + return false; + } + ExternalTable tbl = database.getTableNullable(tblName); + if (tbl == null) { + return false; + } + String remoteDbName = ((ExternalDatabase) tbl.getDatabase()).getRemoteName(); + String remoteTblName = tbl.getRemoteName(); + return jdbcClient.isTableExist(remoteDbName, remoteTblName); + } + + public List listColumns(String remoteDbName, String remoteTblName) { + makeSureInitialized(); + return jdbcClient.getColumnsFromJdbc(remoteDbName, remoteTblName); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java index d078a3e238adbc..1737ec614a6418 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalDatabase.java @@ -30,13 +30,16 @@ public class JdbcExternalDatabase extends ExternalDatabase { * @param id database id. * @param name database name. */ - public JdbcExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.JDBC); + public JdbcExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.JDBC); } @Override - protected JdbcExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new JdbcExternalTable(tblId, tableName, name, (JdbcExternalCatalog) extCatalog); + public JdbcExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new JdbcExternalTable(tblId, localTableName, remoteTableName, (JdbcExternalCatalog) extCatalog, + (JdbcExternalDatabase) db); } public void addTableForTest(JdbcExternalTable tbl) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java index 772c366850b9a4..4bad19b8a6949d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.JdbcResource; import org.apache.doris.catalog.JdbcTable; +import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.qe.AutoCloseConnectContext; @@ -32,6 +33,8 @@ import org.apache.doris.statistics.util.StatisticsUtil; import org.apache.doris.thrift.TTableDescriptor; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.commons.text.StringSubstitutor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -40,6 +43,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; /** * Jdbc external table. @@ -76,11 +80,13 @@ public class JdbcExternalTable extends ExternalTable { * * @param id Table id. * @param name Table name. - * @param dbName Database name. - * @param catalog HMSExternalDataSource. + * @param remoteName Remote table name. + * @param catalog JdbcExternalCatalog. + * @param db JdbcExternalDatabase. */ - public JdbcExternalTable(long id, String name, String dbName, JdbcExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.JDBC_EXTERNAL_TABLE); + public JdbcExternalTable(long id, String name, String remoteName, JdbcExternalCatalog catalog, + JdbcExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.JDBC_EXTERNAL_TABLE); } @Override @@ -105,21 +111,82 @@ public TTableDescriptor toThrift() { @Override public Optional initSchema() { - return Optional.of(new SchemaCacheValue(((JdbcExternalCatalog) catalog).getJdbcClient() - .getColumnsFromJdbc(dbName, name))); + String remoteDbName = ((ExternalDatabase) this.getDatabase()).getRemoteName(); + + // 1. Retrieve remote column information + List columns = ((JdbcExternalCatalog) catalog).listColumns(remoteDbName, remoteName); + if (columns == null || columns.isEmpty()) { + return Optional.empty(); + } + + // 2. Generate local column names from remote names + List remoteColumnNames = columns.stream() + .map(Column::getName) + .collect(Collectors.toList()); + List localColumnNames = Lists.newArrayListWithCapacity(remoteColumnNames.size()); + for (String remoteColName : remoteColumnNames) { + String localName = ((JdbcExternalCatalog) catalog).getIdentifierMapping() + .fromRemoteColumnName(remoteDbName, remoteName, remoteColName); + localColumnNames.add(localName); + } + + // 3. Collect potential conflicts in a case-insensitive scenario + Map> lowerCaseToLocalNames = Maps.newHashMap(); + for (String localColName : localColumnNames) { + String lowerName = localColName.toLowerCase(); + lowerCaseToLocalNames + .computeIfAbsent(lowerName, k -> Lists.newArrayList()) + .add(localColName); + } + + // 4. Check for conflicts + List conflicts = lowerCaseToLocalNames.values().stream() + .filter(names -> names.size() > 1) + .flatMap(List::stream) + .distinct() + .collect(Collectors.toList()); + + if (!conflicts.isEmpty()) { + throw new RuntimeException(String.format( + "Found conflicting column names under case-insensitive conditions. " + + "Conflicting column names: %s in remote table '%s.%s' under catalog '%s'. " + + "Please use meta_names_mapping to handle name mapping.", + String.join(", ", conflicts), remoteDbName, remoteName, catalog.getName())); + } + + // 5. Update column objects with local names + for (int i = 0; i < columns.size(); i++) { + columns.get(i).setName(localColumnNames.get(i)); + } + + // 6. Build remote->local mapping + Map remoteColumnNamesMap = Maps.newHashMap(); + for (int i = 0; i < columns.size(); i++) { + remoteColumnNamesMap.put(localColumnNames.get(i), remoteColumnNames.get(i)); + } + + // 7. Return the SchemaCacheValue + return Optional.of(new JdbcSchemaCacheValue(columns, remoteColumnNamesMap)); } private JdbcTable toJdbcTable() { List schema = getFullSchema(); JdbcExternalCatalog jdbcCatalog = (JdbcExternalCatalog) catalog; - String fullDbName = this.dbName + "." + this.name; - JdbcTable jdbcTable = new JdbcTable(this.id, fullDbName, schema, TableType.JDBC_EXTERNAL_TABLE); - jdbcCatalog.configureJdbcTable(jdbcTable, fullDbName); + String fullTableName = this.dbName + "." + this.name; + JdbcTable jdbcTable = new JdbcTable(this.id, fullTableName, schema, TableType.JDBC_EXTERNAL_TABLE); + jdbcCatalog.configureJdbcTable(jdbcTable, fullTableName); // Set remote properties - jdbcTable.setRemoteDatabaseName(jdbcCatalog.getJdbcClient().getRemoteDatabaseName(this.dbName)); - jdbcTable.setRemoteTableName(jdbcCatalog.getJdbcClient().getRemoteTableName(this.dbName, this.name)); - jdbcTable.setRemoteColumnNames(jdbcCatalog.getJdbcClient().getRemoteColumnNames(this.dbName, this.name)); + jdbcTable.setRemoteDatabaseName(((ExternalDatabase) this.getDatabase()).getRemoteName()); + jdbcTable.setRemoteTableName(this.getRemoteName()); + Map remoteColumnNames = Maps.newHashMap(); + Optional schemaCacheValue = getSchemaCacheValue(); + for (Column column : schema) { + String remoteColumnName = schemaCacheValue.map(value -> ((JdbcSchemaCacheValue) value) + .getremoteColumnName(column.getName())).orElse(column.getName()); + remoteColumnNames.put(column.getName(), remoteColumnName); + } + jdbcTable.setRemoteColumnNames(remoteColumnNames); return jdbcTable; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcIdentifierMapping.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcSchemaCacheValue.java similarity index 51% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcIdentifierMapping.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcSchemaCacheValue.java index 20a74724b3e496..4e21af4fabd99e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcIdentifierMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcSchemaCacheValue.java @@ -17,29 +17,21 @@ package org.apache.doris.datasource.jdbc; -import org.apache.doris.datasource.jdbc.client.JdbcClient; -import org.apache.doris.datasource.mapping.IdentifierMapping; +import org.apache.doris.catalog.Column; +import org.apache.doris.datasource.SchemaCacheValue; -public class JdbcIdentifierMapping extends IdentifierMapping { - private final JdbcClient jdbcClient; +import java.util.List; +import java.util.Map; - public JdbcIdentifierMapping(boolean isLowerCaseMetaNames, String metaNamesMapping, JdbcClient jdbcClient) { - super(isLowerCaseMetaNames, metaNamesMapping); - this.jdbcClient = jdbcClient; - } - - @Override - protected void loadDatabaseNames() { - jdbcClient.getDatabaseNameList(); - } +public class JdbcSchemaCacheValue extends SchemaCacheValue { + private Map remoteColumnNamesMap; - @Override - protected void loadTableNames(String localDbName) { - jdbcClient.getTablesNameList(localDbName); + public JdbcSchemaCacheValue(List schema, Map remoteColumnNamesMap) { + super(schema); + this.remoteColumnNamesMap = remoteColumnNamesMap; } - @Override - protected void loadColumnNames(String localDbName, String localTableName) { - jdbcClient.getColumnsFromJdbc(localDbName, localTableName); + public String getremoteColumnName(String columnName) { + return remoteColumnNamesMap.get(columnName); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java index 1ab32efddcc1ba..51f55ea24ba226 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java @@ -24,7 +24,6 @@ import org.apache.doris.cloud.security.SecurityChecker; import org.apache.doris.common.DdlException; import org.apache.doris.common.util.Util; -import org.apache.doris.datasource.jdbc.JdbcIdentifierMapping; import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema; import com.google.common.collect.ImmutableSet; @@ -63,11 +62,8 @@ public abstract class JdbcClient { protected ClassLoader classLoader = null; protected HikariDataSource dataSource = null; protected boolean isOnlySpecifiedDatabase; - protected boolean isLowerCaseMetaNames; - protected String metaNamesMapping; protected Map includeDatabaseMap; protected Map excludeDatabaseMap; - protected JdbcIdentifierMapping jdbcLowerCaseMetaMatching; public static JdbcClient createJdbcClient(JdbcClientConfig jdbcClientConfig) { String dbType = parseDbType(jdbcClientConfig.getJdbcUrl()); @@ -104,8 +100,6 @@ protected JdbcClient(JdbcClientConfig jdbcClientConfig) { this.catalogName = jdbcClientConfig.getCatalog(); this.jdbcUser = jdbcClientConfig.getUser(); this.isOnlySpecifiedDatabase = Boolean.parseBoolean(jdbcClientConfig.getOnlySpecifiedDatabase()); - this.isLowerCaseMetaNames = Boolean.parseBoolean(jdbcClientConfig.getIsLowerCaseMetaNames()); - this.metaNamesMapping = jdbcClientConfig.getMetaNamesMapping(); this.includeDatabaseMap = Optional.ofNullable(jdbcClientConfig.getIncludeDatabaseMap()).orElse(Collections.emptyMap()); this.excludeDatabaseMap = @@ -114,7 +108,6 @@ protected JdbcClient(JdbcClientConfig jdbcClientConfig) { this.dbType = parseDbType(jdbcUrl); initializeClassLoader(jdbcClientConfig); initializeDataSource(jdbcClientConfig); - this.jdbcLowerCaseMetaMatching = new JdbcIdentifierMapping(isLowerCaseMetaNames, metaNamesMapping, this); } protected void setJdbcDriverSystemProperties() { @@ -174,6 +167,7 @@ public static String parseDbType(String jdbcUrl) { public void closeClient() { dataSource.close(); + dataSource = null; } public Connection getConnection() throws JdbcClientException { @@ -312,10 +306,9 @@ public List getDatabaseNameList() { /** * get all tables of one database */ - public List getTablesNameList(String localDbName) { + public List getTablesNameList(String remoteDbName) { List remoteTablesNames = Lists.newArrayList(); String[] tableTypes = getTableTypes(); - String remoteDbName = getRemoteDatabaseName(localDbName); processTable(remoteDbName, null, tableTypes, (rs) -> { try { while (rs.next()) { @@ -325,14 +318,12 @@ public List getTablesNameList(String localDbName) { throw new JdbcClientException("failed to get all tables for remote database: `%s`", e, remoteDbName); } }); - return filterTableNames(remoteDbName, remoteTablesNames); + return remoteTablesNames; } - public boolean isTableExist(String localDbName, String localTableName) { + public boolean isTableExist(String remoteDbName, String remoteTableName) { final boolean[] isExist = {false}; String[] tableTypes = getTableTypes(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); processTable(remoteDbName, remoteTableName, tableTypes, (rs) -> { try { if (rs.next()) { @@ -349,12 +340,10 @@ public boolean isTableExist(String localDbName, String localTableName) { /** * get all columns of one table */ - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); @@ -381,21 +370,7 @@ public List getColumnsFromJdbc(String localDbName, String localTableName field.isAllowNull(), field.getRemarks(), true, -1)); } - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); - return filterColumnName(remoteDbName, remoteTableName, dorisTableSchema); - } - - public String getRemoteDatabaseName(String localDbname) { - return jdbcLowerCaseMetaMatching.getRemoteDatabaseName(localDbname); - } - - public String getRemoteTableName(String localDbName, String localTableName) { - return jdbcLowerCaseMetaMatching.getRemoteTableName(localDbName, localTableName); - } - - public Map getRemoteColumnNames(String localDbName, String localTableName) { - return jdbcLowerCaseMetaMatching.getRemoteColumnNames(localDbName, localTableName); + return dorisTableSchema; } // protected methods, for subclass to override @@ -454,7 +429,7 @@ protected List filterDatabaseNames(List remoteDbNames) { } filteredDatabaseNames.add(databaseName); } - return jdbcLowerCaseMetaMatching.setDatabaseNameMapping(filteredDatabaseNames); + return filteredDatabaseNames; } protected Set getFilterInternalDatabases() { @@ -465,14 +440,6 @@ protected Set getFilterInternalDatabases() { .build(); } - protected List filterTableNames(String remoteDbName, List remoteTableNames) { - return jdbcLowerCaseMetaMatching.setTableNameMapping(remoteDbName, remoteTableNames); - } - - protected List filterColumnName(String remoteDbName, String remoteTableName, List remoteColumns) { - return jdbcLowerCaseMetaMatching.setColumnNameMapping(remoteDbName, remoteTableName, remoteColumns); - } - protected abstract Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema); protected Type createDecimalOrStringType(int precision, int scale) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java index 7ba393e0d0aae6..086a8a5f393614 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java @@ -87,12 +87,10 @@ protected ResultSet getRemoteColumns(DatabaseMetaData databaseMetaData, String c } @Override - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java index b78589faa77380..f4c331cf35779c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcMySQLClient.java @@ -134,12 +134,10 @@ protected ResultSet getRemoteColumns(DatabaseMetaData databaseMetaData, String c * get all columns of one table */ @Override - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java index 9968de79ab3a7d..adffd06c244e54 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcOracleClient.java @@ -49,12 +49,10 @@ public String getTestQuery() { } @Override - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java index 481e5ea5e490c6..a8e62fc2893f8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java @@ -49,12 +49,10 @@ protected JdbcPostgreSQLClient(JdbcClientConfig jdbcClientConfig) { } @Override - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = null; ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { conn = getConnection(); DatabaseMetaData databaseMetaData = conn.getMetaData(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalDatabase.java index 59a7ace0dca355..32e389f52d3069 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalDatabase.java @@ -23,13 +23,16 @@ public class LakeSoulExternalDatabase extends ExternalDatabase { - public LakeSoulExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.LAKESOUL); + public LakeSoulExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.LAKESOUL); } @Override - protected LakeSoulExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new LakeSoulExternalTable(tblId, tableName, name, (LakeSoulExternalCatalog) catalog); + public LakeSoulExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new LakeSoulExternalTable(tblId, localTableName, remoteTableName, (LakeSoulExternalCatalog) extCatalog, + (LakeSoulExternalDatabase) db); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java index 9dd2f4811e98f0..e5e0447f1cb647 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalTable.java @@ -56,8 +56,9 @@ public class LakeSoulExternalTable extends ExternalTable { public final String tableId; - public LakeSoulExternalTable(long id, String name, String dbName, LakeSoulExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.LAKESOUl_EXTERNAL_TABLE); + public LakeSoulExternalTable(long id, String name, String remoteName, LakeSoulExternalCatalog catalog, + LakeSoulExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.LAKESOUl_EXTERNAL_TABLE); TableInfo tableInfo = getLakeSoulTableInfo(); if (tableInfo == null) { throw new RuntimeException(String.format("LakeSoul table %s.%s does not exist", dbName, name)); @@ -88,9 +89,9 @@ private Type arrowFiledToDorisType(Field field) { return Type.BIGINT; default: throw new IllegalArgumentException("Invalid integer bit width: " - + type.getBitWidth() - + " for LakeSoul table: " - + getTableIdentifier()); + + type.getBitWidth() + + " for LakeSoul table: " + + getTableIdentifier()); } } else if (dt instanceof ArrowType.FloatingPoint) { ArrowType.FloatingPoint type = (ArrowType.FloatingPoint) dt; @@ -101,16 +102,16 @@ private Type arrowFiledToDorisType(Field field) { return Type.DOUBLE; default: throw new IllegalArgumentException("Invalid floating point precision: " - + type.getPrecision() - + " for LakeSoul table: " - + getTableIdentifier()); + + type.getPrecision() + + " for LakeSoul table: " + + getTableIdentifier()); } } else if (dt instanceof ArrowType.Utf8) { return Type.STRING; } else if (dt instanceof ArrowType.Decimal) { ArrowType.Decimal decimalType = (ArrowType.Decimal) dt; return ScalarType.createDecimalType(PrimitiveType.DECIMAL64, decimalType.getPrecision(), - decimalType.getScale()); + decimalType.getScale()); } else if (dt instanceof ArrowType.Date) { return ScalarType.createDateV2Type(); } else if (dt instanceof ArrowType.Timestamp) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java index 363ef351152a39..9199ff985116c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java @@ -17,314 +17,11 @@ package org.apache.doris.datasource.mapping; -import org.apache.doris.catalog.Column; -import org.apache.doris.qe.GlobalVariable; +public interface IdentifierMapping { -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; + String fromRemoteDatabaseName(String remoteDatabaseName); -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; + String fromRemoteTableName(String remoteDatabaseName, String remoteTableName); -public abstract class IdentifierMapping { - private static final Logger LOG = LogManager.getLogger(IdentifierMapping.class); - - private final ObjectMapper mapper = new ObjectMapper(); - private final ConcurrentHashMap localDBToRemoteDB = new ConcurrentHashMap<>(); - private final ConcurrentHashMap> localTableToRemoteTable - = new ConcurrentHashMap<>(); - private final ConcurrentHashMap>> - localColumnToRemoteColumn = new ConcurrentHashMap<>(); - - private final AtomicBoolean dbNamesLoaded = new AtomicBoolean(false); - private final ConcurrentHashMap tableNamesLoadedMap = new ConcurrentHashMap<>(); - private final ConcurrentHashMap> columnNamesLoadedMap - = new ConcurrentHashMap<>(); - - private final boolean isLowerCaseMetaNames; - private final String metaNamesMapping; - - public IdentifierMapping(boolean isLowerCaseMetaNames, String metaNamesMapping) { - this.isLowerCaseMetaNames = isLowerCaseMetaNames; - this.metaNamesMapping = metaNamesMapping; - } - - public List setDatabaseNameMapping(List remoteDatabaseNames) { - JsonNode databasesNode = readAndParseJson(metaNamesMapping, "databases"); - - Map databaseNameMapping = Maps.newTreeMap(); - if (databasesNode.isArray()) { - for (JsonNode node : databasesNode) { - String remoteDatabase = node.path("remoteDatabase").asText(); - String mapping = node.path("mapping").asText(); - databaseNameMapping.put(remoteDatabase, mapping); - } - } - - Map> result = nameListToMapping(remoteDatabaseNames, localDBToRemoteDB, - databaseNameMapping, isLowerCaseMetaNames); - List localDatabaseNames = result.get("localNames"); - List conflictNames = result.get("conflictNames"); - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict database/schema names found when lower_case_meta_names is true: " + conflictNames - + ". Please set lower_case_meta_names to false or" - + " use meta_name_mapping to specify the names."); - } - return localDatabaseNames; - } - - public List setTableNameMapping(String remoteDbName, List remoteTableNames) { - JsonNode tablesNode = readAndParseJson(metaNamesMapping, "tables"); - - Map tableNameMapping = Maps.newTreeMap(); - if (tablesNode.isArray()) { - for (JsonNode node : tablesNode) { - String remoteDatabase = node.path("remoteDatabase").asText(); - if (remoteDbName.equals(remoteDatabase)) { - String remoteTable = node.path("remoteTable").asText(); - String mapping = node.path("mapping").asText(); - tableNameMapping.put(remoteTable, mapping); - } - } - } - - localTableToRemoteTable.putIfAbsent(remoteDbName, new ConcurrentHashMap<>()); - - List localTableNames; - List conflictNames; - - if (GlobalVariable.lowerCaseTableNames == 1) { - Map> result = nameListToMapping(remoteTableNames, - localTableToRemoteTable.get(remoteDbName), - tableNameMapping, true); - localTableNames = result.get("localNames"); - conflictNames = result.get("conflictNames"); - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict table names found in remote database/schema: " + remoteDbName - + " when lower_case_table_names is 1: " + conflictNames - + ". Please use meta_name_mapping to specify the names."); - } - } else { - Map> result = nameListToMapping(remoteTableNames, - localTableToRemoteTable.get(remoteDbName), - tableNameMapping, isLowerCaseMetaNames); - localTableNames = result.get("localNames"); - conflictNames = result.get("conflictNames"); - - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict table names found in remote database/schema: " + remoteDbName - + "when lower_case_meta_names is true: " + conflictNames - + ". Please set lower_case_meta_names to false or" - + " use meta_name_mapping to specify the table names."); - } - } - return localTableNames; - } - - public List setColumnNameMapping(String remoteDbName, String remoteTableName, List remoteColumns) { - JsonNode tablesNode = readAndParseJson(metaNamesMapping, "columns"); - - Map columnNameMapping = Maps.newTreeMap(); - if (tablesNode.isArray()) { - for (JsonNode node : tablesNode) { - String remoteDatabase = node.path("remoteDatabase").asText(); - String remoteTable = node.path("remoteTable").asText(); - if (remoteDbName.equals(remoteDatabase) && remoteTable.equals(remoteTableName)) { - String remoteColumn = node.path("remoteColumn").asText(); - String mapping = node.path("mapping").asText(); - columnNameMapping.put(remoteColumn, mapping); - } - } - } - localColumnToRemoteColumn.putIfAbsent(remoteDbName, new ConcurrentHashMap<>()); - localColumnToRemoteColumn.get(remoteDbName).putIfAbsent(remoteTableName, new ConcurrentHashMap<>()); - - List localColumnNames; - List conflictNames; - - // Get the name from localColumns and save it to List - List remoteColumnNames = Lists.newArrayList(); - for (Column remoteColumn : remoteColumns) { - remoteColumnNames.add(remoteColumn.getName()); - } - - Map> result = nameListToMapping(remoteColumnNames, - localColumnToRemoteColumn.get(remoteDbName).get(remoteTableName), - columnNameMapping, isLowerCaseMetaNames); - localColumnNames = result.get("localNames"); - conflictNames = result.get("conflictNames"); - if (!conflictNames.isEmpty()) { - throw new RuntimeException( - "Conflict column names found in remote database/schema: " + remoteDbName - + " in remote table: " + remoteTableName - + " when lower_case_meta_names is true: " + conflictNames - + ". Please set lower_case_meta_names to false or" - + " use meta_name_mapping to specify the column names."); - } - // Replace the name in remoteColumns with localColumnNames - for (int i = 0; i < remoteColumns.size(); i++) { - remoteColumns.get(i).setName(localColumnNames.get(i)); - } - return remoteColumns; - } - - public String getRemoteDatabaseName(String localDbName) { - return getRequiredMapping(localDBToRemoteDB, localDbName, "database", this::loadDatabaseNamesIfNeeded, - localDbName); - } - - public String getRemoteTableName(String localDbName, String localTableName) { - String remoteDbName = getRemoteDatabaseName(localDbName); - Map tableMap = localTableToRemoteTable.computeIfAbsent(remoteDbName, - k -> new ConcurrentHashMap<>()); - return getRequiredMapping(tableMap, localTableName, "table", () -> loadTableNamesIfNeeded(localDbName), - localTableName); - } - - public Map getRemoteColumnNames(String localDbName, String localTableName) { - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); - ConcurrentHashMap> tableColumnMap - = localColumnToRemoteColumn.computeIfAbsent(remoteDbName, k -> new ConcurrentHashMap<>()); - Map columnMap = tableColumnMap.computeIfAbsent(remoteTableName, k -> new ConcurrentHashMap<>()); - if (columnMap.isEmpty()) { - LOG.info("Column name mapping missing, loading column names for localDbName: {}, localTableName: {}", - localDbName, localTableName); - loadColumnNamesIfNeeded(localDbName, localTableName); - columnMap = tableColumnMap.get(remoteTableName); - } - if (columnMap.isEmpty()) { - LOG.warn("No remote column found for localTableName: {}. Please refresh this catalog.", localTableName); - throw new RuntimeException( - "No remote column found for localTableName: " + localTableName + ". Please refresh this catalog."); - } - return columnMap; - } - - - private void loadDatabaseNamesIfNeeded() { - if (dbNamesLoaded.compareAndSet(false, true)) { - try { - loadDatabaseNames(); - } catch (Exception e) { - dbNamesLoaded.set(false); // Reset on failure - LOG.warn("Error loading database names", e); - } - } - } - - private void loadTableNamesIfNeeded(String localDbName) { - AtomicBoolean isLoaded = tableNamesLoadedMap.computeIfAbsent(localDbName, k -> new AtomicBoolean(false)); - if (isLoaded.compareAndSet(false, true)) { - try { - loadTableNames(localDbName); - } catch (Exception e) { - tableNamesLoadedMap.get(localDbName).set(false); // Reset on failure - LOG.warn("Error loading table names for localDbName: {}", localDbName, e); - } - } - } - - private void loadColumnNamesIfNeeded(String localDbName, String localTableName) { - columnNamesLoadedMap.putIfAbsent(localDbName, new ConcurrentHashMap<>()); - AtomicBoolean isLoaded = columnNamesLoadedMap.get(localDbName) - .computeIfAbsent(localTableName, k -> new AtomicBoolean(false)); - if (isLoaded.compareAndSet(false, true)) { - try { - loadColumnNames(localDbName, localTableName); - } catch (Exception e) { - columnNamesLoadedMap.get(localDbName).get(localTableName).set(false); // Reset on failure - LOG.warn("Error loading column names for localDbName: {}, localTableName: {}", localDbName, - localTableName, e); - } - } - } - - private V getRequiredMapping(Map map, K key, String typeName, Runnable loadIfNeeded, - String entityName) { - if (map.isEmpty() || !map.containsKey(key) || map.get(key) == null) { - LOG.info("{} mapping missing, loading for {}: {}", typeName, typeName, entityName); - loadIfNeeded.run(); - } - V value = map.get(key); - if (value == null) { - LOG.warn("No remote {} found for {}: {}. Please refresh this catalog.", typeName, typeName, entityName); - throw new RuntimeException("No remote " + typeName + " found for " + typeName + ": " + entityName - + ". Please refresh this catalog."); - } - return value; - } - - // Load the database name from the data source. - // In the corresponding getDatabaseNameList(), setDatabaseNameMapping() must be used to update the mapping. - protected abstract void loadDatabaseNames(); - - // Load the table names for the specified database from the data source. - // In the corresponding getTableNameList(), setTableNameMapping() must be used to update the mapping. - protected abstract void loadTableNames(String localDbName); - - // Load the column names for a specified table in a database from the data source. - // In the corresponding getColumnNameList(), setColumnNameMapping() must be used to update the mapping. - protected abstract void loadColumnNames(String localDbName, String localTableName); - - private JsonNode readAndParseJson(String jsonPath, String nodeName) { - JsonNode rootNode; - try { - rootNode = mapper.readTree(jsonPath); - return rootNode.path(nodeName); - } catch (JsonProcessingException e) { - throw new RuntimeException("parse meta_names_mapping property error", e); - } - } - - private Map> nameListToMapping(List remoteNames, - ConcurrentHashMap localNameToRemoteName, - Map nameMapping, boolean isLowerCaseMetaNames) { - List filteredDatabaseNames = Lists.newArrayList(); - Set lowerCaseNames = Sets.newHashSet(); - Map> nameMap = Maps.newHashMap(); - List conflictNames = Lists.newArrayList(); - - for (String name : remoteNames) { - String mappedName = nameMapping.getOrDefault(name, name); - String localName = isLowerCaseMetaNames ? mappedName.toLowerCase() : mappedName; - - // Use computeIfAbsent to ensure atomicity - localNameToRemoteName.computeIfAbsent(localName, k -> name); - - if (isLowerCaseMetaNames && !lowerCaseNames.add(localName)) { - if (nameMap.containsKey(localName)) { - nameMap.get(localName).add(mappedName); - } - } else { - nameMap.putIfAbsent(localName, Lists.newArrayList(Collections.singletonList(mappedName))); - } - - filteredDatabaseNames.add(localName); - } - - for (List conflictNameList : nameMap.values()) { - if (conflictNameList.size() > 1) { - conflictNames.addAll(conflictNameList); - } - } - - Map> result = Maps.newConcurrentMap(); - result.put("localNames", filteredDatabaseNames); - result.put("conflictNames", conflictNames); - return result; - } + String fromRemoteColumnName(String remoteDatabaseName, String remoteTableName, String remoteColumnNames); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/JdbcIdentifierMapping.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/JdbcIdentifierMapping.java new file mode 100644 index 00000000000000..f584c9acee34e4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/JdbcIdentifierMapping.java @@ -0,0 +1,345 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.mapping; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Map; +import java.util.Set; + +public class JdbcIdentifierMapping implements IdentifierMapping { + private static final Logger LOG = LogManager.getLogger(JdbcIdentifierMapping.class); + + private final ObjectMapper mapper = new ObjectMapper(); + private final boolean isLowerCaseTableNames; + private final boolean isLowerCaseMetaNames; + private final String metaNamesMapping; + + public JdbcIdentifierMapping(boolean isLowerCaseTableNames, boolean isLowerCaseMetaNames, String metaNamesMapping) { + this.isLowerCaseTableNames = isLowerCaseTableNames; + this.isLowerCaseMetaNames = isLowerCaseMetaNames; + this.metaNamesMapping = metaNamesMapping; + validateMappings(); + } + + private boolean isMappingInvalid() { + return metaNamesMapping == null || metaNamesMapping.isEmpty(); + } + + @Override + public String fromRemoteDatabaseName(String remoteDatabaseName) { + if (!isLowerCaseMetaNames && isMappingInvalid()) { + return remoteDatabaseName; + } + JsonNode databasesNode = readAndParseJson(metaNamesMapping, "databases"); + + Map databaseNameMapping = Maps.newHashMap(); + if (databasesNode.isArray()) { + for (JsonNode node : databasesNode) { + String remoteDatabase = node.path("remoteDatabase").asText(); + String mapping = applyLowerCaseIfNeeded(node.path("mapping").asText()); + databaseNameMapping.put(remoteDatabase, mapping); + } + } + return getMappedName(remoteDatabaseName, databaseNameMapping); + } + + @Override + public String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) { + if (!isLowerCaseMetaNames && isMappingInvalid()) { + return remoteTableName; + } + JsonNode tablesNode = readAndParseJson(metaNamesMapping, "tables"); + + Map tableNameMapping = Maps.newHashMap(); + if (tablesNode.isArray()) { + for (JsonNode node : tablesNode) { + String remoteDatabase = node.path("remoteDatabase").asText(); + if (remoteDatabaseName.equals(remoteDatabase)) { + String remoteTable = node.path("remoteTable").asText(); + String mapping = applyLowerCaseIfNeeded(node.path("mapping").asText()); + tableNameMapping.put(remoteTable, mapping); + } + } + } + return getMappedName(remoteTableName, tableNameMapping); + } + + @Override + public String fromRemoteColumnName(String remoteDatabaseName, String remoteTableName, String remoteColumnName) { + if (!isLowerCaseMetaNames && isMappingInvalid()) { + return remoteColumnName; + } + JsonNode columnsNode = readAndParseJson(metaNamesMapping, "columns"); + + Map columnNameMapping = Maps.newHashMap(); + if (columnsNode.isArray()) { + for (JsonNode node : columnsNode) { + String remoteDatabase = node.path("remoteDatabase").asText(); + String remoteTable = node.path("remoteTable").asText(); + if (remoteDatabaseName.equals(remoteDatabase) && remoteTableName.equals(remoteTable)) { + String remoteColumn = node.path("remoteColumn").asText(); + String mapping = applyLowerCaseIfNeeded(node.path("mapping").asText()); + columnNameMapping.put(remoteColumn, mapping); + } + } + } + return getMappedName(remoteColumnName, columnNameMapping); + } + + private String getMappedName(String name, Map nameMapping) { + String mappedName = nameMapping.getOrDefault(name, name); + return isLowerCaseMetaNames ? mappedName.toLowerCase() : mappedName; + } + + private JsonNode readAndParseJson(String jsonPath, String nodeName) { + try { + JsonNode rootNode = mapper.readTree(jsonPath); + return rootNode.path(nodeName); + } catch (JsonProcessingException e) { + throw new RuntimeException("JSON format is incorrect, please check the metaNamesMapping property", e); + } + } + + private void validateMappings() { + Map> duplicateErrors = Maps.newLinkedHashMap(); + try { + JsonNode rootNode = mapper.readTree(metaNamesMapping); + + Map> dbMappingCheck = Maps.newHashMap(); + Map>> tableMappingCheck = Maps.newHashMap(); + Map>>> columnMappingCheck = Maps.newHashMap(); + + Set dbKeySet = Sets.newHashSet(); + Map> tableKeySet = Maps.newHashMap(); + Map>> columnKeySet = Maps.newHashMap(); + + validateNode(rootNode.path("databases"), "databases", duplicateErrors, dbMappingCheck, tableMappingCheck, + columnMappingCheck, dbKeySet, tableKeySet, columnKeySet); + validateNode(rootNode.path("tables"), "tables", duplicateErrors, dbMappingCheck, tableMappingCheck, + columnMappingCheck, dbKeySet, tableKeySet, columnKeySet); + validateNode(rootNode.path("columns"), "columns", duplicateErrors, dbMappingCheck, tableMappingCheck, + columnMappingCheck, dbKeySet, tableKeySet, columnKeySet); + + if (!duplicateErrors.isEmpty()) { + StringBuilder errorBuilder = new StringBuilder("Duplicate mapping found:\n"); + duplicateErrors.forEach((key, value) -> { + errorBuilder.append(key).append(":\n"); + value.forEach(error -> errorBuilder.append(" - ").append(error).append("\n")); + }); + throw new RuntimeException(errorBuilder.toString()); + } + } catch (JsonProcessingException e) { + throw new RuntimeException("The JSON format is incorrect, please check the metaNamesMapping property", e); + } + } + + private void validateNode(JsonNode nodes, + String nodeType, + Map> duplicateErrors, + Map> dbMappingCheck, + Map>> tableMappingCheck, + Map>>> columnMappingCheck, + Set dbKeySet, + Map> tableKeySet, + Map>> columnKeySet) { + Map mappingSet = Maps.newHashMap(); + if (nodes.isArray()) { + for (JsonNode node : nodes) { + String remoteKey; + String remoteDb = null; + String remoteTbl = null; + switch (nodeType) { + case "databases": + remoteKey = node.path("remoteDatabase").asText(); + checkDuplicateRemoteDatabaseKey(remoteKey, dbKeySet, duplicateErrors); + break; + case "tables": + remoteDb = node.path("remoteDatabase").asText(); + remoteKey = node.path("remoteTable").asText(); + checkDuplicateRemoteTableKey(remoteDb, remoteKey, tableKeySet, duplicateErrors); + break; + case "columns": + remoteDb = node.path("remoteDatabase").asText(); + remoteTbl = node.path("remoteTable").asText(); + remoteKey = node.path("remoteColumn").asText(); + checkDuplicateRemoteColumnKey(remoteDb, remoteTbl, remoteKey, columnKeySet, duplicateErrors); + break; + default: + throw new IllegalArgumentException("Unknown type: " + nodeType); + } + + String mapping = node.path("mapping").asText(); + + String existed = mappingSet.get(mapping); + if (existed != null) { + duplicateErrors + .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) + .add(String.format("Remote name: %s, duplicate mapping: %s (original: %s)", + remoteKey, mapping, existed)); + } else { + mappingSet.put(mapping, remoteKey); + } + + switch (nodeType) { + case "databases": + if (isLowerCaseMetaNames) { + checkCaseConflictForDatabase(mapping, dbMappingCheck, duplicateErrors, nodeType, remoteKey); + } + break; + case "tables": + if (isLowerCaseMetaNames || isLowerCaseTableNames) { + checkCaseConflictForTable(remoteDb, mapping, tableMappingCheck, duplicateErrors, + nodeType, remoteKey); + } + break; + case "columns": + checkCaseConflictForColumn(remoteDb, remoteTbl, mapping, columnMappingCheck, duplicateErrors, + nodeType, remoteKey); + break; + default: + break; + } + } + } + } + + private void checkDuplicateRemoteDatabaseKey(String remoteDatabase, + Set dbKeySet, + Map> duplicateErrors) { + if (dbKeySet == null) { + return; + } + if (!dbKeySet.add(remoteDatabase)) { + duplicateErrors + .computeIfAbsent("databases", k -> Sets.newLinkedHashSet()) + .add(String.format("Duplicate remoteDatabase found: %s", remoteDatabase)); + } + } + + private void checkDuplicateRemoteTableKey(String remoteDb, + String remoteTable, + Map> tableKeySet, + Map> duplicateErrors) { + if (tableKeySet == null) { + return; + } + Set tables = tableKeySet.computeIfAbsent(remoteDb, k -> Sets.newHashSet()); + if (!tables.add(remoteTable)) { + duplicateErrors + .computeIfAbsent("tables", k -> Sets.newLinkedHashSet()) + .add(String.format("Duplicate remoteTable found in database %s: %s", remoteDb, remoteTable)); + } + } + + private void checkDuplicateRemoteColumnKey(String remoteDb, + String remoteTbl, + String remoteColumn, + Map>> columnKeySet, + Map> duplicateErrors) { + if (columnKeySet == null) { + return; + } + Map> tblMap = columnKeySet.computeIfAbsent(remoteDb, k -> Maps.newHashMap()); + Set columns = tblMap.computeIfAbsent(remoteTbl, k -> Sets.newHashSet()); + if (!columns.add(remoteColumn)) { + duplicateErrors + .computeIfAbsent("columns", k -> Sets.newLinkedHashSet()) + .add(String.format("Duplicate remoteColumn found in database %s, table %s: %s", + remoteDb, remoteTbl, remoteColumn)); + } + } + + private void checkCaseConflictForDatabase(String mapping, + Map> dbMappingCheck, + Map> duplicateErrors, + String nodeType, + String remoteKey) { + if (dbMappingCheck == null) { + return; + } + String lower = mapping.toLowerCase(); + Set variants = dbMappingCheck.computeIfAbsent(lower, k -> Sets.newLinkedHashSet()); + if (!variants.isEmpty() && variants.stream().noneMatch(v -> v.equals(mapping))) { + duplicateErrors + .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) + .add(String.format("Remote name: %s, case-only different mapping found: %s (existing variants: %s)", + remoteKey, mapping, variants)); + } + variants.add(mapping); + } + + private void checkCaseConflictForTable(String remoteDb, + String mapping, + Map>> tableMappingCheck, + Map> duplicateErrors, + String nodeType, + String remoteKey) { + if (tableMappingCheck == null || remoteDb == null) { + return; + } + Map> dbMap = tableMappingCheck.computeIfAbsent(remoteDb, k -> Maps.newHashMap()); + String lower = mapping.toLowerCase(); + Set variants = dbMap.computeIfAbsent(lower, k -> Sets.newLinkedHashSet()); + if (!variants.isEmpty() && variants.stream().noneMatch(v -> v.equals(mapping))) { + duplicateErrors + .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) + .add(String.format("Remote name: %s (database: %s), " + + "case-only different mapping found: %s (existing variants: %s)", + remoteKey, remoteDb, mapping, variants)); + } + variants.add(mapping); + } + + private void checkCaseConflictForColumn(String remoteDb, + String remoteTbl, + String mapping, + Map>>> columnMappingCheck, + Map> duplicateErrors, + String nodeType, + String remoteKey) { + if (columnMappingCheck == null || remoteDb == null || remoteTbl == null) { + return; + } + Map>> dbMap = columnMappingCheck.computeIfAbsent(remoteDb, + k -> Maps.newHashMap()); + Map> tblMap = dbMap.computeIfAbsent(remoteTbl, k -> Maps.newHashMap()); + String lower = mapping.toLowerCase(); + Set variants = tblMap.computeIfAbsent(lower, k -> Sets.newLinkedHashSet()); + + if (!variants.isEmpty() && variants.stream().noneMatch(v -> v.equals(mapping))) { + duplicateErrors + .computeIfAbsent(nodeType, k -> Sets.newLinkedHashSet()) + .add(String.format( + "Remote name: %s (database: %s, table: %s), " + + "case-only different mapping found: %s (existing variants: %s)", + remoteKey, remoteDb, remoteTbl, mapping, variants)); + } + variants.add(mapping); + } + + private String applyLowerCaseIfNeeded(String value) { + return isLowerCaseMetaNames ? value.toLowerCase() : value; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java index 0a100e3495ff3d..7cd38b9d13a007 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalDatabase.java @@ -32,12 +32,16 @@ public class MaxComputeExternalDatabase extends ExternalDatabase columnNameToOdpsColumn = new HashMap(); + private Map columnNameToOdpsColumn = new HashMap(); @Override protected synchronized void makeSureInitialized() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java index 6e4198186e82e3..fffa0a04e42850 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.metacache; import org.apache.doris.common.CacheFactory; +import org.apache.doris.common.Pair; import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.LoadingCache; @@ -27,12 +28,14 @@ import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; public class MetaCache { - private LoadingCache> namesCache; + private LoadingCache>> namesCache; private Map idToName = Maps.newConcurrentMap(); private LoadingCache> metaObjCache; @@ -43,7 +46,7 @@ public MetaCache(String name, OptionalLong expireAfterWriteSec, OptionalLong refreshAfterWriteSec, long maxSize, - CacheLoader> namesCacheLoader, + CacheLoader>> namesCacheLoader, CacheLoader> metaObjCacheLoader, RemovalListener> removalListener) { this.name = name; @@ -71,7 +74,15 @@ public MetaCache(String name, } public List listNames() { - return namesCache.get(""); + return Objects.requireNonNull(namesCache.get("")).stream().map(Pair::value).collect(Collectors.toList()); + } + + public String getRemoteName(String localName) { + return Objects.requireNonNull(namesCache.getIfPresent("")).stream() + .filter(pair -> pair.value().equals(localName)) + .map(Pair::key) + .findFirst() + .orElse(null); } public Optional getMetaObj(String name, long id) { @@ -94,9 +105,9 @@ public void updateCache(String objName, T obj, long id) { metaObjCache.put(objName, Optional.of(obj)); namesCache.asMap().compute("", (k, v) -> { if (v == null) { - return Lists.newArrayList(objName); + return Lists.newArrayList(Pair.of(objName, objName)); } else { - v.add(objName); + v.add(Pair.of(objName, objName)); return v; } }); @@ -121,5 +132,4 @@ public void invalidateAll() { metaObjCache.invalidateAll(); idToName.clear(); } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java index 50265f77463428..fdbad45c5d0af9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalDatabase.java @@ -23,12 +23,15 @@ public class PaimonExternalDatabase extends ExternalDatabase { - public PaimonExternalDatabase(ExternalCatalog extCatalog, Long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.PAIMON); + public PaimonExternalDatabase(ExternalCatalog extCatalog, Long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.PAIMON); } @Override - protected PaimonExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new PaimonExternalTable(tblId, tableName, name, (PaimonExternalCatalog) extCatalog); + public PaimonExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new PaimonExternalTable(tblId, localTableName, remoteTableName, (PaimonExternalCatalog) extCatalog, + (PaimonExternalDatabase) db); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 7b59d879d9301c..488a9fe3339a7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -63,8 +63,9 @@ public class PaimonExternalTable extends ExternalTable implements MvccTable { private final Table paimonTable; - public PaimonExternalTable(long id, String name, String dbName, PaimonExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.PAIMON_EXTERNAL_TABLE); + public PaimonExternalTable(long id, String name, String remoteName, PaimonExternalCatalog catalog, + PaimonExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.PAIMON_EXTERNAL_TABLE); this.paimonTable = catalog.getPaimonTable(dbName, name); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java index 2cf1f57d0e4672..0c11dc8f9400a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalDatabase.java @@ -23,12 +23,15 @@ public class TestExternalDatabase extends ExternalDatabase { - public TestExternalDatabase(ExternalCatalog extCatalog, long id, String name) { - super(extCatalog, id, name, InitDatabaseLog.Type.TEST); + public TestExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.TEST); } @Override - protected TestExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new TestExternalTable(tblId, tableName, name, (TestExternalCatalog) extCatalog); + public TestExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new TestExternalTable(tblId, localTableName, remoteTableName, (TestExternalCatalog) extCatalog, + (TestExternalDatabase) db); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java index 6da0981b97ef54..6d08b10403bb76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalTable.java @@ -33,8 +33,8 @@ public class TestExternalTable extends ExternalTable { private static final Logger LOG = LogManager.getLogger(TestExternalTable.class); - public TestExternalTable(long id, String name, String dbName, TestExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.TEST_EXTERNAL_TABLE); + public TestExternalTable(long id, String name, String remoteName, TestExternalCatalog catalog, TestExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.TEST_EXTERNAL_TABLE); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalDatabase.java index 33652e7e945f41..31ada04eeb68e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalDatabase.java @@ -22,12 +22,16 @@ import org.apache.doris.datasource.InitDatabaseLog.Type; public class TrinoConnectorExternalDatabase extends ExternalDatabase { - public TrinoConnectorExternalDatabase(ExternalCatalog extCatalog, Long id, String name) { - super(extCatalog, id, name, Type.TRINO_CONNECTOR); + public TrinoConnectorExternalDatabase(ExternalCatalog extCatalog, Long id, String name, String remoteName) { + super(extCatalog, id, name, remoteName, Type.TRINO_CONNECTOR); } @Override - protected TrinoConnectorExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) { - return new TrinoConnectorExternalTable(tblId, tableName, name, (TrinoConnectorExternalCatalog) extCatalog); + public TrinoConnectorExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog catalog, + ExternalDatabase db) { + return new TrinoConnectorExternalTable(tblId, localTableName, remoteTableName, + (TrinoConnectorExternalCatalog) extCatalog, + (TrinoConnectorExternalDatabase) db); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalTable.java index 007ad864da3af8..a664d3889241a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalTable.java @@ -71,8 +71,9 @@ public class TrinoConnectorExternalTable extends ExternalTable { - public TrinoConnectorExternalTable(long id, String name, String dbName, TrinoConnectorExternalCatalog catalog) { - super(id, name, catalog, dbName, TableType.TRINO_CONNECTOR_EXTERNAL_TABLE); + public TrinoConnectorExternalTable(long id, String name, String remoteName, TrinoConnectorExternalCatalog catalog, + TrinoConnectorExternalDatabase db) { + super(id, name, remoteName, catalog, db, TableType.TRINO_CONNECTOR_EXTERNAL_TABLE); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java index 07c7abf7ce0c26..a38b5f49fc070f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/constraint/ConstraintPersistTest.java @@ -239,8 +239,8 @@ void addConstraintLogPersistForExternalTableTest() throws Exception { Env.getCurrentEnv().changeCatalog(connectContext, "es"); EsExternalCatalog esCatalog = (EsExternalCatalog) getCatalog("es"); - EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1"); - EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog); + EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1", "es_db1"); + EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog, db); ImmutableList schema = ImmutableList.of(new Column("k1", PrimitiveType.INT)); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); @@ -302,8 +302,8 @@ void dropConstraintLogPersistForExternalTest() throws Exception { Env.getCurrentEnv().changeCatalog(connectContext, "es2"); EsExternalCatalog esCatalog = (EsExternalCatalog) getCatalog("es2"); - EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1"); - EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog); + EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1", "es_db1"); + EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_db1", esCatalog, db); ImmutableList schema = ImmutableList.of(new Column("k1", PrimitiveType.INT)); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index aa5fa313be3c91..9d36d843e9fb7c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -163,15 +163,15 @@ private void createDbAndTableForCatalog(CatalogIf catalog) { schema.add(new Column("k1", PrimitiveType.INT)); if (catalog instanceof HMSExternalCatalog) { HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; - HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hive_db1"); - HMSExternalTable tbl = new HMSExternalTable(10001, "hive_tbl1", "hive_db1", hmsCatalog); + HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hive_db1", "hive_db1"); + HMSExternalTable tbl = new HMSExternalTable(10001, "hive_tbl1", "hive_db1", hmsCatalog, db); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); hmsCatalog.addDatabaseForTest(db); } else if (catalog instanceof EsExternalCatalog) { EsExternalCatalog esCatalog = (EsExternalCatalog) catalog; - EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1"); - EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_tbl1", esCatalog); + EsExternalDatabase db = new EsExternalDatabase(esCatalog, 10002, "es_db1", "es_db1"); + EsExternalTable tbl = new EsExternalTable(10003, "es_tbl1", "es_tbl1", esCatalog, db); tbl.setNewFullSchema(schema); db.addTableForTest(tbl); esCatalog.addDatabaseForTest(db); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java index 2a6342a57affa3..b83b69df26f838 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java @@ -217,7 +217,7 @@ public Table getTable(String dbName, String tblName) { @Mock public ExternalDatabase getDbNullable(String dbName) { if (createdDbs.contains(dbName)) { - return new HMSExternalDatabase(hmsExternalCatalog, RandomUtils.nextLong(), dbName); + return new HMSExternalDatabase(hmsExternalCatalog, RandomUtils.nextLong(), dbName, dbName); } return null; } @@ -228,7 +228,7 @@ public ExternalDatabase getDbNullable(String dbName) { HMSExternalTable getTableNullable(String tableName) { for (Table table : createdTables) { if (table.getTableName().equals(tableName)) { - return new HMSExternalTable(0, tableName, mockedDbName, hmsExternalCatalog); + return new HMSExternalTable(0, tableName, tableName, hmsExternalCatalog, (HMSExternalDatabase) hmsExternalCatalog.getDbNullable(mockedDbName)); } } return null; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java index 46d3e1b897d111..8f268f19426c3e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java @@ -68,7 +68,7 @@ public void init() { new MockUp(HMSExternalCatalog.class) { @Mock public ExternalDatabase getDbNullable(String dbName) { - return new HMSExternalDatabase(mockedCatalog, 0L, "mockedDb"); + return new HMSExternalDatabase(mockedCatalog, 0L, "mockedDb", "mockedDb"); } @Mock diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java index 439f6f2aa7de21..2300ece6253b5f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java @@ -84,7 +84,7 @@ public static void beforeClass() throws Throwable { } else { icebergCatalog.setInitialized(true); } - IcebergExternalDatabase db = new IcebergExternalDatabase(icebergCatalog, 1L, dbName); + IcebergExternalDatabase db = new IcebergExternalDatabase(icebergCatalog, 1L, dbName, dbName); icebergCatalog.addDatabaseForTest(db); // context diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheFalseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheFalseTest.java new file mode 100644 index 00000000000000..acc357d85d684b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheFalseTest.java @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.analysis.CreateCatalogStmt; +import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshCatalogStmt; +import org.apache.doris.analysis.SwitchStmt; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; +import org.apache.doris.qe.GlobalVariable; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ExternalTableNameComparedLowercaseMetaCacheFalseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog + CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"use_meta_cache\" = \"false\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameComparedLowercaseMetaCacheFalseTest$ExternalTableNameComparedLowercaseProvider\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + } + + @Override + protected void beforeCluster() { + Config.lower_case_table_names = 2; + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + DropCatalogStmt stmt = (DropCatalogStmt) parseAndAnalyzeStmt("drop catalog test1"); + env.getCatalogMgr().dropCatalog(stmt); + } + + + @Test + public void testGlobalVariable() { + Assertions.assertEquals(2, GlobalVariable.lowerCaseTableNames); + } + + @Test + public void testGetTableWithOutList() { + RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + String tblName = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table1").getName(); + Assertions.assertEquals("TABLE1", tblName); + String tblName2 = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table2").getName(); + Assertions.assertEquals("TABLE2", tblName2); + } + + @Test + public void testTableNameLowerCase() { + Set tableNames = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNamesWithLock(); + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("TABLE1")); + Assertions.assertTrue(tableNames.contains("TABLE2")); + } + + private void switchTest() throws Exception { + SwitchStmt switchTest = (SwitchStmt) parseAndAnalyzeStmt("switch test1;"); + Env.getCurrentEnv().changeCatalog(connectContext, switchTest.getCatalogName()); + } + + public static class ExternalTableNameComparedLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tblSchemaMap1 = Maps.newHashMap(); + // db1 + tblSchemaMap1.put("TABLE1", Lists.newArrayList( + new Column("siteid", PrimitiveType.INT), + new Column("citycode", PrimitiveType.SMALLINT), + new Column("username", PrimitiveType.VARCHAR), + new Column("pv", PrimitiveType.BIGINT))); + tblSchemaMap1.put("TABLE2", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR), + new Column("k3", PrimitiveType.VARCHAR), + new Column("k4", PrimitiveType.INT), + new Column("k5", PrimitiveType.LARGEINT))); + MOCKED_META.put("db1", tblSchemaMap1); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheTrueTest.java similarity index 82% rename from fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseTest.java rename to fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheTrueTest.java index c48e18cb2711df..ade73f3fef2175 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameComparedLowercaseMetaCacheTrueTest.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.CreateCatalogStmt; import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshCatalogStmt; import org.apache.doris.analysis.SwitchStmt; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; @@ -27,6 +28,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; import org.apache.doris.qe.GlobalVariable; import org.apache.doris.utframe.TestWithFeService; @@ -39,7 +41,7 @@ import java.util.Map; import java.util.Set; -public class ExternalTableNameComparedLowercaseTest extends TestWithFeService { +public class ExternalTableNameComparedLowercaseMetaCacheTrueTest extends TestWithFeService { private static Env env; private ConnectContext rootCtx; @@ -50,8 +52,9 @@ protected void runBeforeAll() throws Exception { // 1. create test catalog CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + " \"type\" = \"test\",\n" + + " \"use_meta_cache\" = \"true\",\n" + " \"catalog_provider.class\" " - + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameComparedLowercaseTest$ExternalTableNameComparedLowercaseProvider\"\n" + + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameComparedLowercaseMetaCacheTrueTest$ExternalTableNameComparedLowercaseProvider\"\n" + ");", rootCtx); env.getCatalogMgr().createCatalog(testCatalog); @@ -77,6 +80,20 @@ public void testGlobalVariable() { Assertions.assertEquals(2, GlobalVariable.lowerCaseTableNames); } + @Test + public void testGetTableWithOutList() { + RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + String tblName = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table1").getName(); + Assertions.assertEquals("TABLE1", tblName); + String tblName2 = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table2").getName(); + Assertions.assertEquals("TABLE2", tblName2); + } + @Test public void testTableNameLowerCase() { Set tableNames = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNamesWithLock(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheFalseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheFalseTest.java new file mode 100644 index 00000000000000..c4e1fee1768cb8 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheFalseTest.java @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.analysis.CreateCatalogStmt; +import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshCatalogStmt; +import org.apache.doris.analysis.SwitchStmt; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; +import org.apache.doris.qe.GlobalVariable; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ExternalTableNameStoredLowercaseMetaCacheFalseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog + CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"use_meta_cache\" = \"false\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameStoredLowercaseMetaCacheFalseTest$ExternalTableNameStoredLowercaseProvider\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + } + + @Override + protected void beforeCluster() { + Config.lower_case_table_names = 1; + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + DropCatalogStmt stmt = (DropCatalogStmt) parseAndAnalyzeStmt("drop catalog test1"); + env.getCatalogMgr().dropCatalog(stmt); + } + + + @Test + public void testGlobalVariable() { + Assertions.assertEquals(1, GlobalVariable.lowerCaseTableNames); + } + + @Test + public void testGetTableWithOutList() { + RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + String tblName = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("TABLE1").getName(); + Assertions.assertEquals("table1", tblName); + String tblName3 = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table3").getName(); + Assertions.assertEquals("table3", tblName3); + } + + @Test + public void testTableNameLowerCase() { + Set tableNames = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNamesWithLock(); + Assertions.assertEquals(3, tableNames.size()); + Assertions.assertTrue(tableNames.contains("table1")); + Assertions.assertTrue(tableNames.contains("table2")); + Assertions.assertTrue(tableNames.contains("table3")); + Assertions.assertFalse(tableNames.contains("TABLE1")); + } + + private void switchTest() throws Exception { + SwitchStmt switchTest = (SwitchStmt) parseAndAnalyzeStmt("switch test1;"); + Env.getCurrentEnv().changeCatalog(connectContext, switchTest.getCatalogName()); + } + + public static class ExternalTableNameStoredLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tblSchemaMap1 = Maps.newHashMap(); + // db1 + tblSchemaMap1.put("table1", Lists.newArrayList( + new Column("siteid", PrimitiveType.INT), + new Column("citycode", PrimitiveType.SMALLINT), + new Column("username", PrimitiveType.VARCHAR), + new Column("pv", PrimitiveType.BIGINT))); + tblSchemaMap1.put("table2", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR), + new Column("k3", PrimitiveType.VARCHAR), + new Column("k4", PrimitiveType.INT), + new Column("k5", PrimitiveType.LARGEINT))); + tblSchemaMap1.put("TABLE3", Lists.newArrayList( + new Column("k1", PrimitiveType.INT), + new Column("k2", PrimitiveType.VARCHAR), + new Column("k3", PrimitiveType.VARCHAR), + new Column("k4", PrimitiveType.INT), + new Column("k5", PrimitiveType.LARGEINT))); + MOCKED_META.put("db1", tblSchemaMap1); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheTrueTest.java similarity index 83% rename from fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseTest.java rename to fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheTrueTest.java index d55e209fa63d9f..d3eca35debb0b1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalTableNameStoredLowercaseMetaCacheTrueTest.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.CreateCatalogStmt; import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.RefreshCatalogStmt; import org.apache.doris.analysis.SwitchStmt; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; @@ -27,6 +28,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.DdlExecutor; import org.apache.doris.qe.GlobalVariable; import org.apache.doris.utframe.TestWithFeService; @@ -39,7 +41,7 @@ import java.util.Map; import java.util.Set; -public class ExternalTableNameStoredLowercaseTest extends TestWithFeService { +public class ExternalTableNameStoredLowercaseMetaCacheTrueTest extends TestWithFeService { private static Env env; private ConnectContext rootCtx; @@ -50,8 +52,9 @@ protected void runBeforeAll() throws Exception { // 1. create test catalog CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt("create catalog test1 properties(\n" + " \"type\" = \"test\",\n" + + " \"use_meta_cache\" = \"true\",\n" + " \"catalog_provider.class\" " - + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameStoredLowercaseTest$ExternalTableNameStoredLowercaseProvider\"\n" + + "= \"org.apache.doris.datasource.lowercase.ExternalTableNameStoredLowercaseMetaCacheTrueTest$ExternalTableNameStoredLowercaseProvider\"\n" + ");", rootCtx); env.getCatalogMgr().createCatalog(testCatalog); @@ -77,6 +80,20 @@ public void testGlobalVariable() { Assertions.assertEquals(1, GlobalVariable.lowerCaseTableNames); } + @Test + public void testGetTableWithOutList() { + RefreshCatalogStmt refreshCatalogStmt = new RefreshCatalogStmt("test1", null); + try { + DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); + } catch (Exception e) { + // Do nothing + } + String tblName = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("TABLE1").getName(); + Assertions.assertEquals("table1", tblName); + String tblName3 = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNullable("table3").getName(); + Assertions.assertEquals("table3", tblName3); + } + @Test public void testTableNameLowerCase() { Set tableNames = env.getCatalogMgr().getCatalog("test1").getDbNullable("db1").getTableNamesWithLock(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/mapping/JdbcIdentifierMappingTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/mapping/JdbcIdentifierMappingTest.java new file mode 100644 index 00000000000000..edf7ab4e29b12a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/mapping/JdbcIdentifierMappingTest.java @@ -0,0 +1,277 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.mapping; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class JdbcIdentifierMappingTest { + private String validJson; + private String invalidJson; + private String duplicateMappingJson; + private String columnConflictJson; + private String tableConflictJson; + + @Before + public void setUp() { + validJson = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"mapping\": \"doris_local\"}\n" + + " ],\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"remoteTable\": \"TABLE_A\", \"mapping\": \"table_a_local\"}\n" + + " ],\n" + + " \"columns\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"remoteTable\": \"TABLE_A\", \"remoteColumn\": \"COLUMN_X\", " + + "\"mapping\": \"column_x_local\"}\n" + + " ]\n" + + "}"; + + invalidJson = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"mapping\": \"doris_local\"}\n" + + " ],\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"remoteTable\": \"TABLE_A\", \"mapping\": \"table_a_local\"}\n" + + " ], // Invalid JSON due to trailing comma\n" + + "}"; + + duplicateMappingJson = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"DORIS\", \"mapping\": \"conflict\"},\n" + + " {\"remoteDatabase\": \"DORIS_DUP\", \"mapping\": \"CONFLICT\"}\n" + + " ]\n" + + "}"; + + columnConflictJson = "{\n" + + " \"columns\": [\n" + + " {\"remoteDatabase\": \"D1\", \"remoteTable\": \"T1\", \"remoteColumn\": \"C1\", \"mapping\": \"custom_col\"},\n" + + " {\"remoteDatabase\": \"D1\", \"remoteTable\": \"T1\", \"remoteColumn\": \"C2\", \"mapping\": \"CUSTOM_COL\"}\n" + + " ]\n" + + "}"; + + tableConflictJson = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"D2\", \"remoteTable\": \"T1\", \"mapping\": \"custom_table\"},\n" + + " {\"remoteDatabase\": \"D2\", \"remoteTable\": \"T2\", \"mapping\": \"CUSTOM_TABLE\"}\n" + + " ]\n" + + "}"; + } + + @Test + public void testIsLowerCaseMetaNamesTrue() { + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(true, false, validJson); + + String databaseName = mapping.fromRemoteDatabaseName("DORIS"); + String tableName = mapping.fromRemoteTableName("DORIS", "TABLE_A"); + String columnName = mapping.fromRemoteColumnName("DORIS", "TABLE_A", "COLUMN_X"); + + Assert.assertEquals("doris_local", databaseName); + Assert.assertEquals("table_a_local", tableName); + Assert.assertEquals("column_x_local", columnName); + } + + @Test + public void testIsLowerCaseMetaNamesFalse() { + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, validJson); + + String databaseName = mapping.fromRemoteDatabaseName("DORIS"); + String tableName = mapping.fromRemoteTableName("DORIS", "TABLE_A"); + String columnName = mapping.fromRemoteColumnName("DORIS", "TABLE_A", "COLUMN_X"); + + Assert.assertEquals("doris_local", databaseName); + Assert.assertEquals("table_a_local", tableName); + Assert.assertEquals("column_x_local", columnName); + } + + @Test(expected = RuntimeException.class) + public void testInvalidJson() { + new JdbcIdentifierMapping(true, false, invalidJson); + } + + @Test + public void testDuplicateMappingWhenLowerCaseMetaNamesTrue() { + try { + new JdbcIdentifierMapping(false, true, duplicateMappingJson); + Assert.fail("Expected RuntimeException due to duplicate mappings"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("Duplicate mapping found")); + } + } + + @Test + public void testDuplicateMappingWhenLowerCaseMetaNamesFalse() { + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, duplicateMappingJson); + + String databaseName1 = mapping.fromRemoteDatabaseName("DORIS"); + String databaseName2 = mapping.fromRemoteDatabaseName("DORIS_DUP"); + + Assert.assertEquals("conflict", databaseName1); + Assert.assertEquals("CONFLICT", databaseName2); + } + + @Test + public void testColumnCaseConflictAlwaysChecked() { + try { + new JdbcIdentifierMapping(false, false, columnConflictJson); + Assert.fail("Expected RuntimeException due to column case-only conflict"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("case-only different mapping")); + } + } + + @Test + public void testTableCaseConflictWhenLowerCaseMetaNamesFalseAndLowerCaseTableNamesFalse() { + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, tableConflictJson); + String tableName1 = mapping.fromRemoteTableName("D2", "T1"); + String tableName2 = mapping.fromRemoteTableName("D2", "T2"); + Assert.assertEquals("custom_table", tableName1); + Assert.assertEquals("CUSTOM_TABLE", tableName2); + } + + @Test + public void testTableCaseConflictWhenLowerCaseMetaNamesTrue() { + try { + new JdbcIdentifierMapping(true, false, tableConflictJson); + Assert.fail("Expected RuntimeException due to table case-only conflict"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("case-only different mapping")); + } + } + + @Test + public void testTableCaseConflictWhenLowerCaseMetaNamesFalseButLowerCaseTableNamesTrue() { + try { + new JdbcIdentifierMapping(false, true, tableConflictJson); + Assert.fail("Expected RuntimeException due to table case-only conflict"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("case-only different mapping")); + } + } + + @Test + public void testUppercaseMappingForDBWhenLowerCaseMetaNamesTrue() { + String json = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"UPPER_DB\", \"mapping\": \"UPPER_LOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, true, json); + Assert.assertEquals("upper_local", mapping.fromRemoteDatabaseName("UPPER_DB")); + } + + @Test + public void testUppercaseMappingForDBWhenLowerCaseMetaNamesFalse() { + String json = "{\n" + + " \"databases\": [\n" + + " {\"remoteDatabase\": \"UPPER_DB\", \"mapping\": \"UPPER_LOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_LOCAL", mapping.fromRemoteDatabaseName("UPPER_DB")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseTableNamesTrue() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(true, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseTableNamesFalse() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseMetaNamesTrue() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, true, json); + Assert.assertEquals("upper_tlocal", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenLowerCaseMetaNamesFalse() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenAllCaseFalse() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_TLOCAL", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForTableWhenAllCaseTrue() { + String json = "{\n" + + " \"tables\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"UPPER_TABLE\", \"mapping\": \"UPPER_TLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(true, true, json); + Assert.assertEquals("upper_tlocal", mapping.fromRemoteTableName("DB", "UPPER_TABLE")); + } + + @Test + public void testUppercaseMappingForColumnWithoutLowerCaseMetaNames() { + String json = "{\n" + + " \"columns\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"TAB\", \"remoteColumn\": \"UPPER_COL\", \"mapping\": \"UPPER_CLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, false, json); + Assert.assertEquals("UPPER_CLOCAL", mapping.fromRemoteColumnName("DB", "TAB", "UPPER_COL")); + } + + @Test + public void testUppercaseMappingForColumnWithLowerCaseMetaNamesTrue() { + String json = "{\n" + + " \"columns\": [\n" + + " {\"remoteDatabase\": \"DB\", \"remoteTable\": \"TAB\", \"remoteColumn\": \"UPPER_COL\", \"mapping\": \"UPPER_CLOCAL\"}\n" + + " ]\n" + + "}"; + JdbcIdentifierMapping mapping = new JdbcIdentifierMapping(false, true, json); + Assert.assertEquals("upper_clocal", mapping.fromRemoteColumnName("DB", "TAB", "UPPER_COL")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java index c875ef6bc2f141..30a233dd1a999e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java @@ -98,7 +98,7 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { List schema = Lists.newArrayList(); schema.add(new Column("k1", PrimitiveType.INT)); - HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db"); + HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db", "hms_db"); Deencapsulation.setField(db, "initialized", true); Deencapsulation.setField(tbl, "objectCreated", true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java index bddb3c8185ae72..0a981dab8a9dcc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java @@ -114,7 +114,7 @@ private void init(HMSExternalCatalog hmsCatalog) { List schema = Lists.newArrayList(); schema.add(new Column("k1", PrimitiveType.INT)); - HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db"); + HMSExternalDatabase db = new HMSExternalDatabase(hmsCatalog, 10000, "hms_db", "hms_db"); Deencapsulation.setField(db, "initialized", true); Deencapsulation.setField(tbl, "objectCreated", true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java index 026b7fb65b069a..1d9ea4bd4eb399 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java @@ -24,8 +24,10 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.Pair; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; +import org.apache.doris.datasource.jdbc.JdbcExternalDatabase; import org.apache.doris.datasource.jdbc.JdbcExternalTable; import mockit.Mock; @@ -117,7 +119,8 @@ public void testSupportAutoAnalyze() { OlapTable table1 = new OlapTable(200, "testTable", schema, null, null, null); Assertions.assertTrue(collector.supportAutoAnalyze(table1)); - ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbcdb", null); + JdbcExternalDatabase jdbcExternalDatabase = new JdbcExternalDatabase(null, 1L, "jdbcdb", "jdbcdb"); + ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbctable", null, jdbcExternalDatabase); Assertions.assertFalse(collector.supportAutoAnalyze(externalTable)); new MockUp() { @@ -126,7 +129,8 @@ public DLAType getDlaType() { return DLAType.ICEBERG; } }; - ExternalTable icebergExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null); + HMSExternalDatabase hmsExternalDatabase = new HMSExternalDatabase(null, 1L, "hmsDb", "hmsDb"); + ExternalTable icebergExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null, hmsExternalDatabase); Assertions.assertFalse(collector.supportAutoAnalyze(icebergExternalTable)); new MockUp() { @@ -135,7 +139,7 @@ public DLAType getDlaType() { return DLAType.HIVE; } }; - ExternalTable hiveExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null); + ExternalTable hiveExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", null, hmsExternalDatabase); Assertions.assertTrue(collector.supportAutoAnalyze(hiveExternalTable)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java index ef1e9ca02970b1..7514d96ddb5e1d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java @@ -30,10 +30,13 @@ import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; +import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.jdbc.JdbcExternalCatalog; +import org.apache.doris.datasource.jdbc.JdbcExternalDatabase; import org.apache.doris.datasource.jdbc.JdbcExternalTable; import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.AnalysisManager; @@ -180,7 +183,7 @@ void testNeedAnalyzeColumn() throws DdlException { schema.add(column); OlapTable table = new OlapTable(200, "testTable", schema, null, null, null); HMSExternalCatalog externalCatalog = new HMSExternalCatalog(); - + HMSExternalDatabase externalDatabase = new HMSExternalDatabase(externalCatalog, 1L, "dbName", "dbName"); // Test olap table auto analyze disabled. Map properties = new HashMap<>(); properties.put(PropertyAnalyzer.PROPERTIES_AUTO_ANALYZE_POLICY, "disable"); @@ -195,7 +198,7 @@ protected synchronized void makeSureInitialized() { }; // Test auto analyze catalog disabled. - HMSExternalTable hmsTable = new HMSExternalTable(1, "name", "dbName", externalCatalog); + HMSExternalTable hmsTable = new HMSExternalTable(1, "name", "name", externalCatalog, externalDatabase); Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(hmsTable, Pair.of("index", column.getName()))); // Test catalog auto analyze enabled. @@ -210,7 +213,7 @@ public TableStatsMeta findTableStatsStatus(long tblId) { // Test external table auto analyze enabled. externalCatalog.getCatalogProperty().addProperty(ExternalCatalog.ENABLE_AUTO_ANALYZE, "false"); - HMSExternalTable hmsTable1 = new HMSExternalTable(1, "name", "dbName", externalCatalog); + HMSExternalTable hmsTable1 = new HMSExternalTable(1, "name", "name", externalCatalog, externalDatabase); externalCatalog.setAutoAnalyzePolicy("dbName", "name", "enable"); Assertions.assertTrue(StatisticsUtil.needAnalyzeColumn(hmsTable1, Pair.of("index", column.getName()))); @@ -246,8 +249,9 @@ protected synchronized void makeSureInitialized() { } }; // Test not supported external table type. - ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbcdb", - new JdbcExternalCatalog(1, "name", "resource", new HashMap<>(), "")); + JdbcExternalCatalog jdbcExternalCatalog = new JdbcExternalCatalog(1, "name", "resource", new HashMap<>(), ""); + JdbcExternalDatabase jdbcExternalDatabase = new JdbcExternalDatabase(jdbcExternalCatalog, 1, "jdbcdb", "jdbcdb"); + ExternalTable externalTable = new JdbcExternalTable(1, "jdbctable", "jdbctable", jdbcExternalCatalog, jdbcExternalDatabase); Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(externalTable, Pair.of("index", column.getName()))); // Test hms external table not hive type. @@ -257,7 +261,7 @@ public DLAType getDlaType() { return DLAType.ICEBERG; } }; - ExternalTable hmsExternalTable = new HMSExternalTable(1, "hmsTable", "hmsDb", externalCatalog); + ExternalTable hmsExternalTable = new HMSExternalTable(1, "hmsTable", "hmsTable", externalCatalog, externalDatabase); Assertions.assertFalse(StatisticsUtil.needAnalyzeColumn(hmsExternalTable, Pair.of("index", column.getName()))); // Test partition first load. @@ -394,7 +398,8 @@ public boolean autoAnalyzeEnabled() { return true; } }; - IcebergExternalTable icebergTable = new IcebergExternalTable(0, "", "", null); + IcebergExternalDatabase icebergDatabase = new IcebergExternalDatabase(null, 1L, "", ""); + IcebergExternalTable icebergTable = new IcebergExternalTable(0, "", "", null, icebergDatabase); Assertions.assertFalse(StatisticsUtil.isLongTimeColumn(icebergTable, Pair.of("index", column.getName()))); // Test table stats meta is null. diff --git a/regression-test/data/external_table_p0/lower_case/test_conflict_name.out b/regression-test/data/external_table_p0/lower_case/test_conflict_name.out new file mode 100644 index 00000000000000..be86d4d3b3ed31 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_conflict_name.out @@ -0,0 +1,5 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_external_conflict_name_db -- + +-- !select_external_conflict_name_tbl -- + diff --git a/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_show_and_select.out b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_show_and_select.out new file mode 100644 index 00000000000000..1332a98aac4f9d --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_show_and_select.out @@ -0,0 +1,49 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_test_cache_false_lower_false1 -- +1 lower + +-- !sql_test_cache_false_lower_false2 -- +1 UPPER + +-- !sql_test_cache_false_lower_false1_insert -- +1 + +-- !sql_test_cache_false_lower_false2_insert -- +1 + +-- !sql_test_cache_false_lower_false1 -- +1 lower + +-- !sql_test_cache_false_lower_false2 -- +1 UPPER + +-- !sql_test_cache_false_lower_false1_insert -- +1 + +-- !sql_test_cache_false_lower_false2_insert -- +1 + +-- !sql_test_cache_false_lower_true1 -- +1 lower + +-- !sql_test_cache_false_lower_true2 -- +1 UPPER + +-- !sql_test_cache_false_lower_true1_insert -- +1 + +-- !sql_test_cache_false_lower_true2_insert -- +1 + +-- !sql_test_cache_true_lower_true1 -- +1 lower + +-- !sql_test_cache_true_lower_true2 -- +1 UPPER + +-- !sql_test_cache_true_lower_true1_insert -- +1 + +-- !sql_test_cache_true_lower_true2_insert -- +1 + diff --git a/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out new file mode 100644 index 00000000000000..6be380ed0d0161 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.out @@ -0,0 +1,241 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_test_cache_false_lower_false_with_conf1_1 -- +1 lower + +-- !sql_test_cache_false_lower_false_with_conf1_2 -- +1 lower + +-- !sql_test_cache_false_lower_false_with_conf1_3 -- +1 UPPER + +-- !sql_test_cache_false_lower_false_with_conf1_4 -- +1 UPPER + +-- !sql_test_cache_false_lower_false_with_conf1_1_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf1_2_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf1_3_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf1_4_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf2_1 -- +1 lower + +-- !sql_test_cache_false_lower_false_with_conf2_2 -- +1 lower + +-- !sql_test_cache_false_lower_false_with_conf2_3 -- +1 UPPER + +-- !sql_test_cache_false_lower_false_with_conf2_4 -- +1 UPPER + +-- !sql_test_cache_false_lower_false_with_conf2_1_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf2_2_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf2_3_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf2_4_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf1_1 -- +1 lower + +-- !sql_test_cache_true_lower_false_with_conf1_2 -- +1 lower + +-- !sql_test_cache_true_lower_false_with_conf1_3 -- +1 UPPER + +-- !sql_test_cache_true_lower_false_with_conf1_4 -- +1 UPPER + +-- !sql_test_cache_true_lower_false_with_conf1_1_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf1_2_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf1_3_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf1_4_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf2_1 -- +1 lower + +-- !sql_test_cache_true_lower_false_with_conf2_2 -- +1 lower + +-- !sql_test_cache_true_lower_false_with_conf2_3 -- +1 UPPER + +-- !sql_test_cache_true_lower_false_with_conf2_4 -- +1 UPPER + +-- !sql_test_cache_true_lower_false_with_conf2_1_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf2_2_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf2_3_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf2_4_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf1_1 -- +1 lower + +-- !sql_test_cache_false_lower_true_with_conf1_2 -- +1 lower + +-- !sql_test_cache_false_lower_true_with_conf1_3 -- +1 UPPER + +-- !sql_test_cache_false_lower_true_with_conf1_4 -- +1 UPPER + +-- !sql_test_cache_false_lower_true_with_conf1_1_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf1_2_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf1_3_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf1_4_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf2_1 -- +1 lower + +-- !sql_test_cache_false_lower_true_with_conf2_2 -- +1 lower + +-- !sql_test_cache_false_lower_true_with_conf2_3 -- +1 UPPER + +-- !sql_test_cache_false_lower_true_with_conf2_4 -- +1 UPPER + +-- !sql_test_cache_false_lower_true_with_conf2_1_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf2_2_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf2_3_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf2_4_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf1_1 -- +1 lower + +-- !sql_test_cache_true_lower_true_with_conf1_2 -- +1 lower + +-- !sql_test_cache_true_lower_true_with_conf1_3 -- +1 UPPER + +-- !sql_test_cache_true_lower_true_with_conf1_4 -- +1 UPPER + +-- !sql_test_cache_true_lower_true_with_conf1_1_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf1_2_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf1_3_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf1_4_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf2_1 -- +1 lower + +-- !sql_test_cache_true_lower_true_with_conf2_2 -- +1 lower + +-- !sql_test_cache_true_lower_true_with_conf2_3 -- +1 UPPER + +-- !sql_test_cache_true_lower_true_with_conf2_4 -- +1 UPPER + +-- !sql_test_cache_true_lower_true_with_conf2_1_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf2_2_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf2_3_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf2_4_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf0_1 -- +1 lower + +-- !sql_test_cache_false_lower_false_with_conf0_2 -- +1 UPPER + +-- !sql_test_cache_false_lower_false_with_conf0_1_insert -- +1 + +-- !sql_test_cache_false_lower_false_with_conf0_2_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf0_1 -- +1 lower + +-- !sql_test_cache_true_lower_false_with_conf0_2 -- +1 UPPER + +-- !sql_test_cache_true_lower_false_with_conf0_1_insert -- +1 + +-- !sql_test_cache_true_lower_false_with_conf0_2_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf0_1 -- +1 lower + +-- !sql_test_cache_false_lower_true_with_conf0_2 -- +1 UPPER + +-- !sql_test_cache_false_lower_true_with_conf0_1_insert -- +1 + +-- !sql_test_cache_false_lower_true_with_conf0_2_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf0_1 -- +1 lower + +-- !sql_test_cache_true_lower_true_with_conf0_2 -- +1 UPPER + +-- !sql_test_cache_true_lower_true_with_conf0_1_insert -- +1 + +-- !sql_test_cache_true_lower_true_with_conf0_2_insert -- +1 + diff --git a/regression-test/data/external_table_p0/lower_case/test_lower_case_mtmv.out b/regression-test/data/external_table_p0/lower_case/test_lower_case_mtmv.out new file mode 100644 index 00000000000000..f958424e65c626 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_lower_case_mtmv.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- + diff --git a/regression-test/data/external_table_p0/lower_case/test_meta_cache_select_without_refresh.out b/regression-test/data/external_table_p0/lower_case/test_meta_cache_select_without_refresh.out new file mode 100644 index 00000000000000..7669a50d30cd21 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_meta_cache_select_without_refresh.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test_meta_cache_lower_true_select_without_refresh_select_table1 -- +1 table1 + +-- !test_meta_cache_lower_false_select_without_refresh_select_table1 -- +1 table1 + +-- !test_meta_cache_lower_false_select_without_refresh_select_table2 -- +1 TABLE2 + diff --git a/regression-test/data/external_table_p0/lower_case/test_meta_names_mapping.out b/regression-test/data/external_table_p0/lower_case/test_meta_names_mapping.out new file mode 100644 index 00000000000000..71551d16891296 --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/test_meta_names_mapping.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_meta_mapping_select_lower -- +1 lowercase 100 + +-- !sql_meta_mapping_select_upper -- +2 UPPERCASE 200 + +-- !sql_meta_mapping_select_lower_lower_case_true -- +1 lowercase 100 + +-- !sql_meta_mapping_select_upper_lower_case_true -- +2 UPPERCASE 200 + diff --git a/regression-test/data/external_table_p0/lower_case/upgrade/load.out b/regression-test/data/external_table_p0/lower_case/upgrade/load.out new file mode 100644 index 00000000000000..bdd9b17d0c6e3d --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/upgrade/load.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_test_upgrade_lower_case_catalog_1 -- +1 lower + +-- !sql_test_upgrade_lower_case_catalog_2 -- +1 UPPER + diff --git a/regression-test/data/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.out b/regression-test/data/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.out new file mode 100644 index 00000000000000..bdd9b17d0c6e3d --- /dev/null +++ b/regression-test/data/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_test_upgrade_lower_case_catalog_1 -- +1 lower + +-- !sql_test_upgrade_lower_case_catalog_2 -- +1 UPPER + diff --git a/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy new file mode 100644 index 00000000000000..6187d47b645564 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_conflict_name", "p0,external,doris,meta_names_mapping") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """drop database if exists internal.external_conflict_name; """ + sql """drop database if exists internal.EXTERNAL_CONFLICT_NAME; """ + sql """create database if not exists internal.external_conflict_name; """ + sql """create database if not exists internal.EXTERNAL_CONFLICT_NAME; """ + sql """create table if not exists internal.external_conflict_name.table_test + (id int, name varchar(20), column_test int) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """drop catalog if exists test_conflict_name """ + sql """ CREATE CATALOG `test_conflict_name` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_conflict_name,EXTERNAL_CONFLICT_NAME" + )""" + + test { + sql """show databases from test_conflict_name""" + exception """Found conflicting database names under case-insensitive conditions. Conflicting remote database names: EXTERNAL_CONFLICT_NAME, external_conflict_name in catalog test_conflict_name. Please use meta_names_mapping to handle name mapping.""" + } + + sql """refresh catalog test_conflict_name""" + + test { + sql """select * from test_conflict_name.external_conflict_name.table_test""" + exception """Found conflicting database names under case-insensitive conditions. Conflicting remote database names: EXTERNAL_CONFLICT_NAME, external_conflict_name in catalog test_conflict_name. Please use meta_names_mapping to handle name mapping.""" + } + + sql """drop database if exists internal.EXTERNAL_CONFLICT_NAME; """ + + sql """refresh catalog test_conflict_name""" + + qt_select_external_conflict_name_db "select * from test_conflict_name.external_conflict_name.table_test" + + sql """create table if not exists internal.external_conflict_name.TABLE_TEST + (id int, name varchar(20), column_test int) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """refresh catalog test_conflict_name""" + + test { + sql """show tables from test_conflict_name.external_conflict_name""" + exception """Found conflicting table names under case-insensitive conditions. Conflicting remote table names: TABLE_TEST, table_test in remote database 'external_conflict_name' under catalog 'test_conflict_name'. Please use meta_names_mapping to handle name mapping.""" + } + + sql """refresh catalog test_conflict_name""" + + test { + sql """select * from test_conflict_name.external_conflict_name.TABLE_TEST""" + exception """Found conflicting table names under case-insensitive conditions. Conflicting remote table names: TABLE_TEST, table_test in remote database 'external_conflict_name' under catalog 'test_conflict_name'. Please use meta_names_mapping to handle name mapping.""" + } + + sql """drop table if exists internal.external_conflict_name.TABLE_TEST; """ + + qt_select_external_conflict_name_tbl "select * from test_conflict_name.external_conflict_name.table_test" + + sql """drop database if exists internal.external_conflict_name; """ + sql """drop database if exists internal.EXTERNAL_CONFLICT_NAME; """ +} \ No newline at end of file diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy new file mode 100644 index 00000000000000..854eb06a8e2755 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lower_case_meta_include", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + String mapping_db = """ + { + "databases": [ + {"remoteDatabase": "external_INCLUDE", "mapping": "external_include_test"}, + {"remoteDatabase": "external_EXCLUDE", "mapping": "external_exclude_test"} + ] + } + """ + + sql """drop database if exists internal.external_INCLUDE; """ + sql """drop database if exists internal.external_EXCLUDE; """ + sql """create database if not exists internal.external_INCLUDE; """ + sql """create database if not exists internal.external_EXCLUDE; """ + + // Test include + sql """drop catalog if exists test_lower_case_include """ + + sql """ CREATE CATALOG `test_lower_case_include` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_INCLUDE" + )""" + + test { + sql """show databases from test_lower_case_include""" + + rowNum 3 + + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_include"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + sql """drop catalog if exists test_lower_case_include """ + + // Test exclude + sql """drop catalog if exists test_lower_case_exclude """ + + sql """ CREATE CATALOG `test_lower_case_exclude` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "exclude_database_list" = "external_EXCLUDE" + )""" + + test { + sql """show databases from test_lower_case_exclude""" + + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_exclude"] + expectedDatabases.each { dbName -> + assertFalse(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + // Test include with mapping + sql """drop catalog if exists test_lower_case_include """ + + sql """ CREATE CATALOG `test_lower_case_include` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + 'meta_names_mapping' = '${mapping_db}', + "include_database_list" = "external_INCLUDE" + )""" + + test { + sql """show databases from test_lower_case_include""" + + rowNum 3 + + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_include_test"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + sql """drop catalog if exists test_lower_case_include """ + + // Test exclude with mapping + sql """drop catalog if exists test_lower_case_exclude """ + + sql """ CREATE CATALOG `test_lower_case_exclude` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + 'meta_names_mapping' = '${mapping_db}', + "exclude_database_list" = "external_EXCLUDE" + )""" + + test { + sql """show databases from test_lower_case_exclude""" + + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_exclude_test"] + expectedDatabases.each { dbName -> + assertFalse(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + sql """drop catalog if exists test_lower_case_exclude """ + sql """drop database if exists internal.external_INCLUDE; """ + sql """drop database if exists internal.external_EXCLUDE; """ +} diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy new file mode 100644 index 00000000000000..d4efd141c2d468 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy @@ -0,0 +1,254 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lower_case_meta_show_and_select", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """drop database if exists internal.external_test_lower; """ + sql """drop database if exists internal.external_test_UPPER; """ + sql """create database if not exists internal.external_test_lower; """ + sql """create database if not exists internal.external_test_UPPER; """ + sql """create table if not exists internal.external_test_lower.lower + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """create table if not exists internal.external_test_lower.UPPER + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_test_lower.lower values(1, 'lower')""" + sql """insert into internal.external_test_lower.UPPER values(1, 'UPPER')""" + + sql """create table if not exists internal.external_test_lower.lower_insert + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """create table if not exists internal.external_test_lower.UPPER_insert + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + // Test for cache false and lower false + sql """drop catalog if exists test_cache_false_lower_false """ + + sql """ CREATE CATALOG `test_cache_false_lower_false` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower,external_test_UPPER" + )""" + + test { + sql """show databases from test_cache_false_lower_false""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_test_lower", "external_test_UPPER"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_cache_false_lower_false.external_test_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "UPPER"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_cache_false_lower_false1 "select * from test_cache_false_lower_false.external_test_lower.lower" + qt_sql_test_cache_false_lower_false2 "select * from test_cache_false_lower_false.external_test_lower.UPPER" + + qt_sql_test_cache_false_lower_false1_insert "insert into internal.external_test_lower.lower_insert select * from test_cache_false_lower_false.external_test_lower.lower" + qt_sql_test_cache_false_lower_false2_insert "insert into internal.external_test_lower.UPPER_insert select * from test_cache_false_lower_false.external_test_lower.UPPER" + + sql """drop catalog if exists test_cache_false_lower_false """ + + // Test for cache true and lower false + sql """drop catalog if exists test_cache_true_lower_false """ + + sql """ CREATE CATALOG `test_cache_true_lower_false` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower,external_test_UPPER" + )""" + + test { + sql """show databases from test_cache_true_lower_false""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_test_lower", "external_test_UPPER"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_cache_true_lower_false.external_test_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "UPPER"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_cache_false_lower_false1 "select * from test_cache_true_lower_false.external_test_lower.lower" + qt_sql_test_cache_false_lower_false2 "select * from test_cache_true_lower_false.external_test_lower.UPPER" + + qt_sql_test_cache_false_lower_false1_insert "insert into internal.external_test_lower.lower_insert select * from test_cache_true_lower_false.external_test_lower.lower" + qt_sql_test_cache_false_lower_false2_insert "insert into internal.external_test_lower.UPPER_insert select * from test_cache_true_lower_false.external_test_lower.UPPER" + + + sql """drop catalog if exists test_cache_true_lower_false """ + + // Test for cache false and lower true + sql """drop catalog if exists test_cache_false_lower_true """ + + sql """ CREATE CATALOG `test_cache_false_lower_true` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower,external_test_UPPER" + )""" + + test { + sql """show databases from test_cache_false_lower_true""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_test_lower", "external_test_upper"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_cache_false_lower_true.external_test_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_cache_false_lower_true1 "select * from test_cache_false_lower_true.external_test_lower.lower" + qt_sql_test_cache_false_lower_true2 "select * from test_cache_false_lower_true.external_test_lower.upper" + + qt_sql_test_cache_false_lower_true1_insert "insert into internal.external_test_lower.lower_insert select * from test_cache_false_lower_true.external_test_lower.lower" + qt_sql_test_cache_false_lower_true2_insert "insert into internal.external_test_lower.UPPER_insert select * from test_cache_false_lower_true.external_test_lower.upper" + + sql """drop catalog if exists test_cache_false_lower_true """ + + // Test for cache true and lower true + sql """drop catalog if exists test_cache_true_lower_true """ + + sql """ CREATE CATALOG `test_cache_true_lower_true` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower,external_test_UPPER" + )""" + + test { + sql """show databases from test_cache_true_lower_true""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_test_lower", "external_test_upper"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_cache_true_lower_true.external_test_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_cache_true_lower_true1 "select * from test_cache_true_lower_true.external_test_lower.lower" + qt_sql_test_cache_true_lower_true2 "select * from test_cache_true_lower_true.external_test_lower.upper" + + qt_sql_test_cache_true_lower_true1_insert "insert into internal.external_test_lower.lower_insert select * from test_cache_true_lower_true.external_test_lower.lower" + qt_sql_test_cache_true_lower_true2_insert "insert into internal.external_test_lower.UPPER_insert select * from test_cache_true_lower_true.external_test_lower.upper" + + sql """drop catalog if exists test_cache_true_lower_true """ + + + sql """drop database if exists internal.external_test_lower; """ + sql """drop database if exists internal.external_test_UPPER; """ +} diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy new file mode 100644 index 00000000000000..c3da24e1f55c1d --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy @@ -0,0 +1,702 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """drop database if exists internal.external_test_lower_with_conf; """ + sql """drop database if exists internal.external_test_UPPER_with_conf; """ + sql """create database if not exists internal.external_test_lower_with_conf; """ + sql """create table if not exists internal.external_test_lower_with_conf.lower_with_conf + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """create table if not exists internal.external_test_lower_with_conf.UPPER_with_conf + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_test_lower_with_conf.lower_with_conf values(1, 'lower')""" + sql """insert into internal.external_test_lower_with_conf.UPPER_with_conf values(1, 'UPPER')""" + + sql """create table if not exists internal.external_test_lower_with_conf.with_conf_insert + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + // Test for cache false and lower false and lower conf 1 + sql """drop catalog if exists test_cache_false_lower_false_with_conf1 """ + + sql """ CREATE CATALOG `test_cache_false_lower_false_with_conf1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "1" + )""" + + qt_sql_test_cache_false_lower_false_with_conf1_1 "select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_2 "select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_3 "select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_4 "select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf1" + qt_sql_test_cache_false_lower_false_with_conf1_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf" + + + test { + sql """show tables from test_cache_false_lower_false_with_conf1.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_false_with_conf1 """ + + // Test for cache false and lower false and lower conf 2 + sql """drop catalog if exists test_cache_false_lower_false_with_conf2 """ + + sql """ CREATE CATALOG `test_cache_false_lower_false_with_conf2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "2" + )""" + + qt_sql_test_cache_false_lower_false_with_conf2_1 "select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_2 "select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_3 "select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_4 "select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf2" + qt_sql_test_cache_false_lower_false_with_conf2_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_false_lower_false_with_conf2.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_false_with_conf2 """ + + + + // Test for cache true and lower false and lower conf 1 + sql """drop catalog if exists test_cache_true_lower_false_with_conf1 """ + + sql """ CREATE CATALOG `test_cache_true_lower_false_with_conf1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "1" + )""" + + qt_sql_test_cache_true_lower_false_with_conf1_1 "select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_2 "select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_3 "select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_4 "select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf1" + qt_sql_test_cache_true_lower_false_with_conf1_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_true_lower_false_with_conf1.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_false_with_conf1 """ + + // Test for cache true and lower false and lower conf 2 + sql """drop catalog if exists test_cache_true_lower_false_with_conf2 """ + + sql """ CREATE CATALOG `test_cache_true_lower_false_with_conf2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "2" + )""" + + qt_sql_test_cache_true_lower_false_with_conf2_1 "select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_2 "select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_3 "select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_4 "select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf2" + qt_sql_test_cache_true_lower_false_with_conf2_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_true_lower_false_with_conf2.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_false_with_conf2 """ + + // Test for cache false and lower true and lower conf 1 + sql """drop catalog if exists test_cache_false_lower_true_with_conf1 """ + + sql """ CREATE CATALOG `test_cache_false_lower_true_with_conf1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "1" + )""" + + qt_sql_test_cache_false_lower_true_with_conf1_1 "select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_2 "select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_3 "select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_4 "select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf1" + qt_sql_test_cache_false_lower_true_with_conf1_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_false_lower_true_with_conf1.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_true_with_conf1 """ + + // Test for cache false and lower true and lower conf 2 + sql """drop catalog if exists test_cache_false_lower_true_with_conf2 """ + + sql """ CREATE CATALOG `test_cache_false_lower_true_with_conf2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "2" + )""" + + qt_sql_test_cache_false_lower_true_with_conf2_1 "select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_2 "select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_3 "select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_4 "select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf2" + qt_sql_test_cache_false_lower_true_with_conf2_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_false_lower_true_with_conf2.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_true_with_conf2 """ + + + // Test for cache true and lower true and lower conf 1 + sql """drop catalog if exists test_cache_true_lower_true_with_conf1 """ + + sql """ CREATE CATALOG `test_cache_true_lower_true_with_conf1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "1" + )""" + + qt_sql_test_cache_true_lower_true_with_conf1_1 "select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_2 "select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_3 "select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_4 "select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf1" + qt_sql_test_cache_true_lower_true_with_conf1_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_true_lower_true_with_conf1.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_true_with_conf1 """ + + // Test for cache true and lower true and lower conf 2 + sql """drop catalog if exists test_cache_true_lower_true_with_conf2 """ + + sql """ CREATE CATALOG `test_cache_true_lower_true_with_conf2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "truee", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "2" + )""" + + qt_sql_test_cache_true_lower_true_with_conf2_1 "select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_2 "select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_3 "select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_4 "select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.LOWER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_3_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.UPPER_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf2" + qt_sql_test_cache_true_lower_true_with_conf2_4_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf.upper_with_conf" + + test { + sql """show tables from test_cache_true_lower_true_with_conf2.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_true_with_conf2 """ + + + // Test for cache false and lower false and lower conf 0 + sql """drop catalog if exists test_cache_false_lower_false_with_conf0 """ + + sql """ CREATE CATALOG `test_cache_false_lower_false_with_conf0` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "0" + )""" + + qt_sql_test_cache_false_lower_false_with_conf0_1 "select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf0" + qt_sql_test_cache_false_lower_false_with_conf0_2 "select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + + sql "refresh catalog test_cache_false_lower_false_with_conf0" + qt_sql_test_cache_false_lower_false_with_conf0_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_false_with_conf0" + qt_sql_test_cache_false_lower_false_with_conf0_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + + sql "refresh catalog test_cache_false_lower_false_with_conf0" + test { + sql "select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_false_with_conf0" + test { + sql "select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.upper_with_conf" + exception "Table [upper_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_false_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_false_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf.upper_with_conf" + exception "Table [upper_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + test { + sql """show tables from test_cache_false_lower_false_with_conf0.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_false_with_conf0 """ + + + // Test for cache true and lower false and lower conf 0 + sql """drop catalog if exists test_cache_true_lower_false_with_conf0 """ + + sql """ CREATE CATALOG `test_cache_true_lower_false_with_conf0` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "0" + )""" + + qt_sql_test_cache_true_lower_false_with_conf0_1 "select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf0" + qt_sql_test_cache_true_lower_false_with_conf0_2 "select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + + sql "refresh catalog test_cache_true_lower_false_with_conf0" + qt_sql_test_cache_true_lower_false_with_conf0_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_false_with_conf0" + qt_sql_test_cache_true_lower_false_with_conf0_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + + + sql "refresh catalog test_cache_true_lower_false_with_conf0" + test { + sql "select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_false_with_conf0" + test { + sql "select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.upper_with_conf" + exception "Table [upper_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_false_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_false_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf.upper_with_conf" + exception "Table [upper_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + test { + sql """show tables from test_cache_true_lower_false_with_conf0.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "UPPER_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_false_with_conf0 """ + + + // Test for cache false and lower true and lower conf 0 + sql """drop catalog if exists test_cache_false_lower_true_with_conf0 """ + + sql """ CREATE CATALOG `test_cache_false_lower_true_with_conf0` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "0" + )""" + + qt_sql_test_cache_false_lower_true_with_conf0_1 "select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf0" + qt_sql_test_cache_false_lower_true_with_conf0_2 "select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_false_lower_true_with_conf0" + qt_sql_test_cache_false_lower_true_with_conf0_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_false_lower_true_with_conf0" + qt_sql_test_cache_false_lower_true_with_conf0_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_false_lower_true_with_conf0" + test { + sql "select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_true_with_conf0" + test { + sql "select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + exception "Table [UPPER_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_true_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_false_lower_true_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + exception "Table [UPPER_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + test { + sql """show tables from test_cache_false_lower_true_with_conf0.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_false_lower_true_with_conf0 """ + + + // Test for cache true and lower true and lower conf 0 + sql """drop catalog if exists test_cache_true_lower_true_with_conf0 """ + + sql """ CREATE CATALOG `test_cache_true_lower_true_with_conf0` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_test_lower_with_conf", + "only_test_lower_case_table_names" = "0" + )""" + + qt_sql_test_cache_true_lower_true_with_conf0_1 "select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf0" + qt_sql_test_cache_true_lower_true_with_conf0_2 "select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf" + + sql "refresh catalog test_cache_true_lower_true_with_conf0" + qt_sql_test_cache_true_lower_true_with_conf0_1_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.lower_with_conf" + sql "refresh catalog test_cache_true_lower_true_with_conf0" + qt_sql_test_cache_true_lower_true_with_conf0_2_insert "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.upper_with_conf" + + + sql "refresh catalog test_cache_true_lower_true_with_conf0" + test { + sql "select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_true_with_conf0" + test { + sql "select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + exception "Table [UPPER_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_true_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.Lower_with_conf" + exception "Table [Lower_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + sql "refresh catalog test_cache_true_lower_true_with_conf0" + test { + sql "insert into internal.external_test_lower_with_conf.with_conf_insert select * from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf.UPPER_with_conf" + exception "Table [UPPER_with_conf] does not exist in database [external_test_lower_with_conf]." + } + + test { + sql """show tables from test_cache_true_lower_true_with_conf0.external_test_lower_with_conf""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower_with_conf", "upper_with_conf"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + sql """drop catalog if exists test_cache_true_lower_true_with_conf0 """ + + sql """drop database if exists internal.external_test_lower_with_conf; """ +} diff --git a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy new file mode 100644 index 00000000000000..40322a22afe737 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_lower_case_mtmv", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """drop database if exists internal.EXTERNAL_LOWER_MTMV; """ + sql """create database if not exists internal.EXTERNAL_LOWER_MTMV;""" + sql """create table if not exists internal.EXTERNAL_LOWER_MTMV.TABLE_TEST + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.EXTERNAL_LOWER_MTMV.TABLE_TEST values(1, 'lower')""" + + sql """drop catalog if exists test_lower_case_mtmv """ + + sql """ CREATE CATALOG `test_lower_case_mtmv` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "EXTERNAL_LOWER_MTMV" + )""" + + + sql """CREATE MATERIALIZED VIEW internal.EXTERNAL_LOWER_MTMV.MTMV_TEST + REFRESH COMPLETE ON SCHEDULE EVERY 1 minute + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + 'replication_num' = '1' + ) + AS SELECT * FROM test_lower_case_mtmv.external_lower_mtmv.table_test;""" + + qt_select """select * from internal.EXTERNAL_LOWER_MTMV.MTMV_TEST""" + + sql """drop catalog if exists test_lower_case_mtmv """ + sql """drop database if exists internal.EXTERNAL_LOWER_MTMV """ +} diff --git a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy new file mode 100644 index 00000000000000..20e6b0f0032e96 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_meta_cache_select_without_refresh", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """ drop database if exists internal.external_lower_select_without_refresh; """ + sql """create database if not exists internal.external_lower_select_without_refresh;""" + + // Test include + sql """drop catalog if exists test_meta_cache_lower_true_select_without_refresh """ + + sql """ CREATE CATALOG `test_meta_cache_lower_true_select_without_refresh` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_lower_select_without_refresh" + )""" + + sql """drop catalog if exists test_meta_cache_lower_false_select_without_refresh """ + + sql """ CREATE CATALOG `test_meta_cache_lower_false_select_without_refresh` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "lower_case_meta_names" = "false", + "only_specified_database" = "true", + "include_database_list" = "external_lower_select_without_refresh" + )""" + + sql """create table if not exists internal.external_lower_select_without_refresh.table1 + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_lower_select_without_refresh.table1 values(1, 'table1')""" + + qt_test_meta_cache_lower_true_select_without_refresh_select_table1 "select * from test_meta_cache_lower_true_select_without_refresh.external_lower_select_without_refresh.table1;" + + qt_test_meta_cache_lower_false_select_without_refresh_select_table1 "select * from test_meta_cache_lower_false_select_without_refresh.external_lower_select_without_refresh.table1;" + + sql """create table if not exists internal.external_lower_select_without_refresh.TABLE2 + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_lower_select_without_refresh.TABLE2 values(1, 'TABLE2')""" + + test { + sql """select * from test_meta_cache_lower_true_select_without_refresh.external_lower_select_without_refresh.table2;""" + + exception "Table [table2] does not exist in database [external_lower_select_without_refresh]." + } + + qt_test_meta_cache_lower_false_select_without_refresh_select_table2 "select * from test_meta_cache_lower_false_select_without_refresh.external_lower_select_without_refresh.TABLE2;" + + sql """drop catalog if exists test_meta_cache_lower_true_select_without_refresh """ + sql """drop catalog if exists test_meta_cache_lower_false_select_without_refresh """ + sql """drop database if exists internal.external_lower_select_without_refresh; """ +} diff --git a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy new file mode 100644 index 00000000000000..1cf48b17c87ab9 --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy @@ -0,0 +1,289 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_meta_names_mapping", "p0,external,doris,meta_names_mapping") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + String validMetaNamesMapping = """ + { + "databases": [ + {"remoteDatabase": "EXTERNAL_META_NAMES_MAPPING", "mapping": "external_meta_names_mapping_upper"}, + {"remoteDatabase": "external_meta_names_mapping", "mapping": "external_meta_names_mapping_lower"} + ], + "tables": [ + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "table_test", "mapping": "table_test_lower"}, + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "TABLE_TEST", "mapping": "table_test_upper"} + ], + "columns": [ + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "TABLE_TEST", "remoteColumn": "column_test", "mapping": "column_test_local"} + ] + } + """ + + sql """drop database if exists internal.external_meta_names_mapping; """ + sql """drop database if exists internal.EXTERNAL_META_NAMES_MAPPING; """ + sql """create database if not exists internal.external_meta_names_mapping; """ + sql """create database if not exists internal.EXTERNAL_META_NAMES_MAPPING; """ + + sql """create table if not exists internal.external_meta_names_mapping.table_test + (id int, name varchar(20), column_test int) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + sql """create table if not exists internal.external_meta_names_mapping.TABLE_TEST + (id int, name varchar(20), column_test int) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + sql """create table if not exists internal.EXTERNAL_META_NAMES_MAPPING.table_test + (id int, name varchar(20), column_test int) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_meta_names_mapping.table_test values(1, 'lowercase', 100);""" + sql """insert into internal.external_meta_names_mapping.TABLE_TEST values(2, 'UPPERCASE', 200);""" + sql """insert into internal.EXTERNAL_META_NAMES_MAPPING.table_test values(3, 'MIXEDCASE', 300);""" + + sql """drop catalog if exists test_valid_meta_names_mapping """ + sql """ CREATE CATALOG `test_valid_meta_names_mapping` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "only_specified_database" = "true", + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING", + "meta_names_mapping" = '${validMetaNamesMapping}' + )""" + + test { + sql """show databases from test_valid_meta_names_mapping""" + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_meta_names_mapping_upper", "external_meta_names_mapping_lower"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_valid_meta_names_mapping.external_meta_names_mapping_lower""" + check { result, ex, startTime, endTime -> + def expectedTables = ["table_test_lower", "table_test_upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + test { + sql """describe test_valid_meta_names_mapping.external_meta_names_mapping_lower.table_test_upper""" + check { result, ex, startTime, endTime -> + def expectedColumns = ["column_test_local"] + expectedColumns.each { columnName -> + assertTrue(result.collect { it[0] }.contains(columnName), "Expected column '${columnName}' not found in result") + } + } + } + + qt_sql_meta_mapping_select_lower "select * from test_valid_meta_names_mapping.external_meta_names_mapping_lower.table_test_lower" + qt_sql_meta_mapping_select_upper "select * from test_valid_meta_names_mapping.external_meta_names_mapping_lower.table_test_upper" + + sql """drop catalog if exists test_valid_meta_names_mapping """ + + sql """ drop catalog if exists test_conflict_meta_names """ + sql """ CREATE CATALOG `test_conflict_meta_names` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING" + )""" + + test { + sql """show databases from test_conflict_meta_names""" + exception """Found conflicting database names under case-insensitive conditions. Conflicting remote database names: EXTERNAL_META_NAMES_MAPPING, external_meta_names_mapping in catalog test_conflict_meta_names. Please use meta_names_mapping to handle name mapping.""" + } + + sql """refresh catalog test_conflict_meta_names""" + + test { + sql """select * from test_conflict_meta_names.external_meta_names_mapping.table_test""" + exception """Found conflicting database names under case-insensitive conditions. Conflicting remote database names: EXTERNAL_META_NAMES_MAPPING, external_meta_names_mapping in catalog test_conflict_meta_names. Please use meta_names_mapping to handle name mapping.""" + } + + String validMetaNamesMapping2 = """ + { + "databases": [ + {"remoteDatabase": "EXTERNAL_META_NAMES_MAPPING", "mapping": "external_meta_names_mapping_upper"}, + {"remoteDatabase": "external_meta_names_mapping", "mapping": "external_meta_names_mapping_lower"} + ] + } + """ + + sql """alter catalog test_conflict_meta_names set properties('meta_names_mapping' = '${validMetaNamesMapping2}')""" + + test { + sql """show tables from test_conflict_meta_names.external_meta_names_mapping_lower""" + exception """Found conflicting table names under case-insensitive conditions. Conflicting remote table names: TABLE_TEST, table_test in remote database 'external_meta_names_mapping' under catalog 'test_conflict_meta_names'. Please use meta_names_mapping to handle name mapping.""" + } + + sql """refresh catalog test_conflict_meta_names""" + + test { + sql """select * from test_conflict_meta_names.external_meta_names_mapping_lower.table_test""" + exception """Found conflicting table names under case-insensitive conditions. Conflicting remote table names: TABLE_TEST, table_test in remote database 'external_meta_names_mapping' under catalog 'test_conflict_meta_names'. Please use meta_names_mapping to handle name mapping.""" + } + + String validMetaNamesMapping3 = """ + { + "databases": [ + {"remoteDatabase": "EXTERNAL_META_NAMES_MAPPING", "mapping": "external_meta_names_mapping_upper"}, + {"remoteDatabase": "external_meta_names_mapping", "mapping": "external_meta_names_mapping_lower"} + ], + "tables": [ + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "TABLE_TEST", "mapping": "table_test_upper"}, + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "table_test", "mapping": "table_test_lower"} + ] + } + """ + + sql """alter catalog test_conflict_meta_names set properties('meta_names_mapping' = '${validMetaNamesMapping3}')""" + + test { + sql """show databases from test_conflict_meta_names""" + check { result, ex, startTime, endTime -> + def expectedDatabases = ["external_meta_names_mapping_upper", "external_meta_names_mapping_lower"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_conflict_meta_names.external_meta_names_mapping_lower""" + check { result, ex, startTime, endTime -> + def expectedTables = ["table_test_lower", "table_test_upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_meta_mapping_select_lower_lower_case_true "select * from test_conflict_meta_names.external_meta_names_mapping_lower.table_test_lower" + qt_sql_meta_mapping_select_upper_lower_case_true "select * from test_conflict_meta_names.external_meta_names_mapping_lower.table_test_upper" + + sql """drop catalog if exists test_conflict_meta_names """ + + String error_mapping_db = """ + { + "databases": [ + {"remoteDatabase": "EXTERNAL_META_NAMES_MAPPING", "mapping": "external_meta_names_mapping_upper"}, + {"remoteDatabase": "EXTERNAL_META_NAMES_MAPPING", "mapping": "external_meta_names_mapping_lower"} + ] + } + """ + + sql """drop catalog if exists test_error_mapping_db """ + + test { + sql """ CREATE CATALOG `test_error_mapping_db` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "only_specified_database" = "true", + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING", + "meta_names_mapping" = '${error_mapping_db}' + )""" + + exception "Duplicate remoteDatabase found: EXTERNAL_META_NAMES_MAPPING" + } + + sql """drop catalog if exists test_error_mapping_db """ + + String error_mapping_tbl = """ + { + "tables": [ + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "TABLE_TEST", "mapping": "table_test_upper"}, + {"remoteDatabase": "external_meta_names_mapping", "remoteTable": "TABLE_TEST", "mapping": "table_test_lower"} + ] + } + """ + + sql """drop catalog if exists test_error_mapping_tbl """ + + test { + sql """ CREATE CATALOG `test_error_mapping_tbl` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "only_specified_database" = "true", + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING", + "meta_names_mapping" = '${error_mapping_tbl}' + )""" + + exception "Duplicate remoteTable found in database external_meta_names_mapping: TABLE_TEST" + } + + sql """drop catalog if exists test_error_mapping_tbl """ + + sql """drop catalog if exists test_alter_error_mapping """ + + sql """ CREATE CATALOG `test_alter_error_mapping` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "only_specified_database" = "true", + "include_database_list" = "external_meta_names_mapping,EXTERNAL_META_NAMES_MAPPING" + )""" + + test { + sql """alter catalog test_alter_error_mapping set properties('meta_names_mapping' = '${error_mapping_db}')""" + exception "Duplicate remoteDatabase found: EXTERNAL_META_NAMES_MAPPING" + } + + test { + sql """alter catalog test_alter_error_mapping set properties('meta_names_mapping' = '${error_mapping_tbl}')""" + exception "Duplicate remoteTable found in database external_meta_names_mapping: TABLE_TEST" + } + + sql """drop catalog if exists test_alter_error_mapping """ + + sql """drop database if exists internal.external_meta_names_mapping; """ + sql """drop database if exists internal.EXTERNAL_META_NAMES_MAPPING; """ +} \ No newline at end of file diff --git a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy new file mode 100644 index 00000000000000..19d7e2db18089a --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_timing_refresh_catalog", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + String mapping = """ + { + "databases": [ + {"remoteDatabase": "external_timing_refresh_catalog", "mapping": "db"} + ], + "tables": [ + {"remoteDatabase": "external_timing_refresh_catalog", "remoteTable": "tbl", "mapping": "table_t"} + ], + "columns": [ + {"remoteDatabase": "external_timing_refresh_catalog", "remoteTable": "tbl", "remoteColumn": "id", "mapping": "id_c"} + ] + } + """ + + sql """drop database if exists internal.external_timing_refresh_catalog; """ + sql """create database if not exists internal.external_timing_refresh_catalog;""" + sql """create table if not exists internal.external_timing_refresh_catalog.tbl + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.external_timing_refresh_catalog.tbl values(1, 'lower')""" + + sql """drop catalog if exists test_timing_refresh_catalog1 """ + + sql """ CREATE CATALOG `test_timing_refresh_catalog1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "metadata_refresh_interval_seconds" = "1", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_timing_refresh_catalog" + )""" + + sql """drop catalog if exists test_timing_refresh_catalog2 """ + + sql """ CREATE CATALOG `test_timing_refresh_catalog2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "metadata_refresh_interval_seconds" = "1", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_timing_refresh_catalog" + )""" + + test { + def catalogName = "test_timing_refresh_catalog1" + for (int i = 0; i < 10; i++) { + sql """ + select * from ${catalogName}.external_timing_refresh_catalog.tbl + """ + Thread.sleep(1000) + } + } + + test { + def catalogName = "test_timing_refresh_catalog2" + for (int i = 0; i < 10; i++) { + sql """ + select * from ${catalogName}.external_timing_refresh_catalog.tbl + """ + Thread.sleep(1000) + } + } + + // with mapping + sql """drop catalog if exists test_timing_refresh_catalog1 """ + + sql """ CREATE CATALOG `test_timing_refresh_catalog1` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "true", + "metadata_refresh_interval_seconds" = "1", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_timing_refresh_catalog", + 'meta_names_mapping' = '${mapping}' + )""" + + sql """drop catalog if exists test_timing_refresh_catalog2 """ + + sql """ CREATE CATALOG `test_timing_refresh_catalog2` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "metadata_refresh_interval_seconds" = "1", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "external_timing_refresh_catalog", + 'meta_names_mapping' = '${mapping}' + )""" + + test { + def catalogName = "test_timing_refresh_catalog1" + for (int i = 0; i < 10; i++) { + sql """ + select id_c from ${catalogName}.db.table_t + """ + Thread.sleep(1000) + } + } + + test { + def catalogName = "test_timing_refresh_catalog2" + for (int i = 0; i < 10; i++) { + sql """ + select id_c from ${catalogName}.db.table_t + """ + Thread.sleep(1000) + } + } + + sql """drop catalog if exists test_timing_refresh_catalog1 """ + sql """drop catalog if exists test_timing_refresh_catalog2 """ + sql """drop database if exists internal.external_timing_refresh_catalog """ +} \ No newline at end of file diff --git a/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy new file mode 100644 index 00000000000000..0ea89d4012d1cb --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_upgrade_lower_case_catalog_prepare", "p0,external,doris,external_docker,external_docker_doris") { + + String jdbcUrl = context.config.jdbcUrl + String jdbcUser = context.config.jdbcUser + String jdbcPassword = context.config.jdbcPassword + String s3_endpoint = getS3Endpoint() + String bucket = getS3BucketName() + String driver_url = "https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar" + + sql """drop database if exists internal.upgrade_lower_case_catalog_lower; """ + sql """drop database if exists internal.upgrade_lower_case_catalog_UPPER; """ + sql """create database if not exists internal.upgrade_lower_case_catalog_lower; """ + sql """create database if not exists internal.upgrade_lower_case_catalog_UPPER; """ + sql """create table if not exists internal.upgrade_lower_case_catalog_lower.lower + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """create table if not exists internal.upgrade_lower_case_catalog_lower.UPPER + (id int, name varchar(20)) + distributed by hash(id) buckets 10 + properties('replication_num' = '1'); + """ + + sql """insert into internal.upgrade_lower_case_catalog_lower.lower values(1, 'lower')""" + sql """insert into internal.upgrade_lower_case_catalog_lower.UPPER values(1, 'UPPER')""" + + // Test for cache false and lower false + sql """drop catalog if exists test_upgrade_lower_case_catalog """ + + sql """ CREATE CATALOG `test_upgrade_lower_case_catalog` PROPERTIES ( + "user" = "${jdbcUser}", + "type" = "jdbc", + "password" = "${jdbcPassword}", + "jdbc_url" = "${jdbcUrl}", + "driver_url" = "${driver_url}", + "driver_class" = "com.mysql.cj.jdbc.Driver", + "use_meta_cache" = "false", + "lower_case_meta_names" = "true", + "only_specified_database" = "true", + "include_database_list" = "upgrade_lower_case_catalog_lower,upgrade_lower_case_catalog_UPPER" + )""" + + test { + sql """show databases from test_upgrade_lower_case_catalog""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["upgrade_lower_case_catalog_lower", "upgrade_lower_case_catalog_upper"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_upgrade_lower_case_catalog_1 "select * from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower.lower" + qt_sql_test_upgrade_lower_case_catalog_2 "select * from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower.upper" + +} diff --git a/regression-test/suites/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.groovy b/regression-test/suites/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.groovy new file mode 100644 index 00000000000000..634e5e5737ef6d --- /dev/null +++ b/regression-test/suites/external_table_p0/lower_case/upgrade/test_upgrade_lower_case_catalog.groovy @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_upgrade_lower_case_catalog", "p0,external,doris,external_docker,external_docker_doris") { + + test { + sql """show databases from test_upgrade_lower_case_catalog""" + + // Verification results include external_test_lower and external_test_UPPER + check { result, ex, startTime, endTime -> + def expectedDatabases = ["upgrade_lower_case_catalog_lower", "upgrade_lower_case_catalog_upper"] + expectedDatabases.each { dbName -> + assertTrue(result.collect { it[0] }.contains(dbName), "Expected database '${dbName}' not found in result") + } + } + } + + test { + sql """show tables from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower""" + + // Verification results include lower and UPPER + check { result, ex, startTime, endTime -> + def expectedTables = ["lower", "upper"] + expectedTables.each { tableName -> + assertTrue(result.collect { it[0] }.contains(tableName), "Expected table '${tableName}' not found in result") + } + } + } + + qt_sql_test_upgrade_lower_case_catalog_1 "select * from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower.lower" + qt_sql_test_upgrade_lower_case_catalog_2 "select * from test_upgrade_lower_case_catalog.upgrade_lower_case_catalog_lower.upper" + +} \ No newline at end of file