Skip to content

[improvement](jdbc catalog) Optimize JdbcCatalog case mapping stability #41510

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zy-kkk merged 4 commits into from
Dec 25, 2024
Merged

[improvement](jdbc catalog) Optimize JdbcCatalog case mapping stability #41510

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a9eafa7
[improvement](jdbc catalog) Optimize JdbcCatalog case mapping stability
zy-kkk Dec 23, 2024
c96964d
fix build
zy-kkk Dec 23, 2024
1019638
fix ut and regression test
zy-kkk Dec 24, 2024
11adf38
add case
zy-kkk Dec 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,4 +220,14 @@ default Optional<TableValuedFunction> getMetaTableFunction(String dbName, String
default Optional<TableValuedFunctionRef> getMetaTableFunctionRef(String dbName, String sourceNameWithMetaName) {
return Optional.empty();
}

// Convert from remote database name to local database name, overridden by subclass if necessary
default String fromRemoteDatabaseName(String remoteDatabaseName) {
return remoteDatabaseName;
}

// Convert from remote table name to local table name, overridden by subclass if necessary
default String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) {
return remoteTableName;
}
}
12 changes: 7 additions & 5 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalDatabase;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand;
Expand Down Expand Up @@ -690,8 +691,8 @@ public boolean externalTableExistInLocal(String dbName, String tableName, String
}

public void registerExternalTableFromEvent(String dbName, String tableName,
String catalogName, long updateTime,
boolean ignoreIfExists) throws DdlException {
String catalogName, long updateTime,
boolean ignoreIfExists) throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
throw new DdlException("No catalog found with name: " + catalogName);
Expand Down Expand Up @@ -721,7 +722,8 @@ public void registerExternalTableFromEvent(String dbName, String tableName,

db.writeLock();
try {
HMSExternalTable namedTable = new HMSExternalTable(tblId, tableName, dbName, (HMSExternalCatalog) catalog);
HMSExternalTable namedTable = ((HMSExternalDatabase) db)
.buildTableForInit(tableName, tableName, tblId, hmsCatalog, (HMSExternalDatabase) db, false);
namedTable.setUpdateTime(updateTime);
db.registerTable(namedTable);
} finally {
Expand Down Expand Up @@ -767,7 +769,7 @@ public void registerExternalDatabaseFromEvent(String dbName, String catalogName)
}

public void addExternalPartitions(String catalogName, String dbName, String tableName,
List<String> partitionNames, long updateTime, boolean ignoreIfNotExists)
List<String> partitionNames, long updateTime, boolean ignoreIfNotExists)
throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
Expand Down Expand Up @@ -802,7 +804,7 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl
}

public void dropExternalPartitions(String catalogName, String dbName, String tableName,
List<String> partitionNames, long updateTime, boolean ignoreIfNotExists)
List<String> partitionNames, long updateTime, boolean ignoreIfNotExists)
throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
Expand Down
196 changes: 163 additions & 33 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
View file
Open in desktop

Large diffs are not rendered by default.

238 changes: 202 additions & 36 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.apache.doris.catalog.Type;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.Config;
import org.apache.doris.common.Pair;
import org.apache.doris.common.ThreadPoolManager;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalTable;
Expand Down Expand Up @@ -302,7 +303,7 @@ public void invalidatePartitionsCache(long catalogId, String dbName, String tabl

public <T> MetaCache<T> buildMetaCache(String name,
OptionalLong expireAfterWriteSec, OptionalLong refreshAfterWriteSec, long maxSize,
CacheLoader<String, List<String>> namesCacheLoader,
CacheLoader<String, List<Pair<String, String>>> namesCacheLoader,
CacheLoader<String, Optional<T>> metaObjCacheLoader,
RemovalListener<String, Optional<T>> removalListener) {
MetaCache<T> metaCache = new MetaCache<>(name, commonRefreshExecutor, expireAfterWriteSec, refreshAfterWriteSec,
Expand Down
26 changes: 23 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,13 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable {
protected long id;
@SerializedName(value = "name")
protected String name;
@SerializedName(value = "remoteName")
protected String remoteName;
Copy link
Contributor

@morningman morningman Dec 4, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remoteName maybe null when upgrading from old version. How to handle it?

@SerializedName(value = "type")
protected TableType type = null;
@SerializedName(value = "timestamp")
protected long timestamp;
// dbName is temporarily retained and will be deleted later. To use dbName, please use db.getFullName()
@SerializedName(value = "dbName")
protected String dbName;
@SerializedName(value = "ta")
Expand All @@ -86,6 +89,7 @@ public class ExternalTable implements TableIf, Writable, GsonPostProcessable {
protected long dbId;
protected boolean objectCreated;
protected ExternalCatalog catalog;
protected ExternalDatabase db;
Copy link
Contributor

@morningman morningman Dec 4, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should also mark dbName as deprecated?


/**
* No args constructor for persist.
Expand All @@ -99,15 +103,19 @@ public ExternalTable() {
*
* @param id Table id.
* @param name Table name.
* @param remoteName Remote table name.
* @param catalog ExternalCatalog this table belongs to.
* @param dbName Name of the db the this table belongs to.
* @param db ExternalDatabase this table belongs to.
* @param type Table type.
*/
public ExternalTable(long id, String name, ExternalCatalog catalog, String dbName, TableType type) {
public ExternalTable(long id, String name, String remoteName, ExternalCatalog catalog, ExternalDatabase db,
TableType type) {
this.id = id;
this.name = name;
this.remoteName = remoteName;
this.catalog = catalog;
this.dbName = dbName;
this.db = db;
this.dbName = db.getFullName();
this.type = type;
this.objectCreated = false;
}
Expand All @@ -116,6 +124,14 @@ public void setCatalog(ExternalCatalog catalog) {
this.catalog = catalog;
}

public void setDb(ExternalDatabase db) {
this.db = db;
}

public void setRemoteName(String remoteName) {
this.remoteName = remoteName;
}

public boolean isView() {
return false;
}
Expand All @@ -141,6 +157,10 @@ public String getName() {
return name;
}

public String getRemoteName() {
return remoteName;
}

@Override
public TableType getType() {
return type;
Expand Down
7 changes: 6 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ public enum Type {
@SerializedName(value = "createDbNames")
private List<String> createDbNames;

@SerializedName(value = "remoteDbNames")
private List<String> remoteDbNames;

@SerializedName(value = "type")
private Type type;

Expand All @@ -77,6 +80,7 @@ public InitCatalogLog() {
refreshDbIds = Lists.newArrayList();
createDbIds = Lists.newArrayList();
createDbNames = Lists.newArrayList();
remoteDbNames = Lists.newArrayList();
type = Type.UNKNOWN;
}

Expand All @@ -85,10 +89,11 @@ public void addRefreshDb(long id) {
refreshDbIds.add(id);
}

public void addCreateDb(long id, String name) {
public void addCreateDb(long id, String name, String remoteName) {
createCount += 1;
createDbIds.add(id);
createDbNames.add(name);
remoteDbNames.add(remoteName);
}

@Override
Expand Down
7 changes: 6 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ public enum Type {
@SerializedName(value = "createTableNames")
private List<String> createTableNames;

@SerializedName(value = "remoteTableNames")
private List<String> remoteTableNames;

@SerializedName(value = "type")
private Type type;

Expand All @@ -82,6 +85,7 @@ public InitDatabaseLog() {
refreshTableIds = Lists.newArrayList();
createTableIds = Lists.newArrayList();
createTableNames = Lists.newArrayList();
remoteTableNames = Lists.newArrayList();
type = Type.UNKNOWN;
}

Expand All @@ -90,10 +94,11 @@ public void addRefreshTable(long id) {
refreshTableIds.add(id);
}

public void addCreateTable(long id, String name) {
public void addCreateTable(long id, String name, String remoteName) {
createCount += 1;
createTableIds.add(id);
createTableNames.add(name);
remoteTableNames.add(remoteName);
}

@Override
Expand Down
12 changes: 8 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalDatabase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,18 @@ public class EsExternalDatabase extends ExternalDatabase<EsExternalTable> {
* @param extCatalog External data source this database belongs to.
* @param id database id.
* @param name database name.
* @param remoteName remote database name.
*/
public EsExternalDatabase(ExternalCatalog extCatalog, long id, String name) {
super(extCatalog, id, name, InitDatabaseLog.Type.ES);
public EsExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) {
super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.ES);
}

@Override
protected EsExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) {
return new EsExternalTable(tblId, tableName, name, (EsExternalCatalog) extCatalog);
public EsExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
ExternalCatalog catalog,
ExternalDatabase db) {
return new EsExternalTable(tblId, localTableName, remoteTableName, (EsExternalCatalog) extCatalog,
(EsExternalDatabase) db);
}

public void addTableForTest(EsExternalTable tbl) {
Expand Down
9 changes: 5 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,12 @@ public class EsExternalTable extends ExternalTable {
*
* @param id Table id.
* @param name Table name.
* @param dbName Database name.
* @param catalog HMSExternalDataSource.
* @param remoteName Remote table name.
* @param catalog EsExternalDataSource.
* @param db Database.
*/
public EsExternalTable(long id, String name, String dbName, EsExternalCatalog catalog) {
super(id, name, catalog, dbName, TableType.ES_EXTERNAL_TABLE);
public EsExternalTable(long id, String name, String remoteName, EsExternalCatalog catalog, EsExternalDatabase db) {
super(id, name, remoteName, catalog, db, TableType.ES_EXTERNAL_TABLE);
}

protected synchronized void makeSureInitialized() {
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ public void registerDatabase(long dbId, String dbName) {
LOG.debug("create database [{}]", dbName);
}

ExternalDatabase<? extends ExternalTable> db = buildDbForInit(dbName, dbId, logType, false);
ExternalDatabase<? extends ExternalTable> db = buildDbForInit(dbName, null, dbId, logType, false);
if (useMetaCache.get()) {
if (isInitialized()) {
metaCache.updateCache(dbName, db, Util.genIdByName(getQualifiedName(dbName)));
Expand Down
12 changes: 8 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalDatabase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,18 @@ public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
* @param extCatalog External catalog this database belongs to.
* @param id database id.
* @param name database name.
* @param remoteName remote database name.
*/
public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name) {
super(extCatalog, id, name, InitDatabaseLog.Type.HMS);
public HMSExternalDatabase(ExternalCatalog extCatalog, long id, String name, String remoteName) {
super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.HMS);
}

@Override
protected HMSExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) {
return new HMSExternalTable(tblId, tableName, name, (HMSExternalCatalog) extCatalog);
public HMSExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
ExternalCatalog catalog,
ExternalDatabase db) {
return new HMSExternalTable(tblId, localTableName, remoteTableName, (HMSExternalCatalog) extCatalog,
(HMSExternalDatabase) db);
}

public void addTableForTest(HMSExternalTable tbl) {
Expand Down
10 changes: 6 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,11 +168,13 @@ public enum DLAType {
*
* @param id Table id.
* @param name Table name.
* @param dbName Database name.
* @param catalog HMSExternalCatalog.
* @param remoteName Remote table name.
* @param catalog HMSExternalDataSource.
* @param db Database.
*/
public HMSExternalTable(long id, String name, String dbName, HMSExternalCatalog catalog) {
super(id, name, catalog, dbName, TableType.HMS_EXTERNAL_TABLE);
public HMSExternalTable(long id, String name, String remoteName, HMSExternalCatalog catalog,
HMSExternalDatabase db) {
super(id, name, remoteName, catalog, db, TableType.HMS_EXTERNAL_TABLE);
}

// Will throw NotSupportedException if not supported hms table.
Expand Down
11 changes: 7 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalDatabase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,16 @@

public class IcebergExternalDatabase extends ExternalDatabase<IcebergExternalTable> {

public IcebergExternalDatabase(ExternalCatalog extCatalog, Long id, String name) {
super(extCatalog, id, name, InitDatabaseLog.Type.ICEBERG);
public IcebergExternalDatabase(ExternalCatalog extCatalog, Long id, String name, String remoteName) {
super(extCatalog, id, name, remoteName, InitDatabaseLog.Type.ICEBERG);
}

@Override
protected IcebergExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) {
return new IcebergExternalTable(tblId, tableName, name, (IcebergExternalCatalog) extCatalog);
public IcebergExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
ExternalCatalog catalog,
ExternalDatabase db) {
return new IcebergExternalTable(tblId, localTableName, remoteTableName, (IcebergExternalCatalog) extCatalog,
(IcebergExternalDatabase) db);
}

public String getLocation() {
Expand Down
5 changes: 3 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,9 @@ public class IcebergExternalTable extends ExternalTable implements MTMVRelatedTa
private boolean isValidRelatedTableCached = false;
private boolean isValidRelatedTable = false;

public IcebergExternalTable(long id, String name, String dbName, IcebergExternalCatalog catalog) {
super(id, name, catalog, dbName, TableType.ICEBERG_EXTERNAL_TABLE);
public IcebergExternalTable(long id, String name, String remoteName, IcebergExternalCatalog catalog,
IcebergExternalDatabase db) {
super(id, name, remoteName, catalog, db, TableType.ICEBERG_EXTERNAL_TABLE);
}

public String getIcebergCatalogType() {
Expand Down
8 changes: 5 additions & 3 deletions ...core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaDatabase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,18 @@ public class ExternalInfoSchemaDatabase extends ExternalDatabase {
* @param dbId The id of this database.
*/
public ExternalInfoSchemaDatabase(ExternalCatalog extCatalog, long dbId) {
super(extCatalog, dbId, InfoSchemaDb.DATABASE_NAME, Type.INFO_SCHEMA_DB);
super(extCatalog, dbId, InfoSchemaDb.DATABASE_NAME, InfoSchemaDb.DATABASE_NAME, Type.INFO_SCHEMA_DB);
}

public static List<String> listTableNames() {
return Lists.newArrayList(SchemaTable.TABLE_MAP.keySet());
}

@Override
protected ExternalTable buildTableForInit(String tableName, long tblId, ExternalCatalog catalog) {
return new ExternalInfoSchemaTable(tblId, tableName, catalog);
public ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId,
ExternalCatalog catalog,
ExternalDatabase db) {
return new ExternalInfoSchemaTable(tblId, localTableName, catalog, db);
}

@Override
Expand Down
6 changes: 3 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/infoschema/ExternalInfoSchemaTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
package org.apache.doris.datasource.infoschema;

import org.apache.doris.analysis.SchemaTableType;
import org.apache.doris.catalog.InfoSchemaDb;
import org.apache.doris.catalog.SchemaTable;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.ExternalDatabase;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.SchemaCacheValue;
import org.apache.doris.thrift.TSchemaTable;
Expand All @@ -31,8 +31,8 @@

public class ExternalInfoSchemaTable extends ExternalTable {

public ExternalInfoSchemaTable(long id, String name, ExternalCatalog catalog) {
super(id, name, catalog, InfoSchemaDb.DATABASE_NAME, TableType.SCHEMA);
public ExternalInfoSchemaTable(long id, String name, ExternalCatalog catalog, ExternalDatabase db) {
super(id, name, name, catalog, db, TableType.SCHEMA);
}

@Override
Expand Down
Loading
Loading