Skip to content

[improvement](external catalog)Optimize the process of refreshing catalog for 2.1 (#39205) #39186

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yiguolei merged 4 commits into from
Aug 17, 2024
Merged

[improvement](external catalog)Optimize the process of refreshing catalog for 2.1 (#39205) #39186

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bcc1f15
fix
wuwenchi Aug 9, 2024
94cfa22
fix
wuwenchi Aug 11, 2024
4b86f10
fix
wuwenchi Aug 11, 2024
b2366f9
fix
wuwenchi Aug 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public void replayRefreshCatalog(CatalogLog log) {
private void refreshCatalogInternal(CatalogIf catalog, boolean invalidCache) {
String catalogName = catalog.getName();
if (!catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
((ExternalCatalog) catalog).onRefresh(invalidCache);
((ExternalCatalog) catalog).onRefreshCache(invalidCache);
LOG.info("refresh catalog {} with invalidCache {}", catalogName, invalidCache);
}
}
Expand Down
9 changes: 9 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,15 @@ public void onRefresh(boolean invalidCache) {
synchronized (this.propLock) {
this.convertedProperties = null;
}

refreshOnlyCatalogCache(invalidCache);
}

public void onRefreshCache(boolean invalidCache) {
refreshOnlyCatalogCache(invalidCache);
}

private void refreshOnlyCatalogCache(boolean invalidCache) {
if (useMetaCache.isPresent()) {
if (useMetaCache.get() && metaCache != null) {
metaCache.invalidateAll();
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ public void createDb(CreateDbStmt stmt) throws DdlException {
catalogDatabase.setProperties(properties);
catalogDatabase.setComment(properties.getOrDefault("comment", ""));
client.createDatabase(catalogDatabase);
catalog.onRefresh(true);
catalog.onRefreshCache(true);
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}
Expand All @@ -146,7 +146,7 @@ public void dropDb(DropDbStmt stmt) throws DdlException {
}
try {
client.dropDatabase(dbName);
catalog.onRefresh(true);
catalog.onRefreshCache(true);
} catch (Exception e) {
throw new RuntimeException(e.getMessage(), e);
}
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ public void createDb(CreateDbStmt stmt) throws DdlException {
}
}
nsCatalog.createNamespace(Namespace.of(dbName), properties);
dorisCatalog.onRefresh(true);
dorisCatalog.onRefreshCache(true);
}

@Override
Expand All @@ -123,7 +123,7 @@ public void dropDb(DropDbStmt stmt) throws DdlException {
}
SupportsNamespaces nsCatalog = (SupportsNamespaces) catalog;
nsCatalog.dropNamespace(Namespace.of(dbName));
dorisCatalog.onRefresh(true);
dorisCatalog.onRefreshCache(true);
}

@Override
Expand Down
5 changes: 5 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,11 @@ public void onRefresh(boolean invalidCache) {
}
}

@Override
public void onRefreshCache(boolean invalidCache) {
onRefresh(invalidCache);
}

@Override
public void onClose() {
super.onClose();
Expand Down
11 changes: 0 additions & 11 deletions fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,18 +155,7 @@ public void testRefreshCatalogLastUpdateTime() throws Exception {
} catch (Exception e) {
// Do nothing
}
Assertions.assertFalse(((ExternalCatalog) test2).isInitialized());
table.makeSureInitialized();
Assertions.assertTrue(((ExternalCatalog) test2).isInitialized());
// table.makeSureInitialized() triggered init method
long l4 = test2.getLastUpdateTime();
Assertions.assertTrue(l4 > l3);
try {
DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt);
} catch (Exception e) {
// Do nothing
}
Assertions.assertFalse(((ExternalCatalog) test2).isInitialized());
}

public static class RefreshCatalogProvider implements TestExternalCatalog.TestCatalogProvider {
Expand Down