From bcc1f15e0c067731652779fe5d9b09485bfa2cc2 Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Fri, 9 Aug 2024 14:36:11 +0800 Subject: [PATCH 1/4] fix --- .../java/org/apache/doris/catalog/RefreshManager.java | 2 +- .../org/apache/doris/datasource/ExternalCatalog.java | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index d017ba7829fd1a..52694e5a5bdffb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -76,7 +76,7 @@ public void replayRefreshCatalog(CatalogLog log) { private void refreshCatalogInternal(CatalogIf catalog, boolean invalidCache) { String catalogName = catalog.getName(); if (!catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { - ((ExternalCatalog) catalog).onRefresh(invalidCache); + ((ExternalCatalog) catalog).onRefreshCache(invalidCache); LOG.info("refresh catalog {} with invalidCache {}", catalogName, invalidCache); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index b6e94c3d39d556..7f8c0c71b51b16 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -388,6 +388,15 @@ public void onRefresh(boolean invalidCache) { synchronized (this.propLock) { this.convertedProperties = null; } + + refreshOnlyCatalogCache(invalidCache); + } + + public void onRefreshCache(boolean invalidCache) { + refreshOnlyCatalogCache(invalidCache); + } + + private void refreshOnlyCatalogCache(boolean invalidCache) { if (useMetaCache.isPresent()) { if (useMetaCache.get() && metaCache != null) { metaCache.invalidateAll(); From 94cfa22e73451bee2dd4ebf77386d3a18e09e708 Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Sun, 11 Aug 2024 09:36:51 +0800 Subject: [PATCH 2/4] fix --- .../apache/doris/datasource/jdbc/JdbcExternalCatalog.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index 73b6639c7b95fd..b7c156b714cb77 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -121,6 +121,14 @@ public void onRefresh(boolean invalidCache) { } } + @Override + public void onRefreshCache(boolean invalidCache) { + super.onRefreshCache(invalidCache); + if (jdbcClient != null) { + jdbcClient.closeClient(); + } + } + @Override public void onClose() { super.onClose(); From 4b86f109990661d26fb105f5e4355662badfa2c6 Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Sun, 11 Aug 2024 12:14:56 +0800 Subject: [PATCH 3/4] fix --- .../doris/datasource/jdbc/JdbcExternalCatalog.java | 5 +---- .../apache/doris/datasource/RefreshCatalogTest.java | 11 ----------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index b7c156b714cb77..80cc0f554f637e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -123,10 +123,7 @@ public void onRefresh(boolean invalidCache) { @Override public void onRefreshCache(boolean invalidCache) { - super.onRefreshCache(invalidCache); - if (jdbcClient != null) { - jdbcClient.closeClient(); - } + onRefresh(invalidCache); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java index 439385993f901b..b51d6b19be507e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/RefreshCatalogTest.java @@ -155,18 +155,7 @@ public void testRefreshCatalogLastUpdateTime() throws Exception { } catch (Exception e) { // Do nothing } - Assertions.assertFalse(((ExternalCatalog) test2).isInitialized()); - table.makeSureInitialized(); Assertions.assertTrue(((ExternalCatalog) test2).isInitialized()); - // table.makeSureInitialized() triggered init method - long l4 = test2.getLastUpdateTime(); - Assertions.assertTrue(l4 > l3); - try { - DdlExecutor.execute(Env.getCurrentEnv(), refreshCatalogStmt); - } catch (Exception e) { - // Do nothing - } - Assertions.assertFalse(((ExternalCatalog) test2).isInitialized()); } public static class RefreshCatalogProvider implements TestExternalCatalog.TestCatalogProvider { From b2366f9b263e8e9427835ffcb18dd05baf01dc5e Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Fri, 16 Aug 2024 10:15:38 +0800 Subject: [PATCH 4/4] fix --- .../org/apache/doris/datasource/hive/HiveMetadataOps.java | 4 ++-- .../apache/doris/datasource/iceberg/IcebergMetadataOps.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java index dcfc6d1ad33f90..e855affc31a9de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java @@ -126,7 +126,7 @@ public void createDb(CreateDbStmt stmt) throws DdlException { catalogDatabase.setProperties(properties); catalogDatabase.setComment(properties.getOrDefault("comment", "")); client.createDatabase(catalogDatabase); - catalog.onRefresh(true); + catalog.onRefreshCache(true); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } @@ -146,7 +146,7 @@ public void dropDb(DropDbStmt stmt) throws DdlException { } try { client.dropDatabase(dbName); - catalog.onRefresh(true); + catalog.onRefreshCache(true); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index a6933f83d76d50..dd4792715e8b1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -107,7 +107,7 @@ public void createDb(CreateDbStmt stmt) throws DdlException { } } nsCatalog.createNamespace(Namespace.of(dbName), properties); - dorisCatalog.onRefresh(true); + dorisCatalog.onRefreshCache(true); } @Override @@ -123,7 +123,7 @@ public void dropDb(DropDbStmt stmt) throws DdlException { } SupportsNamespaces nsCatalog = (SupportsNamespaces) catalog; nsCatalog.dropNamespace(Namespace.of(dbName)); - dorisCatalog.onRefresh(true); + dorisCatalog.onRefreshCache(true); } @Override