From 260975e27ac69ebb2e1c84ba61e28e09a0df399b Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 16 Oct 2024 14:22:37 +0800 Subject: [PATCH 1/2] [fix](catalog) fix filtered database when use_meta_cache=true --- .../doris/datasource/ExternalCatalog.java | 35 ++-- .../doris/datasource/ExternalCatalogTest.java | 164 ++++++++++++++++++ 2 files changed, 183 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 25329a1a829a9d..1791788fd1499f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -349,19 +349,8 @@ private void init() { InitCatalogLog initCatalogLog = new InitCatalogLog(); initCatalogLog.setCatalogId(id); initCatalogLog.setType(logType); - List allDatabases = getFilteredDatabaseNames(); - Map includeDatabaseMap = getIncludeDatabaseMap(); - Map excludeDatabaseMap = getExcludeDatabaseMap(); - for (String dbName : allDatabases) { - if (!dbName.equals(InfoSchemaDb.DATABASE_NAME) && !dbName.equals(MysqlDb.DATABASE_NAME)) { - // Exclude database map take effect with higher priority over include database map - if (!excludeDatabaseMap.isEmpty() && excludeDatabaseMap.containsKey(dbName)) { - continue; - } - if (!includeDatabaseMap.isEmpty() && !includeDatabaseMap.containsKey(dbName)) { - continue; - } - } + List filteredDatabases = getFilteredDatabaseNames(); + for (String dbName : filteredDatabases) { long dbId; if (dbNameToId != null && dbNameToId.containsKey(dbName)) { dbId = dbNameToId.get(dbName); @@ -392,6 +381,20 @@ private List getFilteredDatabaseNames() { allDatabases.add(InfoSchemaDb.DATABASE_NAME); allDatabases.remove(MysqlDb.DATABASE_NAME); allDatabases.add(MysqlDb.DATABASE_NAME); + Map includeDatabaseMap = getIncludeDatabaseMap(); + Map excludeDatabaseMap = getExcludeDatabaseMap(); + allDatabases = allDatabases.stream().filter(dbName -> { + if (!dbName.equals(InfoSchemaDb.DATABASE_NAME) && !dbName.equals(MysqlDb.DATABASE_NAME)) { + // Exclude database map take effect with higher priority over include database map + if (!excludeDatabaseMap.isEmpty() && excludeDatabaseMap.containsKey(dbName)) { + return false; + } + if (!includeDatabaseMap.isEmpty() && !includeDatabaseMap.containsKey(dbName)) { + return false; + } + } + return true; + }).collect(Collectors.toList()); return allDatabases; } @@ -835,15 +838,15 @@ public void registerDatabase(long dbId, String dbName) { throw new NotImplementedException("registerDatabase not implemented"); } - public Map getIncludeDatabaseMap() { + protected Map getIncludeDatabaseMap() { return getSpecifiedDatabaseMap(Resource.INCLUDE_DATABASE_LIST); } - public Map getExcludeDatabaseMap() { + protected Map getExcludeDatabaseMap() { return getSpecifiedDatabaseMap(Resource.EXCLUDE_DATABASE_LIST); } - public Map getSpecifiedDatabaseMap(String catalogPropertyKey) { + private Map getSpecifiedDatabaseMap(String catalogPropertyKey) { String specifiedDatabaseList = catalogProperty.getOrDefault(catalogPropertyKey, ""); Map specifiedDatabaseMap = Maps.newHashMap(); specifiedDatabaseList = specifiedDatabaseList.trim(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java index f527da2725ef6a..972cddec6fe259 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java @@ -17,16 +17,106 @@ package org.apache.doris.datasource; +import org.apache.doris.analysis.CreateCatalogStmt; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.mysql.privilege.Auth; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.util.HashMap; +import java.util.List; +import java.util.Map; public class ExternalCatalogTest extends TestWithFeService { + private static Auth auth; + private static Env env; + private CatalogMgr mgr; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + FeConstants.runningUnitTest = true; + mgr = Env.getCurrentEnv().getCatalogMgr(); + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + auth = env.getAuth(); + // 1. create test catalog + CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt( + "create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_database_list\" = \"db1\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + + testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt( + "create catalog test2 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"exclude_database_list\" = \"db1\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + + testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt( + "create catalog test3 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_database_list\" = \"db1\",\n" + + " \"exclude_database_list\" = \"db1\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + + // use_meta_cache=false + testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt( + "create catalog test4 properties(\n" + + " \"type\" = \"test\",\n" + + " \"use_meta_cache\" = \"false\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_database_list\" = \"db1\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + + testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt( + "create catalog test5 properties(\n" + + " \"type\" = \"test\",\n" + + " \"use_meta_cache\" = \"false\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"exclude_database_list\" = \"db1\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + + testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt( + "create catalog test6 properties(\n" + + " \"type\" = \"test\",\n" + + " \"use_meta_cache\" = \"false\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_database_list\" = \"db1\",\n" + + " \"exclude_database_list\" = \"db1\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + } @Test public void testExternalCatalogAutoAnalyze() throws Exception { @@ -48,4 +138,78 @@ public void testExternalCatalogAutoAnalyze() throws Exception { catalog.modifyCatalogProps(prop); Assertions.assertTrue(catalog.enableAutoAnalyze()); } + + @Test + public void testExternalCatalogFilteredDatabase() throws Exception { + // use_meta_cache=true + TestExternalCatalog ctl = (TestExternalCatalog) mgr.getCatalog("test1"); + List dbNames = ctl.getDbNames(); + System.out.println(dbNames); + Assertions.assertEquals(3, dbNames.size()); + Assertions.assertTrue(!dbNames.contains("db2")); + + ctl = (TestExternalCatalog) mgr.getCatalog("test2"); + dbNames = ctl.getDbNames(); + System.out.println(dbNames); + Assertions.assertEquals(3, dbNames.size()); + Assertions.assertTrue(!dbNames.contains("db1")); + + ctl = (TestExternalCatalog) mgr.getCatalog("test3"); + dbNames = ctl.getDbNames(); + System.out.println(dbNames); + Assertions.assertEquals(2, dbNames.size()); + Assertions.assertTrue(!dbNames.contains("db1")); + Assertions.assertTrue(!dbNames.contains("db2")); + + // use_meta_cache=false + ctl = (TestExternalCatalog) mgr.getCatalog("test4"); + dbNames = ctl.getDbNames(); + System.out.println(dbNames); + Assertions.assertEquals(3, dbNames.size()); + Assertions.assertTrue(!dbNames.contains("db2")); + + ctl = (TestExternalCatalog) mgr.getCatalog("test5"); + dbNames = ctl.getDbNames(); + System.out.println(dbNames); + Assertions.assertEquals(3, dbNames.size()); + Assertions.assertTrue(!dbNames.contains("db1")); + + ctl = (TestExternalCatalog) mgr.getCatalog("test6"); + dbNames = ctl.getDbNames(); + System.out.println(dbNames); + Assertions.assertEquals(2, dbNames.size()); + Assertions.assertTrue(!dbNames.contains("db1")); + Assertions.assertTrue(!dbNames.contains("db2")); + } + + public static class RefreshCatalogProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tblSchemaMap1 = Maps.newHashMap(); + // db1 + tblSchemaMap1.put("tbl11", Lists.newArrayList( + new Column("a11", PrimitiveType.BIGINT), + new Column("a12", PrimitiveType.STRING), + new Column("a13", PrimitiveType.FLOAT))); + tblSchemaMap1.put("tbl12", Lists.newArrayList( + new Column("b21", PrimitiveType.BIGINT), + new Column("b22", PrimitiveType.STRING), + new Column("b23", PrimitiveType.FLOAT))); + MOCKED_META.put("db1", tblSchemaMap1); + // db2 + Map> tblSchemaMap2 = Maps.newHashMap(); + tblSchemaMap2.put("tbl21", Lists.newArrayList( + new Column("c11", PrimitiveType.BIGINT), + new Column("c12", PrimitiveType.STRING), + new Column("c13", PrimitiveType.FLOAT))); + MOCKED_META.put("db2", tblSchemaMap2); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } } From bc5a183809529fd2eb07b31731896e2bc13ecfaa Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 16 Oct 2024 18:23:01 +0800 Subject: [PATCH 2/2] 2 --- .../doris/datasource/ExternalCatalog.java | 5 +-- .../doris/datasource/ExternalCatalogTest.java | 36 +++++++++++++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 1791788fd1499f..34525410834506 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -53,6 +53,7 @@ import org.apache.doris.datasource.operations.ExternalMetadataOps; import org.apache.doris.datasource.paimon.PaimonExternalDatabase; import org.apache.doris.datasource.property.PropertyConverter; +import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.datasource.test.TestExternalDatabase; import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalDatabase; import org.apache.doris.fs.remote.dfs.DFSFileSystem; @@ -678,11 +679,11 @@ protected ExternalDatabase buildDbForInit(String dbName InitCatalogLog.Type logType, boolean checkExists) { // When running ut, disable this check to make ut pass. // Because in ut, the database is not created in remote system. - if (checkExists && !FeConstants.runningUnitTest) { + if (checkExists && (!FeConstants.runningUnitTest || this instanceof TestExternalCatalog)) { try { List dbNames = getDbNames(); if (!dbNames.contains(dbName)) { - dbNames = listDatabaseNames(); + dbNames = getFilteredDatabaseNames(); if (!dbNames.contains(dbName)) { return null; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java index 972cddec6fe259..43348ca8a0e6ef 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java @@ -26,6 +26,8 @@ import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.mysql.privilege.Auth; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.QueryState.MysqlStateType; +import org.apache.doris.qe.StmtExecutor; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.Lists; @@ -141,7 +143,7 @@ public void testExternalCatalogAutoAnalyze() throws Exception { @Test public void testExternalCatalogFilteredDatabase() throws Exception { - // use_meta_cache=true + // 1. use_meta_cache=true TestExternalCatalog ctl = (TestExternalCatalog) mgr.getCatalog("test1"); List dbNames = ctl.getDbNames(); System.out.println(dbNames); @@ -149,6 +151,12 @@ public void testExternalCatalogFilteredDatabase() throws Exception { Assertions.assertTrue(!dbNames.contains("db2")); ctl = (TestExternalCatalog) mgr.getCatalog("test2"); + // before get dbnames + String useDb = "use test2.db3"; + StmtExecutor stmtExecutor = new StmtExecutor(rootCtx, useDb); + stmtExecutor.execute(); + Assertions.assertTrue(rootCtx.getState().getErrorMessage().contains("Unknown database 'db3'")); + dbNames = ctl.getDbNames(); System.out.println(dbNames); Assertions.assertEquals(3, dbNames.size()); @@ -161,7 +169,19 @@ public void testExternalCatalogFilteredDatabase() throws Exception { Assertions.assertTrue(!dbNames.contains("db1")); Assertions.assertTrue(!dbNames.contains("db2")); - // use_meta_cache=false + // use non exist db + useDb = "use test2.db3"; + stmtExecutor = new StmtExecutor(rootCtx, useDb); + stmtExecutor.execute(); + Assertions.assertTrue(rootCtx.getState().getErrorMessage().contains("Unknown database 'db3'")); + + // use exist db + useDb = "use test2.db2"; + stmtExecutor = new StmtExecutor(rootCtx, useDb); + stmtExecutor.execute(); + Assertions.assertEquals(MysqlStateType.OK, rootCtx.getState().getStateType()); + + // 2. use_meta_cache=false ctl = (TestExternalCatalog) mgr.getCatalog("test4"); dbNames = ctl.getDbNames(); System.out.println(dbNames); @@ -180,6 +200,18 @@ public void testExternalCatalogFilteredDatabase() throws Exception { Assertions.assertEquals(2, dbNames.size()); Assertions.assertTrue(!dbNames.contains("db1")); Assertions.assertTrue(!dbNames.contains("db2")); + + // use non exist db + useDb = "use test5.db3"; + stmtExecutor = new StmtExecutor(rootCtx, useDb); + stmtExecutor.execute(); + Assertions.assertTrue(rootCtx.getState().getErrorMessage().contains("Unknown database 'db3'")); + + // use exist db + useDb = "use test5.db2"; + stmtExecutor = new StmtExecutor(rootCtx, useDb); + stmtExecutor.execute(); + Assertions.assertEquals(MysqlStateType.OK, rootCtx.getState().getStateType()); } public static class RefreshCatalogProvider implements TestExternalCatalog.TestCatalogProvider {