Skip to content

[fix](catalog) fix filtered database when use_meta_cache=true #41939

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 2 commits into from
Oct 17, 2024
Merged

[fix](catalog) fix filtered database when use_meta_cache=true #41939

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
260975e
[fix](catalog) fix filtered database when use_meta_cache=true
morningman Oct 16, 2024
bc5a183
2
morningman Oct 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 22 additions & 18 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
import org.apache.doris.datasource.operations.ExternalMetadataOps;
import org.apache.doris.datasource.paimon.PaimonExternalDatabase;
import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.datasource.test.TestExternalCatalog;
import org.apache.doris.datasource.test.TestExternalDatabase;
import org.apache.doris.datasource.trinoconnector.TrinoConnectorExternalDatabase;
import org.apache.doris.fs.remote.dfs.DFSFileSystem;
Expand Down Expand Up @@ -349,19 +350,8 @@ private void init() {
InitCatalogLog initCatalogLog = new InitCatalogLog();
initCatalogLog.setCatalogId(id);
initCatalogLog.setType(logType);
List<String> allDatabases = getFilteredDatabaseNames();
Map<String, Boolean> includeDatabaseMap = getIncludeDatabaseMap();
Map<String, Boolean> excludeDatabaseMap = getExcludeDatabaseMap();
for (String dbName : allDatabases) {
if (!dbName.equals(InfoSchemaDb.DATABASE_NAME) && !dbName.equals(MysqlDb.DATABASE_NAME)) {
// Exclude database map take effect with higher priority over include database map
if (!excludeDatabaseMap.isEmpty() && excludeDatabaseMap.containsKey(dbName)) {
continue;
}
if (!includeDatabaseMap.isEmpty() && !includeDatabaseMap.containsKey(dbName)) {
continue;
}
}
List<String> filteredDatabases = getFilteredDatabaseNames();
for (String dbName : filteredDatabases) {
long dbId;
if (dbNameToId != null && dbNameToId.containsKey(dbName)) {
dbId = dbNameToId.get(dbName);
Expand Down Expand Up @@ -392,6 +382,20 @@ private List<String> getFilteredDatabaseNames() {
allDatabases.add(InfoSchemaDb.DATABASE_NAME);
allDatabases.remove(MysqlDb.DATABASE_NAME);
allDatabases.add(MysqlDb.DATABASE_NAME);
Map<String, Boolean> includeDatabaseMap = getIncludeDatabaseMap();
Map<String, Boolean> excludeDatabaseMap = getExcludeDatabaseMap();
allDatabases = allDatabases.stream().filter(dbName -> {
if (!dbName.equals(InfoSchemaDb.DATABASE_NAME) && !dbName.equals(MysqlDb.DATABASE_NAME)) {
// Exclude database map take effect with higher priority over include database map
if (!excludeDatabaseMap.isEmpty() && excludeDatabaseMap.containsKey(dbName)) {
return false;
}
if (!includeDatabaseMap.isEmpty() && !includeDatabaseMap.containsKey(dbName)) {
return false;
}
}
return true;
}).collect(Collectors.toList());
return allDatabases;
}

Expand Down Expand Up @@ -675,11 +679,11 @@ protected ExternalDatabase<? extends ExternalTable> buildDbForInit(String dbName
InitCatalogLog.Type logType, boolean checkExists) {
// When running ut, disable this check to make ut pass.
// Because in ut, the database is not created in remote system.
if (checkExists && !FeConstants.runningUnitTest) {
if (checkExists && (!FeConstants.runningUnitTest || this instanceof TestExternalCatalog)) {
try {
List<String> dbNames = getDbNames();
if (!dbNames.contains(dbName)) {
dbNames = listDatabaseNames();
dbNames = getFilteredDatabaseNames();
if (!dbNames.contains(dbName)) {
return null;
}
Expand Down Expand Up @@ -835,15 +839,15 @@ public void registerDatabase(long dbId, String dbName) {
throw new NotImplementedException("registerDatabase not implemented");
}

public Map<String, Boolean> getIncludeDatabaseMap() {
protected Map<String, Boolean> getIncludeDatabaseMap() {
return getSpecifiedDatabaseMap(Resource.INCLUDE_DATABASE_LIST);
}

public Map<String, Boolean> getExcludeDatabaseMap() {
protected Map<String, Boolean> getExcludeDatabaseMap() {
return getSpecifiedDatabaseMap(Resource.EXCLUDE_DATABASE_LIST);
}

public Map<String, Boolean> getSpecifiedDatabaseMap(String catalogPropertyKey) {
private Map<String, Boolean> getSpecifiedDatabaseMap(String catalogPropertyKey) {
String specifiedDatabaseList = catalogProperty.getOrDefault(catalogPropertyKey, "");
Map<String, Boolean> specifiedDatabaseMap = Maps.newHashMap();
specifiedDatabaseList = specifiedDatabaseList.trim();
Expand Down
196 changes: 196 additions & 0 deletions fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,108 @@

package org.apache.doris.datasource;

import org.apache.doris.analysis.CreateCatalogStmt;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.FeConstants;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.test.TestExternalCatalog;
import org.apache.doris.mysql.privilege.Auth;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.QueryState.MysqlStateType;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.utframe.TestWithFeService;

import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class ExternalCatalogTest extends TestWithFeService {
private static Auth auth;
private static Env env;
private CatalogMgr mgr;
private ConnectContext rootCtx;

@Override
protected void runBeforeAll() throws Exception {
FeConstants.runningUnitTest = true;
mgr = Env.getCurrentEnv().getCatalogMgr();
rootCtx = createDefaultCtx();
env = Env.getCurrentEnv();
auth = env.getAuth();
// 1. create test catalog
CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt(
"create catalog test1 properties(\n"
+ " \"type\" = \"test\",\n"
+ " \"catalog_provider.class\" "
+ "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n"
+ " \"include_database_list\" = \"db1\"\n"
+ ");",
rootCtx);
env.getCatalogMgr().createCatalog(testCatalog);

testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt(
"create catalog test2 properties(\n"
+ " \"type\" = \"test\",\n"
+ " \"catalog_provider.class\" "
+ "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n"
+ " \"exclude_database_list\" = \"db1\"\n"
+ ");",
rootCtx);
env.getCatalogMgr().createCatalog(testCatalog);

testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt(
"create catalog test3 properties(\n"
+ " \"type\" = \"test\",\n"
+ " \"catalog_provider.class\" "
+ "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n"
+ " \"include_database_list\" = \"db1\",\n"
+ " \"exclude_database_list\" = \"db1\"\n"
+ ");",
rootCtx);
env.getCatalogMgr().createCatalog(testCatalog);

// use_meta_cache=false
testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt(
"create catalog test4 properties(\n"
+ " \"type\" = \"test\",\n"
+ " \"use_meta_cache\" = \"false\",\n"
+ " \"catalog_provider.class\" "
+ "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n"
+ " \"include_database_list\" = \"db1\"\n"
+ ");",
rootCtx);
env.getCatalogMgr().createCatalog(testCatalog);

testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt(
"create catalog test5 properties(\n"
+ " \"type\" = \"test\",\n"
+ " \"use_meta_cache\" = \"false\",\n"
+ " \"catalog_provider.class\" "
+ "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n"
+ " \"exclude_database_list\" = \"db1\"\n"
+ ");",
rootCtx);
env.getCatalogMgr().createCatalog(testCatalog);

testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt(
"create catalog test6 properties(\n"
+ " \"type\" = \"test\",\n"
+ " \"use_meta_cache\" = \"false\",\n"
+ " \"catalog_provider.class\" "
+ "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n"
+ " \"include_database_list\" = \"db1\",\n"
+ " \"exclude_database_list\" = \"db1\"\n"
+ ");",
rootCtx);
env.getCatalogMgr().createCatalog(testCatalog);
}

@Test
public void testExternalCatalogAutoAnalyze() throws Exception {
Expand All @@ -48,4 +140,108 @@ public void testExternalCatalogAutoAnalyze() throws Exception {
catalog.modifyCatalogProps(prop);
Assertions.assertTrue(catalog.enableAutoAnalyze());
}

@Test
public void testExternalCatalogFilteredDatabase() throws Exception {
// 1. use_meta_cache=true
TestExternalCatalog ctl = (TestExternalCatalog) mgr.getCatalog("test1");
List<String> dbNames = ctl.getDbNames();
System.out.println(dbNames);
Assertions.assertEquals(3, dbNames.size());
Assertions.assertTrue(!dbNames.contains("db2"));

ctl = (TestExternalCatalog) mgr.getCatalog("test2");
// before get dbnames
String useDb = "use test2.db3";
StmtExecutor stmtExecutor = new StmtExecutor(rootCtx, useDb);
stmtExecutor.execute();
Assertions.assertTrue(rootCtx.getState().getErrorMessage().contains("Unknown database 'db3'"));

dbNames = ctl.getDbNames();
System.out.println(dbNames);
Assertions.assertEquals(3, dbNames.size());
Assertions.assertTrue(!dbNames.contains("db1"));

ctl = (TestExternalCatalog) mgr.getCatalog("test3");
dbNames = ctl.getDbNames();
System.out.println(dbNames);
Assertions.assertEquals(2, dbNames.size());
Assertions.assertTrue(!dbNames.contains("db1"));
Assertions.assertTrue(!dbNames.contains("db2"));

// use non exist db
useDb = "use test2.db3";
stmtExecutor = new StmtExecutor(rootCtx, useDb);
stmtExecutor.execute();
Assertions.assertTrue(rootCtx.getState().getErrorMessage().contains("Unknown database 'db3'"));

// use exist db
useDb = "use test2.db2";
stmtExecutor = new StmtExecutor(rootCtx, useDb);
stmtExecutor.execute();
Assertions.assertEquals(MysqlStateType.OK, rootCtx.getState().getStateType());

// 2. use_meta_cache=false
ctl = (TestExternalCatalog) mgr.getCatalog("test4");
dbNames = ctl.getDbNames();
System.out.println(dbNames);
Assertions.assertEquals(3, dbNames.size());
Assertions.assertTrue(!dbNames.contains("db2"));

ctl = (TestExternalCatalog) mgr.getCatalog("test5");
dbNames = ctl.getDbNames();
System.out.println(dbNames);
Assertions.assertEquals(3, dbNames.size());
Assertions.assertTrue(!dbNames.contains("db1"));

ctl = (TestExternalCatalog) mgr.getCatalog("test6");
dbNames = ctl.getDbNames();
System.out.println(dbNames);
Assertions.assertEquals(2, dbNames.size());
Assertions.assertTrue(!dbNames.contains("db1"));
Assertions.assertTrue(!dbNames.contains("db2"));

// use non exist db
useDb = "use test5.db3";
stmtExecutor = new StmtExecutor(rootCtx, useDb);
stmtExecutor.execute();
Assertions.assertTrue(rootCtx.getState().getErrorMessage().contains("Unknown database 'db3'"));

// use exist db
useDb = "use test5.db2";
stmtExecutor = new StmtExecutor(rootCtx, useDb);
stmtExecutor.execute();
Assertions.assertEquals(MysqlStateType.OK, rootCtx.getState().getStateType());
}

public static class RefreshCatalogProvider implements TestExternalCatalog.TestCatalogProvider {
public static final Map<String, Map<String, List<Column>>> MOCKED_META;

static {
MOCKED_META = Maps.newHashMap();
Map<String, List<Column>> tblSchemaMap1 = Maps.newHashMap();
// db1
tblSchemaMap1.put("tbl11", Lists.newArrayList(
new Column("a11", PrimitiveType.BIGINT),
new Column("a12", PrimitiveType.STRING),
new Column("a13", PrimitiveType.FLOAT)));
tblSchemaMap1.put("tbl12", Lists.newArrayList(
new Column("b21", PrimitiveType.BIGINT),
new Column("b22", PrimitiveType.STRING),
new Column("b23", PrimitiveType.FLOAT)));
MOCKED_META.put("db1", tblSchemaMap1);
// db2
Map<String, List<Column>> tblSchemaMap2 = Maps.newHashMap();
tblSchemaMap2.put("tbl21", Lists.newArrayList(
new Column("c11", PrimitiveType.BIGINT),
new Column("c12", PrimitiveType.STRING),
new Column("c13", PrimitiveType.FLOAT)));
MOCKED_META.put("db2", tblSchemaMap2);
}

@Override
public Map<String, Map<String, List<Column>>> getMetadata() {
return MOCKED_META;
}
}
}