diff --git a/docs/en/docs/admin-manual/http-actions/fe/query-schema-action.md b/docs/en/docs/admin-manual/http-actions/fe/query-schema-action.md new file mode 100644 index 00000000000000..ccf9bdb3689aa2 --- /dev/null +++ b/docs/en/docs/admin-manual/http-actions/fe/query-schema-action.md @@ -0,0 +1,108 @@ +--- +{ + "title": "Query Schema Action", + "language": "en" +} +--- + + + +# Query Schema Action + + +## Request + +``` +POST /api/query_schema// +``` + +## Description + +The Query Schema Action can return the table creation statement for the given SQL-related table. Can be used to test some query scenarios locally. + +The API was released in version 1.2. + +## Path parameters + +* `` + + Specify the database name. This database will be considered as the default database for the current session, and will be used if the table name in SQL does not qualify the database name. + +## Query parameters + +None + +## Request body + +``` +text/plain + +sql +``` + +* "sql" field is the SQL string. + +## Response + +* Return value + + ``` + CREATE TABLE `tbl1` ( + `k1` int(11) NULL, + `k2` int(11) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ); + + CREATE TABLE `tbl2` ( + `k1` int(11) NULL, + `k2` int(11) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ); + ``` + +## Example + +1. Write the SQL in local file 1.sql + + ``` + select tbl1.k2 from tbl1 join tbl2 on tbl1.k1 = tbl2.k1; + ``` + +2. Use curl to get the table creation statement. + + ``` + curl -X POST -H 'Content-Type: text/plain' -uroot: http://127.0.0.1:8030/api/query_schema/internal/db1 -d@1.sql + ``` \ No newline at end of file diff --git a/docs/sidebars.json b/docs/sidebars.json index 4646c6a1c25468..40e467f6945e14 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -938,7 +938,8 @@ "admin-manual/http-actions/fe/show-proc-action", "admin-manual/http-actions/fe/check-decommission-action", "admin-manual/http-actions/fe/health-action", - "admin-manual/http-actions/fe/check-storage-type-action" + "admin-manual/http-actions/fe/check-storage-type-action", + "admin-manual/http-actions/fe/query-schema-action" ] }, "admin-manual/http-actions/restore-tablet", diff --git a/docs/zh-CN/docs/admin-manual/http-actions/fe/query-schema-action.md b/docs/zh-CN/docs/admin-manual/http-actions/fe/query-schema-action.md new file mode 100644 index 00000000000000..b5855c008d0e5d --- /dev/null +++ b/docs/zh-CN/docs/admin-manual/http-actions/fe/query-schema-action.md @@ -0,0 +1,107 @@ +--- +{ + "title": "Query Schema Action", + "language": "zh-CN" +} +--- + + + +# Query Schema Action + + +## Request + +``` +POST /api/query_schema// +``` + +## Description + +Query Schema Action 可以返回给定的 SQL 有关的表的建表语句。可以用于本地测试一些查询场景。 +该 API 在 1.2 版本中发布。 + +## Path parameters + +* `` + + 指定数据库名称。该数据库会被视为当前session的默认数据库，如果在 SQL 中的表名没有限定数据库名称的话，则使用该数据库。 + +## Query parameters + +无 + +## Request body + +``` +text/plain + +sql +``` + +* sql 字段为具体的 SQL + +## Response + +* 返回结果集 + + ``` + CREATE TABLE `tbl1` ( + `k1` int(11) NULL, + `k2` int(11) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ); + + CREATE TABLE `tbl2` ( + `k1` int(11) NULL, + `k2` int(11) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ); + ``` + +## Example + +1. 在本地文件 1.sql 中写入 SQL + + ``` + select tbl1.k2 from tbl1 join tbl2 on tbl1.k1 = tbl2.k1; + ``` + +2. 使用 curl 命令获取建表语句 + + ``` + curl -X POST -H 'Content-Type: text/plain' -uroot: http://127.0.0.1:8030/api/query_schema/internal/db1 -d@1.sql + ``` \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java index a15512906779f5..9a1aac62168ab2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java @@ -188,7 +188,7 @@ public String getTbl() { public void getTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) throws AnalysisException { // get dbs of statement - queryStmt.getTables(analyzer, tableMap, parentViewNameSet); + queryStmt.getTables(analyzer, false, tableMap, parentViewNameSet); tblName.analyze(analyzer); // disallow external catalog Util.prohibitExternalCatalog(tblName.getCtl(), this.getClass().getSimpleName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java index 7c4079dedd0ca7..03ee4048f98b9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java @@ -454,18 +454,17 @@ private Expr trySubstituteOrdinal(Expr expr, String errorPrefix, } if (pos > resultExprs.size()) { throw new AnalysisException( - errorPrefix + ": ordinal exceeds number of items in select list: " - + expr.toSql()); + errorPrefix + ": ordinal exceeds number of items in select list: " + expr.toSql()); } // Create copy to protect against accidentally shared state. return resultExprs.get((int) pos - 1).clone(); } - public void getWithClauseTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) - throws AnalysisException { + public void getWithClauseTables(Analyzer analyzer, boolean expandView, Map tableMap, + Set parentViewNameSet) throws AnalysisException { if (withClause != null) { - withClause.getTables(analyzer, tableMap, parentViewNameSet); + withClause.getTables(analyzer, expandView, tableMap, parentViewNameSet); } } @@ -542,8 +541,8 @@ public boolean containAlias(Expr expr) { // tmp in child stmt "(select siteid, citycode from tmp)" do not contain with_Clause // so need to check is view name by parentViewNameSet. // issue link: https://github.com/apache/doris/issues/4598 - public abstract void getTables(Analyzer analyzer, Map tables, Set parentViewNameSet) - throws AnalysisException; + public abstract void getTables(Analyzer analyzer, boolean expandView, Map tables, + Set parentViewNameSet) throws AnalysisException; // get TableRefs in this query, including physical TableRefs of this statement and // nested statements of inline views and with_Clause. diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java index 0fa4d0f9c76258..8e4ae4eaa26b4e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -292,18 +292,18 @@ public ExprSubstitutionMap getBaseTblSmap() { } @Override - public void getTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) - throws AnalysisException { - getWithClauseTables(analyzer, tableMap, parentViewNameSet); + public void getTables(Analyzer analyzer, boolean expandView, Map tableMap, + Set parentViewNameSet) throws AnalysisException { + getWithClauseTables(analyzer, expandView, tableMap, parentViewNameSet); for (TableRef tblRef : fromClause) { if (tblRef instanceof InlineViewRef) { // Inline view reference QueryStmt inlineStmt = ((InlineViewRef) tblRef).getViewStmt(); - inlineStmt.getTables(analyzer, tableMap, parentViewNameSet); + inlineStmt.getTables(analyzer, expandView, tableMap, parentViewNameSet); } else if (tblRef instanceof TableValuedFunctionRef) { TableValuedFunctionRef tblFuncRef = (TableValuedFunctionRef) tblRef; tableMap.put(tblFuncRef.getTableFunction().getTable().getId(), - tblFuncRef.getTableFunction().getTable()); + tblFuncRef.getTableFunction().getTable()); } else { String dbName = tblRef.getName().getDb(); String tableName = tblRef.getName().getTbl(); @@ -320,14 +320,19 @@ public void getTables(Analyzer analyzer, Map tableMap, Set getSetOpsResultExprs() { } @Override - public void getTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) - throws AnalysisException { - getWithClauseTables(analyzer, tableMap, parentViewNameSet); + public void getTables(Analyzer analyzer, boolean expandView, Map tableMap, + Set parentViewNameSet) throws AnalysisException { + getWithClauseTables(analyzer, expandView, tableMap, parentViewNameSet); for (SetOperand op : operands) { - op.getQueryStmt().getTables(analyzer, tableMap, parentViewNameSet); + op.getQueryStmt().getTables(analyzer, expandView, tableMap, parentViewNameSet); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java index e9e1e57d105cde..fb4fdbec6a2392 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java @@ -102,8 +102,7 @@ private WithClause(WithClause other) { Preconditions.checkNotNull(other); views = Lists.newArrayList(); for (View view : other.views) { - views.add(new View(view.getName(), view.getQueryStmt().clone(), - view.getOriginalColLabels())); + views.add(new View(view.getName(), view.getQueryStmt().clone(), view.getOriginalColLabels())); } } @@ -113,12 +112,12 @@ public void reset() { } } - public void getTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) - throws AnalysisException { + public void getTables(Analyzer analyzer, boolean expandView, Map tableMap, + Set parentViewNameSet) throws AnalysisException { for (View view : views) { QueryStmt stmt = view.getQueryStmt(); parentViewNameSet.add(view.getName()); - stmt.getTables(analyzer, tableMap, parentViewNameSet); + stmt.getTables(analyzer, expandView, tableMap, parentViewNameSet); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 060a044074a23b..2ab097399ca6ab 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -2703,8 +2703,7 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis if (table.getType() == TableType.VIEW) { View view = (View) table; sb.append("CREATE VIEW `").append(table.getName()).append("` AS ").append(view.getInlineViewDef()); - sb.append(";"); - createTableStmt.add(sb.toString()); + createTableStmt.add(sb + ";"); return; } @@ -3045,7 +3044,7 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis sb.append("\n)"); } - createTableStmt.add(sb.toString()); + createTableStmt.add(sb + ";"); // 2. add partition if (separatePartition && (table instanceof OlapTable) && ((OlapTable) table).getPartitions().size() > 1) { @@ -3077,7 +3076,7 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis sb.append("(\"version_info\" = \""); sb.append(partition.getVisibleVersion()).append("\""); sb.append(");"); - addPartitionStmt.add(sb.toString()); + addPartitionStmt.add(sb + ";"); } } } @@ -3104,7 +3103,7 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis } } sb.append(");"); - createRollupStmt.add(sb.toString()); + createRollupStmt.add(sb + ";"); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java index d4f787132cfb5e..34ebae24153edb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java @@ -17,41 +17,56 @@ package org.apache.doris.httpv2.rest; -import org.apache.doris.common.DdlException; +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.QueryStmt; +import org.apache.doris.analysis.SqlParser; +import org.apache.doris.analysis.SqlScanner; +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.util.SqlParserUtils; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; import org.apache.doris.httpv2.util.ExecutionResultSet; import org.apache.doris.httpv2.util.StatementSubmitter; import org.apache.doris.qe.ConnectContext; import org.apache.doris.system.SystemInfoService; +import com.google.common.base.Joiner; import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; +import java.io.StringReader; import java.lang.reflect.Type; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; /** - * For execute stmt via http + * For execute stmt or get create table stmt via http */ @RestController public class StmtExecutionAction extends RestBaseController { private static final Logger LOG = LogManager.getLogger(StmtExecutionAction.class); private static StatementSubmitter stmtSubmitter = new StatementSubmitter(); - private static final String PARAM_SYNC = "sync"; - private static final String PARAM_LIMIT = "limit"; - private static final long DEFAULT_ROW_LIMIT = 1000; private static final long MAX_ROW_LIMIT = 10000; @@ -59,48 +74,75 @@ public class StmtExecutionAction extends RestBaseController { * Execute a SQL. * Request body: * { - * "stmt" : "select * from tbl1" + * "is_sync": 1, // optional + * "limit" : 1000 // optional + * "stmt" : "select * from tbl1" // required * } */ @RequestMapping(path = "/api/query/{" + NS_KEY + "}/{" + DB_KEY + "}", method = {RequestMethod.POST}) - public Object executeSQL( - @PathVariable(value = NS_KEY) String ns, - @PathVariable(value = DB_KEY) String dbName, - HttpServletRequest request, HttpServletResponse response, - @RequestBody String stmtBody) throws DdlException { + public Object executeSQL(@PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName, + HttpServletRequest request, HttpServletResponse response, @RequestBody String body) { ActionAuthorizationInfo authInfo = checkWithCookie(request, response, false); - if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { - return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now"); - } - - boolean isSync = true; - String syncParam = request.getParameter(PARAM_SYNC); - if (!Strings.isNullOrEmpty(syncParam)) { - isSync = syncParam.equals("1"); - } - - String limitParam = request.getParameter(PARAM_LIMIT); - long limit = DEFAULT_ROW_LIMIT; - if (!Strings.isNullOrEmpty(limitParam)) { - limit = Math.min(Long.valueOf(limitParam), MAX_ROW_LIMIT); + if (ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + ns = InternalCatalog.INTERNAL_CATALOG_NAME; } Type type = new TypeToken() { }.getType(); - StmtRequestBody stmtRequestBody = new Gson().fromJson(stmtBody, type); + StmtRequestBody stmtRequestBody = new Gson().fromJson(body, type); if (Strings.isNullOrEmpty(stmtRequestBody.stmt)) { return ResponseEntityBuilder.badRequest("Missing statement request body"); } - LOG.info("stmt: {}", stmtRequestBody.stmt); + LOG.info("stmt: {}, isSync:{}, limit: {}", stmtRequestBody.stmt, stmtRequestBody.is_sync, + stmtRequestBody.limit); + + ConnectContext.get().changeDefaultCatalog(ns); + ConnectContext.get().setDatabase(getFullDbName(dbName)); + return executeQuery(authInfo, stmtRequestBody.is_sync, stmtRequestBody.limit, stmtRequestBody); + } + + + /** + * Get all create table stmt of a SQL + * + * @param ns + * @param dbName + * @param request + * @param response + * @param sql plain text of sql + * @return plain text of create table stmts + */ + @RequestMapping(path = "/api/query_schema/{" + NS_KEY + "}/{" + DB_KEY + "}", method = {RequestMethod.POST}) + public String querySchema(@PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName, + HttpServletRequest request, HttpServletResponse response, @RequestBody String sql) { + checkWithCookie(request, response, false); + + if (ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + ns = InternalCatalog.INTERNAL_CATALOG_NAME; + } + LOG.info("sql: {}", sql); + ConnectContext.get().changeDefaultCatalog(ns); ConnectContext.get().setDatabase(getFullDbName(dbName)); + return getSchema(sql); + } - // 2. Submit stmt - StatementSubmitter.StmtContext stmtCtx = new StatementSubmitter.StmtContext( - stmtRequestBody.stmt, authInfo.fullUserName, authInfo.password, limit - ); + /** + * Execute a query + * + * @param authInfo + * @param isSync + * @param limit + * @param stmtRequestBody + * @return + */ + @NotNull + private ResponseEntity executeQuery(ActionAuthorizationInfo authInfo, boolean isSync, long limit, + StmtRequestBody stmtRequestBody) { + StatementSubmitter.StmtContext stmtCtx = new StatementSubmitter.StmtContext(stmtRequestBody.stmt, + authInfo.fullUserName, authInfo.password, limit); Future future = stmtSubmitter.submit(stmtCtx); if (isSync) { @@ -119,7 +161,38 @@ public Object executeSQL( } } + @NotNull + private String getSchema(String sql) { + SqlParser parser = new SqlParser(new SqlScanner(new StringReader(sql))); + StatementBase stmt = null; + try { + stmt = SqlParserUtils.getStmt(parser, 0); + if (!(stmt instanceof QueryStmt)) { + return "Only support query stmt"; + } + Analyzer analyzer = new Analyzer(Env.getCurrentEnv(), ConnectContext.get()); + QueryStmt queryStmt = (QueryStmt) stmt; + Map tableMap = Maps.newHashMap(); + Set parentViewNameSet = Sets.newHashSet(); + queryStmt.getTables(analyzer, true, tableMap, parentViewNameSet); + + List createStmts = Lists.newArrayList(); + for (TableIf tbl : tableMap.values()) { + List createTableStmts = Lists.newArrayList(); + Env.getDdlStmt(tbl, createTableStmts, null, null, false, true); + if (!createTableStmts.isEmpty()) { + createStmts.add(createTableStmts.get(0)); + } + } + return Joiner.on("\n\n").join(createStmts); + } catch (Exception e) { + return "Error:" + e.getMessage(); + } + } + private static class StmtRequestBody { + public Boolean is_sync = true; // CHECKSTYLE IGNORE THIS LINE + public Long limit = DEFAULT_ROW_LIMIT; public String stmt; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 2dd7aa501f3e23..31b2c856db6b44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -630,11 +630,11 @@ public void analyze(TQueryOptions tQueryOptions) throws UserException { Set parentViewNameSet = Sets.newHashSet(); if (parsedStmt instanceof QueryStmt) { queryStmt = (QueryStmt) parsedStmt; - queryStmt.getTables(analyzer, tableMap, parentViewNameSet); + queryStmt.getTables(analyzer, false, tableMap, parentViewNameSet); } else if (parsedStmt instanceof CreateTableAsSelectStmt) { CreateTableAsSelectStmt parsedStmt = (CreateTableAsSelectStmt) this.parsedStmt; queryStmt = parsedStmt.getQueryStmt(); - queryStmt.getTables(analyzer, tableMap, parentViewNameSet); + queryStmt.getTables(analyzer, false, tableMap, parentViewNameSet); } else if (parsedStmt instanceof InsertStmt) { InsertStmt insertStmt = (InsertStmt) parsedStmt; insertStmt.getTables(analyzer, tableMap, parentViewNameSet); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index 28c09dc57b8dd4..df6354dff2e172 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -17,15 +17,28 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.ExceptionChecker; +import org.apache.doris.common.util.SqlParserUtils; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSet; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.junit.Assert; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.io.StringReader; +import java.util.List; +import java.util.Map; +import java.util.Set; + /** * test for CreateTableAsSelectStmt. **/ @@ -71,11 +84,10 @@ public void testDecimal() throws Exception { createTableAsSelect(selectFromDecimal); Assertions.assertEquals("CREATE TABLE `select_decimal_table` (\n" + " `userId` varchar(255) NOT NULL,\n" + " `amount_decimal` decimal(10, 2) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" - + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" - + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"," - + "\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showCreateTableByName("select_decimal_table").getResultRows().get(0).get(1)); String selectFromDecimal1 = "create table `test`.`select_decimal_table_1` PROPERTIES(\"replication_num\" = \"1\") " @@ -88,7 +100,7 @@ public void testDecimal() throws Exception { + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"," - + "\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1)); } else { Assertions.assertEquals( @@ -97,7 +109,7 @@ public void testDecimal() throws Exception { + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"," - + "\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1)); } } @@ -119,11 +131,10 @@ public void testVarchar() throws Exception { createTableAsSelect(selectFromVarchar); ShowResultSet showResultSet = showCreateTableByName("select_varchar"); Assertions.assertEquals("CREATE TABLE `select_varchar` (\n" + " `userId` varchar(255) NOT NULL,\n" - + " `username` varchar(255) REPLACE NOT NULL\n" - + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + + " `username` varchar(255) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -138,7 +149,7 @@ public void testFunction() throws Exception { + "DUPLICATE KEY(`_col0`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"" - + ",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + ",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); String selectFromFunction2 = "create table `test`.`select_function_2` PROPERTIES(\"replication_num\" = \"1\") " @@ -153,7 +164,7 @@ public void testFunction() throws Exception { + "DUPLICATE KEY(`_col0`, `_col1`, `_col2`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet2.getResultRows().get(0).get(1)); } @@ -167,17 +178,16 @@ public void testAlias() throws Exception { + "DUPLICATE KEY(`amount`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`amount`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"," - + "\n\"disable_auto_compaction\" = \"false\"\n" + ")", showResultSet1.getResultRows().get(0).get(1)); + + "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); String selectAlias2 = "create table `test`.`select_alias_2` PROPERTIES(\"replication_num\" = \"1\") " + "as select userId as alias_name, username from `test`.`varchar_table`"; createTableAsSelect(selectAlias2); ShowResultSet showResultSet2 = showCreateTableByName("select_alias_2"); Assertions.assertEquals("CREATE TABLE `select_alias_2` (\n" + " `alias_name` varchar(255) NOT NULL,\n" - + " `username` varchar(255) REPLACE NOT NULL\n" - + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`alias_name`)\n" + + " `username` varchar(255) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`alias_name`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`alias_name`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet2.getResultRows().get(0).get(1)); } @@ -193,7 +203,7 @@ public void testJoin() throws Exception { + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); String selectFromJoin1 = "create table `test`.`select_join1` PROPERTIES(\"replication_num\" = \"1\") " + "as select vt.userId as userId1, jt.userId as userId2, vt.username, jt.status " @@ -202,11 +212,10 @@ public void testJoin() throws Exception { ShowResultSet showResultSet1 = showCreateTableByName("select_join1"); Assertions.assertEquals("CREATE TABLE `select_join1` (\n" + " `userId1` varchar(255) NOT NULL,\n" + " `userId2` varchar(255) NOT NULL,\n" + " `username` varchar(255) REPLACE NOT NULL,\n" - + " `status` int(11) REPLACE NOT NULL\n" - + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId1`, `userId2`)\n" + + " `status` int(11) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId1`, `userId2`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId1`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); } @@ -223,7 +232,7 @@ public void testName() throws Exception { + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`user`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`user`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -237,7 +246,7 @@ public void testUnion() throws Exception { + "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\"" - + " = \"false\"\n" + ")", showResultSet.getResultRows().get(0).get(1)); + + " = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @Test @@ -251,7 +260,7 @@ public void testCte() throws Exception { + "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); String selectFromCteAndUnion = "create table `test`.`select_cte_union` PROPERTIES(\"replication_num\" = \"1\")" + "as with source_data as (select 1 as id union all select 2 as id) select * from source_data;"; @@ -261,7 +270,7 @@ public void testCte() throws Exception { + "DUPLICATE KEY(`id`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"," - + "\n\"disable_auto_compaction\" = \"false\"\n" + ")", showResultSet1.getResultRows().get(0).get(1)); + + "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); } @Test @@ -272,13 +281,12 @@ public void testPartition() throws Exception { createTableAsSelect(selectFromPartition); ShowResultSet showResultSet = showCreateTableByName("selectPartition"); Assertions.assertEquals("CREATE TABLE `selectPartition` (\n" + " `userId` varchar(255) NOT NULL,\n" - + " `username` varchar(255) REPLACE NOT NULL\n" - + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + + " `username` varchar(255) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "PARTITION BY LIST(`userId`)\n" + "(PARTITION p1 VALUES IN (\"CA\",\"GB\",\"US\",\"ZH\"))\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -294,6 +302,55 @@ public void testDefaultTimestamp() throws Exception { + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"disable_auto_compaction\" = \"false\"\n" + ")", showResultSet.getResultRows().get(0).get(1)); + + "\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); + } + + @Test + public void testQuerySchema() throws Exception { + connectContext.setDatabase("default_cluster:test"); + String create1 = "create table test.qs1 (k1 int, k2 int) distributed by hash(k1) " + + "buckets 1 properties('replication_num' = '1')"; + String create2 = "create table test.qs2 (k1 int, k2 int) distributed by hash(k1) " + + "buckets 1 properties('replication_num' = '1')"; + createTables(create1, create2); + + String view1 = "create view test.v1 as select qs1.k1 from qs1 join qs2 on qs1.k1 = qs2.k1"; + String view2 = "create view test.v2 as with cte(s1) as (select * from v1) select * from cte"; + + createView(view1); + createView(view2); + + String sql1 = "select * from v1"; + + org.apache.doris.analysis.SqlParser parser = new SqlParser( + new org.apache.doris.analysis.SqlScanner(new StringReader(sql1))); + QueryStmt stmt = (QueryStmt) SqlParserUtils.getStmt(parser, 0); + Map tableMap = Maps.newHashMap(); + Set parentViewNameSet = Sets.newHashSet(); + Analyzer analyzer = new Analyzer(Env.getCurrentEnv(), ConnectContext.get()); + stmt.getTables(analyzer, true, tableMap, parentViewNameSet); + + List createStmts = Lists.newArrayList(); + for (TableIf tbl : tableMap.values()) { + List createTableStmts = Lists.newArrayList(); + Env.getDdlStmt(tbl, createTableStmts, null, null, false, true); + createStmts.add(createTableStmts.get(0)); + if (tbl.getName().equals("qs1")) { + Assert.assertEquals("CREATE TABLE `qs1` (\n" + " `k1` int(11) NULL,\n" + " `k2` int(11) NULL\n" + + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`k1`, `k2`)\n" + "COMMENT 'OLAP'\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n" + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\",\n" + "\"disable_auto_compaction\" = \"false\"\n" + ");", + createTableStmts.get(0)); + } else { + Assert.assertEquals("CREATE TABLE `qs2` (\n" + " `k1` int(11) NULL,\n" + " `k2` int(11) NULL\n" + + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`k1`, `k2`)\n" + "COMMENT 'OLAP'\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n" + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\",\n" + "\"disable_auto_compaction\" = \"false\"\n" + ");", + createTableStmts.get(0)); + } + } + Assert.assertEquals(2, createStmts.size()); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index 4f20dd9866017e..39603385b4f3cf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -197,7 +197,7 @@ public void testSelect(@Mocked QueryStmt queryStmt, minTimes = 0; result = false; - queryStmt.getTables((Analyzer) any, (SortedMap) any, Sets.newHashSet()); + queryStmt.getTables((Analyzer) any, anyBoolean, (SortedMap) any, Sets.newHashSet()); minTimes = 0; queryStmt.getRedirectStatus(); diff --git a/regression-test/data/query/show/test_array_show_create.out b/regression-test/data/query/show/test_array_show_create.out index fd1da0cb7ad8ee..bda800f37e794f 100644 --- a/regression-test/data/query/show/test_array_show_create.out +++ b/regression-test/data/query/show/test_array_show_create.out @@ -1,4 +1,4 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select -- -test_array_show_create CREATE TABLE `test_array_show_create` (\n `k1` int(11) NULL,\n `k2` array NOT NULL,\n `k3` array NOT NULL,\n `k4` array NOT NULL,\n `k5` array NOT NULL,\n `k6` array NULL,\n `k7` array NOT NULL,\n `k8` array NOT NULL,\n `k9` array NOT NULL,\n `k10` array NOT NULL,\n `k11` array NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`k1`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"in_memory" = "false",\n"storage_format" = "V2",\n"disable_auto_compaction" = "false"\n) +test_array_show_create CREATE TABLE `test_array_show_create` (\n `k1` int(11) NULL,\n `k2` array NOT NULL,\n `k3` array NOT NULL,\n `k4` array NOT NULL,\n `k5` array NOT NULL,\n `k6` array NULL,\n `k7` array NOT NULL,\n `k8` array NOT NULL,\n `k9` array NOT NULL,\n `k10` array NOT NULL,\n `k11` array NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`k1`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"in_memory" = "false",\n"storage_format" = "V2",\n"disable_auto_compaction" = "false"\n); diff --git a/regression-test/suites/query/show/test_array_show_create.groovy b/regression-test/suites/query/show/test_array_show_create.groovy index b5692663f460f4..382e2700aeb031 100644 --- a/regression-test/suites/query/show/test_array_show_create.groovy +++ b/regression-test/suites/query/show/test_array_show_create.groovy @@ -42,7 +42,7 @@ suite("test_array_show_create", "query") { PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2" - ) + ); """ // DDL/DML return 1 row and 3 column, the only value is update row count @@ -69,4 +69,4 @@ suite("test_array_show_create", "query") { try_sql("DROP TABLE IF EXISTS ${testTable}") } -} \ No newline at end of file +}