From 7f74946186c4778d78e4222a46d7dcfa5e548d0b Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 22 Aug 2022 19:47:12 +0800 Subject: [PATCH 1/9] [feature](http) support get all create table stmt of a SQL --- .../httpv2/rest/StmtExecutionAction.java | 101 +++++++++++++++--- 1 file changed, 89 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java index d4f787132cfb5e..485fcc5d94b219 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java @@ -17,7 +17,14 @@ package org.apache.doris.httpv2.rest; -import org.apache.doris.common.DdlException; +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.QueryStmt; +import org.apache.doris.analysis.SqlParser; +import org.apache.doris.analysis.SqlScanner; +import org.apache.doris.analysis.StatementBase; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; import org.apache.doris.httpv2.util.ExecutionResultSet; import org.apache.doris.httpv2.util.StatementSubmitter; @@ -25,24 +32,34 @@ import org.apache.doris.system.SystemInfoService; import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; +import java.io.StringReader; import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; /** - * For execute stmt via http + * For execute stmt or get create table stmt via http */ @RestController public class StmtExecutionAction extends RestBaseController { @@ -51,23 +68,24 @@ public class StmtExecutionAction extends RestBaseController { private static final String PARAM_SYNC = "sync"; private static final String PARAM_LIMIT = "limit"; + private static final String PARAM_OPERATION = "operation"; private static final long DEFAULT_ROW_LIMIT = 1000; private static final long MAX_ROW_LIMIT = 10000; + private static final String OPERATION_EXECUTE = "execute"; + private static final String OPERATION_GET_SCHEMA = "get_schema"; + /** * Execute a SQL. * Request body: * { - * "stmt" : "select * from tbl1" + * "stmt" : "select * from tbl1" * } */ @RequestMapping(path = "/api/query/{" + NS_KEY + "}/{" + DB_KEY + "}", method = {RequestMethod.POST}) - public Object executeSQL( - @PathVariable(value = NS_KEY) String ns, - @PathVariable(value = DB_KEY) String dbName, - HttpServletRequest request, HttpServletResponse response, - @RequestBody String stmtBody) throws DdlException { + public Object executeSQL(@PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName, + HttpServletRequest request, HttpServletResponse response, @RequestBody String stmtBody) { ActionAuthorizationInfo authInfo = checkWithCookie(request, response, false); if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { @@ -97,10 +115,39 @@ public Object executeSQL( ConnectContext.get().setDatabase(getFullDbName(dbName)); - // 2. Submit stmt - StatementSubmitter.StmtContext stmtCtx = new StatementSubmitter.StmtContext( - stmtRequestBody.stmt, authInfo.fullUserName, authInfo.password, limit - ); + String operation = request.getParameter(PARAM_OPERATION); + if (Strings.isNullOrEmpty(operation)) { + operation = OPERATION_EXECUTE; + } + + if (operation.equalsIgnoreCase(OPERATION_EXECUTE)) { + return executeQuery(authInfo, isSync, limit, stmtRequestBody); + } else if (operation.equalsIgnoreCase(OPERATION_GET_SCHEMA)) { + return getSchema(stmtRequestBody); + } else { + return ResponseEntityBuilder.badRequest("Unknown operation: " + operation); + } + } + + /** + * return like + * { + * 10000 : "create table xxx", + * 10001 : "create table xxx" + * ... + * } + * + * @param authInfo + * @param isSync + * @param limit + * @param stmtRequestBody + * @return + */ + @NotNull + private ResponseEntity executeQuery(ActionAuthorizationInfo authInfo, boolean isSync, long limit, + StmtRequestBody stmtRequestBody) { + StatementSubmitter.StmtContext stmtCtx = new StatementSubmitter.StmtContext(stmtRequestBody.stmt, + authInfo.fullUserName, authInfo.password, limit); Future future = stmtSubmitter.submit(stmtCtx); if (isSync) { @@ -119,6 +166,36 @@ public Object executeSQL( } } + @NotNull + private ResponseEntity getSchema(StmtRequestBody stmtRequestBody) { + SqlParser parser = new SqlParser(new SqlScanner(new StringReader(stmtRequestBody.stmt))); + StatementBase stmt = null; + try { + stmt = SqlParserUtils.getStmt(parser, 0); + if (!(stmt instanceof QueryStmt)) { + return ResponseEntityBuilder.okWithCommonError("Only support query stmt"); + } + Analyzer analyzer = new Analyzer(Env.getCurrentEnv(), ConnectContext.get()); + QueryStmt queryStmt = (QueryStmt) stmt; + Map tableMap = Maps.newHashMap(); + Set parentViewNameSet = Sets.newHashSet(); + queryStmt.getTables(analyzer, tableMap, parentViewNameSet); + + Map resultMap = new HashMap<>(4); + for (TableIf tbl : tableMap.values()) { + List createTableStmts = Lists.newArrayList(); + Env.getDdlStmt(tbl, createTableStmts, null, null, false, true); + if (!createTableStmts.isEmpty()) { + resultMap.put(tbl.getId(), createTableStmts.get(0)); + } + } + + return ResponseEntityBuilder.ok(resultMap); + } catch (Exception e) { + return ResponseEntityBuilder.okWithCommonError("error happens when parsing the stmt: " + e.getMessage()); + } + } + private static class StmtRequestBody { public String stmt; } From 492123c14a9285110c1fe6fc38d81439ec35b307 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 22 Aug 2022 22:05:40 +0800 Subject: [PATCH 2/9] 2 --- .../org/apache/doris/analysis/InsertStmt.java | 2 +- .../org/apache/doris/analysis/QueryStmt.java | 13 +++--- .../org/apache/doris/analysis/SelectStmt.java | 29 +++++++----- .../doris/analysis/SetOperationStmt.java | 8 ++-- .../org/apache/doris/analysis/WithClause.java | 9 ++-- .../httpv2/rest/StmtExecutionAction.java | 44 +++++++------------ .../org/apache/doris/qe/StmtExecutor.java | 4 +- .../org/apache/doris/qe/StmtExecutorTest.java | 2 +- 8 files changed, 51 insertions(+), 60 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java index a15512906779f5..9a1aac62168ab2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java @@ -188,7 +188,7 @@ public String getTbl() { public void getTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) throws AnalysisException { // get dbs of statement - queryStmt.getTables(analyzer, tableMap, parentViewNameSet); + queryStmt.getTables(analyzer, false, tableMap, parentViewNameSet); tblName.analyze(analyzer); // disallow external catalog Util.prohibitExternalCatalog(tblName.getCtl(), this.getClass().getSimpleName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java index 7c4079dedd0ca7..03ee4048f98b9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java @@ -454,18 +454,17 @@ private Expr trySubstituteOrdinal(Expr expr, String errorPrefix, } if (pos > resultExprs.size()) { throw new AnalysisException( - errorPrefix + ": ordinal exceeds number of items in select list: " - + expr.toSql()); + errorPrefix + ": ordinal exceeds number of items in select list: " + expr.toSql()); } // Create copy to protect against accidentally shared state. return resultExprs.get((int) pos - 1).clone(); } - public void getWithClauseTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) - throws AnalysisException { + public void getWithClauseTables(Analyzer analyzer, boolean expandView, Map tableMap, + Set parentViewNameSet) throws AnalysisException { if (withClause != null) { - withClause.getTables(analyzer, tableMap, parentViewNameSet); + withClause.getTables(analyzer, expandView, tableMap, parentViewNameSet); } } @@ -542,8 +541,8 @@ public boolean containAlias(Expr expr) { // tmp in child stmt "(select siteid, citycode from tmp)" do not contain with_Clause // so need to check is view name by parentViewNameSet. // issue link: https://github.com/apache/doris/issues/4598 - public abstract void getTables(Analyzer analyzer, Map tables, Set parentViewNameSet) - throws AnalysisException; + public abstract void getTables(Analyzer analyzer, boolean expandView, Map tables, + Set parentViewNameSet) throws AnalysisException; // get TableRefs in this query, including physical TableRefs of this statement and // nested statements of inline views and with_Clause. diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java index 0fa4d0f9c76258..8e4ae4eaa26b4e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -292,18 +292,18 @@ public ExprSubstitutionMap getBaseTblSmap() { } @Override - public void getTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) - throws AnalysisException { - getWithClauseTables(analyzer, tableMap, parentViewNameSet); + public void getTables(Analyzer analyzer, boolean expandView, Map tableMap, + Set parentViewNameSet) throws AnalysisException { + getWithClauseTables(analyzer, expandView, tableMap, parentViewNameSet); for (TableRef tblRef : fromClause) { if (tblRef instanceof InlineViewRef) { // Inline view reference QueryStmt inlineStmt = ((InlineViewRef) tblRef).getViewStmt(); - inlineStmt.getTables(analyzer, tableMap, parentViewNameSet); + inlineStmt.getTables(analyzer, expandView, tableMap, parentViewNameSet); } else if (tblRef instanceof TableValuedFunctionRef) { TableValuedFunctionRef tblFuncRef = (TableValuedFunctionRef) tblRef; tableMap.put(tblFuncRef.getTableFunction().getTable().getId(), - tblFuncRef.getTableFunction().getTable()); + tblFuncRef.getTableFunction().getTable()); } else { String dbName = tblRef.getName().getDb(); String tableName = tblRef.getName().getTbl(); @@ -320,14 +320,19 @@ public void getTables(Analyzer analyzer, Map tableMap, Set getSetOpsResultExprs() { } @Override - public void getTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) - throws AnalysisException { - getWithClauseTables(analyzer, tableMap, parentViewNameSet); + public void getTables(Analyzer analyzer, boolean expandView, Map tableMap, + Set parentViewNameSet) throws AnalysisException { + getWithClauseTables(analyzer, expandView, tableMap, parentViewNameSet); for (SetOperand op : operands) { - op.getQueryStmt().getTables(analyzer, tableMap, parentViewNameSet); + op.getQueryStmt().getTables(analyzer, expandView, tableMap, parentViewNameSet); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java index e9e1e57d105cde..fb4fdbec6a2392 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java @@ -102,8 +102,7 @@ private WithClause(WithClause other) { Preconditions.checkNotNull(other); views = Lists.newArrayList(); for (View view : other.views) { - views.add(new View(view.getName(), view.getQueryStmt().clone(), - view.getOriginalColLabels())); + views.add(new View(view.getName(), view.getQueryStmt().clone(), view.getOriginalColLabels())); } } @@ -113,12 +112,12 @@ public void reset() { } } - public void getTables(Analyzer analyzer, Map tableMap, Set parentViewNameSet) - throws AnalysisException { + public void getTables(Analyzer analyzer, boolean expandView, Map tableMap, + Set parentViewNameSet) throws AnalysisException { for (View view : views) { QueryStmt stmt = view.getQueryStmt(); parentViewNameSet.add(view.getName()); - stmt.getTables(analyzer, tableMap, parentViewNameSet); + stmt.getTables(analyzer, expandView, tableMap, parentViewNameSet); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java index 485fcc5d94b219..ab5c202820f2b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java @@ -25,6 +25,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.util.SqlParserUtils; +import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.httpv2.entity.ResponseEntityBuilder; import org.apache.doris.httpv2.util.ExecutionResultSet; import org.apache.doris.httpv2.util.StatementSubmitter; @@ -85,47 +86,31 @@ public class StmtExecutionAction extends RestBaseController { */ @RequestMapping(path = "/api/query/{" + NS_KEY + "}/{" + DB_KEY + "}", method = {RequestMethod.POST}) public Object executeSQL(@PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName, - HttpServletRequest request, HttpServletResponse response, @RequestBody String stmtBody) { + HttpServletRequest request, HttpServletResponse response, @RequestBody String body) { ActionAuthorizationInfo authInfo = checkWithCookie(request, response, false); - if (!ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { - return ResponseEntityBuilder.badRequest("Only support 'default_cluster' now"); - } - - boolean isSync = true; - String syncParam = request.getParameter(PARAM_SYNC); - if (!Strings.isNullOrEmpty(syncParam)) { - isSync = syncParam.equals("1"); - } - - String limitParam = request.getParameter(PARAM_LIMIT); - long limit = DEFAULT_ROW_LIMIT; - if (!Strings.isNullOrEmpty(limitParam)) { - limit = Math.min(Long.valueOf(limitParam), MAX_ROW_LIMIT); + if (ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + ns = InternalCatalog.INTERNAL_CATALOG_NAME; } Type type = new TypeToken() { }.getType(); - StmtRequestBody stmtRequestBody = new Gson().fromJson(stmtBody, type); + StmtRequestBody stmtRequestBody = new Gson().fromJson(body, type); if (Strings.isNullOrEmpty(stmtRequestBody.stmt)) { return ResponseEntityBuilder.badRequest("Missing statement request body"); } - LOG.info("stmt: {}", stmtRequestBody.stmt); + LOG.info("stmt: {}, isSync:{}, limit: {}, operation: {}", stmtRequestBody.stmt, stmtRequestBody.isSync, + stmtRequestBody.limit, stmtRequestBody.operation); + ConnectContext.get().changeDefaultCatalog(ns); ConnectContext.get().setDatabase(getFullDbName(dbName)); - - String operation = request.getParameter(PARAM_OPERATION); - if (Strings.isNullOrEmpty(operation)) { - operation = OPERATION_EXECUTE; - } - - if (operation.equalsIgnoreCase(OPERATION_EXECUTE)) { - return executeQuery(authInfo, isSync, limit, stmtRequestBody); - } else if (operation.equalsIgnoreCase(OPERATION_GET_SCHEMA)) { + if (stmtRequestBody.operation.equalsIgnoreCase(OPERATION_EXECUTE)) { + return executeQuery(authInfo, stmtRequestBody.isSync, stmtRequestBody.limit, stmtRequestBody); + } else if (stmtRequestBody.operation.equalsIgnoreCase(OPERATION_GET_SCHEMA)) { return getSchema(stmtRequestBody); } else { - return ResponseEntityBuilder.badRequest("Unknown operation: " + operation); + return ResponseEntityBuilder.badRequest("Unknown operation: " + stmtRequestBody.operation); } } @@ -179,7 +164,7 @@ private ResponseEntity getSchema(StmtRequestBody stmtRequestBody) { QueryStmt queryStmt = (QueryStmt) stmt; Map tableMap = Maps.newHashMap(); Set parentViewNameSet = Sets.newHashSet(); - queryStmt.getTables(analyzer, tableMap, parentViewNameSet); + queryStmt.getTables(analyzer, true, tableMap, parentViewNameSet); Map resultMap = new HashMap<>(4); for (TableIf tbl : tableMap.values()) { @@ -197,6 +182,9 @@ private ResponseEntity getSchema(StmtRequestBody stmtRequestBody) { } private static class StmtRequestBody { + public Boolean isSync = true; + public Long limit = DEFAULT_ROW_LIMIT; + public String operation = OPERATION_EXECUTE; public String stmt; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 2dd7aa501f3e23..31b2c856db6b44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -630,11 +630,11 @@ public void analyze(TQueryOptions tQueryOptions) throws UserException { Set parentViewNameSet = Sets.newHashSet(); if (parsedStmt instanceof QueryStmt) { queryStmt = (QueryStmt) parsedStmt; - queryStmt.getTables(analyzer, tableMap, parentViewNameSet); + queryStmt.getTables(analyzer, false, tableMap, parentViewNameSet); } else if (parsedStmt instanceof CreateTableAsSelectStmt) { CreateTableAsSelectStmt parsedStmt = (CreateTableAsSelectStmt) this.parsedStmt; queryStmt = parsedStmt.getQueryStmt(); - queryStmt.getTables(analyzer, tableMap, parentViewNameSet); + queryStmt.getTables(analyzer, false, tableMap, parentViewNameSet); } else if (parsedStmt instanceof InsertStmt) { InsertStmt insertStmt = (InsertStmt) parsedStmt; insertStmt.getTables(analyzer, tableMap, parentViewNameSet); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index 4f20dd9866017e..ad2464bad654c4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -197,7 +197,7 @@ public void testSelect(@Mocked QueryStmt queryStmt, minTimes = 0; result = false; - queryStmt.getTables((Analyzer) any, (SortedMap) any, Sets.newHashSet()); + queryStmt.getTables((Analyzer) any, (Boolean) any, (SortedMap) any, Sets.newHashSet()); minTimes = 0; queryStmt.getRedirectStatus(); From 2f303d37c0391ffa6bec85d816568068af8f7ab2 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 22 Aug 2022 23:17:40 +0800 Subject: [PATCH 3/9] 3 --- .../java/org/apache/doris/catalog/Env.java | 9 +-- .../httpv2/rest/StmtExecutionAction.java | 74 ++++++++++--------- 2 files changed, 45 insertions(+), 38 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 060a044074a23b..2ab097399ca6ab 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -2703,8 +2703,7 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis if (table.getType() == TableType.VIEW) { View view = (View) table; sb.append("CREATE VIEW `").append(table.getName()).append("` AS ").append(view.getInlineViewDef()); - sb.append(";"); - createTableStmt.add(sb.toString()); + createTableStmt.add(sb + ";"); return; } @@ -3045,7 +3044,7 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis sb.append("\n)"); } - createTableStmt.add(sb.toString()); + createTableStmt.add(sb + ";"); // 2. add partition if (separatePartition && (table instanceof OlapTable) && ((OlapTable) table).getPartitions().size() > 1) { @@ -3077,7 +3076,7 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis sb.append("(\"version_info\" = \""); sb.append(partition.getVisibleVersion()).append("\""); sb.append(");"); - addPartitionStmt.add(sb.toString()); + addPartitionStmt.add(sb + ";"); } } } @@ -3104,7 +3103,7 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis } } sb.append(");"); - createRollupStmt.add(sb.toString()); + createRollupStmt.add(sb + ";"); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java index ab5c202820f2b2..31923dc817695f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java @@ -32,6 +32,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.system.SystemInfoService; +import com.google.common.base.Joiner; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -50,7 +51,6 @@ import java.io.StringReader; import java.lang.reflect.Type; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -67,21 +67,16 @@ public class StmtExecutionAction extends RestBaseController { private static final Logger LOG = LogManager.getLogger(StmtExecutionAction.class); private static StatementSubmitter stmtSubmitter = new StatementSubmitter(); - private static final String PARAM_SYNC = "sync"; - private static final String PARAM_LIMIT = "limit"; - private static final String PARAM_OPERATION = "operation"; - private static final long DEFAULT_ROW_LIMIT = 1000; private static final long MAX_ROW_LIMIT = 10000; - private static final String OPERATION_EXECUTE = "execute"; - private static final String OPERATION_GET_SCHEMA = "get_schema"; - /** * Execute a SQL. * Request body: * { - * "stmt" : "select * from tbl1" + * "is_sync": 1, // optional + * "limit" : 1000 // optional + * "stmt" : "select * from tbl1" // required * } */ @RequestMapping(path = "/api/query/{" + NS_KEY + "}/{" + DB_KEY + "}", method = {RequestMethod.POST}) @@ -100,27 +95,42 @@ public Object executeSQL(@PathVariable(value = NS_KEY) String ns, @PathVariable( if (Strings.isNullOrEmpty(stmtRequestBody.stmt)) { return ResponseEntityBuilder.badRequest("Missing statement request body"); } - LOG.info("stmt: {}, isSync:{}, limit: {}, operation: {}", stmtRequestBody.stmt, stmtRequestBody.isSync, - stmtRequestBody.limit, stmtRequestBody.operation); + LOG.info("stmt: {}, isSync:{}, limit: {}", stmtRequestBody.stmt, stmtRequestBody.is_sync, + stmtRequestBody.limit); ConnectContext.get().changeDefaultCatalog(ns); ConnectContext.get().setDatabase(getFullDbName(dbName)); - if (stmtRequestBody.operation.equalsIgnoreCase(OPERATION_EXECUTE)) { - return executeQuery(authInfo, stmtRequestBody.isSync, stmtRequestBody.limit, stmtRequestBody); - } else if (stmtRequestBody.operation.equalsIgnoreCase(OPERATION_GET_SCHEMA)) { - return getSchema(stmtRequestBody); - } else { - return ResponseEntityBuilder.badRequest("Unknown operation: " + stmtRequestBody.operation); + return executeQuery(authInfo, stmtRequestBody.is_sync, stmtRequestBody.limit, stmtRequestBody); + } + + + /** + * Get all create table stmt of a SQL + * + * @param ns + * @param dbName + * @param request + * @param response + * @param sql plain text of sql + * @return plain text of create table stmts + */ + @RequestMapping(path = "/api/query_schema/{" + NS_KEY + "}/{" + DB_KEY + "}", method = {RequestMethod.POST}) + public String querySchema(@PathVariable(value = NS_KEY) String ns, @PathVariable(value = DB_KEY) String dbName, + HttpServletRequest request, HttpServletResponse response, @RequestBody String sql) { + checkWithCookie(request, response, false); + + if (ns.equalsIgnoreCase(SystemInfoService.DEFAULT_CLUSTER)) { + ns = InternalCatalog.INTERNAL_CATALOG_NAME; } + LOG.info("sql: {}", sql); + + ConnectContext.get().changeDefaultCatalog(ns); + ConnectContext.get().setDatabase(getFullDbName(dbName)); + return getSchema(sql); } /** - * return like - * { - * 10000 : "create table xxx", - * 10001 : "create table xxx" - * ... - * } + * Execute a query * * @param authInfo * @param isSync @@ -152,13 +162,13 @@ private ResponseEntity executeQuery(ActionAuthorizationInfo authInfo, boolean is } @NotNull - private ResponseEntity getSchema(StmtRequestBody stmtRequestBody) { - SqlParser parser = new SqlParser(new SqlScanner(new StringReader(stmtRequestBody.stmt))); + private String getSchema(String sql) { + SqlParser parser = new SqlParser(new SqlScanner(new StringReader(sql))); StatementBase stmt = null; try { stmt = SqlParserUtils.getStmt(parser, 0); if (!(stmt instanceof QueryStmt)) { - return ResponseEntityBuilder.okWithCommonError("Only support query stmt"); + return "Only support query stmt"; } Analyzer analyzer = new Analyzer(Env.getCurrentEnv(), ConnectContext.get()); QueryStmt queryStmt = (QueryStmt) stmt; @@ -166,25 +176,23 @@ private ResponseEntity getSchema(StmtRequestBody stmtRequestBody) { Set parentViewNameSet = Sets.newHashSet(); queryStmt.getTables(analyzer, true, tableMap, parentViewNameSet); - Map resultMap = new HashMap<>(4); + List createStmts = Lists.newArrayList(); for (TableIf tbl : tableMap.values()) { List createTableStmts = Lists.newArrayList(); Env.getDdlStmt(tbl, createTableStmts, null, null, false, true); if (!createTableStmts.isEmpty()) { - resultMap.put(tbl.getId(), createTableStmts.get(0)); + createStmts.add(createTableStmts.get(0)); } } - - return ResponseEntityBuilder.ok(resultMap); + return Joiner.on("\n").join(createStmts); } catch (Exception e) { - return ResponseEntityBuilder.okWithCommonError("error happens when parsing the stmt: " + e.getMessage()); + return "Error:" + e.getMessage(); } } private static class StmtRequestBody { - public Boolean isSync = true; + public Boolean is_sync = true; // CHECKSTYLE IGNORE THIS LINE public Long limit = DEFAULT_ROW_LIMIT; - public String operation = OPERATION_EXECUTE; public String stmt; } } From b5dbc2fd13c25b83abf95ca7eb24b5e988bc48c8 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 22 Aug 2022 23:40:50 +0800 Subject: [PATCH 4/9] 5 --- .../http-actions/fe/query-schema-action.md | 108 ++++++++++++++++++ .../http-actions/fe/query-schema-action.md | 107 +++++++++++++++++ .../httpv2/rest/StmtExecutionAction.java | 2 +- 3 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 docs/en/docs/admin-manual/http-actions/fe/query-schema-action.md create mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/query-schema-action.md diff --git a/docs/en/docs/admin-manual/http-actions/fe/query-schema-action.md b/docs/en/docs/admin-manual/http-actions/fe/query-schema-action.md new file mode 100644 index 00000000000000..ccf9bdb3689aa2 --- /dev/null +++ b/docs/en/docs/admin-manual/http-actions/fe/query-schema-action.md @@ -0,0 +1,108 @@ +--- +{ + "title": "Query Schema Action", + "language": "en" +} +--- + + + +# Query Schema Action + + +## Request + +``` +POST /api/query_schema// +``` + +## Description + +The Query Schema Action can return the table creation statement for the given SQL-related table. Can be used to test some query scenarios locally. + +The API was released in version 1.2. + +## Path parameters + +* `` + + Specify the database name. This database will be considered as the default database for the current session, and will be used if the table name in SQL does not qualify the database name. + +## Query parameters + +None + +## Request body + +``` +text/plain + +sql +``` + +* "sql" field is the SQL string. + +## Response + +* Return value + + ``` + CREATE TABLE `tbl1` ( + `k1` int(11) NULL, + `k2` int(11) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ); + + CREATE TABLE `tbl2` ( + `k1` int(11) NULL, + `k2` int(11) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ); + ``` + +## Example + +1. Write the SQL in local file 1.sql + + ``` + select tbl1.k2 from tbl1 join tbl2 on tbl1.k1 = tbl2.k1; + ``` + +2. Use curl to get the table creation statement. + + ``` + curl -X POST -H 'Content-Type: text/plain' -uroot: http://127.0.0.1:8030/api/query_schema/internal/db1 -d@1.sql + ``` \ No newline at end of file diff --git a/docs/zh-CN/docs/admin-manual/http-actions/fe/query-schema-action.md b/docs/zh-CN/docs/admin-manual/http-actions/fe/query-schema-action.md new file mode 100644 index 00000000000000..b5855c008d0e5d --- /dev/null +++ b/docs/zh-CN/docs/admin-manual/http-actions/fe/query-schema-action.md @@ -0,0 +1,107 @@ +--- +{ + "title": "Query Schema Action", + "language": "zh-CN" +} +--- + + + +# Query Schema Action + + +## Request + +``` +POST /api/query_schema// +``` + +## Description + +Query Schema Action 可以返回给定的 SQL 有关的表的建表语句。可以用于本地测试一些查询场景。 +该 API 在 1.2 版本中发布。 + +## Path parameters + +* `` + + 指定数据库名称。该数据库会被视为当前session的默认数据库，如果在 SQL 中的表名没有限定数据库名称的话，则使用该数据库。 + +## Query parameters + +无 + +## Request body + +``` +text/plain + +sql +``` + +* sql 字段为具体的 SQL + +## Response + +* 返回结果集 + + ``` + CREATE TABLE `tbl1` ( + `k1` int(11) NULL, + `k2` int(11) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ); + + CREATE TABLE `tbl2` ( + `k1` int(11) NULL, + `k2` int(11) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`k1`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false" + ); + ``` + +## Example + +1. 在本地文件 1.sql 中写入 SQL + + ``` + select tbl1.k2 from tbl1 join tbl2 on tbl1.k1 = tbl2.k1; + ``` + +2. 使用 curl 命令获取建表语句 + + ``` + curl -X POST -H 'Content-Type: text/plain' -uroot: http://127.0.0.1:8030/api/query_schema/internal/db1 -d@1.sql + ``` \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java index 31923dc817695f..34ebae24153edb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StmtExecutionAction.java @@ -184,7 +184,7 @@ private String getSchema(String sql) { createStmts.add(createTableStmts.get(0)); } } - return Joiner.on("\n").join(createStmts); + return Joiner.on("\n\n").join(createStmts); } catch (Exception e) { return "Error:" + e.getMessage(); } From 6ce63a14d30e007d518598d2d75b21fea7fa7d31 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 23 Aug 2022 00:12:04 +0800 Subject: [PATCH 5/9] fix ut --- .../analysis/CreateTableAsSelectStmtTest.java | 45 +++++++++---------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index 28c09dc57b8dd4..626b6ca2f9c655 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -71,11 +71,10 @@ public void testDecimal() throws Exception { createTableAsSelect(selectFromDecimal); Assertions.assertEquals("CREATE TABLE `select_decimal_table` (\n" + " `userId` varchar(255) NOT NULL,\n" + " `amount_decimal` decimal(10, 2) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" - + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" - + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"," - + "\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showCreateTableByName("select_decimal_table").getResultRows().get(0).get(1)); String selectFromDecimal1 = "create table `test`.`select_decimal_table_1` PROPERTIES(\"replication_num\" = \"1\") " @@ -88,7 +87,7 @@ public void testDecimal() throws Exception { + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"," - + "\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1)); } else { Assertions.assertEquals( @@ -97,7 +96,7 @@ public void testDecimal() throws Exception { + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"," - + "\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1)); } } @@ -119,11 +118,10 @@ public void testVarchar() throws Exception { createTableAsSelect(selectFromVarchar); ShowResultSet showResultSet = showCreateTableByName("select_varchar"); Assertions.assertEquals("CREATE TABLE `select_varchar` (\n" + " `userId` varchar(255) NOT NULL,\n" - + " `username` varchar(255) REPLACE NOT NULL\n" - + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + + " `username` varchar(255) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -138,7 +136,7 @@ public void testFunction() throws Exception { + "DUPLICATE KEY(`_col0`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"" - + ",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + ",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); String selectFromFunction2 = "create table `test`.`select_function_2` PROPERTIES(\"replication_num\" = \"1\") " @@ -153,7 +151,7 @@ public void testFunction() throws Exception { + "DUPLICATE KEY(`_col0`, `_col1`, `_col2`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`_col0`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet2.getResultRows().get(0).get(1)); } @@ -167,17 +165,16 @@ public void testAlias() throws Exception { + "DUPLICATE KEY(`amount`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`amount`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"," - + "\n\"disable_auto_compaction\" = \"false\"\n" + ")", showResultSet1.getResultRows().get(0).get(1)); + + "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); String selectAlias2 = "create table `test`.`select_alias_2` PROPERTIES(\"replication_num\" = \"1\") " + "as select userId as alias_name, username from `test`.`varchar_table`"; createTableAsSelect(selectAlias2); ShowResultSet showResultSet2 = showCreateTableByName("select_alias_2"); Assertions.assertEquals("CREATE TABLE `select_alias_2` (\n" + " `alias_name` varchar(255) NOT NULL,\n" - + " `username` varchar(255) REPLACE NOT NULL\n" - + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`alias_name`)\n" + + " `username` varchar(255) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`alias_name`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`alias_name`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet2.getResultRows().get(0).get(1)); } @@ -193,7 +190,7 @@ public void testJoin() throws Exception { + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); String selectFromJoin1 = "create table `test`.`select_join1` PROPERTIES(\"replication_num\" = \"1\") " + "as select vt.userId as userId1, jt.userId as userId2, vt.username, jt.status " @@ -202,11 +199,10 @@ public void testJoin() throws Exception { ShowResultSet showResultSet1 = showCreateTableByName("select_join1"); Assertions.assertEquals("CREATE TABLE `select_join1` (\n" + " `userId1` varchar(255) NOT NULL,\n" + " `userId2` varchar(255) NOT NULL,\n" + " `username` varchar(255) REPLACE NOT NULL,\n" - + " `status` int(11) REPLACE NOT NULL\n" - + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId1`, `userId2`)\n" + + " `status` int(11) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId1`, `userId2`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId1`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); } @@ -223,7 +219,7 @@ public void testName() throws Exception { + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`user`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`user`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -237,7 +233,7 @@ public void testUnion() throws Exception { + "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\"" - + " = \"false\"\n" + ")", showResultSet.getResultRows().get(0).get(1)); + + " = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @Test @@ -251,7 +247,7 @@ public void testCte() throws Exception { + "DUPLICATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); String selectFromCteAndUnion = "create table `test`.`select_cte_union` PROPERTIES(\"replication_num\" = \"1\")" + "as with source_data as (select 1 as id union all select 2 as id) select * from source_data;"; @@ -261,7 +257,7 @@ public void testCte() throws Exception { + "DUPLICATE KEY(`id`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`id`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\"," - + "\n\"disable_auto_compaction\" = \"false\"\n" + ")", showResultSet1.getResultRows().get(0).get(1)); + + "\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); } @Test @@ -272,13 +268,12 @@ public void testPartition() throws Exception { createTableAsSelect(selectFromPartition); ShowResultSet showResultSet = showCreateTableByName("selectPartition"); Assertions.assertEquals("CREATE TABLE `selectPartition` (\n" + " `userId` varchar(255) NOT NULL,\n" - + " `username` varchar(255) REPLACE NOT NULL\n" - + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + + " `username` varchar(255) REPLACE NOT NULL\n" + ") ENGINE=OLAP\n" + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "PARTITION BY LIST(`userId`)\n" + "(PARTITION p1 VALUES IN (\"CA\",\"GB\",\"US\",\"ZH\"))\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" - + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ")", + + "\"storage_format\" = \"V2\",\n\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } From 98327f01683e807c1a91c19a5acbf81a9d5cc4bd Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 23 Aug 2022 00:22:53 +0800 Subject: [PATCH 6/9] add sidebar --- docs/sidebars.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/sidebars.json b/docs/sidebars.json index 4646c6a1c25468..40e467f6945e14 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -938,7 +938,8 @@ "admin-manual/http-actions/fe/show-proc-action", "admin-manual/http-actions/fe/check-decommission-action", "admin-manual/http-actions/fe/health-action", - "admin-manual/http-actions/fe/check-storage-type-action" + "admin-manual/http-actions/fe/check-storage-type-action", + "admin-manual/http-actions/fe/query-schema-action" ] }, "admin-manual/http-actions/restore-tablet", From c1ea805344bdbed413d19ace678144f4a532a34e Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 23 Aug 2022 07:30:57 +0800 Subject: [PATCH 7/9] fix ut --- .../src/test/java/org/apache/doris/qe/StmtExecutorTest.java | 2 +- regression-test/data/query/show/test_array_show_create.out | 2 +- .../suites/query/show/test_array_show_create.groovy | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java index ad2464bad654c4..39603385b4f3cf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/StmtExecutorTest.java @@ -197,7 +197,7 @@ public void testSelect(@Mocked QueryStmt queryStmt, minTimes = 0; result = false; - queryStmt.getTables((Analyzer) any, (Boolean) any, (SortedMap) any, Sets.newHashSet()); + queryStmt.getTables((Analyzer) any, anyBoolean, (SortedMap) any, Sets.newHashSet()); minTimes = 0; queryStmt.getRedirectStatus(); diff --git a/regression-test/data/query/show/test_array_show_create.out b/regression-test/data/query/show/test_array_show_create.out index fd1da0cb7ad8ee..bda800f37e794f 100644 --- a/regression-test/data/query/show/test_array_show_create.out +++ b/regression-test/data/query/show/test_array_show_create.out @@ -1,4 +1,4 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select -- -test_array_show_create CREATE TABLE `test_array_show_create` (\n `k1` int(11) NULL,\n `k2` array NOT NULL,\n `k3` array NOT NULL,\n `k4` array NOT NULL,\n `k5` array NOT NULL,\n `k6` array NULL,\n `k7` array NOT NULL,\n `k8` array NOT NULL,\n `k9` array NOT NULL,\n `k10` array NOT NULL,\n `k11` array NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`k1`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"in_memory" = "false",\n"storage_format" = "V2",\n"disable_auto_compaction" = "false"\n) +test_array_show_create CREATE TABLE `test_array_show_create` (\n `k1` int(11) NULL,\n `k2` array NOT NULL,\n `k3` array NOT NULL,\n `k4` array NOT NULL,\n `k5` array NOT NULL,\n `k6` array NULL,\n `k7` array NOT NULL,\n `k8` array NOT NULL,\n `k9` array NOT NULL,\n `k10` array NOT NULL,\n `k11` array NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`k1`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY HASH(`k1`) BUCKETS 1\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"in_memory" = "false",\n"storage_format" = "V2",\n"disable_auto_compaction" = "false"\n); diff --git a/regression-test/suites/query/show/test_array_show_create.groovy b/regression-test/suites/query/show/test_array_show_create.groovy index b5692663f460f4..382e2700aeb031 100644 --- a/regression-test/suites/query/show/test_array_show_create.groovy +++ b/regression-test/suites/query/show/test_array_show_create.groovy @@ -42,7 +42,7 @@ suite("test_array_show_create", "query") { PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "storage_format" = "V2" - ) + ); """ // DDL/DML return 1 row and 3 column, the only value is update row count @@ -69,4 +69,4 @@ suite("test_array_show_create", "query") { try_sql("DROP TABLE IF EXISTS ${testTable}") } -} \ No newline at end of file +} From 2d862959a5f381c6cf42434499d183f44259911d Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 23 Aug 2022 07:50:53 +0800 Subject: [PATCH 8/9] add ut --- .../analysis/CreateTableAsSelectStmtTest.java | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index 626b6ca2f9c655..08abd29e0a6dd4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -17,15 +17,28 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.ExceptionChecker; +import org.apache.doris.common.util.SqlParserUtils; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ShowResultSet; import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.junit.Assert; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.io.StringReader; +import java.util.List; +import java.util.Map; +import java.util.Set; + /** * test for CreateTableAsSelectStmt. **/ @@ -291,4 +304,53 @@ public void testDefaultTimestamp() throws Exception { + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\",\n" + "\"disable_auto_compaction\" = \"false\"\n" + ")", showResultSet.getResultRows().get(0).get(1)); } + + @Test + public void testQuerySchema() throws Exception { + connectContext.setDatabase("default_cluster:test"); + String create1 = "create table test.qs1 (k1 int, k2 int) distributed by hash(k1) " + + "buckets 1 properties('replication_num' = '1')"; + String create2 = "create table test.qs2 (k1 int, k2 int) distributed by hash(k1) " + + "buckets 1 properties('replication_num' = '1')"; + createTables(create1, create2); + + String view1 = "create view test.v1 as select qs1.k1 from qs1 join qs2 on qs1.k1 = qs2.k1"; + String view2 = "create view test.v2 as with cte(s1) as (select * from v1) select * from cte"; + + createView(view1); + createView(view2); + + String sql1 = "select * from v1"; + + org.apache.doris.analysis.SqlParser parser = new SqlParser( + new org.apache.doris.analysis.SqlScanner(new StringReader(sql1))); + QueryStmt stmt = (QueryStmt) SqlParserUtils.getStmt(parser, 0); + Map tableMap = Maps.newHashMap(); + Set parentViewNameSet = Sets.newHashSet(); + Analyzer analyzer = new Analyzer(Env.getCurrentEnv(), ConnectContext.get()); + stmt.getTables(analyzer, true, tableMap, parentViewNameSet); + + List createStmts = Lists.newArrayList(); + for (TableIf tbl : tableMap.values()) { + List createTableStmts = Lists.newArrayList(); + Env.getDdlStmt(tbl, createTableStmts, null, null, false, true); + createStmts.add(createTableStmts.get(0)); + if (tbl.getName().equals("qs1")) { + Assert.assertEquals("CREATE TABLE `qs1` (\n" + " `k1` int(11) NULL,\n" + " `k2` int(11) NULL\n" + + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`k1`, `k2`)\n" + "COMMENT 'OLAP'\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n" + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\",\n" + "\"disable_auto_compaction\" = \"false\"\n" + ");", + createTableStmts.get(0)); + } else { + Assert.assertEquals("CREATE TABLE `qs2` (\n" + " `k1` int(11) NULL,\n" + " `k2` int(11) NULL\n" + + ") ENGINE=OLAP\n" + "DUPLICATE KEY(`k1`, `k2`)\n" + "COMMENT 'OLAP'\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 1\n" + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\",\n" + "\"disable_auto_compaction\" = \"false\"\n" + ");", + createTableStmts.get(0)); + } + } + Assert.assertEquals(2, createStmts.size()); + } } From 61a3e42192cb9d514b1f62dc1f3936259163c080 Mon Sep 17 00:00:00 2001 From: morningman Date: Wed, 24 Aug 2022 09:03:21 +0800 Subject: [PATCH 9/9] rebase --- .../org/apache/doris/analysis/CreateTableAsSelectStmtTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index 08abd29e0a6dd4..df6354dff2e172 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -302,7 +302,7 @@ public void testDefaultTimestamp() throws Exception { + "AGGREGATE KEY(`userId`)\n" + "COMMENT 'OLAP'\n" + "DISTRIBUTED BY HASH(`userId`) BUCKETS 10\n" + "PROPERTIES (\n" + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + "\"in_memory\" = \"false\",\n" + "\"storage_format\" = \"V2\",\n" - + "\"disable_auto_compaction\" = \"false\"\n" + ")", showResultSet.getResultRows().get(0).get(1)); + + "\"disable_auto_compaction\" = \"false\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @Test