diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 5112e6b5a94f65..ac7e6f841b198e 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -602,6 +602,11 @@ under the License. lombok + + hu.webarticum + tree-printer + + diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 615b9459279142..1d8c5ec013e384 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -238,7 +238,7 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALL, KW_ALTER, KW_A KW_ELSE, KW_ENABLE, KW_END, KW_ENGINE, KW_ENGINES, KW_ENTER, KW_ERRORS, KW_EVENTS, KW_EXCEPT, KW_EXCLUDE, KW_EXISTS, KW_EXPORT, KW_EXTERNAL, KW_EXTRACT, KW_FALSE, KW_FEATURE, KW_FOLLOWER, KW_FOLLOWING, KW_FREE, KW_FROM, KW_FILE, KW_FILTER, KW_FIRST, KW_FLOAT, KW_FOR, KW_FORCE, KW_FORMAT, KW_FRONTEND, KW_FRONTENDS, KW_FULL, KW_FUNCTION, KW_FUNCTIONS, - KW_GLOBAL, KW_GRANT, KW_GRANTS, KW_GROUP, KW_GROUPING, + KW_GLOBAL, KW_GRANT, KW_GRANTS, KW_GRAPH, KW_GROUP, KW_GROUPING, KW_HASH, KW_HAVING, KW_HDFS, KW_HELP,KW_HLL, KW_HLL_UNION, KW_HOUR, KW_HUB, KW_IDENTIFIED, KW_IF, KW_IN, KW_INDEX, KW_INDEXES, KW_INFILE, KW_INSTALL, KW_INNER, KW_INSERT, KW_INT, KW_INTERMEDIATE, KW_INTERSECT, KW_INTERVAL, KW_INTO, KW_IS, KW_ISNULL, KW_ISOLATION, @@ -252,7 +252,7 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALL, KW_ALTER, KW_A KW_PARTITION, KW_PARTITIONS, KW_PASSWORD, KW_PATH, KW_PAUSE, KW_PIPE, KW_PRECEDING, KW_PLUGIN, KW_PLUGINS, KW_PRIMARY, - KW_PROC, KW_PROCEDURE, KW_PROCESSLIST, KW_PROPERTIES, KW_PROPERTY, + KW_PROC, KW_PROCEDURE, KW_PROCESSLIST, KW_PROFILE, KW_PROPERTIES, KW_PROPERTY, KW_QUERY, KW_QUOTA, KW_RANDOM, KW_RANGE, KW_READ, KW_RECOVER, KW_REGEXP, KW_RELEASE, KW_RENAME, KW_REPAIR, KW_REPEATABLE, KW_REPOSITORY, KW_REPOSITORIES, KW_REPLACE, KW_REPLACE_IF_NOT_NULL, KW_REPLICA, KW_RESOURCE, KW_RESOURCES, KW_RESTORE, KW_RETURNS, KW_RESUME, KW_REVOKE, @@ -478,7 +478,7 @@ nonterminal IndexDef.IndexType opt_index_type; nonterminal ShowAlterStmt.AlterType opt_alter_type; nonterminal Boolean opt_builtin; -nonterminal Boolean opt_verbose; +nonterminal ExplainOptions opt_explain_options; nonterminal Boolean opt_tmp; @@ -2494,6 +2494,10 @@ show_param ::= {: RESULT = new ShowTransactionStmt(dbName, parser.where); :} + | KW_QUERY KW_PROFILE STRING_LITERAL:queryIdPath + {: + RESULT = new ShowQueryProfileStmt(queryIdPath); + :} ; opt_tmp ::= @@ -2633,13 +2637,17 @@ opt_builtin ::= :} ; -opt_verbose ::= +opt_explain_options ::= {: - RESULT = false; + RESULT = new ExplainOptions(false, false); :} | KW_VERBOSE {: - RESULT = true; + RESULT = new ExplainOptions(true, false); + :} + | KW_GRAPH + {: + RESULT = new ExplainOptions(false, true); :} ; @@ -2653,14 +2661,14 @@ describe_stmt ::= {: RESULT = new DescribeStmt(table, true); :} - | describe_command opt_verbose:isVerbose query_stmt:query + | describe_command opt_explain_options:options query_stmt:query {: - query.setIsExplain(true, isVerbose); + query.setIsExplain(options); RESULT = query; :} | describe_command insert_stmt:stmt {: - stmt.getQueryStmt().setIsExplain(true); + stmt.getQueryStmt().setIsExplain(new ExplainOptions(true, false)); RESULT = stmt; :} ; @@ -4735,6 +4743,8 @@ keyword ::= {: RESULT = id; :} | KW_PATH:id {: RESULT = id; :} + | KW_PROFILE:id + {: RESULT = id; :} | KW_FUNCTION:id {: RESULT = id; :} | KW_END:id @@ -4753,6 +4763,8 @@ keyword ::= {: RESULT = id; :} | KW_GLOBAL:id {: RESULT = id; :} + | KW_GRAPH:id + {: RESULT = id; :} | KW_HASH:id {: RESULT = id; :} | KW_HELP:id diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExplainOptions.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExplainOptions.java new file mode 100644 index 00000000000000..543fd5bbae6421 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExplainOptions.java @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +public class ExplainOptions { + + private boolean isVerbose; + private boolean isGraph; + + public ExplainOptions(boolean isVerbose, boolean isGraph) { + this.isVerbose = isVerbose; + this.isGraph = isGraph; + } + + public boolean isVerbose() { + return isVerbose; + } + + public boolean isGraph() { + return isGraph; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java index a53b2e3bae3825..43875f573e4981 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java @@ -509,12 +509,12 @@ public AssertNumRowsElement getAssertNumRowsElement() { return assertNumRowsElement; } - public void setIsExplain(boolean isExplain) { - this.isExplain = isExplain; + public void setIsExplain(ExplainOptions options) { + this.explainOptions = options; } public boolean isExplain() { - return isExplain; + return this.explainOptions != null; } public boolean hasLimitClause() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java new file mode 100644 index 00000000000000..ae6a7ffb6eafbb --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowQueryProfileStmt.java @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.UserException; +import org.apache.doris.qe.ShowResultSetMetaData; + +import com.google.common.base.Strings; + +// For stmt like: +// show query profile "/"; # list all saving query ids +// show query profile "/e0f7390f5363419e-b416a2a79996083e" # show graph of fragments of the query +// show query profile "/e0f7390f5363419e-b416a2a79996083e/0" # show instance list of the specified fragment +// show query profile "/e0f7390f5363419e-b416a2a79996083e/0/e0f7390f5363419e-b416a2a799960906" # show graph of the instance +public class ShowQueryProfileStmt extends ShowStmt { + // This should be same as ProfileManager.PROFILE_HEADERS + private static final ShowResultSetMetaData META_DATA_QUERYIDS = + ShowResultSetMetaData.builder() + .addColumn(new Column("QueryId", ScalarType.createVarchar(128))) + .addColumn(new Column("User", ScalarType.createVarchar(128))) + .addColumn(new Column("DefaultDb", ScalarType.createVarchar(128))) + .addColumn(new Column("SQL", ScalarType.createVarchar(65535))) + .addColumn(new Column("QueryType", ScalarType.createVarchar(128))) + .addColumn(new Column("StartTime", ScalarType.createVarchar(128))) + .addColumn(new Column("EndTime", ScalarType.createVarchar(128))) + .addColumn(new Column("TotalTime", ScalarType.createVarchar(128))) + .addColumn(new Column("QueryState", ScalarType.createVarchar(128))) + .build(); + + private static final ShowResultSetMetaData META_DATA_FRAGMENTS = + ShowResultSetMetaData.builder() + .addColumn(new Column("Fragments", ScalarType.createVarchar(65535))) + .build(); + private static final ShowResultSetMetaData META_DATA_INSTANCES = + ShowResultSetMetaData.builder() + .addColumn(new Column("Instances", ScalarType.createVarchar(128))) + .addColumn(new Column("Host", ScalarType.createVarchar(64))) + .addColumn(new Column("ActiveTime", ScalarType.createVarchar(64))) + .build(); + private static final ShowResultSetMetaData META_DATA_SINGLE_INSTANCE = + ShowResultSetMetaData.builder() + .addColumn(new Column("Instance", ScalarType.createVarchar(65535))) + .build(); + + public enum PathType { + QUERY_IDS, + FRAGMETNS, + INSTANCES, + SINGLE_INSTANCE + } + + private String queryIdPath; + private PathType pathType; + + private String queryId = ""; + private String fragmentId = ""; + private String instanceId = ""; + + public ShowQueryProfileStmt(String queryIdPath) { + this.queryIdPath = queryIdPath; + } + + public PathType getPathType() { + return pathType; + } + + public String getQueryId() { + return queryId; + } + + public String getFragmentId() { + return fragmentId; + } + + public String getInstanceId() { + return instanceId; + } + + @Override + public void analyze(Analyzer analyzer) throws UserException { + super.analyze(analyzer); + if (Strings.isNullOrEmpty(queryIdPath)) { + // list all query ids + pathType = PathType.QUERY_IDS; + return; + } + + if (!queryIdPath.startsWith("/")) { + throw new AnalysisException("Query path must starts with '/'"); + } + pathType = PathType.QUERY_IDS; + String[] parts = queryIdPath.split("/"); + if (parts.length > 4) { + throw new AnalysisException("Query path must in format '/queryId/fragmentId/instanceId'"); + } + + for (int i = 0; i < parts.length; i++) { + switch (i) { + case 0: + pathType = PathType.QUERY_IDS; + continue; + case 1: + queryId = parts[i]; + pathType = PathType.FRAGMETNS; + break; + case 2: + fragmentId = parts[i]; + pathType = PathType.INSTANCES; + break; + case 3: + instanceId = parts[i]; + pathType = PathType.SINGLE_INSTANCE; + break; + default: + break; + } + } + } + + @Override + public String toSql() { + StringBuilder sb = new StringBuilder("SHOW QUERY PROFILE ").append(queryIdPath); + return sb.toString(); + } + + @Override + public String toString() { + return toSql(); + } + + @Override + public ShowResultSetMetaData getMetaData() { + switch (pathType) { + case QUERY_IDS: + return META_DATA_QUERYIDS; + case FRAGMETNS: + return META_DATA_FRAGMENTS; + case INSTANCES: + return META_DATA_INSTANCES; + case SINGLE_INSTANCE: + return META_DATA_SINGLE_INSTANCE; + default: + return null; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java index 4fb5f64dd32e3f..79047ac111731d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StatementBase.java @@ -35,10 +35,8 @@ public abstract class StatementBase implements ParseNode { private String clusterName; - // True if this QueryStmt is the top level query from an EXPLAIN - protected boolean isExplain = false; - // True if the describe_stmt print verbose information, if `isVerbose` is true, `isExplain` must be set to true. - protected boolean isVerbose = false; + // Set this variable if this QueryStmt is the top level query from an EXPLAIN + protected ExplainOptions explainOptions = null; ///////////////////////////////////////// // BEGIN: Members that need to be reset() @@ -60,7 +58,7 @@ protected StatementBase() { } */ protected StatementBase(StatementBase other) { analyzer = other.analyzer; - isExplain = other.isExplain; + explainOptions = other.explainOptions; } /** @@ -72,7 +70,6 @@ protected StatementBase(StatementBase other) { */ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { if (isAnalyzed()) return; - if (isExplain) analyzer.setIsExplain(); this.analyzer = analyzer; if (Strings.isNullOrEmpty(analyzer.getClusterName())) { ErrorReport.reportAnalysisException(ErrorCode.ERR_CLUSTER_NO_SELECT_CLUSTER); @@ -80,14 +77,33 @@ public void analyze(Analyzer analyzer) throws AnalysisException, UserException { this.clusterName = analyzer.getClusterName(); } - public Analyzer getAnalyzer() { return analyzer; } - public boolean isAnalyzed() { return analyzer != null; } - public void setIsExplain(boolean isExplain, boolean isVerbose) { this.isExplain = isExplain; this.isVerbose = isVerbose;} - public boolean isExplain() { return isExplain; } - public boolean isVerbose() { return isVerbose; } + public Analyzer getAnalyzer() { + return analyzer; + } + + public boolean isAnalyzed() { + return analyzer != null; + } + + public void setIsExplain(ExplainOptions options) { + this.explainOptions = options; + } + + public boolean isExplain() { + return this.explainOptions != null; + } + + public boolean isVerbose() { + return explainOptions != null && explainOptions.isVerbose(); + } + + public ExplainOptions getExplainOptions() { + return explainOptions; + } + /* * Print SQL syntax corresponding to this node. - * + * * @see org.apache.doris.parser.ParseNode#toSql() */ @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/CounterNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/CounterNode.java new file mode 100644 index 00000000000000..86f2d1a0104aa3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/CounterNode.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.profile; + +import org.apache.doris.common.Pair; +import org.apache.doris.common.TreeNode; + +public class CounterNode extends TreeNode { + private Pair counter; + + public void setCounter(String key, String value) { + counter = Pair.create(key, value); + } + + public String toTree(int indent) { + StringBuilder sb = new StringBuilder(); + sb.append(debugString(indent)); + for (CounterNode node : getChildren()) { + sb.append("\n").append(node.debugString(indent + 4)); + } + return sb.toString(); + } + + public String debugString(int indent) { + if (counter == null) { + return printIndent(indent) + " - Counters:"; + } + return printIndent(indent) + " - " + counter.first + ": " + counter.second; + } + + private String printIndent(int indent) { + String res = ""; + for (int i = 0; i < indent; i++) { + res += " "; + } + return res; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecNodeNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecNodeNode.java new file mode 100644 index 00000000000000..4cc12b1be7b3d9 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecNodeNode.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.profile; + +public class ExecNodeNode extends ProfileTreeNode { + + public ExecNodeNode(String name, String id) { + super(name, id); + } + + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/PlanTreeBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/PlanTreeBuilder.java new file mode 100644 index 00000000000000..39c92c840905b7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/PlanTreeBuilder.java @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.profile; + +import org.apache.doris.common.UserException; +import org.apache.doris.planner.DataSink; +import org.apache.doris.planner.ExchangeNode; +import org.apache.doris.planner.PlanFragment; +import org.apache.doris.planner.PlanNode; +import org.apache.doris.planner.PlanNodeId; +import org.apache.doris.thrift.TExplainLevel; + +import com.clearspring.analytics.util.Lists; + +import java.util.List; + +public class PlanTreeBuilder { + + private List fragments; + private PlanTreeNode treeRoot; + private List sinkNodes = Lists.newArrayList(); + private List exchangeNodes = Lists.newArrayList(); + + public PlanTreeBuilder(List fragments) { + this.fragments = fragments; + } + + public PlanTreeNode getTreeRoot() { + return treeRoot; + } + + public void build() throws UserException { + buildFragmentPlans(); + assembleFragmentPlans(); + } + + private void buildFragmentPlans() { + int i = 0; + for (PlanFragment fragment : fragments) { + DataSink sink = fragment.getSink(); + PlanTreeNode sinkNode = null; + if (sink != null) { + StringBuilder sb = new StringBuilder(); + sb.append("[").append(sink.getExchNodeId().asInt()).append(": ").append(sink.getClass().getSimpleName()).append("]"); + sb.append("\nFragment: ").append(fragment.getId().asInt()); + sb.append("\n").append(sink.getExplainString("", TExplainLevel.BRIEF)); + sinkNode = new PlanTreeNode(sink.getExchNodeId(), sb.toString()); + if (i == 0) { + // sink of first fragment, set it as tree root + treeRoot = sinkNode; + } else { + sinkNodes.add(sinkNode); + } + } + + PlanNode planRoot = fragment.getPlanRoot(); + if (planRoot != null) { + buildForPlanNode(planRoot, sinkNode); + } + i++; + } + } + + private void assembleFragmentPlans() throws UserException { + for (PlanTreeNode sender : sinkNodes) { + if (sender == treeRoot) { + // This is the result sink, skip it + continue; + } + PlanNodeId senderId = sender.getId(); + PlanTreeNode exchangeNode = findExchangeNode(senderId); + if (exchangeNode == null) { + throw new UserException("Failed to find exchange node for sender id: " + senderId.asInt()); + } + + exchangeNode.addChild(sender); + } + } + + private PlanTreeNode findExchangeNode(PlanNodeId senderId) { + for (PlanTreeNode exchangeNode : exchangeNodes) { + if (exchangeNode.getId().equals(senderId)) { + return exchangeNode; + } + } + return null; + } + + private void buildForPlanNode(PlanNode planNode, PlanTreeNode parent) { + PlanTreeNode node = new PlanTreeNode(planNode.getId(), planNode.toString()); + + if (parent != null) { + parent.addChild(node); + } + + if (planNode.getPlanNodeName().equals(ExchangeNode.EXCHANGE_NODE) + || planNode.getPlanNodeName().equals(ExchangeNode.MERGING_EXCHANGE_NODE)) { + exchangeNodes.add(node); + } else { + // Do not traverse children of exchange node, + // They will be visited in other fragments. + for (PlanNode child : planNode.getChildren()) { + buildForPlanNode(child, node); + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/PlanTreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/PlanTreeNode.java new file mode 100644 index 00000000000000..def1976e46c872 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/PlanTreeNode.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.profile; + +import org.apache.doris.common.TreeNode; +import org.apache.doris.planner.PlanNodeId; + +public class PlanTreeNode extends TreeNode { + private PlanNodeId id; + private String explainStr; + + public PlanTreeNode(PlanNodeId id, String explainStr) { + this.id = id; + this.explainStr = explainStr; + } + + public PlanNodeId getId() { + return id; + } + + public String getExplainStr() { + return explainStr; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/PlanTreePrinter.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/PlanTreePrinter.java new file mode 100644 index 00000000000000..a4d9b6d3b5b10f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/PlanTreePrinter.java @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.profile; + +import hu.webarticum.treeprinter.BorderTreeNodeDecorator; +import hu.webarticum.treeprinter.SimpleTreeNode; +import hu.webarticum.treeprinter.TraditionalTreePrinter; + +public class PlanTreePrinter { + + public static String printPlanExplanation(PlanTreeNode root) { + SimpleTreeNode rootNode = buildNode(root); + StringBuilder sb = new StringBuilder(); + new TraditionalTreePrinter().print(new BorderTreeNodeDecorator(rootNode), sb); + return sb.toString(); + } + + private static SimpleTreeNode buildNode(PlanTreeNode planNode) { + SimpleTreeNode node = new SimpleTreeNode(planNode.getExplainStr()); + for (PlanTreeNode child : planNode.getChildren()) { + node.addChild(buildNode(child)); + } + return node; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeBuilder.java new file mode 100644 index 00000000000000..e4a34e38bc21bd --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeBuilder.java @@ -0,0 +1,349 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.profile; + +import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; +import org.apache.doris.common.util.Counter; +import org.apache.doris.common.util.RuntimeProfile; +import org.apache.doris.thrift.TUnit; + +import org.apache.commons.lang3.tuple.ImmutableTriple; +import org.apache.commons.lang3.tuple.Triple; + +import com.clearspring.analytics.util.Lists; +import com.google.common.collect.Maps; + +import java.util.Formatter; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * This class is used to build a tree from the query runtime profile + * It will build tree for the entire query, and also tree for each instance, + * So that user can view the profile tree by query id or by instance id. + * + * Each runtime profile of a query should be built once and be read every where. + */ +public class ProfileTreeBuilder { + + private static final String PROFILE_NAME_QUERY = "Query"; + private static final String PROFILE_NAME_EXECUTION = "Execution Profile"; + private static final String PROFILE_NAME_DATA_STREAM_SENDER = "DataStreamSender"; + private static final String PROFILE_NAME_DATA_BUFFER_SENDER = "DataBufferSender"; + private static final String PROFILE_NAME_BLOCK_MGR = "BlockMgr"; + private static final String PROFILE_NAME_BUFFER_POOL = "Buffer pool"; + private static final String PROFILE_NAME_EXCHANGE_NODE = "EXCHANGE_NODE"; + public static final String DATA_BUFFER_SENDER_ID = "-1"; + public static final String UNKNOWN_ID = "-2"; + + private RuntimeProfile profileRoot; + + // auxiliary structure to link different fragments + private List exchangeNodes = Lists.newArrayList(); + private List senderNodes = Lists.newArrayList(); + + // fragment id -> instance id -> instance tree root + private Map> instanceTreeMap = Maps.newHashMap(); + // fragment id -> (instance id, instance host, instance active time) + private Map>> instanceActiveTimeMap = Maps.newHashMap(); + + // the tree root of the entire query profile tree + private ProfileTreeNode fragmentTreeRoot; + + // Match string like: + // EXCHANGE_NODE (id=3):(Active: 103.899ms, % non-child: 2.27%) + // Extract "EXCHANGE_NODE" and "3" + private static final String EXEC_NODE_NAME_ID_PATTERN_STR = "^(.*) .*id=([0-9]+).*"; + private static final Pattern EXEC_NODE_NAME_ID_PATTERN; + + // Match string like: + // Fragment 0: + // Extract "0" + private static final String FRAGMENT_ID_PATTERN_STR = "^Fragment ([0-9]+).*"; + private static final Pattern FRAGMENT_ID_PATTERN; + + // Match string like: + // Instance e0f7390f5363419e-b416a2a7999608b6 (host=TNetworkAddress(hostname:192.168.1.1, port:9060)):(Active: 1s858ms, % non-child: 0.02%) + // Extract "e0f7390f5363419e-b416a2a7999608b6", "192.168.1.1", "9060" + private static final String INSTANCE_PATTERN_STR = "^Instance (.*) \\(.*hostname:(.*), port:([0-9]+).*"; + private static final Pattern INSTANCE_PATTERN; + + static { + EXEC_NODE_NAME_ID_PATTERN = Pattern.compile(EXEC_NODE_NAME_ID_PATTERN_STR); + FRAGMENT_ID_PATTERN = Pattern.compile(FRAGMENT_ID_PATTERN_STR); + INSTANCE_PATTERN = Pattern.compile(INSTANCE_PATTERN_STR); + } + + public ProfileTreeBuilder(RuntimeProfile root) { + this.profileRoot = root; + } + + public ProfileTreeNode getFragmentTreeRoot() { + return fragmentTreeRoot; + } + + public ProfileTreeNode getInstanceTreeRoot(String fragmentId, String instanceId) { + if (!instanceTreeMap.containsKey(fragmentId)) { + return null; + } + return instanceTreeMap.get(fragmentId).get(instanceId); + } + + public List> getInstanceList(String fragmentId) { + return instanceActiveTimeMap.get(fragmentId); + } + + public void build() throws UserException { + reset(); + unwrapProfile(); + analyzeAndBuildFragmentTrees(); + assembleFragmentTrees(); + } + + private void reset() { + exchangeNodes.clear(); + senderNodes.clear(); + instanceTreeMap.clear(); + instanceActiveTimeMap.clear(); + fragmentTreeRoot = null; + } + + private void unwrapProfile() throws UserException { + while(true) { + if (profileRoot.getName().startsWith(PROFILE_NAME_QUERY)) { + List> children = profileRoot.getChildList(); + boolean find = false; + for (Pair pair : children) { + if (pair.first.getName().startsWith(PROFILE_NAME_EXECUTION)) { + this.profileRoot = pair.first; + find = true; + break; + } + } + if (!find) { + throw new UserException("Invalid profile. Expected " + PROFILE_NAME_EXECUTION + + " in " + PROFILE_NAME_QUERY); + } + } else { + break; + } + } + } + + private void analyzeAndBuildFragmentTrees() throws UserException { + List> childrenFragment = profileRoot.getChildList(); + for (Pair pair : childrenFragment) { + analyzeAndBuildFragmentTree(pair.first); + } + } + + private void analyzeAndBuildFragmentTree(RuntimeProfile fragmentProfile) throws UserException { + String fragmentId = getFragmentId(fragmentProfile); + List> fragmentChildren = fragmentProfile.getChildList(); + if (fragmentChildren.isEmpty()) { + throw new UserException("Empty instance in fragment: " + fragmentProfile.getName()); + } + + // 1. Get max active time of instances in this fragment + List> instanceIdAndActiveTimeList = Lists.newArrayList(); + long maxActiveTimeNs = 0; + for (Pair pair : fragmentChildren) { + Triple instanceIdAndActiveTime = getInstanceIdHostAndActiveTime(pair.first); + maxActiveTimeNs = Math.max(instanceIdAndActiveTime.getRight(), maxActiveTimeNs); + instanceIdAndActiveTimeList.add(instanceIdAndActiveTime); + } + instanceActiveTimeMap.put(fragmentId, instanceIdAndActiveTimeList); + + // 2. Build tree for all fragments + // All instance in a fragment are same, so use first instance to build the fragment tree + RuntimeProfile instanceProfile = fragmentChildren.get(0).first; + ProfileTreeNode instanceTreeRoot = buildSingleInstanceTree(instanceProfile, fragmentId, null); + instanceTreeRoot.setMaxInstanceActiveTime(RuntimeProfile.printCounter(maxActiveTimeNs, TUnit.TIME_NS)); + if (instanceTreeRoot.id.equals(DATA_BUFFER_SENDER_ID)) { + fragmentTreeRoot = instanceTreeRoot; + } + + // 2. Build tree for each single instance + int i = 0; + Map instanceTrees = Maps.newHashMap(); + for (Pair pair : fragmentChildren) { + String instanceId = instanceIdAndActiveTimeList.get(i).getLeft(); + ProfileTreeNode singleInstanceTreeNode = buildSingleInstanceTree(pair.first, fragmentId, instanceId); + instanceTrees.put(instanceId, singleInstanceTreeNode); + i++; + } + this.instanceTreeMap.put(fragmentId, instanceTrees); + } + + // If instanceId is null, which means this profile tree node is for bulding the entire fragment tree. + // So that we need to add sender and exchange node to the auxiliary structure. + private ProfileTreeNode buildSingleInstanceTree(RuntimeProfile instanceProfile, String fragmentId, + String instanceId) throws UserException { + List> instanceChildren = instanceProfile.getChildList(); + ProfileTreeNode senderNode = null; + ProfileTreeNode execNode = null; + for (Pair pair : instanceChildren) { + RuntimeProfile profile = pair.first; + if (profile.getName().startsWith(PROFILE_NAME_DATA_STREAM_SENDER) + || profile.getName().startsWith(PROFILE_NAME_DATA_BUFFER_SENDER)) { + senderNode = buildTreeNode(profile, null, fragmentId, instanceId); + if (instanceId == null) { + senderNodes.add(senderNode); + } + } else if (profile.getName().startsWith(PROFILE_NAME_BLOCK_MGR) + || profile.getName().startsWith(PROFILE_NAME_BUFFER_POOL)) { + // skip BlockMgr nad Buffer pool profile + continue; + } else { + // This should be an ExecNode profile + execNode = buildTreeNode(profile, null, fragmentId, instanceId); + } + } + if (senderNode == null || execNode == null) { + throw new UserException("Invalid instance profile, without sender or exec node: " + instanceProfile); + } + senderNode.addChild(execNode); + execNode.setParentNode(senderNode); + + senderNode.setFragmentAndInstanceId(fragmentId, instanceId); + execNode.setFragmentAndInstanceId(fragmentId, instanceId); + + return senderNode; + } + + private ProfileTreeNode buildTreeNode(RuntimeProfile profile, ProfileTreeNode root, + String fragmentId, String instanceId) { + String name = profile.getName(); + if (name.startsWith(PROFILE_NAME_BUFFER_POOL)) { + // skip Buffer pool, and buffer pool does not has child + return null; + } + boolean isDataBufferSender = name.startsWith(PROFILE_NAME_DATA_BUFFER_SENDER); + Matcher m = EXEC_NODE_NAME_ID_PATTERN.matcher(name); + String extractName; + String extractId; + if ((!m.find() && !isDataBufferSender) || m.groupCount() != 2) { + // DataStreamBuffer name like: "DataBufferSender (dst_fragment_instance_id=d95356f9219b4831-986b4602b41683ca):" + // So it has no id. + // Other profile should has id like: + // EXCHANGE_NODE (id=3):(Active: 103.899ms, % non-child: 2.27%) + // HASH_JOIN_NODE (id=2):(Active: 972.329us, % non-child: 0.00%) + extractName = name; + extractId = UNKNOWN_ID; + } else { + extractName = isDataBufferSender ? PROFILE_NAME_DATA_BUFFER_SENDER : m.group(1); + extractId = isDataBufferSender ? DATA_BUFFER_SENDER_ID : m.group(2); + } + Counter activeCounter = profile.getCounterTotalTime(); + ExecNodeNode node = new ExecNodeNode(extractName, extractId); + node.setActiveTime(RuntimeProfile.printCounter(activeCounter.getValue(), activeCounter.getType())); + try (Formatter fmt = new Formatter()) { + node.setNonChild(fmt.format("%.2f", profile.getLocalTimePercent()).toString()); + } + CounterNode rootCounterNode = new CounterNode(); + buildCounterNode(profile, RuntimeProfile.ROOT_COUNTER, rootCounterNode); + node.setCounterNode(rootCounterNode); + + if (root != null) { + root.addChild(node); + node.setParentNode(root); + } + + if (node.name.equals(PROFILE_NAME_EXCHANGE_NODE) && instanceId == null) { + exchangeNodes.add(node); + } + + // The children in profile is reversed, so traverse it from last to first + List> children = profile.getChildList(); + for (int i = children.size() - 1; i >= 0; i--) { + Pair pair = children.get(i); + ProfileTreeNode execNode = buildTreeNode(pair.first, node, fragmentId, instanceId); + if (execNode != null) { + // For buffer pool profile, buildTreeNode will return null + execNode.setFragmentAndInstanceId(fragmentId, instanceId); + } + } + return node; + } + + private void buildCounterNode(RuntimeProfile profile, String counterName, CounterNode root) { + Map> childCounterMap = profile.getChildCounterMap(); + Set childCounterSet = childCounterMap.get(counterName); + if (childCounterSet == null) { + return; + } + + Map counterMap = profile.getCounterMap(); + for (String childCounterName : childCounterSet) { + Counter counter = counterMap.get(childCounterName); + CounterNode counterNode = new CounterNode(); + if (root != null) { + root.addChild(counterNode); + } + counterNode.setCounter(childCounterName, RuntimeProfile.printCounter(counter.getValue(), counter.getType())); + buildCounterNode(profile, childCounterName, counterNode); + } + return; + } + + private void assembleFragmentTrees() throws UserException { + for (ProfileTreeNode senderNode : senderNodes) { + if (senderNode.id.equals(DATA_BUFFER_SENDER_ID)) { + // this is result sender, skip it. + continue; + } + ProfileTreeNode exchangeNode = findExchangeNode(senderNode.id); + exchangeNode.addChild(senderNode); + senderNode.setParentNode(exchangeNode); + } + } + + private ProfileTreeNode findExchangeNode(String senderId) throws UserException { + for (ProfileTreeNode node : exchangeNodes) { + if (node.id.equals(senderId)) { + return node; + } + } + throw new UserException("Failed to find fragment for sender id: " + senderId); + } + + private String getFragmentId(RuntimeProfile fragmentProfile) throws UserException { + String name = fragmentProfile.getName(); + Matcher m = FRAGMENT_ID_PATTERN.matcher(name); + if (!m.find() || m.groupCount() != 1) { + throw new UserException("Invalid fragment profile name: " + name); + } + return m.group(1); + } + + private Triple getInstanceIdHostAndActiveTime(RuntimeProfile instanceProfile) + throws UserException { + long activeTimeNs = instanceProfile.getCounterTotalTime().getValue(); + String name = instanceProfile.getName(); + Matcher m = INSTANCE_PATTERN.matcher(name); + if (!m.find() || m.groupCount() != 3) { + throw new UserException("Invalid instance profile name: " + name); + } + return new ImmutableTriple<>(m.group(1), m.group(2) + ":" + m.group(3), activeTimeNs); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeNode.java new file mode 100644 index 00000000000000..e68e9e71f0efd6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreeNode.java @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.profile; + +import org.apache.doris.common.TreeNode; + +import com.google.common.base.Strings; + +public class ProfileTreeNode extends TreeNode { + + protected String name; + protected String id; + protected CounterNode counterNode; + protected String activeTime; + protected String nonChild; + + protected String fragmentId = ""; + protected String instanceId = ""; + + // This is used to record the max activeTime of all instances in a fragment. + // Usually recorded on the Sender node. + protected String maxInstanceActiveTime = ""; + + protected ProfileTreeNode parentNode; + + protected ProfileTreeNode(String name, String id) { + this.name = name; + this.id = id; + } + + public void setParentNode(ProfileTreeNode parentNode) { + this.parentNode = parentNode; + } + + public ProfileTreeNode getParentNode() { + return parentNode; + } + + public void setCounterNode(CounterNode counterNode) { + this.counterNode = counterNode; + } + + public CounterNode getCounterNode() { + return counterNode; + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public void setActiveTime(String activeTime) { + this.activeTime = activeTime; + } + + public String getActiveTime() { + return activeTime; + } + + public void setNonChild(String nonChild) { + this.nonChild = nonChild; + } + + public String getNonChild() { + return nonChild; + } + + public String getIdentity() { + if (id.equals(ProfileTreeBuilder.UNKNOWN_ID)) { + return "[" + name + "]"; + } + return "[" + id + ": " + name + "]"; + } + + public void setFragmentAndInstanceId(String fragmentId, String instanceId) { + this.fragmentId = fragmentId; + this.instanceId = instanceId; + } + + public void setMaxInstanceActiveTime(String maxInstanceActiveTime) { + this.maxInstanceActiveTime = maxInstanceActiveTime; + } + + public String getMaxInstanceActiveTime() { + return maxInstanceActiveTime; + } + + public String debugTree(int indent, ProfileTreePrinter.PrintLevel level) { + StringBuilder sb = new StringBuilder(printIndent(indent)); + sb.append(debugString(indent, level)); + if (!getChildren().isEmpty()) { + int childSize = getChildren().size(); + for (int i = 0; i < childSize; i++) { + ProfileTreeNode node = getChild(i); + sb.append("\n").append(node.debugTree(indent + 4, level)); + } + } + return sb.toString(); + } + + public String debugString(int indent, ProfileTreePrinter.PrintLevel level) { + String indentStr = printIndent(indent); + StringBuilder sb = new StringBuilder(); + sb.append(indentStr).append(getIdentity()).append("\n"); + if (level == ProfileTreePrinter.PrintLevel.FRAGMENT) { + sb.append(indentStr).append("Fragment: ").append(fragmentId).append("\n"); + if (!Strings.isNullOrEmpty(maxInstanceActiveTime)) { + sb.append(indentStr).append("MaxActiveTime: ").append(maxInstanceActiveTime).append("\n"); + } + } + if (level == ProfileTreePrinter.PrintLevel.INSTANCE) { + sb.append("(Active: ").append(activeTime).append(", "); + sb.append("non-child: ").append(nonChild).append(")").append("\n"); + // print counters + sb.append(counterNode.toTree(indent + 1)); + } + return sb.toString(); + } + + private String printIndent(int indent) { + String res = ""; + for (int i = 0; i < indent; i++) { + res += " "; + } + return res; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreePrinter.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreePrinter.java new file mode 100644 index 00000000000000..19d2f6a62f00fd --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ProfileTreePrinter.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.profile; + +import hu.webarticum.treeprinter.BorderTreeNodeDecorator; +import hu.webarticum.treeprinter.SimpleTreeNode; +import hu.webarticum.treeprinter.TraditionalTreePrinter; + +public class ProfileTreePrinter { + + public static enum PrintLevel { + FRAGMENT, + INSTANCE + } + + // Fragment tree only print the entire query plan tree with node name + // and some other brief info. + public static String printFragmentTree(ProfileTreeNode root) { + SimpleTreeNode rootNode = buildNode(root, PrintLevel.FRAGMENT); + StringBuilder sb = new StringBuilder(); + new TraditionalTreePrinter().print(new BorderTreeNodeDecorator(rootNode), sb); + return sb.toString(); + } + + // Instance tree will print the details of the tree of a single instance + public static String printInstanceTree(ProfileTreeNode root) { + SimpleTreeNode rootNode = buildNode(root, PrintLevel.INSTANCE); + StringBuilder sb = new StringBuilder(); + new TraditionalTreePrinter().print(new BorderTreeNodeDecorator(rootNode), sb); + return sb.toString(); + } + + private static SimpleTreeNode buildNode(ProfileTreeNode profileNode, PrintLevel level) { + SimpleTreeNode node = new SimpleTreeNode(profileNode.debugString(0, level)); + for (ProfileTreeNode child : profileNode.getChildren()) { + node.addChild(buildNode(child, level)); + } + return node; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java index decb1646f225fd..db1cc6212733f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java @@ -17,10 +17,16 @@ package org.apache.doris.common.util; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.profile.ProfileTreeBuilder; +import org.apache.doris.common.profile.ProfileTreeNode; +import org.apache.doris.common.profile.ProfileTreePrinter; + import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.commons.lang3.tuple.Triple; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -64,10 +70,12 @@ public class ProfileManager { public static final ArrayList PROFILE_HEADERS = new ArrayList( Arrays.asList(QUERY_ID, USER, DEFAULT_DB, SQL_STATEMENT, QUERY_TYPE, START_TIME, END_TIME, TOTAL_TIME, QUERY_STATE)); - + private class ProfileElement { - public Map infoStrings = Maps.newHashMap(); - public String profileContent; + public Map infoStrings = Maps.newHashMap(); + public String profileContent = ""; + public ProfileTreeBuilder builder = null; + public String errMsg = ""; } // only protect profileDeque; profileMap is concurrent, no need to protect @@ -99,6 +107,17 @@ public ProfileElement createElement(RuntimeProfile profile) { for (String header : PROFILE_HEADERS) { element.infoStrings.put(header, summaryProfile.getInfoString(header)); } + + ProfileTreeBuilder builder = new ProfileTreeBuilder(profile); + try { + builder.build(); + } catch (Exception e) { + element.errMsg = e.getMessage(); + LOG.warn("failed to build profile tree", e); + return element; + } + + element.builder = builder; element.profileContent = profile.toString(); return element; } @@ -164,4 +183,70 @@ public String getProfile(String queryID) { readLock.unlock(); } } + + public String getFragmentProfileTreeString(String queryID) { + readLock.lock(); + try { + ProfileElement element = profileMap.get(queryID); + if (element == null || element.builder == null) { + return null; + } + ProfileTreeBuilder builder = element.builder; + return builder.getFragmentTreeRoot().debugTree(0, ProfileTreePrinter.PrintLevel.INSTANCE); + } catch (Exception e) { + LOG.warn("failed to get profile tree", e); + return null; + } finally { + readLock.unlock(); + } + } + + public ProfileTreeNode getFragmentProfileTree(String queryID) throws AnalysisException { + ProfileTreeNode tree; + readLock.lock(); + try { + ProfileElement element = profileMap.get(queryID); + if (element == null || element.builder == null) { + throw new AnalysisException("failed to get fragment profile tree. err: " + + (element == null ? "not found" : element.errMsg)); + } + return element.builder.getFragmentTreeRoot(); + } finally { + readLock.unlock(); + } + } + + public List> getFragmentInstanceList(String queryID, String fragmentId) throws AnalysisException { + ProfileTreeBuilder builder; + readLock.lock(); + try { + ProfileElement element = profileMap.get(queryID); + if (element == null || element.builder == null) { + throw new AnalysisException("failed to get instance list. err: " + + (element == null ? "not found" : element.errMsg)); + } + builder = element.builder; + } finally { + readLock.unlock(); + } + + return builder.getInstanceList(fragmentId); + } + + public ProfileTreeNode getInstanceProfileTree(String queryID, String fragmentId, String instanceId) throws AnalysisException { + ProfileTreeBuilder builder; + readLock.lock(); + try { + ProfileElement element = profileMap.get(queryID); + if (element == null || element.builder == null) { + throw new AnalysisException("failed to get instance profile tree. err: " + + (element == null ? "not found" : element.errMsg)); + } + builder = element.builder; + } finally { + readLock.unlock(); + } + + return builder.getInstanceTreeRoot(fragmentId, instanceId); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java index 83e8d2b27c6aa4..bafece39cd7a34 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/RuntimeProfile.java @@ -24,13 +24,13 @@ import org.apache.doris.thrift.TRuntimeProfileTree; import org.apache.doris.thrift.TUnit; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.util.Collections; import java.util.Comparator; import java.util.Formatter; @@ -47,7 +47,7 @@ */ public class RuntimeProfile { private static final Logger LOG = LogManager.getLogger(RuntimeProfile.class); - private static String ROOT_COUNTER = ""; + public static String ROOT_COUNTER = ""; private Counter counterTotalTime; private double localTimePercent; @@ -62,7 +62,7 @@ public class RuntimeProfile { private LinkedList> childList = Lists.newLinkedList(); private String name; - + public RuntimeProfile(String name) { this(); this.name = name; @@ -73,7 +73,11 @@ public RuntimeProfile() { this.localTimePercent = 0; this.counterMap.put("TotalTime", counterTotalTime); } - + + public String getName() { + return name; + } + public Counter getCounterTotalTime() { return counterTotalTime; } @@ -86,12 +90,20 @@ public List> getChildList() { return childList; } - public Map getChildMap () { + public Map getChildMap() { return childMap; } + public Map> getChildCounterMap() { + return childCounterMap; + } + + public double getLocalTimePercent() { + return localTimePercent; + } + public Counter addCounter(String name, TUnit type, String parentCounterName) { - Counter counter = this.counterMap.get(name); + Counter counter = this.counterMap.get(name); if (counter != null) { return counter; } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java index 975f2b1852b978..8fcba78d7be1ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java @@ -29,13 +29,13 @@ import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TPlanNodeType; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.util.ArrayList; import java.util.List; @@ -256,16 +256,20 @@ protected String getDisplayLabelDetail() { } @Override - protected String getNodeExplainString(String detailPrefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String detailPrefix, TExplainLevel detailLevel) { StringBuilder output = new StringBuilder(); String nameDetail = getDisplayLabelDetail(); if (nameDetail != null) { output.append(detailPrefix + nameDetail + "\n"); } + if (detailLevel == TExplainLevel.BRIEF) { + return output.toString(); + } + if (aggInfo.getAggregateExprs() != null && aggInfo.getMaterializedAggregateExprs().size() > 0) { output.append(detailPrefix + "output: ").append( - getExplainString(aggInfo.getAggregateExprs()) + "\n"); + getExplainString(aggInfo.getAggregateExprs()) + "\n"); } // TODO: group by can be very long. Break it into multiple lines output.append(detailPrefix + "group by: ").append( diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java index ab6903cf1fbf2d..d06e4dc05e6c6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticEvalNode.java @@ -30,15 +30,15 @@ import org.apache.doris.thrift.TPlanNodeType; import org.apache.doris.thrift.TQueryOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.google.common.base.Joiner; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.List; /** @@ -199,10 +199,12 @@ protected void toThrift(TPlanNode msg) { } } - protected String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + @Override + public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + if (detailLevel == TExplainLevel.BRIEF) { + return ""; + } StringBuilder output = new StringBuilder(); - // output.append(String.format("%s%s", prefix, getDisplayLabel())); - // output.append("\n"); output.append(prefix + "functions: "); List strings = Lists.newArrayList(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java index cd492448efeb47..86b978d5220de6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java @@ -47,7 +47,10 @@ public AssertNumRowsNode(PlanNodeId id, PlanNode input, AssertNumRowsElement ass } @Override - protected String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + if (detailLevel == TExplainLevel.BRIEF) { + return ""; + } StringBuilder output = new StringBuilder() .append(prefix + "assert number of rows: ") .append(assertion).append(" ").append(desiredNumOfRows).append("\n"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java index e3e7dc59610a49..7c5c65133eecf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/BrokerScanNode.java @@ -51,14 +51,14 @@ import org.apache.doris.thrift.TScanRangeLocation; import org.apache.doris.thrift.TScanRangeLocations; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Collections; @@ -538,13 +538,15 @@ public List getScanRangeLocations(long maxScanRangeLength) } @Override - protected String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { StringBuilder output = new StringBuilder(); if (!isLoad()) { BrokerTable brokerTable = (BrokerTable) targetTable; output.append(prefix).append("TABLE: ").append(brokerTable.getName()).append("\n"); - output.append(prefix).append("PATH: ") - .append(Joiner.on(",").join(brokerTable.getPaths())).append("\",\n"); + if (detailLevel != TExplainLevel.BRIEF) { + output.append(prefix).append("PATH: ") + .append(Joiner.on(",").join(brokerTable.getPaths())).append("\",\n"); + } } output.append(prefix).append("BROKER: ").append(brokerDesc.getName()).append("\n"); return output.toString(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/CrossJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/CrossJoinNode.java index cc0eb95a4232bb..9246549bc3e294 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/CrossJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/CrossJoinNode.java @@ -23,11 +23,11 @@ import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TPlanNodeType; -import com.google.common.base.MoreObjects; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.google.common.base.MoreObjects; + /** * Cross join between left child and right child. */ @@ -84,7 +84,10 @@ protected void toThrift(TPlanNode msg) { } @Override - protected String getNodeExplainString(String detailPrefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String detailPrefix, TExplainLevel detailLevel) { + if (detailLevel == TExplainLevel.BRIEF) { + return ""; + } StringBuilder output = new StringBuilder().append(detailPrefix + "cross join:" + "\n"); if (!conjuncts.isEmpty()) { output.append(detailPrefix + "predicates: ").append(getExplainString(conjuncts) + "\n"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java index ecd807a37bd38a..d075946a3d781e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java @@ -41,6 +41,9 @@ import org.apache.doris.thrift.TScanRangeLocation; import org.apache.doris.thrift.TScanRangeLocations; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import com.google.common.collect.HashMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -48,9 +51,6 @@ import com.google.common.collect.Range; import com.google.common.collect.Sets; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -300,11 +300,14 @@ private Collection partitionPrune(PartitionInfo partitionInfo) throws Anal } @Override - protected String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { StringBuilder output = new StringBuilder(); - output.append(prefix).append("TABLE: ").append(table.getName()).append("\n"); + if (detailLevel == TExplainLevel.BRIEF) { + return output.toString(); + } + if (null != sortColumn) { output.append(prefix).append("SORT COLUMN: ").append(sortColumn).append("\n"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java index 350921f53771ca..a19aa43c91af72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExchangeNode.java @@ -50,6 +50,9 @@ public class ExchangeNode extends PlanNode { private static final Logger LOG = LogManager.getLogger(ExchangeNode.class); + public static final String EXCHANGE_NODE = "EXCHANGE"; + public static final String MERGING_EXCHANGE_NODE = "MERGING-EXCHANGE"; + // The parameters based on which sorted input streams are merged by this // exchange node. Null if this exchange does not merge sorted streams private SortInfo mergeInfo; @@ -64,7 +67,7 @@ public class ExchangeNode extends PlanNode { * need to compute the cardinality here. */ public ExchangeNode(PlanNodeId id, PlanNode inputNode, boolean copyConjuncts) { - super(id, inputNode, "EXCHANGE"); + super(id, inputNode, EXCHANGE_NODE); offset = 0; children.add(inputNode); if (!copyConjuncts) { @@ -101,7 +104,7 @@ public void init(Analyzer analyzer) throws UserException { public void setMergeInfo(SortInfo info, long offset) { this.mergeInfo = info; this.offset = offset; - this.planNodeName = "MERGING-EXCHANGE"; + this.planNodeName = MERGING_EXCHANGE_NODE; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ExportSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ExportSink.java index dcddbca033cf46..01229e5b34bbb8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ExportSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ExportSink.java @@ -48,6 +48,9 @@ public ExportSink(String exportPath, String columnSeparator, public String getExplainString(String prefix, TExplainLevel explainLevel) { StringBuilder sb = new StringBuilder(); sb.append(prefix + "EXPORT SINK\n"); + if (explainLevel == TExplainLevel.BRIEF) { + return sb.toString(); + } sb.append(prefix + " path=" + exportPath + "\n"); sb.append(prefix + " columnSeparator=" + StringEscapeUtils.escapeJava(columnSeparator) + "\n"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java index e847bfe5f1a1b5..93ed8cf804a6c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HashJoinNode.java @@ -35,13 +35,13 @@ import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TPlanNodeType; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -302,12 +302,17 @@ protected void toThrift(TPlanNode msg) { } @Override - protected String getNodeExplainString(String detailPrefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String detailPrefix, TExplainLevel detailLevel) { String distrModeStr = - (distrMode != DistributionMode.NONE) ? (" (" + distrMode.toString() + ")") : ""; + (distrMode != DistributionMode.NONE) ? (" (" + distrMode.toString() + ")") : ""; StringBuilder output = new StringBuilder() - .append(detailPrefix).append("join op: ").append(joinOp.toString()).append(distrModeStr).append("\n") - .append(detailPrefix).append("hash predicates:\n") + .append(detailPrefix).append("join op: ").append(joinOp.toString()).append(distrModeStr).append("\n"); + + if (detailLevel == TExplainLevel.BRIEF) { + return output.toString(); + } + + output.append(detailPrefix).append("hash predicates:\n") .append(detailPrefix).append("colocate: ").append(isColocate).append(isColocate ? "" : ", reason: " + colocateReason).append("\n"); for (BinaryPredicate eqJoinPredicate : eqJoinConjuncts) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MergeJoinNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MergeJoinNode.java index 2ac7e8fe97e299..a7758746a27448 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/MergeJoinNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MergeJoinNode.java @@ -125,7 +125,7 @@ protected void toThrift(TPlanNode msg) { } @Override - protected String getNodeExplainString(String detailPrefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String detailPrefix, TExplainLevel detailLevel) { String distrModeStr = (distrMode != DistributionMode.NONE) ? (" (" + distrMode.toString() + ")") : ""; StringBuilder output = new StringBuilder().append( diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MergeNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MergeNode.java index 1f7f9d487953d8..629cfa9cd88e9b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/MergeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MergeNode.java @@ -30,8 +30,10 @@ import org.apache.doris.thrift.TMergeNode; import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TPlanNodeType; + import com.google.common.base.Preconditions; import com.google.common.collect.Lists; + import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -171,7 +173,10 @@ protected void toThrift(TPlanNode msg) { } @Override - protected String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + if (detailLevel == TExplainLevel.BRIEF) { + return ""; + } StringBuilder output = new StringBuilder(); // A MergeNode may have predicates if a union is used inside an inline view, // and the enclosing select stmt has predicates referring to the inline view. diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlScanNode.java index e54875e035a15a..945542be1a3f0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlScanNode.java @@ -32,13 +32,13 @@ import org.apache.doris.thrift.TPlanNodeType; import org.apache.doris.thrift.TScanRangeLocations; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import com.google.common.base.Joiner; import com.google.common.base.MoreObjects; import com.google.common.collect.Lists; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.util.ArrayList; import java.util.List; @@ -75,9 +75,12 @@ public void finalize(Analyzer analyzer) throws UserException { } @Override - protected String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { StringBuilder output = new StringBuilder(); output.append(prefix).append("TABLE: ").append(tblName).append("\n"); + if (detailLevel == TExplainLevel.BRIEF) { + return output.toString(); + } output.append(prefix).append("Query: ").append(getMysqlQueryStr()).append("\n"); return output.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlTableSink.java index d1b7edc7eceefa..2104a70465f19a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/MysqlTableSink.java @@ -42,7 +42,10 @@ public MysqlTableSink(MysqlTable mysqlTable) { @Override public String getExplainString(String prefix, TExplainLevel explainLevel) { - return null; + StringBuilder sb = new StringBuilder(); + sb.append("MYSQL TABLE SINK\n"); + sb.append("host: ").append(host).append("\n"); + return sb.toString(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java index 7e3772972a7f34..918a56781857c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OdbcScanNode.java @@ -34,13 +34,13 @@ import org.apache.doris.thrift.TPlanNodeType; import org.apache.doris.thrift.TScanRangeLocations; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import com.google.common.base.Joiner; import com.google.common.base.MoreObjects; import com.google.common.collect.Lists; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.util.ArrayList; import java.util.List; @@ -94,10 +94,13 @@ public void finalize(Analyzer analyzer) throws UserException { } @Override - protected String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { StringBuilder output = new StringBuilder(); output.append(prefix).append("TABLE: ").append(tblName).append("\n"); output.append(prefix).append("TABLE TYPE: ").append(odbcType.toString()).append("\n"); + if (detailLevel == TExplainLevel.BRIEF) { + return output.toString(); + } output.append(prefix).append("QUERY: ").append(getOdbcQueryStr()).append("\n"); return output.toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 58b100e7019dab..b2c4a61bcae4b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -65,6 +65,9 @@ import org.apache.doris.thrift.TScanRangeLocation; import org.apache.doris.thrift.TScanRangeLocations; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import com.google.common.base.Joiner; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; @@ -73,9 +76,6 @@ import com.google.common.collect.Maps; import com.google.common.collect.Range; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -549,11 +549,15 @@ public List getScanRangeLocations(long maxScanRangeLength) } @Override - protected String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { StringBuilder output = new StringBuilder(); output.append(prefix).append("TABLE: ").append(olapTable.getName()).append("\n"); + if (detailLevel == TExplainLevel.BRIEF) { + return output.toString(); + } + if (null != sortColumn) { output.append(prefix).append("SORT COLUMN: ").append(sortColumn).append("\n"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index af20dadbbf40ff..bb3cf103d9348b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -134,6 +134,9 @@ public void complete() throws UserException { public String getExplainString(String prefix, TExplainLevel explainLevel) { StringBuilder strBuilder = new StringBuilder(); strBuilder.append(prefix + "OLAP TABLE SINK\n"); + if (explainLevel == TExplainLevel.BRIEF) { + return strBuilder.toString(); + } strBuilder.append(prefix + " TUPLE ID: " + tupleDescriptor.getId() + "\n"); strBuilder.append(prefix + " " + DataPartition.RANDOM.getExplainString(explainLevel)); return strBuilder.toString(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java index 1fa6acd02acc01..27ada66860a9e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java @@ -246,6 +246,8 @@ public boolean isPartitioned() { return (dataPartition.getType() != TPartitionType.UNPARTITIONED); } + public PlanFragmentId getId() { return fragmentId; } + public PlanFragment getDestFragment() { if (destNode == null) return null; return destNode.getFragment(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java index 45b36aec65831c..1f4b4dc69c2512 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java @@ -30,15 +30,15 @@ import org.apache.doris.thrift.TPlan; import org.apache.doris.thrift.TPlanNode; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import com.google.common.base.Preconditions; import com.google.common.base.Predicates; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.math.LongMath; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -113,6 +113,10 @@ abstract public class PlanNode extends TreeNode { protected boolean compactData; protected int numInstances; + public String getPlanNodeName() { + return planNodeName; + } + protected PlanNode(PlanNodeId id, ArrayList tupleIds, String planNodeName) { this.id = id; this.limit = -1; @@ -383,7 +387,7 @@ protected final String getExplainString(String rootPrefix, String prefix, TExpla * Subclass should override this function. * Each line should be prefix by detailPrefix. */ - protected String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { return ""; } @@ -632,4 +636,13 @@ public void appendTrace(StringBuilder sb) { sb.append(")"); } } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("[").append(getId().asInt()).append(": ").append(getPlanNodeName()).append("]"); + sb.append("\nFragment: ").append(getFragmentId().asInt()).append("]"); + sb.append("\n").append(getNodeExplainString("", TExplainLevel.BRIEF)); + return sb.toString(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java index e17319083ce9cd..6f66cdc0e1c53a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java @@ -18,6 +18,7 @@ package org.apache.doris.planner; import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.ExplainOptions; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.InsertStmt; import org.apache.doris.analysis.QueryStmt; @@ -28,6 +29,8 @@ import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.UserException; +import org.apache.doris.common.profile.PlanTreeBuilder; +import org.apache.doris.common.profile.PlanTreePrinter; import org.apache.doris.rewrite.mvrewrite.MVSelectFailedException; import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TQueryOptions; @@ -115,7 +118,22 @@ private void setResultExprScale(Analyzer analyzer, ArrayList outputExprs) /** * Return combined explain string for all plan fragments. */ - public String getExplainString(List fragments, TExplainLevel explainLevel) { + public String getExplainString(List fragments, ExplainOptions explainOptions) { + Preconditions.checkNotNull(explainOptions); + if (explainOptions.isGraph()) { + // print the plan graph + PlanTreeBuilder builder = new PlanTreeBuilder(fragments); + try { + builder.build(); + } catch (UserException e) { + LOG.warn("Failed to build explain plan tree", e); + return e.getMessage(); + } + return PlanTreePrinter.printPlanExplanation(builder.getTreeRoot()); + } + + // print text plan + TExplainLevel explainLevel = explainOptions.isVerbose() ? TExplainLevel.VERBOSE : TExplainLevel.NORMAL; StringBuilder str = new StringBuilder(); for (int i = 0; i < fragments.size(); ++i) { PlanFragment fragment = fragments.get(i); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java index e7c3255189ced1..b70af9e8a9c949 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RepeatNode.java @@ -33,11 +33,11 @@ import org.apache.doris.thrift.TPlanNodeType; import org.apache.doris.thrift.TRepeatNode; +import org.apache.commons.collections.CollectionUtils; + import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; -import org.apache.commons.collections.CollectionUtils; - import java.util.ArrayList; import java.util.BitSet; import java.util.Collections; @@ -182,13 +182,16 @@ protected String debugString() { } @Override - protected String getNodeExplainString(String detailPrefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String detailPrefix, TExplainLevel detailLevel) { + if (detailLevel == TExplainLevel.BRIEF) { + return ""; + } StringBuilder output = new StringBuilder(); output.append(detailPrefix + "repeat: repeat "); output.append(repeatSlotIdList.size() - 1); output.append(" lines "); output.append(repeatSlotIdList); - output.append("\n" ); + output.append("\n"); if (CollectionUtils.isNotEmpty(outputTupleDesc.getSlots())) { output.append(detailPrefix + "generate: "); output.append(outputTupleDesc.getSlots().stream().map(slot -> "`" + slot.getColumn().getName() + "`") diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java index 83219af9a386ea..c2d30bcb168f08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SelectNode.java @@ -73,7 +73,10 @@ public void computeStats(Analyzer analyzer) { } @Override - protected String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + if (detailLevel == TExplainLevel.BRIEF) { + return ""; + } StringBuilder output = new StringBuilder(); if (!conjuncts.isEmpty()) { output.append(prefix + "predicates: " + getExplainString(conjuncts) + "\n"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java index 9d2b959daf690b..502d4ec72db5db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java @@ -35,9 +35,15 @@ import org.apache.doris.thrift.TPlanNode; import org.apache.doris.thrift.TPlanNodeType; import org.apache.doris.thrift.TUnionNode; + +import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; + import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -337,7 +343,11 @@ protected void toThrift(TPlanNode msg, TPlanNodeType nodeType) { } @Override - protected String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + if (detailLevel == TExplainLevel.BRIEF) { + return ""; + } + StringBuilder output = new StringBuilder(); // A SetOperationNode may have predicates if a union is set operation inside an inline view, // and the enclosing select stmt has predicates referring to the inline view. @@ -346,7 +356,7 @@ protected String getNodeExplainString(String prefix, TExplainLevel detailLevel) } if (CollectionUtils.isNotEmpty(constExprLists_)) { output.append(prefix).append("constant exprs: ").append("\n"); - for(List exprs : constExprLists_) { + for (List exprs : constExprLists_) { output.append(prefix).append(" ").append(exprs.stream().map(Expr::toSql) .collect(Collectors.joining(" | "))).append("\n"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java index 97c2c5b9a8fbf9..93c139e43e9408 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SortNode.java @@ -31,14 +31,14 @@ import org.apache.doris.thrift.TSortInfo; import org.apache.doris.thrift.TSortNode; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import com.google.common.base.Joiner; import com.google.common.base.MoreObjects; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.util.Iterator; import java.util.List; @@ -170,7 +170,11 @@ protected void toThrift(TPlanNode msg) { } @Override - protected String getNodeExplainString(String detailPrefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String detailPrefix, TExplainLevel detailLevel) { + if (detailLevel == TExplainLevel.BRIEF) { + return ""; + } + StringBuilder output = new StringBuilder(); output.append(detailPrefix + "order by: "); Iterator expr = info.getOrderingExprs().iterator(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java index 112fa22bffb29f..edb935dc7f3313 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadScanNode.java @@ -39,12 +39,12 @@ import org.apache.doris.thrift.TScanRangeLocations; import org.apache.doris.thrift.TUniqueId; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.nio.charset.Charset; import java.util.List; import java.util.Map; @@ -188,7 +188,7 @@ public List getScanRangeLocations(long maxScanRangeLength) public int getNumInstances() { return 1; } @Override - protected String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { return "StreamLoadScanNode"; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 7cc08bdaaa4846..189704a36467ce 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -51,6 +51,7 @@ import org.apache.doris.analysis.ShowPluginsStmt; import org.apache.doris.analysis.ShowProcStmt; import org.apache.doris.analysis.ShowProcesslistStmt; +import org.apache.doris.analysis.ShowQueryProfileStmt; import org.apache.doris.analysis.ShowRepositoriesStmt; import org.apache.doris.analysis.ShowResourcesStmt; import org.apache.doris.analysis.ShowRestoreStmt; @@ -109,10 +110,14 @@ import org.apache.doris.common.proc.RollupProcDir; import org.apache.doris.common.proc.SchemaChangeProcDir; import org.apache.doris.common.proc.TabletsProcDir; +import org.apache.doris.common.profile.ProfileTreeNode; +import org.apache.doris.common.profile.ProfileTreePrinter; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; import org.apache.doris.common.util.OrderByPair; +import org.apache.doris.common.util.ProfileManager; +import org.apache.doris.common.util.RuntimeProfile; import org.apache.doris.load.DeleteHandler; import org.apache.doris.load.ExportJob; import org.apache.doris.load.ExportMgr; @@ -125,17 +130,19 @@ import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.TUnit; import org.apache.doris.transaction.GlobalTransactionMgr; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.apache.commons.lang3.tuple.Triple; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.io.BufferedReader; import java.io.InputStream; import java.io.InputStreamReader; @@ -263,6 +270,8 @@ public ShowResultSet execute() throws AnalysisException { handleShowTransaction(); } else if (stmt instanceof ShowPluginsStmt) { handleShowPlugins(); + } else if (stmt instanceof ShowQueryProfileStmt) { + handleShowQueryProfile(); } else { handleEmtpy(); } @@ -1612,6 +1621,54 @@ private void handleShowPlugins() throws AnalysisException { List> rows = Catalog.getCurrentPluginMgr().getPluginShowInfos(); resultSet = new ShowResultSet(pluginsStmt.getMetaData(), rows); } + + private void handleShowQueryProfile() throws AnalysisException { + ShowQueryProfileStmt showStmt = (ShowQueryProfileStmt) stmt; + ShowQueryProfileStmt.PathType pathType = showStmt.getPathType(); + List> rows = Lists.newArrayList(); + switch (pathType) { + case QUERY_IDS: + rows = ProfileManager.getInstance().getAllQueries(); + break; + case FRAGMETNS: { + ProfileTreeNode treeRoot = ProfileManager.getInstance().getFragmentProfileTree(showStmt.getQueryId()); + if (treeRoot == null) { + throw new AnalysisException("Failed to get fragment tree for query: " + showStmt.getQueryId()); + } + List row = Lists.newArrayList(ProfileTreePrinter.printFragmentTree(treeRoot)); + rows.add(row); + break; + } + case INSTANCES: { + List> instanceList + = ProfileManager.getInstance().getFragmentInstanceList(showStmt.getQueryId(), showStmt.getFragmentId()); + if (instanceList == null) { + throw new AnalysisException("Failed to get instance list for fragment: " + showStmt.getFragmentId()); + } + for (Triple triple : instanceList) { + List row = Lists.newArrayList(triple.getLeft(), triple.getMiddle(), + RuntimeProfile.printCounter(triple.getRight(), TUnit.TIME_NS)); + rows.add(row); + } + break; + } + case SINGLE_INSTANCE: { + ProfileTreeNode treeRoot = ProfileManager.getInstance().getInstanceProfileTree(showStmt.getQueryId(), + showStmt.getFragmentId(), showStmt.getInstanceId()); + if (treeRoot == null) { + throw new AnalysisException("Failed to get instance tree for instance: " + showStmt.getInstanceId()); + } + List row = Lists.newArrayList(ProfileTreePrinter.printInstanceTree(treeRoot)); + rows.add(row); + break; + } + default: + break; + } + + resultSet = new ShowResultSet(showStmt.getMetaData(), rows); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 0a991effd77a0d..07ecb11a521d39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.CreateTableAsSelectStmt; import org.apache.doris.analysis.DdlStmt; import org.apache.doris.analysis.EnterStmt; +import org.apache.doris.analysis.ExplainOptions; import org.apache.doris.analysis.ExportStmt; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.InsertStmt; @@ -79,7 +80,6 @@ import org.apache.doris.rewrite.mvrewrite.MVSelectFailedException; import org.apache.doris.rpc.RpcException; import org.apache.doris.task.LoadEtlTask; -import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TQueryOptions; import org.apache.doris.thrift.TQueryType; import org.apache.doris.thrift.TUniqueId; @@ -504,9 +504,8 @@ public void analyze(TQueryOptions tQueryOptions) throws UserException { private void analyzeAndGenerateQueryPlan(TQueryOptions tQueryOptions) throws UserException { parsedStmt.analyze(analyzer); if (parsedStmt instanceof QueryStmt || parsedStmt instanceof InsertStmt) { - boolean isExplain = parsedStmt.isExplain(); - boolean isVerbose = parsedStmt.isVerbose(); // Apply expr and subquery rewrites. + ExplainOptions explainOptions = parsedStmt.getExplainOptions(); boolean reAnalyze = false; ExprRewriter rewriter = analyzer.getExprRewriter(); @@ -541,7 +540,7 @@ private void analyzeAndGenerateQueryPlan(TQueryOptions tQueryOptions) throws Use if (LOG.isTraceEnabled()) { LOG.trace("rewrittenStmt: " + parsedStmt.toSql()); } - if (isExplain) parsedStmt.setIsExplain(isExplain, isVerbose); + if (explainOptions != null) parsedStmt.setIsExplain(explainOptions); } } plannerProfile.setQueryAnalysisFinishTime(); @@ -732,7 +731,7 @@ private void handleQueryStmt() throws Exception { QueryDetailQueue.addOrUpdateQueryDetail(queryDetail); if (queryStmt.isExplain()) { - String explainString = planner.getExplainString(planner.getFragments(), queryStmt.isVerbose() ? TExplainLevel.VERBOSE: TExplainLevel.NORMAL.NORMAL); + String explainString = planner.getExplainString(planner.getFragments(), queryStmt.getExplainOptions()); handleExplainStmt(explainString); return; } @@ -813,7 +812,7 @@ private void handleInsertStmt() throws Exception { } if (insertStmt.getQueryStmt().isExplain()) { - String explainString = planner.getExplainString(planner.getFragments(), TExplainLevel.VERBOSE); + String explainString = planner.getExplainString(planner.getFragments(), new ExplainOptions(true, false)); handleExplainStmt(explainString); return; } diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex index 5abcf97f233ce1..2f760d3eb98d64 100644 --- a/fe/fe-core/src/main/jflex/sql_scanner.flex +++ b/fe/fe-core/src/main/jflex/sql_scanner.flex @@ -205,6 +205,7 @@ import org.apache.doris.qe.SqlModeHelper; keywordMap.put("global", new Integer(SqlParserSymbols.KW_GLOBAL)); keywordMap.put("grant", new Integer(SqlParserSymbols.KW_GRANT)); keywordMap.put("grants", new Integer(SqlParserSymbols.KW_GRANTS)); + keywordMap.put("graph", new Integer(SqlParserSymbols.KW_GRAPH)); keywordMap.put("group", new Integer(SqlParserSymbols.KW_GROUP)); keywordMap.put("grouping", new Integer(SqlParserSymbols.KW_GROUPING)); keywordMap.put("hash", new Integer(SqlParserSymbols.KW_HASH)); @@ -288,6 +289,7 @@ import org.apache.doris.qe.SqlModeHelper; keywordMap.put("proc", new Integer(SqlParserSymbols.KW_PROC)); keywordMap.put("procedure", new Integer(SqlParserSymbols.KW_PROCEDURE)); keywordMap.put("processlist", new Integer(SqlParserSymbols.KW_PROCESSLIST)); + keywordMap.put("profile", new Integer(SqlParserSymbols.KW_PROFILE)); keywordMap.put("properties", new Integer(SqlParserSymbols.KW_PROPERTIES)); keywordMap.put("property", new Integer(SqlParserSymbols.KW_PROPERTY)); keywordMap.put("query", new Integer(SqlParserSymbols.KW_QUERY)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/DistributedPlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/DistributedPlannerTest.java index 6c480b3933248e..9bf0ed8d2c6a25 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/DistributedPlannerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/DistributedPlannerTest.java @@ -19,19 +19,17 @@ import org.apache.doris.analysis.CreateDbStmt; import org.apache.doris.analysis.CreateTableStmt; +import org.apache.doris.analysis.ExplainOptions; import org.apache.doris.analysis.TupleId; import org.apache.doris.catalog.Catalog; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.StmtExecutor; -import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.utframe.UtFrameUtils; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import mockit.Expectations; -import mockit.Injectable; -import mockit.Mocked; + import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.junit.After; @@ -44,6 +42,10 @@ import java.util.Set; import java.util.UUID; +import mockit.Expectations; +import mockit.Injectable; +import mockit.Mocked; + public class DistributedPlannerTest { private static String runningDir = "fe/mocked/DemoTest/" + UUID.randomUUID().toString() + "/"; private static ConnectContext ctx; @@ -134,7 +136,7 @@ public void testExplicitlyBroadcastJoin() throws Exception { stmtExecutor.execute(); Planner planner = stmtExecutor.planner(); List fragments = planner.getFragments(); - String plan = planner.getExplainString(fragments, TExplainLevel.NORMAL); + String plan = planner.getExplainString(fragments, new ExplainOptions(false, false)); Assert.assertEquals(1, StringUtils.countMatches(plan, "INNER JOIN (BROADCAST)")); sql = "explain select * from db1.tbl1 join [SHUFFLE] db1.tbl2 on tbl1.k1 = tbl2.k3"; @@ -142,7 +144,7 @@ public void testExplicitlyBroadcastJoin() throws Exception { stmtExecutor.execute(); planner = stmtExecutor.planner(); fragments = planner.getFragments(); - plan = planner.getExplainString(fragments, TExplainLevel.NORMAL); + plan = planner.getExplainString(fragments, new ExplainOptions(false, false)); Assert.assertEquals(1, StringUtils.countMatches(plan, "INNER JOIN (PARTITIONED)")); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java index e54aad69ca5a6a..eb843fc589138a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java @@ -17,15 +17,16 @@ package org.apache.doris.planner; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.doris.analysis.CreateDbStmt; import org.apache.doris.analysis.CreateTableStmt; +import org.apache.doris.analysis.ExplainOptions; import org.apache.doris.catalog.Catalog; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.StmtExecutor; -import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.utframe.UtFrameUtils; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; import org.junit.After; import org.junit.Assert; import org.junit.BeforeClass; @@ -78,7 +79,7 @@ public void testSetOperation() throws Exception { stmtExecutor1.execute(); Planner planner1 = stmtExecutor1.planner(); List fragments1 = planner1.getFragments(); - String plan1 = planner1.getExplainString(fragments1, TExplainLevel.NORMAL); + String plan1 = planner1.getExplainString(fragments1, new ExplainOptions(false, false)); Assert.assertEquals(1, StringUtils.countMatches(plan1, "UNION")); String sql2 = "explain select * from db1.tbl1 where k1='a' and k4=1\n" + "union distinct\n" @@ -103,7 +104,7 @@ public void testSetOperation() throws Exception { stmtExecutor2.execute(); Planner planner2 = stmtExecutor2.planner(); List fragments2 = planner2.getFragments(); - String plan2 = planner2.getExplainString(fragments2, TExplainLevel.NORMAL); + String plan2 = planner2.getExplainString(fragments2, new ExplainOptions(false, false)); Assert.assertEquals(4, StringUtils.countMatches(plan2, "UNION")); // intersect @@ -119,7 +120,7 @@ public void testSetOperation() throws Exception { stmtExecutor3.execute(); Planner planner3 = stmtExecutor3.planner(); List fragments3 = planner3.getFragments(); - String plan3 = planner3.getExplainString(fragments3, TExplainLevel.NORMAL); + String plan3 = planner3.getExplainString(fragments3, new ExplainOptions(false, false)); Assert.assertEquals(1, StringUtils.countMatches(plan3, "INTERSECT")); String sql4 = "explain select * from db1.tbl1 where k1='a' and k4=1\n" + "intersect distinct\n" @@ -145,7 +146,7 @@ public void testSetOperation() throws Exception { stmtExecutor4.execute(); Planner planner4 = stmtExecutor4.planner(); List fragments4 = planner4.getFragments(); - String plan4 = planner4.getExplainString(fragments4, TExplainLevel.NORMAL); + String plan4 = planner4.getExplainString(fragments4, new ExplainOptions(false, false)); Assert.assertEquals(3, StringUtils.countMatches(plan4, "INTERSECT")); // except @@ -161,7 +162,7 @@ public void testSetOperation() throws Exception { stmtExecutor5.execute(); Planner planner5 = stmtExecutor5.planner(); List fragments5 = planner5.getFragments(); - String plan5 = planner5.getExplainString(fragments5, TExplainLevel.NORMAL); + String plan5 = planner5.getExplainString(fragments5, new ExplainOptions(false, false)); Assert.assertEquals(1, StringUtils.countMatches(plan5, "EXCEPT")); String sql6 = "select * from db1.tbl1 where k1='a' and k4=1\n" @@ -176,7 +177,7 @@ public void testSetOperation() throws Exception { stmtExecutor6.execute(); Planner planner6 = stmtExecutor6.planner(); List fragments6 = planner6.getFragments(); - String plan6 = planner6.getExplainString(fragments6, TExplainLevel.NORMAL); + String plan6 = planner6.getExplainString(fragments6, new ExplainOptions(false, false)); Assert.assertEquals(1, StringUtils.countMatches(plan6, "EXCEPT")); String sql7 = "select * from db1.tbl1 where k1='a' and k4=1\n" @@ -191,7 +192,7 @@ public void testSetOperation() throws Exception { stmtExecutor7.execute(); Planner planner7 = stmtExecutor7.planner(); List fragments7 = planner7.getFragments(); - String plan7 = planner7.getExplainString(fragments7, TExplainLevel.NORMAL); + String plan7 = planner7.getExplainString(fragments7, new ExplainOptions(false, false)); Assert.assertEquals(1, StringUtils.countMatches(plan7, "EXCEPT")); // mixed @@ -207,7 +208,7 @@ public void testSetOperation() throws Exception { stmtExecutor8.execute(); Planner planner8 = stmtExecutor8.planner(); List fragments8 = planner8.getFragments(); - String plan8 = planner8.getExplainString(fragments8, TExplainLevel.NORMAL); + String plan8 = planner8.getExplainString(fragments8, new ExplainOptions(false, false)); Assert.assertEquals(1, StringUtils.countMatches(plan8, "UNION")); Assert.assertEquals(1, StringUtils.countMatches(plan8, "INTERSECT")); Assert.assertEquals(1, StringUtils.countMatches(plan8, "EXCEPT")); @@ -236,7 +237,7 @@ public void testSetOperation() throws Exception { stmtExecutor9.execute(); Planner planner9 = stmtExecutor9.planner(); List fragments9 = planner9.getFragments(); - String plan9 = planner9.getExplainString(fragments9, TExplainLevel.NORMAL); + String plan9 = planner9.getExplainString(fragments9, new ExplainOptions(false, false)); Assert.assertEquals(2, StringUtils.countMatches(plan9, "UNION")); Assert.assertEquals(3, StringUtils.countMatches(plan9, "INTERSECT")); Assert.assertEquals(2, StringUtils.countMatches(plan9, "EXCEPT")); @@ -325,7 +326,7 @@ public void testWithStmtSoltIsAllowNull() throws Exception { stmtExecutor1.execute(); Planner planner1 = stmtExecutor1.planner(); List fragments1 = planner1.getFragments(); - String plan1 = planner1.getExplainString(fragments1, TExplainLevel.VERBOSE); + String plan1 = planner1.getExplainString(fragments1, new ExplainOptions(false, false)); Assert.assertEquals(3, StringUtils.countMatches(plan1, "nullIndicatorBit=0")); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/DorisAssert.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/DorisAssert.java index d3d87804692310..d54e10a320e882 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/DorisAssert.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/DorisAssert.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.CreateMaterializedViewStmt; import org.apache.doris.analysis.CreateTableStmt; import org.apache.doris.analysis.DropTableStmt; +import org.apache.doris.analysis.ExplainOptions; import org.apache.doris.analysis.SqlParser; import org.apache.doris.analysis.SqlScanner; import org.apache.doris.analysis.StatementBase; @@ -36,7 +37,6 @@ import org.apache.doris.qe.QueryState; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.system.SystemInfoService; -import org.apache.doris.thrift.TExplainLevel; import org.apache.commons.lang.StringUtils; import org.junit.Assert; @@ -169,7 +169,7 @@ private String internalExecute(String sql) throws Exception { } } Planner planner = stmtExecutor.planner(); - String explainString = planner.getExplainString(planner.getFragments(), TExplainLevel.NORMAL); + String explainString = planner.getExplainString(planner.getFragments(), new ExplainOptions(false, false)); System.out.println(explainString); return explainString; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java index 8f5ed06326ea94..ab34e204893ae2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java @@ -18,6 +18,7 @@ package org.apache.doris.utframe; import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.ExplainOptions; import org.apache.doris.analysis.SqlParser; import org.apache.doris.analysis.SqlScanner; import org.apache.doris.analysis.StatementBase; @@ -35,7 +36,6 @@ import org.apache.doris.qe.StmtExecutor; import org.apache.doris.system.Backend; import org.apache.doris.system.SystemInfoService; -import org.apache.doris.thrift.TExplainLevel; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.utframe.MockedBackendFactory.DefaultBeThriftServiceImpl; import org.apache.doris.utframe.MockedBackendFactory.DefaultHeartbeatServiceImpl; @@ -44,12 +44,12 @@ import org.apache.doris.utframe.MockedFrontend.FeStartException; import org.apache.doris.utframe.MockedFrontend.NotInitException; -import org.apache.commons.io.FileUtils; - import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import org.apache.commons.io.FileUtils; + import java.io.File; import java.io.IOException; import java.io.StringReader; @@ -215,7 +215,7 @@ public static String getSQLPlanOrErrorMsg(ConnectContext ctx, String queryStr) t stmtExecutor.execute(); if (ctx.getState().getStateType() != QueryState.MysqlStateType.ERR) { Planner planner = stmtExecutor.planner(); - return planner.getExplainString(planner.getFragments(), TExplainLevel.NORMAL); + return planner.getExplainString(planner.getFragments(), new ExplainOptions(false, false)); } else { return ctx.getState().getErrorMessage(); } diff --git a/fe/pom.xml b/fe/pom.xml index 4ceb929915770a..9419c871ce4ed7 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -671,6 +671,13 @@ under the License. provided + + + hu.webarticum + tree-printer + 1.2 + + diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index b872eb255c33cb..34ef5a29d5ec6c 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -189,6 +189,7 @@ enum TStmtType { // level of verboseness for "explain" output // TODO: should this go somewhere else? enum TExplainLevel { + BRIEF, NORMAL, VERBOSE }