From 32986c99311d3f85dd76403b99e37f9144d2dfc5 Mon Sep 17 00:00:00 2001 From: kangkaisen Date: Fri, 20 Jul 2018 15:39:53 +0800 Subject: [PATCH 1/2] Improve cardinality, avgRowSize, numNodes stat info in OlapScanNode --- .../apache/doris/planner/OlapScanNode.java | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java index 578c3fc693ab16..b56b161e82a089 100644 --- a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -40,6 +40,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.UserException; import org.apache.doris.service.FrontendOptions; import org.apache.doris.system.Backend; @@ -93,9 +94,12 @@ public class OlapScanNode extends ScanNode { private long totalTabletsNum = 0; private long selectedIndexId = -1; private int selectedPartitionNum = 0; + private long totalBytes = 0; boolean isFinalized = false; + private HashSet scanBackendIds = new HashSet<>(); + /** * Constructs node to scan given data files of table 'tbl'. */ @@ -143,9 +147,23 @@ public void finalize(Analyzer analyzer) throws UserException { throw new UserException(e.getMessage()); } + computeStats(analyzer); isFinalized = true; } + @Override + public void computeStats(Analyzer analyzer) { + if (cardinality > 0) { + avgRowSize = totalBytes / (float) cardinality; + cardinality = cardinality / FeConstants.default_replication_num; //the cardinality should be single replica + if (hasLimit()) { + cardinality = Math.min(cardinality, limit); + } + numNodes = scanBackendIds.size(); + } + } + + // private void analyzeVectorizedConjuncts(Analyzer analyzer) throws InternalException { // for (SlotDescriptor slot : desc.getSlots()) { // for (Expr conjunct : conjuncts) { @@ -447,6 +465,13 @@ private void addScanRangeLocations(Partition partition, scanRangeLocations.addToLocations(scanRangeLocation); paloRange.addToHosts(new TNetworkAddress(ip, port)); tabletIsNull = false; + + //for CBO + if (replica.getRowCount() != -1) { + cardinality += replica.getRowCount(); + totalBytes += replica.getDataSize(); + } + scanBackendIds.add(backend.getId()); } if (tabletIsNull) { throw new UserException(tabletId + "have no alive replicas"); @@ -599,6 +624,18 @@ protected String getNodeExplainString(String prefix, TExplainLevel detailLevel) "buckets=%s/%s", selectedTabletsNum, totalTabletsNum)); output.append("\n"); + output.append(prefix).append(String.format( + "cardinality=%s", cardinality)); + output.append("\n"); + + output.append(prefix).append(String.format( + "avgRowSize=%s", avgRowSize)); + output.append("\n"); + + output.append(prefix).append(String.format( + "numNodes=%s", numNodes)); + output.append("\n"); + return output.toString(); } From c2af9f437ff86658f1623b14cac98bca3cadddb0 Mon Sep 17 00:00:00 2001 From: kangkaisen Date: Wed, 7 Nov 2018 10:56:40 +0800 Subject: [PATCH 2/2] Compute cardinality only for one replica --- fe/src/main/java/org/apache/doris/planner/OlapScanNode.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java index b56b161e82a089..8c4b71a51d6e63 100644 --- a/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -40,7 +40,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; -import org.apache.doris.common.FeConstants; import org.apache.doris.common.UserException; import org.apache.doris.service.FrontendOptions; import org.apache.doris.system.Backend; @@ -155,7 +154,6 @@ public void finalize(Analyzer analyzer) throws UserException { public void computeStats(Analyzer analyzer) { if (cardinality > 0) { avgRowSize = totalBytes / (float) cardinality; - cardinality = cardinality / FeConstants.default_replication_num; //the cardinality should be single replica if (hasLimit()) { cardinality = Math.min(cardinality, limit); } @@ -452,6 +450,7 @@ private void addScanRangeLocations(Partition partition, Collections.shuffle(replicas); boolean tabletIsNull = true; + boolean collectedStat = false; for (Replica replica : replicas) { Backend backend = Catalog.getCurrentSystemInfo().getBackend(replica.getBackendId()); if (backend == null) { @@ -467,9 +466,10 @@ private void addScanRangeLocations(Partition partition, tabletIsNull = false; //for CBO - if (replica.getRowCount() != -1) { + if (!collectedStat && replica.getRowCount() != -1) { cardinality += replica.getRowCount(); totalBytes += replica.getDataSize(); + collectedStat = true; } scanBackendIds.add(backend.getId()); }