From 9334ad6edd3ecdaec99fb31fcb9f2587c57ccca9 Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Wed, 31 Jul 2024 14:45:27 +0800 Subject: [PATCH 1/2] revert --- .../apache/doris/datasource/FileScanNode.java | 1 - .../iceberg/source/IcebergScanNode.java | 13 +++++----- .../apache/doris/nereids/NereidsPlanner.java | 24 +------------------ .../apache/doris/planner/OriginalPlanner.java | 16 ------------- 4 files changed, 8 insertions(+), 46 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java index 11a7b13024e804..a83a51dad7df1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java @@ -71,7 +71,6 @@ public abstract class FileScanNode extends ExternalScanNode { protected long totalPartitionNum = 0; protected long readPartitionNum = 0; protected long fileSplitSize; - public long rowCount = 0; public FileScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, StatisticalType statisticalType, boolean needCheckColumnPriv) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 6ea58014003659..0d5ff45c3e2748 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -76,6 +76,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -210,12 +211,6 @@ private List doGetSplits() throws UserException { HashSet partitionPathSet = new HashSet<>(); boolean isPartitionedTable = icebergTable.spec().isPartitioned(); - long rowCount = getCountFromSnapshot(); - if (getPushDownAggNoGroupingOp().equals(TPushAggOp.COUNT) && rowCount >= 0) { - this.rowCount = rowCount; - return new ArrayList<>(); - } - CloseableIterable fileScanTasks = TableScanUtil.splitFiles(scan.planFiles(), splitSize); try (CloseableIterable combinedScanTasks = TableScanUtil.planTasks(fileScanTasks, splitSize, 1, 0)) { @@ -269,6 +264,12 @@ private List doGetSplits() throws UserException { throw new UserException(e.getMessage(), e.getCause()); } + TPushAggOp aggOp = getPushDownAggNoGroupingOp(); + if (aggOp.equals(TPushAggOp.COUNT) && getCountFromSnapshot() >= 0) { + // we can create a special empty split and skip the plan process + return Collections.singletonList(splits.get(0)); + } + readPartitionNum = partitionPathSet.size(); return splits; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 12bb8c3623c1dc..67f4bb73b1886a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -26,7 +26,6 @@ import org.apache.doris.common.NereidsException; import org.apache.doris.common.Pair; import org.apache.doris.common.profile.SummaryProfile; -import org.apache.doris.datasource.iceberg.source.IcebergScanNode; import org.apache.doris.mysql.FieldInfo; import org.apache.doris.nereids.CascadesContext.Lock; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -54,18 +53,14 @@ import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache; -import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; -import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalSqlCache; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.Planner; import org.apache.doris.planner.RuntimeFilter; import org.apache.doris.planner.ScanNode; -import org.apache.doris.qe.CommonResultSet; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ResultSet; -import org.apache.doris.qe.ResultSetMetaData; import org.apache.doris.qe.SessionVariable; import com.google.common.annotations.VisibleForTesting; @@ -75,7 +70,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.function.Function; @@ -572,23 +566,7 @@ public Optional handleQueryInFe(StatementBase parsedStmt) { } } - if (physicalPlan instanceof PhysicalResultSink - && physicalPlan.child(0) instanceof PhysicalHashAggregate && !getScanNodes().isEmpty() - && getScanNodes().get(0) instanceof IcebergScanNode) { - List columns = Lists.newArrayList(); - NamedExpression output = physicalPlan.getOutput().get(0); - columns.add(new Column(output.getName(), output.getDataType().toCatalogDataType())); - if (((IcebergScanNode) getScanNodes().get(0)).rowCount > 0) { - ResultSetMetaData metadata = new CommonResultSet.CommonResultSetMetaData(columns); - ResultSet resultSet = new CommonResultSet(metadata, Collections.singletonList( - Lists.newArrayList(String.valueOf(((IcebergScanNode) getScanNodes().get(0)).rowCount)))); - // only support one iceberg scan node and one count, e.g. select count(*) from icetbl; - return Optional.of(resultSet); - } - return Optional.empty(); - } else { - return Optional.empty(); - } + return Optional.empty(); } private void setFormatOptions() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index 70e442546c3be5..2657232db89786 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -41,7 +41,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.FormatOptions; import org.apache.doris.common.UserException; -import org.apache.doris.datasource.iceberg.source.IcebergScanNode; import org.apache.doris.nereids.PlannerHook; import org.apache.doris.qe.CommonResultSet; import org.apache.doris.qe.ConnectContext; @@ -642,21 +641,6 @@ public Optional handleQueryInFe(StatementBase parsedStmt) { List columns = new ArrayList<>(selectItems.size()); List columnLabels = parsedSelectStmt.getColLabels(); List data = new ArrayList<>(); - if ((singleNodePlanner.getScanNodes().size() > 0 && singleNodePlanner.getScanNodes().get(0) - instanceof IcebergScanNode) && (((IcebergScanNode) getScanNodes().get(0)).rowCount > 0)) { - SelectListItem item = selectItems.get(0); - Expr expr = item.getExpr(); - String columnName = columnLabels.get(0); - columns.add(new Column(columnName, expr.getType())); - data.add(String.valueOf(((IcebergScanNode) getScanNodes().get(0)).rowCount)); - ResultSetMetaData metadata = new CommonResultSet.CommonResultSetMetaData(columns); - ResultSet resultSet = new CommonResultSet(metadata, Collections.singletonList(data)); - // only support one iceberg scan node and one count, e.g. select count(*) from icetbl; - return Optional.of(resultSet); - } - if (!parsedSelectStmt.getTableRefs().isEmpty()) { - return Optional.empty(); - } FormatOptions options = FormatOptions.getDefault(); for (int i = 0; i < selectItems.size(); i++) { SelectListItem item = selectItems.get(i); From c54b976c1d651d4d99da7823dccff241418dc272 Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Thu, 1 Aug 2024 14:54:05 +0800 Subject: [PATCH 2/2] fix --- .../apache/doris/datasource/iceberg/source/IcebergScanNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 0d5ff45c3e2748..df6d4d6702ade8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -267,7 +267,7 @@ private List doGetSplits() throws UserException { TPushAggOp aggOp = getPushDownAggNoGroupingOp(); if (aggOp.equals(TPushAggOp.COUNT) && getCountFromSnapshot() >= 0) { // we can create a special empty split and skip the plan process - return Collections.singletonList(splits.get(0)); + return splits.isEmpty() ? splits : Collections.singletonList(splits.get(0)); } readPartitionNum = partitionPathSet.size();