From 52f2642f76ceaf872c34c73bc0293cad1d35ace2 Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Thu, 7 Dec 2023 23:46:45 +0800 Subject: [PATCH 1/6] [Performance](point query)Opimize partition prune for point query --- .../apache/doris/planner/OlapScanNode.java | 30 +++++++- .../org/apache/doris/planner/RangeMap.java | 73 +++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/RangeMap.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 0683e54082a401..20bf7682f1199d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -28,6 +28,7 @@ import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.InPredicate; import org.apache.doris.analysis.IntLiteral; +import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.PartitionNames; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotId; @@ -51,6 +52,7 @@ import org.apache.doris.catalog.Partition.PartitionState; import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.ScalarType; @@ -92,6 +94,7 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Range; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -104,6 +107,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -199,6 +203,8 @@ public class OlapScanNode extends ScanNode { private boolean shouldColoScan = false; + private RangeMap partitionRangeMapByLiteral; + // Constructs node to scan given data files of table 'tbl'. public OlapScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName) { super(id, desc, planNodeName, StatisticalType.OLAP_SCAN_NODE); @@ -674,7 +680,29 @@ private Collection partitionPrune(PartitionInfo partitionInfo, } else { keyItemMap = partitionInfo.getIdToItem(false); } - + if (isPointQuery() && partitionInfo.getPartitionColumns().size() == 1) { + // short circuit, a quick path to find partition + // TODO handle new partition + ColumnRange filterRange = columnNameToRange.get(partitionInfo.getPartitionColumns().get(0).getName()); + LiteralExpr lowerBound = filterRange.getRangeSet().get().asRanges().stream() + .findFirst().get().lowerEndpoint().getValue(); + LiteralExpr upperBound = filterRange.getRangeSet().get().asRanges().stream() + .findFirst().get().upperEndpoint().getValue(); + Range filterRangeValue = Range.closed(lowerBound, upperBound); + if (partitionRangeMapByLiteral == null) { + partitionRangeMapByLiteral = new RangeMap<>(); + for (Entry entry : keyItemMap.entrySet()) { + Range range = entry.getValue().getItems(); + LiteralExpr partitionLowerBound = (LiteralExpr) range.lowerEndpoint().getKeys().get(0); + LiteralExpr partitionUpperBound = (LiteralExpr) range.upperEndpoint().getKeys().get(0); + Range partitionRange = Range.closedOpen(partitionLowerBound, partitionUpperBound); + partitionRangeMapByLiteral.put(partitionRange, entry.getKey()); + } + } + List partitionList = partitionRangeMapByLiteral.getOverlappingRangeValues(filterRangeValue); + LOG.debug("pick partitionList {} for point query", partitionList); + return partitionList; + } if (partitionInfo.getType() == PartitionType.RANGE) { partitionPruner = new RangePartitionPrunerV2(keyItemMap, partitionInfo.getPartitionColumns(), columnNameToRange); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RangeMap.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RangeMap.java new file mode 100644 index 00000000000000..c8fd07f940d1b6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RangeMap.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.planner; + +import com.google.common.collect.Range; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.NavigableMap; +import java.util.TreeMap; +import java.util.stream.Collectors; + +public class RangeMap, V> { + + private final NavigableMap, V> rangeMap = new TreeMap<>(new RangeComparator()); + + public void put(Range range, V value) { + rangeMap.put(range, value); + } + + public List getOverlappingRangeValues(Range searchRange) { + return getOverlappingRanges(searchRange).stream() + .map(Map.Entry::getValue) + .collect(Collectors.toList()); + } + + public List, V>> getOverlappingRanges(Range searchRange) { + List, V>> overlappingRanges = new ArrayList<>(); + + // Find the possible starting point for the search + Map.Entry, V> floorEntry = rangeMap.floorEntry(searchRange); + Map.Entry, V> ceilingEntry = rangeMap.ceilingEntry(searchRange); + + // Start iterating from the earlier of the floor or ceiling entry + Map.Entry, V> startEntry = (floorEntry != null) ? floorEntry : ceilingEntry; + if (startEntry == null) { + return overlappingRanges; + } + + for (Map.Entry, V> entry : rangeMap.tailMap(startEntry.getKey()).entrySet()) { + if (entry.getKey().lowerEndpoint().compareTo(searchRange.upperEndpoint()) > 0) { + break; // No more overlapping ranges possible + } + if (entry.getKey().isConnected(searchRange) && !entry.getKey().intersection(searchRange).isEmpty()) { + overlappingRanges.add(entry); + } + } + return overlappingRanges; + } + + private static class RangeComparator> implements java.util.Comparator> { + @Override + public int compare(Range r1, Range r2) { + return r1.lowerEndpoint().compareTo(r2.lowerEndpoint()); + } + } +} From 019085ecaa85ae8a4c82551c44f1c8cb6dd9a2ae Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Mon, 11 Dec 2023 11:28:30 +0800 Subject: [PATCH 2/6] refactor --- .../apache/doris/planner/OlapScanNode.java | 30 ++----- .../PartitionPruneV2ForShortCircuitPlan.java | 90 +++++++++++++++++++ .../doris/planner/PartitionPrunerV2Base.java | 7 ++ 3 files changed, 105 insertions(+), 22 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPruneV2ForShortCircuitPlan.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 20bf7682f1199d..4af2b5b0391c60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -52,7 +52,6 @@ import org.apache.doris.catalog.Partition.PartitionState; import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionItem; -import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.Replica; import org.apache.doris.catalog.ScalarType; @@ -94,7 +93,6 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Range; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -107,7 +105,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -203,7 +200,9 @@ public class OlapScanNode extends ScanNode { private boolean shouldColoScan = false; - private RangeMap partitionRangeMapByLiteral; + // cached for prepared statement to quickly prune partition + private final PartitionPruneV2ForShortCircuitPlan cachedPartitionPruner = + new PartitionPruneV2ForShortCircuitPlan(); // Constructs node to scan given data files of table 'tbl'. public OlapScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName) { @@ -680,30 +679,17 @@ private Collection partitionPrune(PartitionInfo partitionInfo, } else { keyItemMap = partitionInfo.getIdToItem(false); } - if (isPointQuery() && partitionInfo.getPartitionColumns().size() == 1) { + if (partitionInfo.getType() == PartitionType.RANGE + && isPointQuery() && partitionInfo.getPartitionColumns().size() == 1) { // short circuit, a quick path to find partition - // TODO handle new partition ColumnRange filterRange = columnNameToRange.get(partitionInfo.getPartitionColumns().get(0).getName()); LiteralExpr lowerBound = filterRange.getRangeSet().get().asRanges().stream() .findFirst().get().lowerEndpoint().getValue(); LiteralExpr upperBound = filterRange.getRangeSet().get().asRanges().stream() .findFirst().get().upperEndpoint().getValue(); - Range filterRangeValue = Range.closed(lowerBound, upperBound); - if (partitionRangeMapByLiteral == null) { - partitionRangeMapByLiteral = new RangeMap<>(); - for (Entry entry : keyItemMap.entrySet()) { - Range range = entry.getValue().getItems(); - LiteralExpr partitionLowerBound = (LiteralExpr) range.lowerEndpoint().getKeys().get(0); - LiteralExpr partitionUpperBound = (LiteralExpr) range.upperEndpoint().getKeys().get(0); - Range partitionRange = Range.closedOpen(partitionLowerBound, partitionUpperBound); - partitionRangeMapByLiteral.put(partitionRange, entry.getKey()); - } - } - List partitionList = partitionRangeMapByLiteral.getOverlappingRangeValues(filterRangeValue); - LOG.debug("pick partitionList {} for point query", partitionList); - return partitionList; - } - if (partitionInfo.getType() == PartitionType.RANGE) { + cachedPartitionPruner.update(keyItemMap, lowerBound, upperBound); + partitionPruner = cachedPartitionPruner; + } else if (partitionInfo.getType() == PartitionType.RANGE) { partitionPruner = new RangePartitionPrunerV2(keyItemMap, partitionInfo.getPartitionColumns(), columnNameToRange); } else if (partitionInfo.getType() == PartitionType.LIST) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPruneV2ForShortCircuitPlan.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPruneV2ForShortCircuitPlan.java new file mode 100644 index 00000000000000..441c955cdd82f1 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPruneV2ForShortCircuitPlan.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.planner; + +import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.common.AnalysisException; + +import com.google.common.collect.Range; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Collection; +import java.util.Map; +import java.util.Map.Entry; + +public class PartitionPruneV2ForShortCircuitPlan extends PartitionPrunerV2Base { + private static final Logger LOG = LogManager.getLogger(PartitionPruneV2ForShortCircuitPlan.class); + // map to record literal range to find specific partition + private RangeMap partitionRangeMapByLiteral = new RangeMap<>(); + // last timestamp partitionRangeMapByLiteral updated + private long lastPartitionRangeMapUpdateTimestampS = 0; + LiteralExpr lowerBoundLiteral; + LiteralExpr upperBoundLiteral; + + PartitionPruneV2ForShortCircuitPlan() { + super(); + } + + public boolean update(Map keyItemMap, LiteralExpr lowerBound, LiteralExpr upperBound) { + this.lowerBoundLiteral = lowerBound; + this.upperBoundLiteral = upperBound; + // interval to update partitionRangeMapByLiteral + long partitionRangeMapUpdateIntervalS = 10; + if (System.currentTimeMillis() - lastPartitionRangeMapUpdateTimestampS + > partitionRangeMapUpdateIntervalS * 1000) { + partitionRangeMapByLiteral = new RangeMap<>(); + // recalculate map + for (Entry entry : keyItemMap.entrySet()) { + Range range = entry.getValue().getItems(); + LiteralExpr partitionLowerBound = (LiteralExpr) range.lowerEndpoint().getKeys().get(0); + LiteralExpr partitionUpperBound = (LiteralExpr) range.upperEndpoint().getKeys().get(0); + Range partitionRange = Range.closedOpen(partitionLowerBound, partitionUpperBound); + partitionRangeMapByLiteral.put(partitionRange, entry.getKey()); + } + LOG.debug("update partitionRangeMapByLiteral"); + this.lastPartitionRangeMapUpdateTimestampS = System.currentTimeMillis(); + return true; + } + return false; + } + + @Override + public Collection prune() throws AnalysisException { + Range filterRangeValue = Range.closed(lowerBoundLiteral, upperBoundLiteral); + return partitionRangeMapByLiteral.getOverlappingRangeValues(filterRangeValue); + } + + @Override + void genSingleColumnRangeMap() { + } + + @Override + FinalFilters getFinalFilters(ColumnRange columnRange, + Column column) throws AnalysisException { + throw new AnalysisException("Not implemented"); + } + + @Override + Collection pruneMultipleColumnPartition(Map columnToFilters) throws AnalysisException { + throw new AnalysisException("Not implemented"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPrunerV2Base.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPrunerV2Base.java index 376e2a4c7f064c..1d9f163ca804e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPrunerV2Base.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPrunerV2Base.java @@ -48,6 +48,13 @@ public abstract class PartitionPrunerV2Base implements PartitionPruner { // currently only used for list partition private Map.Entry defaultPartition; + // Only called in PartitionPruneV2ByShortCircuitPlan constructor + PartitionPrunerV2Base() { + this.idToPartitionItem = null; + this.partitionColumns = null; + this.columnNameToRange = null; + } + public PartitionPrunerV2Base(Map idToPartitionItem, List partitionColumns, Map columnNameToRange) { From aba915b870e7cc6a9b3934c61424c5262ab68fd3 Mon Sep 17 00:00:00 2001 From: eldenmoon <15605149486@163.com> Date: Mon, 11 Dec 2023 12:48:39 +0800 Subject: [PATCH 3/6] add case --- .../test_point_query_partition.out | 31 +++++ .../test_point_query_partition.groovy | 117 ++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 regression-test/data/point_query_p0/test_point_query_partition.out create mode 100644 regression-test/suites/point_query_p0/test_point_query_partition.groovy diff --git a/regression-test/data/point_query_p0/test_point_query_partition.out b/regression-test/data/point_query_p0/test_point_query_partition.out new file mode 100644 index 00000000000000..baf4f03a441688 --- /dev/null +++ b/regression-test/data/point_query_p0/test_point_query_partition.out @@ -0,0 +1,31 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !point_select -- +1 a + +-- !point_select -- +2 b + +-- !point_select -- +11 d + +-- !point_select -- +-1 c + +-- !point_select -- +11 d + +-- !point_select -- + +-- !point_select -- + +-- !point_select -- +33 f + +-- !point_select -- +45 g + +-- !point_select -- + +-- !point_select -- +999 h + diff --git a/regression-test/suites/point_query_p0/test_point_query_partition.groovy b/regression-test/suites/point_query_p0/test_point_query_partition.groovy new file mode 100644 index 00000000000000..b8bc829d128ea6 --- /dev/null +++ b/regression-test/suites/point_query_p0/test_point_query_partition.groovy @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.math.BigDecimal; + +suite("test_point_query_partition") { + def user = context.config.jdbcUser + def password = context.config.jdbcPassword + def realDb = "regression_test_serving_p0" + def tableName = realDb + ".tbl_point_query_partition" + sql "CREATE DATABASE IF NOT EXISTS ${realDb}" + + // Parse url + String jdbcUrl = context.config.jdbcUrl + String urlWithoutSchema = jdbcUrl.substring(jdbcUrl.indexOf("://") + 3) + def sql_ip = urlWithoutSchema.substring(0, urlWithoutSchema.indexOf(":")) + def sql_port + if (urlWithoutSchema.indexOf("/") >= 0) { + // e.g: jdbc:mysql://locahost:8080/?a=b + sql_port = urlWithoutSchema.substring(urlWithoutSchema.indexOf(":") + 1, urlWithoutSchema.indexOf("/")) + } else { + // e.g: jdbc:mysql://locahost:8080 + sql_port = urlWithoutSchema.substring(urlWithoutSchema.indexOf(":") + 1) + } + // set server side prepared statement url + def prepare_url = "jdbc:mysql://" + sql_ip + ":" + sql_port + "/" + realDb + "?&useServerPrepStmts=true" + + def generateString = {len -> + def str = "" + for (int i = 0; i < len; i++) { + str += "a" + } + return str + } + + def nprep_sql = { sql_str -> + def url_without_prep = "jdbc:mysql://" + sql_ip + ":" + sql_port + "/" + realDb + connect(user = user, password = password, url = url_without_prep) { + sql sql_str + } + } + + sql """DROP TABLE IF EXISTS ${tableName}""" + sql """ + CREATE TABLE IF NOT EXISTS ${tableName} ( + `k1` int(11) NULL COMMENT "", + `value` text NULL COMMENT "" + ) ENGINE=OLAP + UNIQUE KEY(`k1`) + PARTITION BY RANGE(`k1`) + ( + PARTITION `p1` VALUES LESS THAN ("1"), + PARTITION `p2` VALUES LESS THAN ("10"), + PARTITION `p3` VALUES LESS THAN ("30"), + PARTITION `p4` VALUES LESS THAN ("40"), + PARTITION `p5` VALUES LESS THAN ("1000") + ) + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "store_row_column" = "true", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "storage_format" = "V2") + """ + + sql """INSERT INTO ${tableName} VALUES (1, 'a')""" + sql """INSERT INTO ${tableName} VALUES (2, 'b')""" + sql """INSERT INTO ${tableName} VALUES (-1, 'c')""" + sql """INSERT INTO ${tableName} VALUES (11, 'd')""" + sql """INSERT INTO ${tableName} VALUES (15, 'e')""" + sql """INSERT INTO ${tableName} VALUES (33, 'f')""" + sql """INSERT INTO ${tableName} VALUES (45, 'g')""" + sql """INSERT INTO ${tableName} VALUES (999, 'h')""" + def result1 = connect(user=user, password=password, url=prepare_url) { + def stmt = prepareStatement "select * from ${tableName} where k1 = ?" + assertEquals(stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement); + stmt.setInt(1, 1) + qe_point_select stmt + stmt.setInt(1, 2) + qe_point_select stmt + stmt.setInt(1, 11) + qe_point_select stmt + stmt.setInt(1, -1) + qe_point_select stmt + stmt.setInt(1, 11) + qe_point_select stmt + stmt.setInt(1, 12) + qe_point_select stmt + stmt.setInt(1, 34) + qe_point_select stmt + stmt.setInt(1, 33) + qe_point_select stmt + stmt.setInt(1, 45) + qe_point_select stmt + stmt.setInt(1, 666) + qe_point_select stmt + stmt.setInt(1, 999) + qe_point_select stmt + // stmt.setInt(1, 1000) + // qe_point_select stmt + } +} \ No newline at end of file From 8062c8efb30f150107a39c18274744cbb8debb53 Mon Sep 17 00:00:00 2001 From: eldenmoon <15605149486@163.com> Date: Mon, 11 Dec 2023 14:06:48 +0800 Subject: [PATCH 4/6] fix no partition found case --- .../src/main/java/org/apache/doris/qe/PointQueryExec.java | 7 +++++++ .../data/point_query_p0/test_point_query_partition.out | 2 ++ .../point_query_p0/test_point_query_partition.groovy | 4 ++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExec.java b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExec.java index 50c422a1979340..06308fd6341987 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExec.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExec.java @@ -121,6 +121,9 @@ void setScanRangeLocations() throws Exception { OlapScanNode planRoot = getPlanRoot(); // compute scan range List locations = planRoot.lazyEvaluateRangeLocations(); + if (planRoot.getScanTabletIds().isEmpty()) { + return; + } Preconditions.checkState(planRoot.getScanTabletIds().size() == 1); this.tabletID = planRoot.getScanTabletIds().get(0); @@ -167,6 +170,10 @@ public void cancel(Types.PPlanFragmentCancelReason cancelReason) { @Override public RowBatch getNext() throws Exception { setScanRangeLocations(); + // No partition/tablet found return emtpy row batch + if (candidateBackends == null || candidateBackends.isEmpty()) { + return new RowBatch(); + } Iterator backendIter = candidateBackends.iterator(); RowBatch rowBatch = null; int tryCount = 0; diff --git a/regression-test/data/point_query_p0/test_point_query_partition.out b/regression-test/data/point_query_p0/test_point_query_partition.out index baf4f03a441688..cd22e6c93ec4bb 100644 --- a/regression-test/data/point_query_p0/test_point_query_partition.out +++ b/regression-test/data/point_query_p0/test_point_query_partition.out @@ -29,3 +29,5 @@ -- !point_select -- 999 h +-- !point_select -- + diff --git a/regression-test/suites/point_query_p0/test_point_query_partition.groovy b/regression-test/suites/point_query_p0/test_point_query_partition.groovy index b8bc829d128ea6..7b5966db0c1915 100644 --- a/regression-test/suites/point_query_p0/test_point_query_partition.groovy +++ b/regression-test/suites/point_query_p0/test_point_query_partition.groovy @@ -111,7 +111,7 @@ suite("test_point_query_partition") { qe_point_select stmt stmt.setInt(1, 999) qe_point_select stmt - // stmt.setInt(1, 1000) - // qe_point_select stmt + stmt.setInt(1, 1000) + qe_point_select stmt } } \ No newline at end of file From 3b8a89d4c1e3fd66d754f483b4413641ec51a0a4 Mon Sep 17 00:00:00 2001 From: eldenmoon <15605149486@163.com> Date: Mon, 18 Dec 2023 21:45:20 +0800 Subject: [PATCH 5/6] modify ac review --- .../apache/doris/planner/OlapScanNode.java | 23 ++++++++++--------- .../PartitionPruneV2ForShortCircuitPlan.java | 14 +++++------ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 4af2b5b0391c60..54ccc40e9e2b28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -201,6 +201,7 @@ public class OlapScanNode extends ScanNode { private boolean shouldColoScan = false; // cached for prepared statement to quickly prune partition + // only used in short circuit plan at present private final PartitionPruneV2ForShortCircuitPlan cachedPartitionPruner = new PartitionPruneV2ForShortCircuitPlan(); @@ -679,17 +680,17 @@ private Collection partitionPrune(PartitionInfo partitionInfo, } else { keyItemMap = partitionInfo.getIdToItem(false); } - if (partitionInfo.getType() == PartitionType.RANGE - && isPointQuery() && partitionInfo.getPartitionColumns().size() == 1) { - // short circuit, a quick path to find partition - ColumnRange filterRange = columnNameToRange.get(partitionInfo.getPartitionColumns().get(0).getName()); - LiteralExpr lowerBound = filterRange.getRangeSet().get().asRanges().stream() - .findFirst().get().lowerEndpoint().getValue(); - LiteralExpr upperBound = filterRange.getRangeSet().get().asRanges().stream() - .findFirst().get().upperEndpoint().getValue(); - cachedPartitionPruner.update(keyItemMap, lowerBound, upperBound); - partitionPruner = cachedPartitionPruner; - } else if (partitionInfo.getType() == PartitionType.RANGE) { + if (partitionInfo.getType() == PartitionType.RANGE) { + if (isPointQuery() && partitionInfo.getPartitionColumns().size() == 1) { + // short circuit, a quick path to find partition + ColumnRange filterRange = columnNameToRange.get(partitionInfo.getPartitionColumns().get(0).getName()); + LiteralExpr lowerBound = filterRange.getRangeSet().get().asRanges().stream() + .findFirst().get().lowerEndpoint().getValue(); + LiteralExpr upperBound = filterRange.getRangeSet().get().asRanges().stream() + .findFirst().get().upperEndpoint().getValue(); + cachedPartitionPruner.update(keyItemMap); + return cachedPartitionPruner.prune(lowerBound, upperBound); + } partitionPruner = new RangePartitionPrunerV2(keyItemMap, partitionInfo.getPartitionColumns(), columnNameToRange); } else if (partitionInfo.getType() == PartitionType.LIST) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPruneV2ForShortCircuitPlan.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPruneV2ForShortCircuitPlan.java index 441c955cdd82f1..17188d1c763c44 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPruneV2ForShortCircuitPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPruneV2ForShortCircuitPlan.java @@ -37,16 +37,12 @@ public class PartitionPruneV2ForShortCircuitPlan extends PartitionPrunerV2Base { private RangeMap partitionRangeMapByLiteral = new RangeMap<>(); // last timestamp partitionRangeMapByLiteral updated private long lastPartitionRangeMapUpdateTimestampS = 0; - LiteralExpr lowerBoundLiteral; - LiteralExpr upperBoundLiteral; PartitionPruneV2ForShortCircuitPlan() { super(); } - public boolean update(Map keyItemMap, LiteralExpr lowerBound, LiteralExpr upperBound) { - this.lowerBoundLiteral = lowerBound; - this.upperBoundLiteral = upperBound; + public boolean update(Map keyItemMap) { // interval to update partitionRangeMapByLiteral long partitionRangeMapUpdateIntervalS = 10; if (System.currentTimeMillis() - lastPartitionRangeMapUpdateTimestampS @@ -67,10 +63,14 @@ public boolean update(Map keyItemMap, LiteralExpr lowerBoun return false; } + public Collection prune(LiteralExpr lowerBound, LiteralExpr upperBound) throws AnalysisException { + Range filterRangeValue = Range.closed(lowerBound, upperBound); + return partitionRangeMapByLiteral.getOverlappingRangeValues(filterRangeValue); + } + @Override public Collection prune() throws AnalysisException { - Range filterRangeValue = Range.closed(lowerBoundLiteral, upperBoundLiteral); - return partitionRangeMapByLiteral.getOverlappingRangeValues(filterRangeValue); + throw new AnalysisException("Not implemented"); } @Override From 742016c5eec10bd80e897d2340a1365dfc39d60b Mon Sep 17 00:00:00 2001 From: eldenmoon <15605149486@163.com> Date: Mon, 18 Dec 2023 23:32:00 +0800 Subject: [PATCH 6/6] rename --- .../doris/planner/PartitionPruneV2ForShortCircuitPlan.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPruneV2ForShortCircuitPlan.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPruneV2ForShortCircuitPlan.java index 17188d1c763c44..8d39ed4d4fcbd6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPruneV2ForShortCircuitPlan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionPruneV2ForShortCircuitPlan.java @@ -36,7 +36,7 @@ public class PartitionPruneV2ForShortCircuitPlan extends PartitionPrunerV2Base { // map to record literal range to find specific partition private RangeMap partitionRangeMapByLiteral = new RangeMap<>(); // last timestamp partitionRangeMapByLiteral updated - private long lastPartitionRangeMapUpdateTimestampS = 0; + private long lastPartitionRangeMapUpdateTimestampMs = 0; PartitionPruneV2ForShortCircuitPlan() { super(); @@ -45,7 +45,7 @@ public class PartitionPruneV2ForShortCircuitPlan extends PartitionPrunerV2Base { public boolean update(Map keyItemMap) { // interval to update partitionRangeMapByLiteral long partitionRangeMapUpdateIntervalS = 10; - if (System.currentTimeMillis() - lastPartitionRangeMapUpdateTimestampS + if (System.currentTimeMillis() - lastPartitionRangeMapUpdateTimestampMs > partitionRangeMapUpdateIntervalS * 1000) { partitionRangeMapByLiteral = new RangeMap<>(); // recalculate map @@ -57,7 +57,7 @@ public boolean update(Map keyItemMap) { partitionRangeMapByLiteral.put(partitionRange, entry.getKey()); } LOG.debug("update partitionRangeMapByLiteral"); - this.lastPartitionRangeMapUpdateTimestampS = System.currentTimeMillis(); + this.lastPartitionRangeMapUpdateTimestampMs = System.currentTimeMillis(); return true; } return false;