From 60772d327b6464eb854dbea9d4ad3f325521a28f Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 21 Jun 2024 23:42:20 +0800 Subject: [PATCH 1/7] [feature](sql-block-rule) sql block rule support external table create sql_block_rule r1 properties("partition_num" = "100", "global" = "true"); --- .../org/apache/doris/datasource/FileScanNode.java | 7 +------ .../doris/datasource/hive/source/HiveScanNode.java | 4 ++-- .../doris/datasource/hudi/source/HudiScanNode.java | 4 ++-- .../datasource/iceberg/source/IcebergScanNode.java | 2 +- .../maxcompute/source/MaxComputeScanNode.java | 2 +- .../datasource/paimon/source/PaimonScanNode.java | 2 +- .../org/apache/doris/planner/OlapScanNode.java | 14 ++------------ .../java/org/apache/doris/planner/ScanNode.java | 11 +++++++++++ .../java/org/apache/doris/qe/StmtExecutor.java | 11 ++++++----- .../org/apache/doris/qe/cache/CacheAnalyzer.java | 2 +- 10 files changed, 28 insertions(+), 31 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java index bb6865582fce63..20d8add5e4b415 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java @@ -69,7 +69,6 @@ public abstract class FileScanNode extends ExternalScanNode { protected long inputSplitsNum = 0; protected long totalFileSize = 0; protected long totalPartitionNum = 0; - protected long readPartitionNum = 0; protected long fileSplitSize; public long rowCount = 0; @@ -116,7 +115,7 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { } output.append("inputSplitNum=").append(inputSplitsNum).append(", totalFileSize=") .append(totalFileSize).append(", scanRanges=").append(scanRangeLocations.size()).append("\n"); - output.append(prefix).append("partition=").append(readPartitionNum).append("/").append(totalPartitionNum) + output.append(prefix).append("partition=").append(selectedPartitionNum).append("/").append(totalPartitionNum) .append("\n"); if (detailLevel == TExplainLevel.VERBOSE) { @@ -299,8 +298,4 @@ protected int getBlockIndex(BlockLocation[] blkLocations, long offset) { long fileLength = last.getOffset() + last.getLength() - 1L; throw new IllegalArgumentException(String.format("Offset %d is outside of file (0..%d)", offset, fileLength)); } - - public long getReadPartitionNum() { - return this.readPartitionNum; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java index 0214ecc464238a..f306eabd460e43 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java @@ -180,7 +180,7 @@ protected List getPartitions() throws AnalysisException { partitionItems = selectedPartitions.selectedPartitions.values(); } Preconditions.checkNotNull(partitionItems); - this.readPartitionNum = partitionItems.size(); + this.selectedPartitionNum = partitionItems.size(); // get partitions from cache List> partitionValuesList = Lists.newArrayListWithCapacity(partitionItems.size()); @@ -197,7 +197,7 @@ protected List getPartitions() throws AnalysisException { hmsTable.getRemoteTable().getSd().getInputFormat(), hmsTable.getRemoteTable().getSd().getLocation(), null, Maps.newHashMap()); this.totalPartitionNum = 1; - this.readPartitionNum = 1; + this.selectedPartitionNum = 1; resPartitions.add(dummyPartition); } if (ConnectContext.get().getExecutor() != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 82e21bcdd17214..0fc6b35511a198 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -280,7 +280,7 @@ private List getPrunedPartitions( partitionValues.getSingleColumnRangeMap(), true); Collection filteredPartitionIds = pruner.prune(); - this.readPartitionNum = filteredPartitionIds.size(); + this.selectedPartitionNum = filteredPartitionIds.size(); // 3. get partitions from cache String dbName = hmsTable.getDbName(); String tblName = hmsTable.getName(); @@ -305,7 +305,7 @@ private List getPrunedPartitions( hmsTable.getRemoteTable().getSd().getInputFormat(), hmsTable.getRemoteTable().getSd().getLocation(), null, Maps.newHashMap()); this.totalPartitionNum = 1; - this.readPartitionNum = 1; + this.selectedPartitionNum = 1; return Lists.newArrayList(dummyPartition); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index d5d3360845c793..c169abda90f35d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -278,7 +278,7 @@ private List doGetSplits() throws UserException { return Collections.singletonList(splits.get(0)); } - readPartitionNum = partitionPathSet.size(); + selectedPartitionNum = partitionPathSet.size(); return splits; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java index c17a1d3e3f0d48..6521ecd3101e53 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java @@ -197,7 +197,7 @@ private List getPrunedPartitionSpecs() throws AnalysisException { partitionValues.getSingleColumnRangeMap(), false); Collection filteredPartitionIds = pruner.prune(); - this.readPartitionNum = filteredPartitionIds.size(); + this.selectedPartitionNum = filteredPartitionIds.size(); // get partitions from cache Map partitionIdToNameMap = partitionValues.getPartitionIdToNameMap(); filteredPartitionIds.forEach(id -> result.add(partitionIdToNameMap.get(id))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index 2956d61d90925e..b5edcdee089285 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -263,7 +263,7 @@ public List getSplits() throws UserException { } splitStats.add(splitStat); } - this.readPartitionNum = selectedPartitionValues.size(); + this.selectedPartitionNum = selectedPartitionValues.size(); // TODO: get total partition number return splits; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 64479cdc855880..f928feaaf73ead 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -163,10 +163,8 @@ public class OlapScanNode extends ScanNode { private boolean canTurnOnPreAggr = true; private boolean forceOpenPreAgg = false; private OlapTable olapTable = null; - private long selectedTabletsNum = 0; private long totalTabletsNum = 0; private long selectedIndexId = -1; - private int selectedPartitionNum = 0; private Collection selectedPartitionIds = Lists.newArrayList(); private long totalBytes = 0; // tablet id to single replica bytes @@ -295,14 +293,6 @@ public void setForceOpenPreAgg(boolean forceOpenPreAgg) { this.forceOpenPreAgg = forceOpenPreAgg; } - public Integer getSelectedPartitionNum() { - return selectedPartitionNum; - } - - public Long getSelectedTabletsNum() { - return selectedTabletsNum; - } - public SortInfo getSortInfo() { return sortInfo; } @@ -1215,7 +1205,7 @@ private void computeTabletInfo() throws UserException { } totalTabletsNum += selectedTable.getTablets().size(); - selectedTabletsNum += tablets.size(); + selectedSplitNum += tablets.size(); addScanRangeLocations(partition, tablets); } } @@ -1377,7 +1367,7 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { .collect(Collectors.joining(",")); output.append(prefix).append(String.format("partitions=%s/%s (%s)", selectedPartitionNum, olapTable.getPartitions().size(), selectedPartitions)).append("\n"); - output.append(prefix).append(String.format("tablets=%s/%s", selectedTabletsNum, totalTabletsNum)); + output.append(prefix).append(String.format("tablets=%s/%s", selectedSplitNum, totalTabletsNum)); // We print up to 3 tablet, and we print "..." if the number is more than 3 if (scanTabletIds.size() > 3) { List firstTenTabletIds = scanTabletIds.subList(0, 3); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index f53f69505da5bd..e4c1a8be3373cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -99,6 +99,9 @@ public abstract class ScanNode extends PlanNode implements SplitGenerator { protected PartitionInfo partitionsInfo = null; protected SplitAssignment splitAssignment = null; + protected long selectedPartitionNum = 0; + protected long selectedSplitNum = 0; + // create a mapping between output slot's id and project expr Map outputSlotToProjectExpr = new HashMap<>(); @@ -747,6 +750,7 @@ public boolean shouldUseOneInstance(ConnectContext ctx) { return hasLimit() && getLimit() < limitRowsForSingleInstance && conjuncts.isEmpty(); } +<<<<<<< HEAD // In cloud mode, meta read lock is not enough to keep a snapshot of the partition versions. // After all scan node are collected, it is possible to gain a snapshot of the partition version. public static void setVisibleVersionForOlapScanNodes(List scanNodes) throws UserException { @@ -833,4 +837,11 @@ public boolean useTopnFilter() { return !topnFilterSortNodes.isEmpty(); } + public long getSelectedPartitionNum() { + return selectedPartitionNum; + } + + public long getSelectedSplitNum() { + return selectedSplitNum; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 6eaeb559b7343c..060c2a89319c93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -125,6 +125,7 @@ import org.apache.doris.common.util.SqlParserUtils; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.FileScanNode; import org.apache.doris.datasource.jdbc.client.JdbcClientException; import org.apache.doris.datasource.tvf.source.TVFScanNode; import org.apache.doris.load.EtlJobType; @@ -653,13 +654,13 @@ public void checkBlockRulesByScan(Planner planner) throws AnalysisException { } List scanNodeList = planner.getScanNodes(); for (ScanNode scanNode : scanNodeList) { - if (scanNode instanceof OlapScanNode) { - OlapScanNode olapScanNode = (OlapScanNode) scanNode; + if (scanNode instanceof OlapScanNode || scanNode instanceof FileScanNode) { Env.getCurrentEnv().getSqlBlockRuleMgr().checkLimitations( - olapScanNode.getSelectedPartitionNum().longValue(), - olapScanNode.getSelectedTabletsNum(), - olapScanNode.getCardinality(), + scanNode.getSelectedPartitionNum(), + scanNode.getSelectedSplitNum(), + scanNode.getCardinality(), context.getQualifiedUser()); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java index 17be5c42e226bd..67917a07e4ef2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java @@ -714,7 +714,7 @@ private CacheTable buildCacheTableForOlapScanNode(OlapScanNode node) { private CacheTable buildCacheTableForHiveScanNode(HiveScanNode node) { CacheTable cacheTable = new CacheTable(); cacheTable.table = node.getTargetTable(); - cacheTable.partitionNum = node.getReadPartitionNum(); + cacheTable.partitionNum = node.getSelectedPartitionNum(); cacheTable.latestPartitionTime = cacheTable.table.getUpdateTime(); TableIf tableIf = cacheTable.table; DatabaseIf database = tableIf.getDatabase(); From 80b69228d81603427c0d20c7fb66bc619186130d Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 30 Jun 2024 23:03:17 +0800 Subject: [PATCH 2/7] format --- fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java | 1 - 1 file changed, 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index e4c1a8be3373cc..224f85b37a567f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -750,7 +750,6 @@ public boolean shouldUseOneInstance(ConnectContext ctx) { return hasLimit() && getLimit() < limitRowsForSingleInstance && conjuncts.isEmpty(); } -<<<<<<< HEAD // In cloud mode, meta read lock is not enough to keep a snapshot of the partition versions. // After all scan node are collected, it is possible to gain a snapshot of the partition version. public static void setVisibleVersionForOlapScanNodes(List scanNodes) throws UserException { From 907607d11fe335aa2a3ca1b529442540b578fc60 Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 30 Jun 2024 23:49:36 +0800 Subject: [PATCH 3/7] fix fe ut --- .../test/java/org/apache/doris/planner/ColocatePlanTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/ColocatePlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/ColocatePlanTest.java index 5110ea2fde1580..68279489bfc1e5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/ColocatePlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/ColocatePlanTest.java @@ -161,7 +161,7 @@ public void sqlAggWithColocateTable() throws Exception { Assert.assertTrue(scanNodeList.get(0) instanceof OlapScanNode); OlapScanNode olapScanNode = (OlapScanNode) scanNodeList.get(0); Assert.assertEquals(olapScanNode.getSelectedPartitionIds().size(), 2); - long selectedTablet = Deencapsulation.getField(olapScanNode, "selectedTabletsNum"); + long selectedTablet = Deencapsulation.getField(olapScanNode, "selectedSplitNum"); Assert.assertEquals(selectedTablet, 2); List instanceInfo = coordinator.getFragmentInstanceInfos(); From 8e14a3db72af47d9d78f5c73be05fee1d6972470 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 1 Jul 2024 22:44:49 +0800 Subject: [PATCH 4/7] 1 --- .../hive/test_external_sql_block_rule.out | 13 +++++ .../hive/test_external_sql_block_rule.groovy | 56 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 regression-test/data/external_table_p0/hive/test_external_sql_block_rule.out create mode 100644 regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy diff --git a/regression-test/data/external_table_p0/hive/test_external_sql_block_rule.out b/regression-test/data/external_table_p0/hive/test_external_sql_block_rule.out new file mode 100644 index 00000000000000..a09d46dbea3efb --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_external_sql_block_rule.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql01 -- +6179 21 4 20 5.00 4600.10 0.05 0.00 A F 1994-06-05 1994-07-27 1994-06-26 COLLECT COD MAIL silent deposits. furiously us chicago +6273 2552 1 51 33.00 31384.65 0.04 0.08 R F 1995-04-23 1995-05-02 1995-05-13 DELIVER IN PERSON TRUCK ges. unusual, pending packages accordi us chicago +8645 7554 4 53 34.00 32403.70 0.03 0.03 N O 1996-12-29 1997-01-25 1997-01-16 TAKE BACK RETURN FOB ackages are carefully above the jp tokyo +5121 7580 6 79 2.00 1958.14 0.04 0.07 R F 1992-08-10 1992-06-28 1992-08-11 NONE FOB final, regular account us washington +2883 2592 1 91 33.00 32705.97 0.08 0.07 R F 1995-02-26 1995-03-04 1995-03-01 NONE RAIL s. final i cn shanghai +4452 7650 2 149 47.00 49309.58 0.01 0.06 A F 1994-10-08 1994-08-09 1994-10-09 TAKE BACK RETURN TRUCK ts. slyly regular cour us washington +807 5150 7 149 19.00 19933.66 0.08 0.05 A F 1994-02-10 1994-02-20 1994-03-06 NONE SHIP ns haggle quickly across the furi cn beijing +4102 5176 5 175 32.00 34405.44 0.08 0.01 N O 1996-05-14 1996-04-29 1996-05-29 NONE RAIL the even requests; regular pinto us washington +2117 2680 6 179 27.00 29137.59 0.09 0.08 N O 1997-06-30 1997-06-27 1997-07-11 TAKE BACK RETURN REG AIR the carefully ironic ideas cn shanghai +548 7683 3 182 21.00 22725.78 0.03 0.08 A F 1995-01-13 1994-12-18 1995-01-25 NONE AIR ideas. special accounts above the furiou cn beijing + diff --git a/regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy b/regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy new file mode 100644 index 00000000000000..db08ea1dfba4d2 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_external_sql_block_rule", "external_docker,hive,external_docker_hive,p0,external") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test.") + return; + } + + String hivePrefix = "hive2"; + String catalog_name = "test_${hivePrefix}_serde_prop" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + + sql """drop catalog if exists ${catalog_name} """ + + sql """CREATE CATALOG ${catalog_name} PROPERTIES ( + 'type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}', + 'hadoop.username' = 'hive' + );""" + + sql "use ${catalog_name}.`default`"; + qt_sql01 """select * from parquet_partition_table order by l_linenumber limit 10;""" + + sql """drop sql_block_rule if exists external_hive_partition""" + sql """create sql_block_rule external_hive_partition properties("partition_num" = "3", "global" = "false");""" + sql """drop user if exists external_block_user1""" + sql """create user external_block_user1;""" + sql """SET PROPERTY FOR 'external_block_user1' 'sql_block_rules' = 'external_hive_partition';""" + sql """grant all on *.*.* to external_block_user1;""" + + // login as external_block_user1 + def result1 = connect(user = 'external_block_user1', password = '', url = context.config.jdbcUrl) { + test { + sql """select * from ${catalog_name}.`default`.parquet_partition_table order by l_linenumber limit 10;""" + exception """sql hits sql block rule: external_hive_partition, reach partition_num : 3""" + } + } +} + From 18bcc4d8d65beda574007c587c19551f7b78d7f4 Mon Sep 17 00:00:00 2001 From: morningman Date: Tue, 2 Jul 2024 18:24:23 +0800 Subject: [PATCH 5/7] 1 --- .../java/org/apache/doris/datasource/FileQueryScanNode.java | 6 +++--- .../main/java/org/apache/doris/datasource/FileScanNode.java | 3 +-- .../apache/doris/datasource/hudi/source/HudiScanNode.java | 2 +- .../java/org/apache/doris/planner/FileLoadScanNode.java | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index af42093442bc7d..89acd8b87afd5c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -326,7 +326,7 @@ public void createScanRangeLocations() throws UserException { if (splitAssignment.getSampleSplit() == null && !(getLocationType() == TFileType.FILE_STREAM)) { return; } - inputSplitsNum = numApproximateSplits(); + selectedSplitNum = numApproximateSplits(); TFileType locationType; FileSplit fileSplit = (FileSplit) splitAssignment.getSampleSplit(); @@ -336,7 +336,7 @@ public void createScanRangeLocations() throws UserException { } else { locationType = getLocationType(fileSplit.getPath().toString()); } - totalFileSize = fileSplit.getLength() * inputSplitsNum; + totalFileSize = fileSplit.getLength() * selectedSplitNum; long maxWaitTime = ConnectContext.get().getSessionVariable().getFetchSplitsMaxWaitTime(); // Not accurate, only used to estimate concurrency. int numSplitsPerBE = numApproximateSplits() / backendPolicy.numBackends(); @@ -364,7 +364,7 @@ public void createScanRangeLocations() throws UserException { if (ConnectContext.get().getExecutor() != null) { ConnectContext.get().getExecutor().getSummaryProfile().setGetSplitsFinishTime(); } - inputSplitsNum = inputSplits.size(); + selectedSplitNum = inputSplits.size(); if (inputSplits.isEmpty() && !(getLocationType() == TFileType.FILE_STREAM)) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java index 20d8add5e4b415..3f36e5c37db3a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java @@ -66,7 +66,6 @@ public abstract class FileScanNode extends ExternalScanNode { public static final long DEFAULT_SPLIT_SIZE = 8 * 1024 * 1024; // 8MB // For explain - protected long inputSplitsNum = 0; protected long totalFileSize = 0; protected long totalPartitionNum = 0; protected long fileSplitSize; @@ -113,7 +112,7 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { if (isBatchMode()) { output.append("(approximate)"); } - output.append("inputSplitNum=").append(inputSplitsNum).append(", totalFileSize=") + output.append("inputSplitNum=").append(selectedSplitNum).append(", totalFileSize=") .append(totalFileSize).append(", scanRanges=").append(scanRangeLocations.size()).append("\n"); output.append(prefix).append("partition=").append(selectedPartitionNum).append("/").append(totalPartitionNum) .append("\n"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 0fc6b35511a198..111f0877f3a82b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -497,7 +497,7 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { return super.getNodeExplainString(prefix, detailLevel); } else { return super.getNodeExplainString(prefix, detailLevel) - + String.format("%shudiNativeReadSplits=%d/%d\n", prefix, noLogsSplitNum.get(), inputSplitsNum); + + String.format("%shudiNativeReadSplits=%d/%d\n", prefix, noLogsSplitNum.get(), selectedSplitNum); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/FileLoadScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/FileLoadScanNode.java index 1fb5b9ddd3751d..4047c8f30b3609 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/FileLoadScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/FileLoadScanNode.java @@ -206,7 +206,7 @@ public void finalize(Analyzer analyzer) throws UserException { LoadScanProvider scanProvider = scanProviders.get(i); finalizeParamsForLoad(context, analyzer); createScanRangeLocations(context, scanProvider, localBackendPolicy); - this.inputSplitsNum += scanProvider.getInputSplitNum(); + this.selectedSplitNum += scanProvider.getInputSplitNum(); this.totalFileSize += scanProvider.getInputFileSize(); } } From bd634ddf4eac87a983874f03c1aed11f26f06fe3 Mon Sep 17 00:00:00 2001 From: morningman Date: Sun, 7 Jul 2024 00:00:45 +0800 Subject: [PATCH 6/7] 3 --- .../hive/test_external_sql_block_rule.out | 2 +- .../hive/test_external_sql_block_rule.groovy | 26 +++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/regression-test/data/external_table_p0/hive/test_external_sql_block_rule.out b/regression-test/data/external_table_p0/hive/test_external_sql_block_rule.out index a09d46dbea3efb..534f11b0e4441d 100644 --- a/regression-test/data/external_table_p0/hive/test_external_sql_block_rule.out +++ b/regression-test/data/external_table_p0/hive/test_external_sql_block_rule.out @@ -5,8 +5,8 @@ 8645 7554 4 53 34.00 32403.70 0.03 0.03 N O 1996-12-29 1997-01-25 1997-01-16 TAKE BACK RETURN FOB ackages are carefully above the jp tokyo 5121 7580 6 79 2.00 1958.14 0.04 0.07 R F 1992-08-10 1992-06-28 1992-08-11 NONE FOB final, regular account us washington 2883 2592 1 91 33.00 32705.97 0.08 0.07 R F 1995-02-26 1995-03-04 1995-03-01 NONE RAIL s. final i cn shanghai -4452 7650 2 149 47.00 49309.58 0.01 0.06 A F 1994-10-08 1994-08-09 1994-10-09 TAKE BACK RETURN TRUCK ts. slyly regular cour us washington 807 5150 7 149 19.00 19933.66 0.08 0.05 A F 1994-02-10 1994-02-20 1994-03-06 NONE SHIP ns haggle quickly across the furi cn beijing +4452 7650 2 149 47.00 49309.58 0.01 0.06 A F 1994-10-08 1994-08-09 1994-10-09 TAKE BACK RETURN TRUCK ts. slyly regular cour us washington 4102 5176 5 175 32.00 34405.44 0.08 0.01 N O 1996-05-14 1996-04-29 1996-05-29 NONE RAIL the even requests; regular pinto us washington 2117 2680 6 179 27.00 29137.59 0.09 0.08 N O 1997-06-30 1997-06-27 1997-07-11 TAKE BACK RETURN REG AIR the carefully ironic ideas cn shanghai 548 7683 3 182 21.00 22725.78 0.03 0.08 A F 1995-01-13 1994-12-18 1995-01-25 NONE AIR ideas. special accounts above the furiou cn beijing diff --git a/regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy b/regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy index db08ea1dfba4d2..907b70714357cf 100644 --- a/regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy +++ b/regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy @@ -23,7 +23,7 @@ suite("test_external_sql_block_rule", "external_docker,hive,external_docker_hive } String hivePrefix = "hive2"; - String catalog_name = "test_${hivePrefix}_serde_prop" + String catalog_name = "test_${hivePrefix}_external_sql_block_rule"; String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") @@ -36,15 +36,30 @@ suite("test_external_sql_block_rule", "external_docker,hive,external_docker_hive );""" sql "use ${catalog_name}.`default`"; - qt_sql01 """select * from parquet_partition_table order by l_linenumber limit 10;""" + qt_sql01 """select * from parquet_partition_table order by l_linenumber,l_orderkey limit 10;""" sql """drop sql_block_rule if exists external_hive_partition""" sql """create sql_block_rule external_hive_partition properties("partition_num" = "3", "global" = "false");""" + sql """drop sql_block_rule if exists external_hive_partition2""" + sql """create sql_block_rule external_hive_partition2 properties("tablet_num" = "3", "global" = "false");""" + sql """drop sql_block_rule if exists external_hive_partition3""" + sql """create sql_block_rule external_hive_partition3 properties("cardinality" = "3", "global" = "false");""" + // create 3 users sql """drop user if exists external_block_user1""" sql """create user external_block_user1;""" sql """SET PROPERTY FOR 'external_block_user1' 'sql_block_rules' = 'external_hive_partition';""" sql """grant all on *.*.* to external_block_user1;""" + sql """drop user if exists external_block_user2""" + sql """create user external_block_user2;""" + sql """SET PROPERTY FOR 'external_block_user2' 'sql_block_rules' = 'external_hive_partition2';""" + sql """grant all on *.*.* to external_block_user2;""" + + sql """drop user if exists external_block_user3""" + sql """create user external_block_user3;""" + sql """SET PROPERTY FOR 'external_block_user3' 'sql_block_rules' = 'external_hive_partition3';""" + sql """grant all on *.*.* to external_block_user3;""" + // login as external_block_user1 def result1 = connect(user = 'external_block_user1', password = '', url = context.config.jdbcUrl) { test { @@ -52,5 +67,12 @@ suite("test_external_sql_block_rule", "external_docker,hive,external_docker_hive exception """sql hits sql block rule: external_hive_partition, reach partition_num : 3""" } } + // login as external_block_user2 + def result2 = connect(user = 'external_block_user2', password = '', url = context.config.jdbcUrl) { + test { + sql """select * from ${catalog_name}.`default`.parquet_partition_table order by l_linenumber limit 10;""" + exception """sql hits sql block rule: external_hive_partition2, reach tablet_num : 3""" + } + } } From eff8312f254d7d1296f0f73872f5af1e61c438e7 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 8 Jul 2024 23:16:42 +0800 Subject: [PATCH 7/7] 1 --- .../hive/test_external_sql_block_rule.groovy | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy b/regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy index 907b70714357cf..37e7993b1cf2ea 100644 --- a/regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy +++ b/regression-test/suites/external_table_p0/hive/test_external_sql_block_rule.groovy @@ -74,5 +74,12 @@ suite("test_external_sql_block_rule", "external_docker,hive,external_docker_hive exception """sql hits sql block rule: external_hive_partition2, reach tablet_num : 3""" } } + // login as external_block_user3 + def result3 = connect(user = 'external_block_user3', password = '', url = context.config.jdbcUrl) { + test { + sql """select * from ${catalog_name}.`default`.parquet_partition_table order by l_linenumber limit 10;""" + exception """sql hits sql block rule: external_hive_partition3, reach cardinality : 3""" + } + } }