From f148fab59c9480e30e1ca7e37f0e5f54366baf27 Mon Sep 17 00:00:00 2001 From: Jibing Li Date: Thu, 27 Jun 2024 14:25:45 +0800 Subject: [PATCH] Use real base index id to fetch stats cache. --- .../doris/nereids/stats/StatsCalculator.java | 6 ++-- .../nereids/trees/plans/algebra/OlapScan.java | 6 ---- .../LogicalDeferMaterializeOlapScan.java | 5 ---- .../trees/plans/logical/LogicalOlapScan.java | 8 ----- .../PhysicalDeferMaterializeOlapScan.java | 5 ---- .../plans/physical/PhysicalOlapScan.java | 8 ----- .../doris/statistics/StatisticsCache.java | 29 +++++++++++++++++++ 7 files changed, 32 insertions(+), 35 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java index bf78fe2a0bf735..07277f032fd0f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java @@ -350,7 +350,7 @@ private void adjustColStats(CatalogRelation catalogRelation, SlotReference slot, private ColumnStatistic getColumnStatsFromTableCache(CatalogRelation catalogRelation, SlotReference slot) { long idxId = -1; if (catalogRelation instanceof OlapScan) { - idxId = ((OlapScan) catalogRelation).getSelectedIndexIdForMV(); + idxId = ((OlapScan) catalogRelation).getSelectedIndexId(); } return getColumnStatistic(catalogRelation.getTable(), slot.getName(), idxId); } @@ -359,7 +359,7 @@ private ColumnStatistic getColumnStatsFromPartitionCache(CatalogRelation catalog List partitionNames) { long idxId = -1; if (catalogRelation instanceof OlapScan) { - idxId = ((OlapScan) catalogRelation).getSelectedIndexIdForMV(); + idxId = ((OlapScan) catalogRelation).getSelectedIndexId(); } return getColumnStatistic(catalogRelation.getTable(), slot.getName(), idxId, partitionNames); } @@ -434,7 +434,7 @@ private Statistics computeOlapScan(LogicalOlapScan olapScan) { for (Slot slot : olapScan.getOutput()) { if (isVisibleSlotReference(slot)) { ColumnStatistic cache = getColumnStatistic(olapTable, slot.getName(), - olapScan.getSelectedIndexIdForMV()); + olapScan.getSelectedIndexId()); rowCount = Math.max(rowCount, cache.count); } builder.putColumnStatistics(slot, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OlapScan.java index 65f3639462182a..d5fe7c2341369d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/OlapScan.java @@ -28,12 +28,6 @@ public interface OlapScan { long getSelectedIndexId(); - /** - * if this is mv, return selectedIndexId, o.w -1 - * @return -1 or selectedIndexId - */ - long getSelectedIndexIdForMV(); - List getSelectedPartitionIds(); List getSelectedTabletIds(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java index bff416c19d2610..77da6537dd7d09 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalDeferMaterializeOlapScan.java @@ -93,11 +93,6 @@ public long getSelectedIndexId() { return logicalOlapScan.getSelectedIndexId(); } - @Override - public long getSelectedIndexIdForMV() { - return logicalOlapScan.getSelectedIndexIdForMV(); - } - @Override public List getSelectedPartitionIds() { return logicalOlapScan.getSelectedPartitionIds(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index f14cd661d95612..65e4710836d7e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -333,14 +333,6 @@ public long getSelectedIndexId() { return selectedIndexId; } - @Override - public long getSelectedIndexIdForMV() { - if (getTable().getBaseIndexId() != selectedIndexId) { - return selectedIndexId; - } - return -1; - } - public boolean isIndexSelected() { return indexSelected; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeOlapScan.java index 1acfb8dc4c04b6..f82bd6dbec5342 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDeferMaterializeOlapScan.java @@ -90,11 +90,6 @@ public long getSelectedIndexId() { return physicalOlapScan.getSelectedIndexId(); } - @Override - public long getSelectedIndexIdForMV() { - return physicalOlapScan.getSelectedIndexIdForMV(); - } - @Override public List getSelectedPartitionIds() { return physicalOlapScan.getSelectedPartitionIds(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java index 1839f46c552eb3..76713a51e29209 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java @@ -90,14 +90,6 @@ public long getSelectedIndexId() { return selectedIndexId; } - @Override - public long getSelectedIndexIdForMV() { - if (getTable().getBaseIndexId() != selectedIndexId) { - return selectedIndexId; - } - return -1; - } - @Override public List getSelectedTabletIds() { return selectedTabletIds; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java index d333ecdb96d406..a922e8619a4710 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java @@ -18,6 +18,8 @@ package org.apache.doris.statistics; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.ClientPool; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; @@ -91,6 +93,12 @@ public ColumnStatistic getColumnStatistics(long catalogId, long dbId, long tblId if (ctx != null && ctx.getSessionVariable().internalSession) { return ColumnStatistic.UNKNOWN; } + // Need to change base index id to -1 for OlapTable. + try { + idxId = changeBaseIndexId(catalogId, dbId, tblId, idxId); + } catch (Exception e) { + return ColumnStatistic.UNKNOWN; + } StatisticsCacheKey k = new StatisticsCacheKey(catalogId, dbId, tblId, idxId, colName); try { CompletableFuture> f = columnStatisticsCache.get(k); @@ -109,6 +117,12 @@ public PartitionColumnStatistic getPartitionColumnStatistics(long catalogId, lon if (ctx != null && ctx.getSessionVariable().internalSession) { return PartitionColumnStatistic.UNKNOWN; } + // Need to change base index id to -1 for OlapTable. + try { + idxId = changeBaseIndexId(catalogId, dbId, tblId, idxId); + } catch (Exception e) { + return PartitionColumnStatistic.UNKNOWN; + } PartitionColumnStatisticCacheKey k = new PartitionColumnStatisticCacheKey( catalogId, dbId, tblId, idxId, partName, colName); try { @@ -122,6 +136,21 @@ public PartitionColumnStatistic getPartitionColumnStatistics(long catalogId, lon return PartitionColumnStatistic.UNKNOWN; } + // Base index id should be set to -1 for OlapTable. Because statistics tables use -1 for base index. + // TODO: Need to use the real index id in statistics table in later version. + private long changeBaseIndexId(long catalogId, long dbId, long tblId, long idxId) { + if (idxId != -1) { + TableIf table = StatisticsUtil.findTable(catalogId, dbId, tblId); + if (table instanceof OlapTable) { + OlapTable olapTable = (OlapTable) table; + if (idxId == olapTable.getBaseIndexId()) { + idxId = -1; + } + } + } + return idxId; + } + public Histogram getHistogram(long ctlId, long dbId, long tblId, String colName) { return getHistogram(ctlId, dbId, tblId, -1, colName).orElse(null); }