Skip to content

[opt](nereids) tabe row count priority: user injected > BE report > analyzed #40529

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
englefly merged 1 commit into from
Sep 9, 2024
Merged

[opt](nereids) tabe row count priority: user injected > BE report > analyzed #40529

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 27 additions & 23 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -364,19 +364,28 @@ private void checkIfUnknownStatsUsedAsKey(StatisticsBuilder builder) {
}
}

private Statistics computeOlapScan(OlapScan olapScan) {
private double getOlapTableRowCount(OlapScan olapScan) {
OlapTable olapTable = olapScan.getTable();
double tableRowCount = olapTable.getRowCountForIndex(olapScan.getSelectedIndexId(), true);
if (tableRowCount <= 0) {
AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(olapScan.getTable().getId());
if (tableMeta != null) {
// create-view after analyzing, we may get -1 for this view row count
tableRowCount = Math.max(1, tableMeta.getRowCount(olapScan.getSelectedIndexId()));
} else {
tableRowCount = 1;
AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
TableStatsMeta tableMeta = analysisManager.findTableStatsStatus(olapScan.getTable().getId());
double rowCount = -1;
if (tableMeta != null && tableMeta.userInjected) {
rowCount = tableMeta.getRowCount(olapScan.getSelectedIndexId());
} else {
rowCount = olapTable.getRowCountForIndex(olapScan.getSelectedIndexId(), true);
if (rowCount == -1) {
if (tableMeta != null) {
rowCount = tableMeta.getRowCount(olapScan.getSelectedIndexId());
}
}
}
return rowCount;
}

private Statistics computeOlapScan(OlapScan olapScan) {
OlapTable olapTable = olapScan.getTable();
double tableRowCount = getOlapTableRowCount(olapScan);
tableRowCount = Math.max(1, tableRowCount);

if (olapScan.getSelectedIndexId() != olapScan.getTable().getBaseIndexId() || olapTable instanceof MTMV) {
// mv is selected, return its estimated stats
Expand Down Expand Up @@ -441,10 +450,13 @@ private Statistics computeOlapScan(OlapScan olapScan) {
}
}

boolean useTableLevelStats = true;
if (olapScan.getSelectedPartitionIds().size() < olapScan.getTable().getPartitionNum()) {
// partition pruned
// try to use selected partition stats, if failed, fall back to table stats
double selectedPartitionsRowCount = getSelectedPartitionRowCount(olapScan);
if (selectedPartitionsRowCount > 0) {
if (selectedPartitionsRowCount >= 0) {
useTableLevelStats = false;
List<String> selectedPartitionNames = new ArrayList<>(olapScan.getSelectedPartitionIds().size());
olapScan.getSelectedPartitionIds().forEach(id -> {
selectedPartitionNames.add(olapScan.getTable().getPartition(id).getName());
Expand All @@ -458,19 +470,11 @@ private Statistics computeOlapScan(OlapScan olapScan) {
}
checkIfUnknownStatsUsedAsKey(builder);
builder.setRowCount(selectedPartitionsRowCount + deltaRowCount);
} else {
// if partition row count is invalid (-1), fallback to table stats
for (SlotReference slot : visibleOutputSlots) {
ColumnStatistic cache = getColumnStatsFromTableCache((CatalogRelation) olapScan, slot);
ColumnStatisticBuilder colStatsBuilder = new ColumnStatisticBuilder(cache);
colStatsBuilder.setCount(tableRowCount);
colStatsBuilder.normalizeAvgSizeByte(slot);
builder.putColumnStatistics(slot, colStatsBuilder.build());
}
checkIfUnknownStatsUsedAsKey(builder);
builder.setRowCount(tableRowCount + deltaRowCount);
}
} else {
}
// 1. no partition is pruned, or
// 2. fall back to table stats
if (useTableLevelStats) {
// get table level stats
for (SlotReference slot : visibleOutputSlots) {
ColumnStatistic cache = getColumnStatsFromTableCache((CatalogRelation) olapScan, slot);
Expand Down