Skip to content

[fix](nereids)filter estimation for slot=unknown #39592

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
englefly merged 3 commits into from
Aug 26, 2024
Merged

[fix](nereids)filter estimation for slot=unknown #39592

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ee33463
filter estimation for slot=unknown
englefly Aug 20, 2024
ca6b1c0
fmt
englefly Aug 21, 2024
b1b7fd4
fmt
englefly Aug 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 20 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,14 +331,28 @@ private Statistics estimateEqualTo(ComparisonPredicate cp, ColumnStatistic stats
ColumnStatistic statsForRight,
EstimationContext context) {
double selectivity;
double ndv = statsForLeft.ndv;
double val = statsForRight.maxValue;
if (val > statsForLeft.maxValue || val < statsForLeft.minValue) {
selectivity = 0.0;
if (statsForLeft.isUnKnown) {
selectivity = DEFAULT_INEQUALITY_COEFFICIENT;
} else {
selectivity = StatsMathUtil.minNonNaN(1.0, 1.0 / ndv);
double ndv = statsForLeft.ndv;
if (statsForRight.isUnKnown) {
if (ndv >= 1.0) {
selectivity = 1.0 / ndv;
} else {
selectivity = DEFAULT_INEQUALITY_COEFFICIENT;
}
} else {
double val = statsForRight.maxValue;
if (val > statsForLeft.maxValue || val < statsForLeft.minValue) {
selectivity = 0.0;
} else if (ndv >= 1.0) {
selectivity = StatsMathUtil.minNonNaN(1.0, 1.0 / ndv);
} else {
selectivity = DEFAULT_INEQUALITY_COEFFICIENT;
}
selectivity = getNotNullSelectivity(statsForLeft, selectivity);
}
}
selectivity = getNotNullSelectivity(statsForLeft, selectivity);
Statistics equalStats = context.statistics.withSel(selectivity);
Expression left = cp.left();
equalStats.addColumnStats(left, statsForRight);
Expand Down
75 changes: 75 additions & 0 deletions fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Left;
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
Expand Down Expand Up @@ -177,6 +178,80 @@ public void testRelatedAnd() {
Assertions.assertEquals(10, aStatsEst.ndv);
}

@Test
public void knownEqualToUnknown() {
SlotReference ym = new SlotReference("a", new VarcharType(7));
double rowCount = 404962.0;
double ndv = 14.0;
ColumnStatistic ymStats = new ColumnStatisticBuilder()
.setCount(rowCount)
.setNdv(ndv)
.setMinExpr(new StringLiteral("2023-07"))
.setMinValue(14126741000630328.000000)
.setMaxExpr(new StringLiteral("2024-08"))
.setMaxValue(14126741017407544.000000)
.setAvgSizeByte(7)
.build();
Statistics stats = new StatisticsBuilder()
.setRowCount(404962).putColumnStatistics(ym, ymStats)
.build();

EqualTo predicate = new EqualTo(ym,
new Left(new org.apache.doris.nereids.trees.expressions.literal.StringLiteral("2024-08-14"),
new IntegerLiteral(7))
);
FilterEstimation filterEstimation = new FilterEstimation();
Statistics outStats = filterEstimation.estimate(predicate, stats);
Assertions.assertEquals(rowCount / ndv, outStats.getRowCount());
}

@Test
public void knownEqualToUnknownWithLittleNdv() {
SlotReference ym = new SlotReference("a", new VarcharType(7));
double rowCount = 404962.0;
double ndv = 0.5;
ColumnStatistic ymStats = new ColumnStatisticBuilder()
.setCount(rowCount)
.setNdv(ndv)
.setMinExpr(new StringLiteral("2023-07"))
.setMinValue(14126741000630328.000000)
.setMaxExpr(new StringLiteral("2024-08"))
.setMaxValue(14126741017407544.000000)
.setAvgSizeByte(7)
.build();
Statistics stats = new StatisticsBuilder()
.setRowCount(404962).putColumnStatistics(ym, ymStats)
.build();

EqualTo predicate = new EqualTo(ym,
new Left(new org.apache.doris.nereids.trees.expressions.literal.StringLiteral("2024-08-14"),
new IntegerLiteral(7))
);
FilterEstimation filterEstimation = new FilterEstimation();
Statistics outStats = filterEstimation.estimate(predicate, stats);
Assertions.assertEquals(rowCount * FilterEstimation.DEFAULT_INEQUALITY_COEFFICIENT,
outStats.getRowCount());
}

@Test
public void unknownEqualToUnknown() {
SlotReference ym = new SlotReference("a", new VarcharType(7));
ColumnStatistic ymStats = ColumnStatistic.UNKNOWN;
double rowCount = 404962.0;
Statistics stats = new StatisticsBuilder()
.setRowCount(rowCount).putColumnStatistics(ym, ymStats)
.build();

EqualTo predicate = new EqualTo(ym,
new Left(new org.apache.doris.nereids.trees.expressions.literal.StringLiteral("2024-08-14"),
new IntegerLiteral(7))
);
FilterEstimation filterEstimation = new FilterEstimation();
Statistics outStats = filterEstimation.estimate(predicate, stats);
Assertions.assertEquals(rowCount * FilterEstimation.DEFAULT_INEQUALITY_COEFFICIENT,
outStats.getRowCount());
}

// a > 500 and b < 100 or a = c
@Test
public void test1() {
Expand Down