Skip to content

[Feature](materialized-view) support visitBitmapUnion mv rewrite #29200

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BiteTheDDDDt merged 2 commits into from
Dec 28, 2023
Merged

[Feature](materialized-view) support visitBitmapUnion mv rewrite #29200

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
47251f7
support visitBitmapUnion rewrite
BiteTheDDDDt Dec 28, 2023
5dc3391
add case
BiteTheDDDDt Dec 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 77 additions & 24 deletions .../java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@
import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
import org.apache.doris.nereids.trees.expressions.functions.combinator.MergeCombinator;
import org.apache.doris.nereids.trees.expressions.functions.combinator.StateCombinator;
import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapHash;
import org.apache.doris.nereids.trees.expressions.functions.scalar.HllHash;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmap;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmapWithCheck;
Expand Down Expand Up @@ -1282,6 +1281,66 @@ public Expression visitCount(Count count, RewriteContext context) {
return count;
}

/**
* bitmap_union(to_bitmap(col)) ->
* bitmap_union(mva_BITMAP_UNION__to_bitmap_with_check(col))
*/
@Override
public Expression visitBitmapUnion(BitmapUnion bitmapUnion, RewriteContext context) {
Expression result = visitAggregateFunction(bitmapUnion, context);
if (result != bitmapUnion) {
return result;
}
if (bitmapUnion.child() instanceof ToBitmap) {
ToBitmap toBitmap = (ToBitmap) bitmapUnion.child();
Optional<Slot> slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(toBitmap.child());
if (slotOpt.isPresent()) {
String bitmapUnionColumn = normalizeName(CreateMaterializedViewStmt
.mvColumnBuilder(AggregateType.BITMAP_UNION, CreateMaterializedViewStmt
.mvColumnBuilder(new ToBitmapWithCheck(toBitmap.child()).toSql())));

Column mvColumn = context.checkContext.getColumn(bitmapUnionColumn);
// has bitmap_union column
if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) {

Slot bitmapUnionSlot = context.checkContext.scan.getOutputByIndex(context.checkContext.index)
.stream().filter(s -> bitmapUnionColumn.equalsIgnoreCase(normalizeName(s.getName())))
.findFirst().orElseThrow(
() -> new AnalysisException("cannot find bitmap union slot when select mv"));

context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionSlot);
context.exprRewriteMap.projectExprMap.put(toBitmap, bitmapUnionSlot);
BitmapUnion newBitmapUnion = new BitmapUnion(bitmapUnionSlot);
context.exprRewriteMap.aggFuncMap.put(bitmapUnion, newBitmapUnion);
return newBitmapUnion;
}
}
} else {
Expression child = bitmapUnion.child();
String bitmapUnionColumn = normalizeName(CreateMaterializedViewStmt.mvColumnBuilder(
AggregateType.BITMAP_UNION, CreateMaterializedViewStmt.mvColumnBuilder(child.toSql())));

Column mvColumn = context.checkContext.getColumn(bitmapUnionColumn);
// has bitmap_union column
if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) {

Slot bitmapUnionSlot = context.checkContext.scan.getOutputByIndex(context.checkContext.index)
.stream().filter(s -> bitmapUnionColumn.equalsIgnoreCase(normalizeName(s.getName())))
.findFirst()
.orElseThrow(() -> new AnalysisException("cannot find bitmap union slot when select mv"));
if (child instanceof Slot) {
context.exprRewriteMap.slotMap.put((Slot) child, bitmapUnionSlot);
}
context.exprRewriteMap.projectExprMap.put(child, bitmapUnionSlot);
BitmapUnion newBitmapUnion = new BitmapUnion(bitmapUnionSlot);
context.exprRewriteMap.aggFuncMap.put(bitmapUnion, newBitmapUnion);
return newBitmapUnion;
}
}

return bitmapUnion;
}

/**
* bitmap_union_count(to_bitmap(col)) -> bitmap_union_count(mva_BITMAP_UNION__to_bitmap_with_check(col))
*/
Expand Down Expand Up @@ -1318,32 +1377,26 @@ public Expression visitBitmapUnionCount(BitmapUnionCount bitmapUnionCount, Rewri
return newBitmapUnionCount;
}
}
} else if (bitmapUnionCount.child() instanceof BitmapHash) {
BitmapHash bitmapHash = (BitmapHash) bitmapUnionCount.child();
Optional<Slot> slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(bitmapHash.child());
if (slotOpt.isPresent()) {
String bitmapUnionCountColumn = normalizeName(
CreateMaterializedViewStmt.mvColumnBuilder(AggregateType.BITMAP_UNION,
CreateMaterializedViewStmt.mvColumnBuilder(bitmapHash.toSql())));

Column mvColumn = context.checkContext.getColumn(bitmapUnionCountColumn);
// has bitmap_union_count column
if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) {
} else {
Expression child = bitmapUnionCount.child();
String bitmapUnionCountColumn = normalizeName(CreateMaterializedViewStmt.mvColumnBuilder(
AggregateType.BITMAP_UNION, CreateMaterializedViewStmt.mvColumnBuilder(child.toSql())));

Slot bitmapUnionCountSlot = context.checkContext.scan
.getOutputByIndex(context.checkContext.index)
.stream()
.filter(s -> bitmapUnionCountColumn.equalsIgnoreCase(normalizeName(s.getName())))
.findFirst()
.orElseThrow(() -> new AnalysisException(
"cannot find bitmap union count slot when select mv"));
Column mvColumn = context.checkContext.getColumn(bitmapUnionCountColumn);
// has bitmap_union_count column
if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) {

context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionCountSlot);
context.exprRewriteMap.projectExprMap.put(bitmapHash, bitmapUnionCountSlot);
BitmapUnionCount newBitmapUnionCount = new BitmapUnionCount(bitmapUnionCountSlot);
context.exprRewriteMap.aggFuncMap.put(bitmapUnionCount, newBitmapUnionCount);
return newBitmapUnionCount;
Slot bitmapUnionCountSlot = context.checkContext.scan.getOutputByIndex(context.checkContext.index)
.stream().filter(s -> bitmapUnionCountColumn.equalsIgnoreCase(normalizeName(s.getName())))
.findFirst().orElseThrow(
() -> new AnalysisException("cannot find bitmap union count slot when select mv"));
if (child instanceof Slot) {
context.exprRewriteMap.slotMap.put((Slot) child, bitmapUnionCountSlot);
}
context.exprRewriteMap.projectExprMap.put(child, bitmapUnionCountSlot);
BitmapUnionCount newBitmapUnionCount = new BitmapUnionCount(bitmapUnionCountSlot);
context.exprRewriteMap.aggFuncMap.put(bitmapUnionCount, newBitmapUnionCount);
return newBitmapUnionCount;
}
}

Expand Down
3 changes: 3 additions & 0 deletions regression-test/data/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,6 @@
-- !select_mv --
1 2

-- !select_mv --
1 2

6 changes: 6 additions & 0 deletions regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,4 +47,10 @@ suite ("testBitmapUnionInQuery") {
contains "(user_tags_mv)"
}
qt_select_mv "select user_id, bitmap_union_count(to_bitmap(tag_id)) a from user_tags group by user_id having a>1 order by a;"

explain {
sql("select user_id, bitmap_count(bitmap_union(to_bitmap(tag_id))) a from user_tags group by user_id having a>1 order by a;")
contains "(user_tags_mv)"
}
qt_select_mv "select user_id, bitmap_count(bitmap_union(to_bitmap(tag_id))) a from user_tags group by user_id having a>1 order by a;"
}