Skip to content

branch-3.0: [fix](mtmv) Fix nest mtmv rewrite fail when bottom mtmv cache is invalid #48222 #49129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dataroaring merged 1 commit into from
Mar 19, 2025
Merged

branch-3.0: [fix](mtmv) Fix nest mtmv rewrite fail when bottom mtmv cache is invalid #48222 #49129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,10 +332,6 @@ public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws Ana
}
}

public MTMVCache getCache() {
return cache;
}

public Map<String, String> getMvProperties() {
readMvLock();
try {
Expand Down
55 changes: 46 additions & 9 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.doris.catalog.MTMV;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.mtmv.MTMVCache;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.DataTrait;
Expand All @@ -36,6 +37,7 @@
import org.apache.doris.nereids.trees.plans.algebra.OlapScan;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
Expand All @@ -46,6 +48,8 @@
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONObject;

import java.util.ArrayList;
Expand All @@ -62,6 +66,8 @@
*/
public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan {

private static final Logger LOG = LogManager.getLogger(LogicalOlapScan.class);

///////////////////////////////////////////////////////////////////////////
// Members for materialized index.
///////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -527,9 +533,15 @@ AGGREGATE KEY (siteid,citycode,username)
Set<Slot> outputSet = Utils.fastToImmutableSet(getOutputSet());
if (getTable() instanceof MTMV) {
MTMV mtmv = (MTMV) getTable();
MTMVCache cache = mtmv.getCache();
MTMVCache cache;
try {
cache = mtmv.getOrGenerateCache(ConnectContext.get());
} catch (AnalysisException e) {
LOG.warn(String.format("LogicalOlapScan computeUnique fail, mv name is %s", mtmv.getName()), e);
return;
}
// Maybe query specified index, should not calc, such as select count(*) from t1 index col_index
if (cache == null || this.getSelectedIndexId() != this.getTable().getBaseIndexId()) {
if (this.getSelectedIndexId() != this.getTable().getBaseIndexId()) {
return;
}
Plan originalPlan = cache.getOriginalPlan();
Expand All @@ -554,9 +566,15 @@ AGGREGATE KEY (siteid,citycode,username)
public void computeUniform(DataTrait.Builder builder) {
if (getTable() instanceof MTMV) {
MTMV mtmv = (MTMV) getTable();
MTMVCache cache = mtmv.getCache();
MTMVCache cache;
try {
cache = mtmv.getOrGenerateCache(ConnectContext.get());
} catch (AnalysisException e) {
LOG.warn(String.format("LogicalOlapScan computeUniform fail, mv name is %s", mtmv.getName()), e);
return;
}
// Maybe query specified index, should not calc, such as select count(*) from t1 index col_index
if (cache == null || this.getSelectedIndexId() != this.getTable().getBaseIndexId()) {
if (this.getSelectedIndexId() != this.getTable().getBaseIndexId()) {
return;
}
Plan originalPlan = cache.getOriginalPlan();
Expand All @@ -569,9 +587,15 @@ public void computeUniform(DataTrait.Builder builder) {
public void computeEqualSet(DataTrait.Builder builder) {
if (getTable() instanceof MTMV) {
MTMV mtmv = (MTMV) getTable();
MTMVCache cache = mtmv.getCache();
MTMVCache cache;
try {
cache = mtmv.getOrGenerateCache(ConnectContext.get());
} catch (AnalysisException e) {
LOG.warn(String.format("LogicalOlapScan computeEqualSet fail, mv name is %s", mtmv.getName()), e);
return;
}
// Maybe query specified index, should not calc, such as select count(*) from t1 index col_index
if (cache == null || this.getSelectedIndexId() != this.getTable().getBaseIndexId()) {
if (this.getSelectedIndexId() != this.getTable().getBaseIndexId()) {
return;
}
Plan originalPlan = cache.getOriginalPlan();
Expand All @@ -584,9 +608,15 @@ public void computeEqualSet(DataTrait.Builder builder) {
public void computeFd(DataTrait.Builder builder) {
if (getTable() instanceof MTMV) {
MTMV mtmv = (MTMV) getTable();
MTMVCache cache = mtmv.getCache();
MTMVCache cache;
try {
cache = mtmv.getOrGenerateCache(ConnectContext.get());
} catch (AnalysisException e) {
LOG.warn(String.format("LogicalOlapScan computeFd fail, mv name is %s", mtmv.getName()), e);
return;
}
// Maybe query specified index, should not calc, such as select count(*) from t1 index col_index
if (cache == null || this.getSelectedIndexId() != this.getTable().getBaseIndexId()) {
if (this.getSelectedIndexId() != this.getTable().getBaseIndexId()) {
return;
}
Plan originalPlan = cache.getOriginalPlan();
Expand All @@ -599,7 +629,14 @@ Map<Slot, Slot> constructReplaceMap(MTMV mtmv) {
Map<Slot, Slot> replaceMap = new HashMap<>();
// Need remove invisible column, and then mapping them
List<Slot> originOutputs = new ArrayList<>();
for (Slot originSlot : mtmv.getCache().getOriginalPlan().getOutput()) {
MTMVCache cache;
try {
cache = mtmv.getOrGenerateCache(ConnectContext.get());
} catch (AnalysisException e) {
LOG.warn(String.format("LogicalOlapScan constructReplaceMap fail, mv name is %s", mtmv.getName()), e);
return replaceMap;
}
for (Slot originSlot : cache.getOriginalPlan().getOutput()) {
if (!(originSlot instanceof SlotReference) || (((SlotReference) originSlot).isVisible())) {
originOutputs.add(originSlot);
}
Expand Down