diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java index 12b1fedcd67d31..aad6ab45479cad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java @@ -826,6 +826,7 @@ private void modifyViewDef(Database db, View view, String inlineViewDef, long sq db.registerTable(view); AlterViewInfo alterViewInfo = new AlterViewInfo(db.getId(), view.getId(), inlineViewDef, newFullSchema, sqlMode, comment); + Env.getCurrentEnv().getMtmvService().alterView(new BaseTableInfo(view)); Env.getCurrentEnv().getEditLog().logModifyViewDef(alterViewInfo); LOG.info("modify view[{}] definition to {}", viewName, inlineViewDef); } finally { @@ -863,7 +864,7 @@ public void replayModifyViewDef(AlterViewInfo alterViewInfo) throws MetaNotFound db.unregisterTable(viewName); db.registerTable(view); - + Env.getCurrentEnv().getMtmvService().alterView(new BaseTableInfo(view)); LOG.info("replay modify view[{}] definition to {}", viewName, inlineViewDef); } finally { view.writeUnlock(); @@ -1236,6 +1237,7 @@ public AlterHandler getSystemHandler() { public void processAlterMTMV(AlterMTMV alterMTMV, boolean isReplay) { TableNameInfo tbl = alterMTMV.getMvName(); MTMV mtmv = null; + boolean alterSuccess = true; try { Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(tbl.getDb()); mtmv = (MTMV) db.getTableOrMetaException(tbl.getTbl(), TableType.MATERIALIZED_VIEW); @@ -1250,7 +1252,8 @@ public void processAlterMTMV(AlterMTMV alterMTMV, boolean isReplay) { mtmv.alterMvProperties(alterMTMV.getMvProperties()); break; case ADD_TASK: - mtmv.addTaskResult(alterMTMV.getTask(), alterMTMV.getRelation(), alterMTMV.getPartitionSnapshots(), + alterSuccess = mtmv.addTaskResult(alterMTMV.getTask(), alterMTMV.getRelation(), + alterMTMV.getPartitionSnapshots(), isReplay); // If it is not a replay thread, it means that the current service is already a new version // and does not require compatibility @@ -1265,7 +1268,7 @@ public void processAlterMTMV(AlterMTMV alterMTMV, boolean isReplay) { Env.getCurrentEnv().getMtmvService().alterJob(mtmv, isReplay); } // 4. log it and replay it in the follower - if (!isReplay) { + if (!isReplay && alterSuccess) { Env.getCurrentEnv().getEditLog().logAlterMTMV(alterMTMV); } } catch (UserException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 14571133e0d055..b4791bf06a0834 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -84,6 +84,7 @@ public class MTMV extends OlapTable { private MTMVRefreshSnapshot refreshSnapshot; // Should update after every fresh, not persist private MTMVCache cache; + private long schemaChangeVersion; // For deserialization public MTMV() { @@ -179,13 +180,26 @@ public MTMVStatus alterStatus(MTMVStatus newStatus) { writeMvLock(); try { // only can update state, refresh state will be change by add task + this.schemaChangeVersion++; return this.status.updateStateAndDetail(newStatus); } finally { writeMvUnlock(); } } - public void addTaskResult(MTMVTask task, MTMVRelation relation, + public void processBaseViewChange(String schemaChangeDetail) { + writeMvLock(); + try { + this.schemaChangeVersion++; + this.status.setState(MTMVState.SCHEMA_CHANGE); + this.status.setSchemaChangeDetail(schemaChangeDetail); + this.refreshSnapshot = new MTMVRefreshSnapshot(); + } finally { + writeMvUnlock(); + } + } + + public boolean addTaskResult(MTMVTask task, MTMVRelation relation, Map partitionSnapshots, boolean isReplay) { MTMVCache mtmvCache = null; boolean needUpdateCache = false; @@ -209,6 +223,14 @@ public void addTaskResult(MTMVTask task, MTMVRelation relation, } writeMvLock(); try { + if (!isReplay && task.getMtmvSchemaChangeVersion() != this.schemaChangeVersion) { + LOG.warn( + "addTaskResult failed, schemaChangeVersion has changed. " + + "mvName: {}, taskId: {}, taskSchemaChangeVersion: {}, " + + "mvSchemaChangeVersion: {}", + name, task.getTaskId(), task.getMtmvSchemaChangeVersion(), this.schemaChangeVersion); + return false; + } if (task.getStatus() == TaskStatus.SUCCESS) { this.status.setState(MTMVState.NORMAL); this.status.setSchemaChangeDetail(null); @@ -224,6 +246,7 @@ public void addTaskResult(MTMVTask task, MTMVRelation relation, this.refreshSnapshot.updateSnapshots(partitionSnapshots, getPartitionNames()); Env.getCurrentEnv().getMtmvService() .refreshComplete(this, relation, task); + return true; } finally { writeMvUnlock(); } @@ -372,6 +395,15 @@ public MTMVRefreshSnapshot getRefreshSnapshot() { return refreshSnapshot; } + public long getSchemaChangeVersion() { + readMvLock(); + try { + return schemaChangeVersion; + } finally { + readMvUnlock(); + } + } + /** * generateMvPartitionDescs * diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 4868caefd5ea1e..ebfccd631d95a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -121,6 +121,7 @@ import org.apache.doris.common.util.Util; import org.apache.doris.datasource.es.EsRepository; import org.apache.doris.event.DropPartitionEvent; +import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.trees.plans.commands.DropCatalogRecycleBinCommand.IdType; @@ -989,8 +990,9 @@ private void dropTableInternal(Database db, Table table, boolean isView, boolean } finally { table.writeUnlock(); } - - Env.getCurrentEnv().getMtmvService().dropTable(table); + if (table instanceof OlapTable) { + Env.getCurrentEnv().getMtmvService().dropTable(table); + } if (Config.isCloudMode()) { ((CloudGlobalTransactionMgr) Env.getCurrentGlobalTransactionMgr()) .clearTableLastTxnId(db.getId(), table.getId()); @@ -1017,6 +1019,9 @@ public boolean unprotectDropTable(Database db, Table table, boolean isForceDrop, if (table instanceof MTMV) { Env.getCurrentEnv().getMtmvService().dropJob((MTMV) table, isReplay); } + if (table instanceof View) { + Env.getCurrentEnv().getMtmvService().dropView(new BaseTableInfo(table)); + } Env.getCurrentEnv().getAnalysisManager().removeTableStats(table.getId()); Env.getCurrentEnv().getDictionaryManager().dropTableDictionaries(db.getName(), table.getName()); Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentInternalCatalog().getId(), db.getId(), table.getId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index a92754e8aee3ed..d99d560ae51e95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -24,7 +24,6 @@ import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.TableIf; -import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; @@ -59,7 +58,6 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.trees.plans.commands.UpdateMvByPartitionCommand; -import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo; import org.apache.doris.qe.AuditLogHelper; import org.apache.doris.qe.ConnectContext; @@ -90,7 +88,6 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; public class MTMVTask extends AbstractTask { private static final Logger LOG = LogManager.getLogger(MTMVTask.class); @@ -158,6 +155,7 @@ public enum MTMVTaskRefreshMode { private MTMVRelation relation; private StmtExecutor executor; private Map partitionSnapshots; + private long mtmvSchemaChangeVersion; private Map snapshots = Maps.newHashMap(); @@ -182,6 +180,7 @@ public void run() throws JobException { if (LOG.isDebugEnabled()) { LOG.debug("mtmv task run, taskId: {}", super.getTaskId()); } + mtmvSchemaChangeVersion = mtmv.getSchemaChangeVersion(); ConnectContext ctx = MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK); try { if (LOG.isDebugEnabled()) { @@ -193,10 +192,10 @@ public void run() throws JobException { } // Every time a task is run, the relation is regenerated because baseTables and baseViews may change, // such as deleting a table and creating a view with the same name - Set tablesInPlan = MTMVPlanUtil.getBaseTableFromQuery(mtmv.getQuerySql(), ctx); - this.relation = MTMVPlanUtil.generateMTMVRelation(tablesInPlan, ctx); + Pair, Set> tablesInPlan = MTMVPlanUtil.getBaseTableFromQuery(mtmv.getQuerySql(), ctx); + this.relation = MTMVPlanUtil.generateMTMVRelation(tablesInPlan.first, tablesInPlan.second); beforeMTMVRefresh(); - List tableIfs = Lists.newArrayList(tablesInPlan); + List tableIfs = Lists.newArrayList(tablesInPlan.first); tableIfs.sort(Comparator.comparing(TableIf::getId)); MTMVRefreshContext context; @@ -207,7 +206,7 @@ public void run() throws JobException { // if mtmv is schema_change, check if column type has changed // If it's not in the schema_change state, the column type definitely won't change. if (MTMVState.SCHEMA_CHANGE.equals(mtmv.getStatus().getState())) { - checkColumnTypeIfChange(mtmv, ctx); + MTMVPlanUtil.ensureMTMVQueryUsable(mtmv, ctx); } if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE) { MTMVRelatedTableIf relatedTable = mtmv.getMvPartitionInfo().getRelatedTable(); @@ -254,7 +253,8 @@ public void run() throws JobException { .subList(start, Math.min(end, needRefreshPartitions.size()))); // need get names before exec Map execPartitionSnapshots = MTMVPartitionUtil - .generatePartitionSnapshots(context, relation.getBaseTablesOneLevel(), execPartitionNames); + .generatePartitionSnapshots(context, relation.getBaseTablesOneLevelAndFromView(), + execPartitionNames); try { executeWithRetry(execPartitionNames, tableWithPartKey); } catch (Exception e) { @@ -277,39 +277,6 @@ public void run() throws JobException { } } - private void checkColumnTypeIfChange(MTMV mtmv, ConnectContext ctx) throws JobException { - List currentColumnsDefinition = MTMVPlanUtil.generateColumnsBySql(mtmv.getQuerySql(), ctx, - mtmv.getMvPartitionInfo().getPartitionCol(), - mtmv.getDistributionColumnNames(), null, mtmv.getTableProperty().getProperties()); - List currentColumns = currentColumnsDefinition.stream() - .map(ColumnDefinition::translateToCatalogStyle) - .collect(Collectors.toList()); - List originalColumns = mtmv.getBaseSchema(true); - if (currentColumns.size() != originalColumns.size()) { - throw new JobException(String.format( - "column length not equals, please check whether columns of base table have changed, " - + "original length is: %s, current length is: %s", - originalColumns.size(), currentColumns.size())); - } - for (int i = 0; i < originalColumns.size(); i++) { - if (!isTypeLike(originalColumns.get(i).getType(), currentColumns.get(i).getType())) { - throw new JobException(String.format( - "column type not same, please check whether columns of base table have changed, " - + "column name is: %s, original type is: %s, current type is: %s", - originalColumns.get(i).getName(), originalColumns.get(i).getType().toSql(), - currentColumns.get(i).getType().toSql())); - } - } - } - - private boolean isTypeLike(Type type, Type typeOther) { - if (type.isStringType()) { - return typeOther.isStringType(); - } else { - return type.equals(typeOther); - } - } - private void executeWithRetry(Set execPartitionNames, Map tableWithPartKey) throws Exception { int retryCount = 0; @@ -336,7 +303,7 @@ private void executeWithRetry(Set execPartitionNames, Map= retryTime) { @@ -361,7 +328,7 @@ private void exec(Set refreshPartitionNames, // if SELF_MANAGE mv, only have default partition, will not have partitionItem, so we give empty set UpdateMvByPartitionCommand command = UpdateMvByPartitionCommand .from(mtmv, mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE - ? refreshPartitionNames : Sets.newHashSet(), tableWithPartKey); + ? refreshPartitionNames : Sets.newHashSet(), tableWithPartKey, statementContext); try { executor = new StmtExecutor(ctx, new LogicalPlanAdapter(command, ctx.getStatementContext())); ctx.setExecutor(executor); @@ -482,7 +449,7 @@ public void before() throws JobException { * @throws DdlException */ private void beforeMTMVRefresh() throws AnalysisException, DdlException { - for (BaseTableInfo tableInfo : relation.getBaseTablesOneLevel()) { + for (BaseTableInfo tableInfo : relation.getBaseTablesOneLevelAndFromView()) { TableIf tableIf = MTMVUtil.getTable(tableInfo); if (tableIf instanceof MTMVBaseTableIf) { MTMVBaseTableIf baseTableIf = (MTMVBaseTableIf) tableIf; @@ -645,7 +612,7 @@ public List calculateNeedRefreshPartitions(MTMVRefreshContext context) // check if data is fresh // We need to use a newly generated relationship and cannot retrieve it using mtmv.getRelation() // to avoid rebuilding the baseTable and causing a change in the tableId - boolean fresh = MTMVPartitionUtil.isMTMVSync(context, relation.getBaseTablesOneLevel(), + boolean fresh = MTMVPartitionUtil.isMTMVSync(context, relation.getBaseTablesOneLevelAndFromView(), mtmv.getExcludedTriggerTables()); if (fresh) { return Lists.newArrayList(); @@ -656,13 +623,17 @@ public List calculateNeedRefreshPartitions(MTMVRefreshContext context) } // We need to use a newly generated relationship and cannot retrieve it using mtmv.getRelation() // to avoid rebuilding the baseTable and causing a change in the tableId - return MTMVPartitionUtil.getMTMVNeedRefreshPartitions(context, relation.getBaseTablesOneLevel()); + return MTMVPartitionUtil.getMTMVNeedRefreshPartitions(context, relation.getBaseTablesOneLevelAndFromView()); } public MTMVTaskContext getTaskContext() { return taskContext; } + public long getMtmvSchemaChangeVersion() { + return mtmvSchemaChangeVersion; + } + @Override public String toString() { return "MTMVTask{" diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVAnalyzeQueryInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVAnalyzeQueryInfo.java new file mode 100644 index 00000000000000..f9ac4f18c1b588 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVAnalyzeQueryInfo.java @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; + +import java.util.List; + +public class MTMVAnalyzeQueryInfo { + private MTMVRelation relation; + private MTMVPartitionInfo mvPartitionInfo; + private List columnDefinitions; + + public MTMVAnalyzeQueryInfo(List columnDefinitions, MTMVPartitionInfo mvPartitionInfo, + MTMVRelation relation) { + this.columnDefinitions = columnDefinitions; + this.mvPartitionInfo = mvPartitionInfo; + this.relation = relation; + } + + public List getColumnDefinitions() { + return columnDefinitions; + } + + public MTMVPartitionInfo getMvPartitionInfo() { + return mvPartitionInfo; + } + + public MTMVRelation getRelation() { + return relation; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java index b2ef26db1edbd0..1885d10ae452f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java @@ -111,4 +111,18 @@ public interface MTMVHookService { * @param info */ void cancelMTMVTask(CancelMTMVTaskInfo info) throws DdlException, MetaNotFoundException, JobException; + + /** + * Triggered when baseView is altered + * + * @param baseViewInfo + */ + void alterView(BaseTableInfo baseViewInfo); + + /** + * Triggered when baseView is dropped + * + * @param baseViewInfo + */ + void dropView(BaseTableInfo baseViewInfo); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java index e449b84641450a..1edf590f310341 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java @@ -216,4 +216,13 @@ private MTMVJob getJobByMTMV(MTMV mtmv) { return (MTMVJob) Env.getCurrentEnv().getJobManager().getJob(mtmv.getId()); } + @Override + public void alterView(BaseTableInfo baseViewInfo) { + + } + + @Override + public void dropView(BaseTableInfo baseViewInfo) { + + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java index d34580f6608336..9247b716497967 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java @@ -26,6 +26,7 @@ import com.google.gson.annotations.SerializedName; import java.util.List; +import java.util.Objects; /** * MTMVPartitionInfo @@ -129,6 +130,22 @@ public int getRelatedColPos() throws AnalysisException { partitionColumns)); } + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + MTMVPartitionInfo that = (MTMVPartitionInfo) o; + return partitionType == that.partitionType && Objects.equals(relatedTable, that.relatedTable) + && Objects.equals(relatedCol, that.relatedCol) && Objects.equals(partitionCol, + that.partitionCol) && Objects.equals(expr, that.expr); + } + + @Override + public int hashCode() { + return Objects.hash(partitionType, relatedTable, relatedCol, partitionCol, expr); + } + // toString() is not easy to find where to call the method public String toInfoString() { return "MTMVPartitionInfo{" @@ -136,6 +153,7 @@ public String toInfoString() { + ", relatedTable=" + relatedTable + ", relatedCol='" + relatedCol + '\'' + ", partitionCol='" + partitionCol + '\'' + + ", expr='" + expr + '\'' + '}'; } @@ -150,6 +168,7 @@ public String toNameString() { + ", relatedTable=" + relatedTable.getTableName() + ", relatedCol='" + relatedCol + '\'' + ", partitionCol='" + partitionCol + '\'' + + ", expr='" + expr + '\'' + '}'; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index 74cf6396eabee6..590cbf4951af83 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -198,7 +198,7 @@ public static boolean isMTMVSync(MTMV mtmv) { return false; } try { - return isMTMVSync(MTMVRefreshContext.buildContext(mtmv), mtmvRelation.getBaseTablesOneLevel(), + return isMTMVSync(MTMVRefreshContext.buildContext(mtmv), mtmvRelation.getBaseTablesOneLevelAndFromView(), Sets.newHashSet()); } catch (AnalysisException e) { LOG.warn("isMTMVSync failed: ", e); @@ -253,7 +253,7 @@ private static List getPartitionUnSyncTables(MTMVRefreshContext context, MTMV mtmv = context.getMtmv(); Set relatedPartitionNames = context.getPartitionMappings().get(partitionName); List res = Lists.newArrayList(); - for (BaseTableInfo baseTableInfo : mtmv.getRelation().getBaseTablesOneLevel()) { + for (BaseTableInfo baseTableInfo : mtmv.getRelation().getBaseTablesOneLevelAndFromView()) { TableIf table = MTMVUtil.getTable(baseTableInfo); if (!(table instanceof MTMVRelatedTableIf)) { continue; @@ -610,11 +610,11 @@ private static Map getPartitionVersions(MTMV mtmv) throws Analysis private static Map getTableVersions(MTMV mtmv) { Map res = Maps.newHashMap(); MTMVRelation relation = mtmv.getRelation(); - if (relation == null || relation.getBaseTablesOneLevel() == null) { + if (relation == null || relation.getBaseTablesOneLevelAndFromView() == null) { return res; } List olapTables = Lists.newArrayList(); - for (BaseTableInfo baseTableInfo : relation.getBaseTablesOneLevel()) { + for (BaseTableInfo baseTableInfo : relation.getBaseTablesOneLevelAndFromView()) { TableIf table = null; try { table = MTMVUtil.getTable(baseTableInfo); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index c734e7497f9549..ac49d57194e889 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -17,32 +17,53 @@ package org.apache.doris.mtmv; +import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeNameFormat; +import org.apache.doris.common.Pair; +import org.apache.doris.common.UserException; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.job.exception.JobException; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.analyzer.UnboundResultSink; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; +import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo; +import org.apache.doris.nereids.trees.plans.commands.info.DistributionDescriptor; +import org.apache.doris.nereids.trees.plans.commands.info.MTMVPartitionDefinition; import org.apache.doris.nereids.trees.plans.commands.info.SimpleColumnDefinition; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalSink; import org.apache.doris.nereids.types.AggStateType; import org.apache.doris.nereids.types.CharType; import org.apache.doris.nereids.types.DataType; @@ -54,6 +75,7 @@ import org.apache.doris.nereids.types.coercion.CharacterType; import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -150,27 +172,38 @@ private static void setCatalogAndDb(ConnectContext ctx, MTMV mtmv) { ctx.setDatabase(databaseIf.get().getFullName()); } - public static MTMVRelation generateMTMVRelation(Set tablesInPlan, ConnectContext ctx) { + public static MTMVRelation generateMTMVRelation(Set tablesInPlan, Set oneLevelTablesInPlan) { Set oneLevelTables = Sets.newHashSet(); Set allLevelTables = Sets.newHashSet(); Set oneLevelViews = Sets.newHashSet(); + Set allLevelViews = Sets.newHashSet(); + Set oneLevelTablesAndFromView = Sets.newHashSet(); for (TableIf table : tablesInPlan) { BaseTableInfo baseTableInfo = new BaseTableInfo(table); if (table.getType() == TableType.VIEW) { - // TODO reopen it after we support mv on view - // oneLevelViews.add(baseTableInfo); + allLevelViews.add(baseTableInfo); } else { - oneLevelTables.add(baseTableInfo); + oneLevelTablesAndFromView.add(baseTableInfo); allLevelTables.add(baseTableInfo); if (table instanceof MTMV) { allLevelTables.addAll(((MTMV) table).getRelation().getBaseTables()); } } } - return new MTMVRelation(allLevelTables, oneLevelTables, oneLevelViews); + for (TableIf table : oneLevelTablesInPlan) { + BaseTableInfo baseTableInfo = new BaseTableInfo(table); + if (table.getType() == TableType.VIEW) { + oneLevelViews.add(baseTableInfo); + } else { + oneLevelTables.add(baseTableInfo); + } + } + return new MTMVRelation(allLevelTables, oneLevelTables, oneLevelTablesAndFromView, allLevelViews, + oneLevelViews); } - public static Set getBaseTableFromQuery(String querySql, ConnectContext ctx) { + // return allLevelTables:oneLevelTables + public static Pair, Set> getBaseTableFromQuery(String querySql, ConnectContext ctx) { List statements; try { statements = new NereidsParser().parseSQL(querySql); @@ -185,7 +218,8 @@ public static Set getBaseTableFromQuery(String querySql, ConnectContext try { NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); planner.planWithLock(logicalPlan, PhysicalProperties.ANY, ExplainLevel.ANALYZED_PLAN); - return Sets.newHashSet(ctx.getStatementContext().getTables().values()); + return Pair.of(Sets.newHashSet(ctx.getStatementContext().getTables().values()), + Sets.newHashSet(ctx.getStatementContext().getOneLevelTables().values())); } finally { ctx.setStatementContext(original); } @@ -296,6 +330,7 @@ public static List generateColumns(Plan plan, ConnectContext c /** * generate DataType by Slot + * * @param s * @param i * @param ctx @@ -340,4 +375,216 @@ public static DataType getDataType(Slot s, int i, ConnectContext ctx, String par } return dataType; } + + public static MTMVAnalyzeQueryInfo analyzeQueryWithSql(MTMV mtmv, ConnectContext ctx) throws UserException { + String querySql = mtmv.getQuerySql(); + MTMVPartitionInfo mvPartitionInfo = mtmv.getMvPartitionInfo(); + MTMVPartitionDefinition mtmvPartitionDefinition = new MTMVPartitionDefinition(); + mtmvPartitionDefinition.setPartitionCol(mvPartitionInfo.getPartitionCol()); + mtmvPartitionDefinition.setPartitionType(mvPartitionInfo.getPartitionType()); + Expr expr = mvPartitionInfo.getExpr(); + if (expr != null) { + mtmvPartitionDefinition.setFunctionCallExpression(new NereidsParser().parseExpression(expr.toSql())); + } + List keys = mtmv.getBaseSchema().stream() + .filter(Column::isKey) + .map(Column::getName) + .collect(Collectors.toList()); + List statements; + try { + statements = new NereidsParser().parseSQL(querySql); + } catch (Exception e) { + throw new ParseException("Nereids parse failed. " + e.getMessage()); + } + StatementBase parsedStmt = statements.get(0); + LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + DistributionInfo defaultDistributionInfo = mtmv.getDefaultDistributionInfo(); + DistributionDescriptor distribution = new DistributionDescriptor(defaultDistributionInfo.getType().equals( + DistributionInfoType.HASH), defaultDistributionInfo.getAutoBucket(), + defaultDistributionInfo.getBucketNum(), Lists.newArrayList(mtmv.getDistributionColumnNames())); + return analyzeQuery(ctx, mtmv.getMvProperties(), querySql, mtmvPartitionDefinition, distribution, null, + mtmv.getTableProperty().getProperties(), keys, logicalPlan); + } + + public static MTMVAnalyzeQueryInfo analyzeQuery(ConnectContext ctx, Map mvProperties, + String querySql, + MTMVPartitionDefinition mvPartitionDefinition, DistributionDescriptor distribution, + List simpleColumnDefinitions, Map properties, List keys, + LogicalPlan + logicalQuery) throws UserException { + try (StatementContext statementContext = ctx.getStatementContext()) { + NereidsPlanner planner = new NereidsPlanner(statementContext); + // this is for expression column name infer when not use alias + LogicalSink logicalSink = new UnboundResultSink<>(logicalQuery); + // Should not make table without data to empty relation when analyze the related table, + // so add disable rules + Set tempDisableRules = ctx.getSessionVariable().getDisableNereidsRuleNames(); + ctx.getSessionVariable().setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); + statementContext.invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + Plan plan; + try { + // must disable constant folding by be, because be constant folding may return wrong type + ctx.getSessionVariable().setVarOnce(SessionVariable.ENABLE_FOLD_CONSTANT_BY_BE, "false"); + plan = planner.planWithLock(logicalSink, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); + } finally { + // after operate, roll back the disable rules + ctx.getSessionVariable().setDisableNereidsRules(String.join(",", tempDisableRules)); + statementContext.invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + } + // can not contain Random function + analyzeExpressions(planner.getAnalyzedPlan(), mvProperties); + // can not contain partition or tablets + boolean containTableQueryOperator = MaterializedViewUtils.containTableQueryOperator( + planner.getAnalyzedPlan()); + if (containTableQueryOperator) { + throw new AnalysisException("can not contain invalid expression"); + } + + Set baseTables = Sets.newHashSet(statementContext.getTables().values()); + Set oneLevelTables = Sets.newHashSet(statementContext.getOneLevelTables().values()); + for (TableIf table : baseTables) { + if (table.isTemporary()) { + throw new AnalysisException("do not support create materialized view on temporary table (" + + Util.getTempTableDisplayName(table.getName()) + ")"); + } + } + MTMVRelation relation = generateMTMVRelation(baseTables, oneLevelTables); + MTMVPartitionInfo mvPartitionInfo = mvPartitionDefinition.analyzeAndTransferToMTMVPartitionInfo(planner); + List columns = MTMVPlanUtil.generateColumns(plan, ctx, mvPartitionInfo.getPartitionCol(), + (distribution == null || CollectionUtils.isEmpty(distribution.getCols())) ? Sets.newHashSet() + : Sets.newHashSet(distribution.getCols()), + simpleColumnDefinitions, properties); + analyzeKeys(keys, properties, columns); + // analyze column + final boolean finalEnableMergeOnWrite = false; + Set keysSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + keysSet.addAll(keys); + validateColumns(columns, keysSet, finalEnableMergeOnWrite); + return new MTMVAnalyzeQueryInfo(columns, mvPartitionInfo, relation); + } + } + + /** + * validate column name + */ + private static void validateColumns(List columns, Set keysSet, + boolean finalEnableMergeOnWrite) throws UserException { + Set colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + for (ColumnDefinition col : columns) { + if (!colSets.add(col.getName())) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, col.getName()); + } + if (col.getType().isVarBinaryType()) { + throw new AnalysisException("MTMV do not support varbinary type : " + col.getName()); + } + col.validate(true, keysSet, Sets.newHashSet(), finalEnableMergeOnWrite, KeysType.DUP_KEYS); + } + } + + private static void analyzeKeys(List keys, Map properties, List columns) { + boolean enableDuplicateWithoutKeysByDefault = false; + try { + if (properties != null) { + enableDuplicateWithoutKeysByDefault = + PropertyAnalyzer.analyzeEnableDuplicateWithoutKeysByDefault(properties); + } + } catch (Exception e) { + throw new AnalysisException(e.getMessage(), e.getCause()); + } + if (keys.isEmpty() && !enableDuplicateWithoutKeysByDefault) { + keys = Lists.newArrayList(); + int keyLength = 0; + for (ColumnDefinition column : columns) { + DataType type = column.getType(); + Type catalogType = column.getType().toCatalogDataType(); + keyLength += catalogType.getIndexSize(); + if (keys.size() >= FeConstants.shortkey_max_column_count + || keyLength > FeConstants.shortkey_maxsize_bytes) { + if (keys.isEmpty() && type.isStringLikeType()) { + keys.add(column.getName()); + column.setIsKey(true); + } + break; + } + if (column.getAggType() != null) { + break; + } + if (!catalogType.couldBeShortKey()) { + break; + } + keys.add(column.getName()); + column.setIsKey(true); + if (type.isVarcharType()) { + break; + } + } + } + } + + private static void analyzeExpressions(Plan plan, Map mvProperties) { + boolean enableNondeterministicFunction = Boolean.parseBoolean( + mvProperties.get(PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION)); + if (enableNondeterministicFunction) { + return; + } + List functionCollectResult = MaterializedViewUtils.extractNondeterministicFunction(plan); + if (!CollectionUtils.isEmpty(functionCollectResult)) { + throw new AnalysisException(String.format( + "can not contain nonDeterministic expression, the expression is %s. " + + "Should add 'enable_nondeterministic_function' = 'true' property " + + "when create materialized view if you know the property real meaning entirely", + functionCollectResult.stream().map(Expression::toString).collect(Collectors.joining(",")))); + } + } + + public static void ensureMTMVQueryUsable(MTMV mtmv, ConnectContext ctx) throws JobException { + MTMVAnalyzeQueryInfo mtmvAnalyzedQueryInfo; + try { + mtmvAnalyzedQueryInfo = MTMVPlanUtil.analyzeQueryWithSql(mtmv, ctx); + } catch (Exception e) { + throw new JobException(e.getMessage(), e); + } + checkColumnIfChange(mtmv, mtmvAnalyzedQueryInfo.getColumnDefinitions()); + checkMTMVPartitionInfo(mtmv, mtmvAnalyzedQueryInfo.getMvPartitionInfo()); + } + + private static void checkMTMVPartitionInfo(MTMV mtmv, MTMVPartitionInfo analyzedMvPartitionInfo) + throws JobException { + MTMVPartitionInfo originalMvPartitionInfo = mtmv.getMvPartitionInfo(); + if (!analyzedMvPartitionInfo.equals(originalMvPartitionInfo)) { + throw new JobException("async materialized view partition info changed, analyzed: %s, original: %s", + analyzedMvPartitionInfo.toInfoString(), originalMvPartitionInfo.toInfoString()); + } + } + + private static void checkColumnIfChange(MTMV mtmv, List analyzedColumnDefinitions) + throws JobException { + List analyzedColumns = analyzedColumnDefinitions.stream() + .map(ColumnDefinition::translateToCatalogStyle) + .collect(Collectors.toList()); + List originalColumns = mtmv.getBaseSchema(true); + if (analyzedColumns.size() != originalColumns.size()) { + throw new JobException(String.format( + "column length not equals, please check whether columns of base table have changed, " + + "original length is: %s, current length is: %s", + originalColumns.size(), analyzedColumns.size())); + } + for (int i = 0; i < originalColumns.size(); i++) { + if (!isTypeLike(originalColumns.get(i).getType(), analyzedColumns.get(i).getType())) { + throw new JobException(String.format( + "column type not same, please check whether columns of base table have changed, " + + "column name is: %s, original type is: %s, current type is: %s", + originalColumns.get(i).getName(), originalColumns.get(i).getType().toSql(), + analyzedColumns.get(i).getType().toSql())); + } + } + } + + private static boolean isTypeLike(Type type, Type typeOther) { + if (type.isStringType()) { + return typeOther.isStringType(); + } else { + return type.equals(typeOther); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java index 490fc8ca2fe4c2..cae7d26c77809f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshPartitionSnapshot.java @@ -152,19 +152,19 @@ private void compatibleTables(MTMV mtmv) throws Exception { return; } MTMVRelation relation = mtmv.getRelation(); - if (relation == null || CollectionUtils.isEmpty(relation.getBaseTablesOneLevel())) { + if (relation == null || CollectionUtils.isEmpty(relation.getBaseTablesOneLevelAndFromView())) { return; } for (Entry entry : tables.entrySet()) { Optional tableInfo = getByTableId(entry.getKey(), - relation.getBaseTablesOneLevel()); + relation.getBaseTablesOneLevelAndFromView()); if (tableInfo.isPresent()) { tablesInfo.put(tableInfo.get(), entry.getValue()); } else { String msg = String.format( "Failed to get table info based on id during compatibility process, " + "tableId: %s, relationTables: %s", - entry.getKey(), relation.getBaseTablesOneLevel()); + entry.getKey(), relation.getBaseTablesOneLevelAndFromView()); LOG.warn(msg); throw new Exception(msg); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java index 148d2d008843d6..7e784932e7be57 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelation.java @@ -25,22 +25,36 @@ import java.util.Set; public class MTMVRelation { - // if mtmv query sql is `select * from view1`; - // and `view1` query sql is `select * from table1 join table2` - // then baseTables will include: `table1` and `table2` - // baseViews will include `view1` + // t1 => v1 => v2 + // t2 => mv1 + // mv1 join v2 => mv2 + // + // data of mv2 is: + // + // baseTables => mv1,t1,t2 + // baseTablesOneLevelAndFromView => mv1,t1 + // baseTablesOneLevel => mv1 + // baseViews => v2,v1 + // baseViewsOneLevel => v2 @SerializedName("bt") private Set baseTables; @SerializedName("bv") private Set baseViews; @SerializedName("btol") private Set baseTablesOneLevel; + @SerializedName("btolafv") + private Set baseTablesOneLevelAndFromView; + @SerializedName("bvol") + private Set baseViewsOneLevel; public MTMVRelation(Set baseTables, Set baseTablesOneLevel, - Set baseViews) { + Set baseTablesOneLevelAndFromView, Set baseViews, + Set baseViewsOneLevel) { this.baseTables = baseTables; this.baseTablesOneLevel = baseTablesOneLevel; + this.baseTablesOneLevelAndFromView = baseTablesOneLevelAndFromView; this.baseViews = baseViews; + this.baseViewsOneLevel = baseViewsOneLevel; } public Set getBaseTables() { @@ -52,6 +66,16 @@ public Set getBaseTablesOneLevel() { return baseTablesOneLevel == null ? baseTables : baseTablesOneLevel; } + public Set getBaseTablesOneLevelAndFromView() { + // For compatibility, previously created MTMV may not have baseTablesOneLevelAndFromView + return CollectionUtils.isEmpty(baseTablesOneLevelAndFromView) ? baseTablesOneLevel + : baseTablesOneLevelAndFromView; + } + + public Set getBaseViewsOneLevel() { + return baseViewsOneLevel; + } + public Set getBaseViews() { return baseViews; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java index d8a2146210bd31..2cb4f2ffd35fa0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java @@ -59,19 +59,28 @@ public class MTMVRelationManager implements MTMVHookService { private static final Logger LOG = LogManager.getLogger(MTMVRelationManager.class); // when - // create mv1 as select * from table1; + // create v1 as select * from table1 + // create v2 as select * from v1 + // create mv1 as select * from v1; // create mv2 as select * from mv1; - // `tableMTMVs` will have 3 pair: table1 ==> mv1,mv1==>mv2, table1 ==> mv2 - // `tableMTMVsOneLevel` will have 2 pair: table1 ==> mv1,mv1==>mv2 + // `tableMTMVs` will have 3 pair: table1 ==> mv1, mv1==>mv2, table1 ==> mv2 + // `tableMTMVsOneLevelAndFromView` will have 2 pair: table1 ==> mv1, mv1==>mv2 + // `viewMTMVs` will have 2 pair: v1 ==> mv1, v2 ==> mv1 private final Map> tableMTMVs = Maps.newConcurrentMap(); - private final Map> tableMTMVsOneLevel = Maps.newConcurrentMap(); + private final Map> tableMTMVsOneLevelAndFromView = Maps.newConcurrentMap(); + // view => mtmv + private final Map> viewMTMVs = Maps.newConcurrentMap(); public Set getMtmvsByBaseTable(BaseTableInfo table) { return tableMTMVs.getOrDefault(table, ImmutableSet.of()); } - public Set getMtmvsByBaseTableOneLevel(BaseTableInfo table) { - return tableMTMVsOneLevel.getOrDefault(table, ImmutableSet.of()); + public Set getMtmvsByBaseView(BaseTableInfo table) { + return viewMTMVs.getOrDefault(table, ImmutableSet.of()); + } + + public Set getMtmvsByBaseTableOneLevelAndFromView(BaseTableInfo table) { + return tableMTMVsOneLevelAndFromView.getOrDefault(table, ImmutableSet.of()); } /** @@ -149,11 +158,18 @@ private Set getOrCreateMTMVs(BaseTableInfo baseTableInfo) { return tableMTMVs.get(baseTableInfo); } - private Set getOrCreateMTMVsOneLevel(BaseTableInfo baseTableInfo) { - if (!tableMTMVsOneLevel.containsKey(baseTableInfo)) { - tableMTMVsOneLevel.put(baseTableInfo, Sets.newConcurrentHashSet()); + private Set getOrCreateMTMVsView(BaseTableInfo baseTableInfo) { + if (!viewMTMVs.containsKey(baseTableInfo)) { + viewMTMVs.put(baseTableInfo, Sets.newConcurrentHashSet()); + } + return viewMTMVs.get(baseTableInfo); + } + + private Set getOrCreateMTMVsOneLevelAndFromView(BaseTableInfo baseTableInfo) { + if (!tableMTMVsOneLevelAndFromView.containsKey(baseTableInfo)) { + tableMTMVsOneLevelAndFromView.put(baseTableInfo, Sets.newConcurrentHashSet()); } - return tableMTMVsOneLevel.get(baseTableInfo); + return tableMTMVsOneLevelAndFromView.get(baseTableInfo); } public void refreshMTMVCache(MTMVRelation relation, BaseTableInfo mtmvInfo) { @@ -167,8 +183,8 @@ private void addMTMV(MTMVRelation relation, BaseTableInfo mtmvInfo) { return; } addMTMVTables(relation.getBaseTables(), mtmvInfo); - addMTMVTables(relation.getBaseViews(), mtmvInfo); - addMTMVTablesOneLevel(relation.getBaseTablesOneLevel(), mtmvInfo); + addMTMVViews(relation.getBaseViews(), mtmvInfo); + addMTMVTablesOneLevelAndFromView(relation.getBaseTablesOneLevelAndFromView(), mtmvInfo); } private void addMTMVTables(Set baseTables, BaseTableInfo mtmvInfo) { @@ -180,12 +196,21 @@ private void addMTMVTables(Set baseTables, BaseTableInfo mtmvInfo } } - private void addMTMVTablesOneLevel(Set baseTables, BaseTableInfo mtmvInfo) { + private void addMTMVViews(Set baseTables, BaseTableInfo mtmvInfo) { + if (CollectionUtils.isEmpty(baseTables)) { + return; + } + for (BaseTableInfo baseTableInfo : baseTables) { + getOrCreateMTMVsView(baseTableInfo).add(mtmvInfo); + } + } + + private void addMTMVTablesOneLevelAndFromView(Set baseTables, BaseTableInfo mtmvInfo) { if (CollectionUtils.isEmpty(baseTables)) { return; } for (BaseTableInfo baseTableInfo : baseTables) { - getOrCreateMTMVsOneLevel(baseTableInfo).add(mtmvInfo); + getOrCreateMTMVsOneLevelAndFromView(baseTableInfo).add(mtmvInfo); } } @@ -193,7 +218,10 @@ private void removeMTMV(BaseTableInfo mtmvInfo) { for (Set sets : tableMTMVs.values()) { sets.remove(mtmvInfo); } - for (Set sets : tableMTMVsOneLevel.values()) { + for (Set sets : viewMTMVs.values()) { + sets.remove(mtmvInfo); + } + for (Set sets : tableMTMVsOneLevelAndFromView.values()) { sets.remove(mtmvInfo); } } @@ -286,8 +314,47 @@ public void cancelMTMVTask(CancelMTMVTaskInfo info) { } + /** + * update mtmv status to `SCHEMA_CHANGE` and drop snapshot + * + * @param baseViewInfo + */ + @Override + public void alterView(BaseTableInfo baseViewInfo) { + processBaseViewChange(baseViewInfo, "The base view has been updated:"); + } + + /** + * update mtmv status to `SCHEMA_CHANGE` and drop snapshot + * + * @param baseViewInfo + */ + @Override + public void dropView(BaseTableInfo baseViewInfo) { + processBaseViewChange(baseViewInfo, "The base view has been dropped:"); + } + + private void processBaseViewChange(BaseTableInfo baseViewInfo, String msgPrefix) { + Set mtmvsByBaseView = getMtmvsByBaseView(baseViewInfo); + LOG.info("processBaseViewChange, baseViewInfo: {}, mtmvsByBaseView: {}", baseViewInfo, mtmvsByBaseView); + if (CollectionUtils.isEmpty(mtmvsByBaseView)) { + return; + } + for (BaseTableInfo mtmvInfo : mtmvsByBaseView) { + MTMV mtmv = null; + try { + mtmv = MTMVUtil.getMTMV(mtmvInfo); + } catch (AnalysisException e) { + LOG.warn(e); + continue; + } + String schemaChangeDetail = msgPrefix + baseViewInfo; + mtmv.processBaseViewChange(schemaChangeDetail); + } + } + private void processBaseTableChange(BaseTableInfo baseTableInfo, String msgPrefix) { - Set mtmvsByBaseTable = getMtmvsByBaseTableOneLevel(baseTableInfo); + Set mtmvsByBaseTable = getMtmvsByBaseTableOneLevelAndFromView(baseTableInfo); if (CollectionUtils.isEmpty(mtmvsByBaseTable)) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index 751362be92dfff..c95e1262281579 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -91,7 +91,7 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne } try { if (MTMVPartitionUtil.isMTMVPartitionSync(refreshContext, partition.getName(), - mtmvRelation.getBaseTablesOneLevel(), + mtmvRelation.getBaseTablesOneLevelAndFromView(), forceConsistent ? ImmutableSet.of() : mtmv.getQueryRewriteConsistencyRelaxedTables())) { res.add(partition); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java index 96e7226a20e310..5053b2a046746e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java @@ -122,6 +122,22 @@ public void alterTable(BaseTableInfo oldTableInfo, Optional newTa } } + public void dropView(BaseTableInfo baseViewInfo) { + Objects.requireNonNull(baseViewInfo, "baseViewInfo can not be null"); + LOG.info("dropView, view: {}", baseViewInfo); + for (MTMVHookService mtmvHookService : hooks.values()) { + mtmvHookService.dropView(baseViewInfo); + } + } + + public void alterView(BaseTableInfo baseViewInfo) { + Objects.requireNonNull(baseViewInfo, "baseViewInfo can not be null"); + LOG.info("alterView, view: {}", baseViewInfo); + for (MTMVHookService mtmvHookService : hooks.values()) { + mtmvHookService.alterView(baseViewInfo); + } + } + public void refreshComplete(MTMV mtmv, MTMVRelation cache, MTMVTask task) { Objects.requireNonNull(mtmv, "mtmv can not be null"); Objects.requireNonNull(task, "task can not be null"); @@ -180,7 +196,7 @@ public void processEvent(Event event) throws EventException { } catch (AnalysisException e) { throw new EventException(e); } - Set mtmvs = relationManager.getMtmvsByBaseTableOneLevel( + Set mtmvs = relationManager.getMtmvsByBaseTableOneLevelAndFromView( new BaseTableInfo(table)); for (BaseTableInfo baseTableInfo : mtmvs) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java index fa0a948f3decc4..01a08f823c51c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java @@ -85,6 +85,14 @@ public static MTMVRelatedTableIf getRelatedTable(BaseTableInfo baseTableInfo) { return (MTMVRelatedTableIf) relatedTable; } + public static MTMV getMTMV(BaseTableInfo baseTableInfo) throws AnalysisException { + TableIf table = getTable(baseTableInfo); + if (!(table instanceof MTMV)) { + throw new AnalysisException(String.format("table is not MTMV, table: %s", baseTableInfo)); + } + return (MTMV) table; + } + public static MTMV getMTMV(long dbId, long mtmvId) throws DdlException, MetaNotFoundException { Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbId); return (MTMV) db.getTableOrMetaException(mtmvId, TableType.MATERIALIZED_VIEW); @@ -97,7 +105,7 @@ public static MTMV getMTMV(long dbId, long mtmvId) throws DdlException, MetaNotF * @return */ public static boolean mtmvContainsExternalTable(MTMV mtmv) { - Set baseTables = mtmv.getRelation().getBaseTablesOneLevel(); + Set baseTables = mtmv.getRelation().getBaseTablesOneLevelAndFromView(); for (BaseTableInfo baseTableInfo : baseTables) { if (!baseTableInfo.isInternalTable()) { return true; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index efb53b66822427..6980765049f37b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -272,8 +272,8 @@ public void toMemo() { } } - public TableCollectAndHookInitializer newTableCollector() { - return new TableCollectAndHookInitializer(this); + public TableCollectAndHookInitializer newTableCollector(boolean firstLevel) { + return new TableCollectAndHookInitializer(this, firstLevel); } public Analyzer newAnalyzer() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 28e7a477f9d91f..81d5108ce2c3d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -394,7 +394,7 @@ protected void collectAndLockTable(boolean showPlanProcess) { if (LOG.isDebugEnabled()) { LOG.debug("Start collect and lock table"); } - keepOrShowPlanProcess(showPlanProcess, () -> cascadesContext.newTableCollector().collect()); + keepOrShowPlanProcess(showPlanProcess, () -> cascadesContext.newTableCollector(true).collect()); statementContext.lock(); cascadesContext.setCteContext(new CTEContext()); NereidsTracer.logImportantTime("EndCollectAndLockTables"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 746efa3cf952cd..5829a0c9b56993 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -188,6 +188,14 @@ public enum TableFrom { // tables in this query directly private final Map, TableIf> tables = Maps.newHashMap(); + // onelevel tables in this query directly, + // if + // create v1 as select * from t1 + // create v2 as select * from v1 + // current query is: select * from v2 join t2 + // oneLevelTables will have two data: v2, t2, + // tables will have 4 data: t1, v1, v2, t2 + private final Map, TableIf> oneLevelTables = Maps.newHashMap(); // tables maybe used by mtmv rewritten in this query, // this contains mvs which use table in tables and the tables in mvs // such as @@ -275,6 +283,8 @@ public enum TableFrom { private boolean isInsert = false; + private Optional>> mvRefreshPredicates = Optional.empty(); + public StatementContext() { this(ConnectContext.get(), null, 0); } @@ -357,6 +367,10 @@ public Map, TableIf> getTables() { return tables; } + public Map, TableIf> getOneLevelTables() { + return oneLevelTables; + } + public Set getCandidateMTMVs() { return candidateMTMVs; } @@ -994,4 +1008,13 @@ public void setIsInsert(boolean isInsert) { public boolean isInsert() { return isInsert; } + + public Optional>> getMvRefreshPredicates() { + return mvRefreshPredicates; + } + + public void setMvRefreshPredicates( + Map> mvRefreshPredicates) { + this.mvRefreshPredicates = Optional.of(mvRefreshPredicates); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TableCollectAndHookInitializer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TableCollectAndHookInitializer.java index 01ce6687ecbbce..02b2dc76fb527a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TableCollectAndHookInitializer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/TableCollectAndHookInitializer.java @@ -33,21 +33,21 @@ */ public class TableCollectAndHookInitializer extends AbstractBatchJobExecutor { - public static final List COLLECT_JOBS = buildCollectTableJobs(); + public final List collectJobs; /** * constructor of Analyzer. For view, we only do bind relation since other analyze step will do by outer Analyzer. * * @param cascadesContext current context for analyzer */ - public TableCollectAndHookInitializer(CascadesContext cascadesContext) { + public TableCollectAndHookInitializer(CascadesContext cascadesContext, boolean firstLevel) { super(cascadesContext); - + collectJobs = buildCollectTableJobs(firstLevel); } @Override public List getJobs() { - return COLLECT_JOBS; + return collectJobs; } /** @@ -57,17 +57,17 @@ public void collect() { execute(); } - private static List buildCollectTableJobs() { + private static List buildCollectTableJobs(boolean firstLevel) { return notTraverseChildrenOf( ImmutableSet.of(LogicalView.class), - TableCollectAndHookInitializer::buildCollectorJobs + () -> TableCollectAndHookInitializer.buildCollectorJobs(firstLevel) ); } - private static List buildCollectorJobs() { + private static List buildCollectorJobs(boolean firstLevel) { return jobs( topDown(new AddInitMaterializationHook()), - topDown(new CollectRelation()) + topDown(new CollectRelation(firstLevel)) ); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectOneLevelRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectOneLevelRelation.java index 5a99d865558f24..69be32122f8159 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectOneLevelRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectOneLevelRelation.java @@ -27,7 +27,9 @@ * collect one level relations */ public class CollectOneLevelRelation extends CollectRelation { - public CollectOneLevelRelation() {} + public CollectOneLevelRelation() { + super(true); + } protected void collectMTMVCandidates(TableIf table, CascadesContext cascadesContext) { //do nothing diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java index 53650234610f80..914220f177edbb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java @@ -67,7 +67,11 @@ public class CollectRelation implements AnalysisRuleFactory { private static final Logger LOG = LogManager.getLogger(CollectRelation.class); - public CollectRelation() {} + private boolean firstLevel; + + public CollectRelation(boolean firstLevel) { + this.firstLevel = firstLevel; + } @Override public List buildRules() { @@ -104,7 +108,7 @@ private CTEContext collectFromCte( LogicalPlan parsedCtePlan = (LogicalPlan) aliasQuery.child(); CascadesContext innerCascadesCtx = CascadesContext.newContextWithCteContext( cascadesContext, parsedCtePlan, outerCteCtx); - innerCascadesCtx.newTableCollector().collect(); + innerCascadesCtx.newTableCollector(true).collect(); LogicalPlan analyzedCtePlan = (LogicalPlan) innerCascadesCtx.getRewritePlan(); // cteId is not used in CollectTable stage CTEId cteId = new CTEId(0); @@ -124,7 +128,7 @@ private Plan collectFromAny(MatchingContext ctx) { CascadesContext subqueryContext = CascadesContext.newContextWithCteContext( ctx.cascadesContext, subqueryExpr.getQueryPlan(), ctx.cteContext); subqueryContext.keepOrShowPlanProcess(ctx.cascadesContext.showPlanProcess(), - () -> subqueryContext.newTableCollector().collect()); + () -> subqueryContext.newTableCollector(true).collect()); ctx.cascadesContext.addPlanProcesses(subqueryContext.getPlanProcesses()); } }); @@ -189,8 +193,11 @@ private void collectFromUnboundRelation(CascadesContext cascadesContext, if (cascadesContext.getRewritePlan() instanceof UnboundDictionarySink) { table = ((UnboundDictionarySink) cascadesContext.getRewritePlan()).getDictionary(); } else { - table = cascadesContext.getConnectContext().getStatementContext() - .getAndCacheTable(tableQualifier, tableFrom, unboundRelation); + StatementContext statementContext = cascadesContext.getConnectContext().getStatementContext(); + table = statementContext.getAndCacheTable(tableQualifier, tableFrom, unboundRelation); + if (firstLevel) { + statementContext.getOneLevelTables().put(tableQualifier, table); + } } if (LOG.isDebugEnabled()) { LOG.debug("collect table {} from {}", nameParts, tableFrom); @@ -281,7 +288,7 @@ protected void parseAndCollectFromView(List tableQualifier, View view, C CascadesContext viewContext = CascadesContext.initContext( parentContext.getStatementContext(), parsedViewPlan, PhysicalProperties.ANY); viewContext.keepOrShowPlanProcess(parentContext.showPlanProcess(), - () -> viewContext.newTableCollector().collect()); + () -> viewContext.newTableCollector(false).collect()); parentContext.addPlanProcesses(viewContext.getPlanProcesses()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index cb62eaf42f194a..5c273ebfb0173f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.constraint.TableIdentifier; +import org.apache.doris.common.DdlException; import org.apache.doris.common.Pair; import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.mtmv.BaseTableInfo; @@ -677,8 +678,14 @@ public Void visitLogicalRelation(LogicalRelation relation, IncrementCheckerConte table.getName())); return null; } - Set partitionColumnSet = new HashSet<>( - relatedTable.getPartitionColumns(MvccUtil.getSnapshotFromContext(relatedTable))); + Set partitionColumnSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + try { + partitionColumnSet.addAll(relatedTable.getPartitionColumnNames( + MvccUtil.getSnapshotFromContext(relatedTable))); + } catch (DdlException e) { + context.addFailReason(e.getMessage()); + return null; + } Column mvReferenceColumn = contextPartitionColumn.getOriginalColumn().get(); Expr definExpr = mvReferenceColumn.getDefineExpr(); if (definExpr instanceof SlotRef) { @@ -687,7 +694,7 @@ public Void visitLogicalRelation(LogicalRelation relation, IncrementCheckerConte mvReferenceColumn = referenceRollupColumn; } } - if (partitionColumnSet.contains(mvReferenceColumn) + if (partitionColumnSet.contains(mvReferenceColumn.getName()) && (!mvReferenceColumn.isAllowNull() || relatedTable.isPartitionColumnAllowNull())) { context.addTableColumn(table, mvReferenceColumn); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index 27758175527731..8411bf14085e21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -32,6 +32,7 @@ import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVRelatedTableIf; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundTableSinkCreator; @@ -101,16 +102,16 @@ public boolean isForceDropPartition() { * @param mv materialize view * @param partitionNames update partitions in mv and tables * @param tableWithPartKey the partitions key for different table + * @param statementContext statementContext * @return command */ public static UpdateMvByPartitionCommand from(MTMV mv, Set partitionNames, - Map tableWithPartKey) throws UserException { + Map tableWithPartKey, StatementContext statementContext) throws UserException { NereidsParser parser = new NereidsParser(); Map> predicates = constructTableWithPredicates(mv, partitionNames, tableWithPartKey); List parts = constructPartsForMv(partitionNames); Plan plan = parser.parseSingle(mv.getQuerySql()); - plan = plan.accept(new PredicateAdder(), new PredicateAddContext(predicates)); if (plan instanceof Sink) { plan = plan.child(0); } @@ -120,6 +121,7 @@ public static UpdateMvByPartitionCommand from(MTMV mv, Set partitionName LOG.debug("MTMVTask plan for mvName: {}, partitionNames: {}, plan: {}", mv.getName(), partitionNames, sink.treeString()); } + statementContext.setMvRefreshPredicates(predicates); return new UpdateMvByPartitionCommand(sink); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index 11947d2da84cf5..ff2f6e4c378cb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -24,18 +24,14 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.PartitionType; -import org.apache.doris.catalog.TableIf; -import org.apache.doris.catalog.Type; -import org.apache.doris.catalog.View; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; -import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeNameFormat; import org.apache.doris.common.UserException; import org.apache.doris.common.util.DynamicPartitionUtil; import org.apache.doris.common.util.PropertyAnalyzer; -import org.apache.doris.common.util.Util; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.mtmv.MTMVAnalyzeQueryInfo; import org.apache.doris.mtmv.MTMVPartitionInfo; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; import org.apache.doris.mtmv.MTMVPartitionUtil; @@ -47,30 +43,20 @@ import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.commands.info.BaseViewInfo.AnalyzerForCreateView; import org.apache.doris.nereids.trees.plans.commands.info.BaseViewInfo.PlanSlotFinder; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; -import org.apache.doris.nereids.trees.plans.logical.LogicalSink; -import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; -import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.util.Utils; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.SessionVariable; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; -import org.apache.commons.collections.CollectionUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -78,7 +64,6 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -155,12 +140,8 @@ public void analyze(ConnectContext ctx) throws Exception { throw new AnalysisException(message); } analyzeProperties(); - analyzeQuery(ctx, this.mvProperties); - // analyze column - final boolean finalEnableMergeOnWrite = false; - Set keysSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); - keysSet.addAll(keys); - validateColumns(this.columns, keysSet, finalEnableMergeOnWrite); + analyzeQuery(ctx); + this.partitionDesc = generatePartitionDesc(ctx); if (distribution == null) { throw new AnalysisException("Create async materialized view should contain distribution desc"); } @@ -185,21 +166,6 @@ public void analyze(ConnectContext ctx) throws Exception { setTableInformation(ctx); } - /**validate column name*/ - public void validateColumns(List columns, Set keysSet, - boolean finalEnableMergeOnWrite) throws UserException { - Set colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); - for (ColumnDefinition col : columns) { - if (!colSets.add(col.getName())) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, col.getName()); - } - if (col.getType().isVarBinaryType()) { - throw new AnalysisException("MTMV do not support varbinary type : " + col.getName()); - } - col.validate(true, keysSet, Sets.newHashSet(), finalEnableMergeOnWrite, KeysType.DUP_KEYS); - } - } - private void rewriteQuerySql(ConnectContext ctx) { analyzeAndFillRewriteSqlMap(querySql, ctx); querySql = BaseViewInfo.rewriteSql(ctx.getStatementContext().getIndexInSqlToString(), querySql); @@ -241,98 +207,13 @@ private void analyzeProperties() { /** * analyzeQuery */ - public void analyzeQuery(ConnectContext ctx, Map mvProperties) { - try (StatementContext statementContext = ctx.getStatementContext()) { - NereidsPlanner planner = new NereidsPlanner(statementContext); - // this is for expression column name infer when not use alias - LogicalSink logicalSink = new UnboundResultSink<>(logicalQuery); - // Should not make table without data to empty relation when analyze the related table, - // so add disable rules - Set tempDisableRules = ctx.getSessionVariable().getDisableNereidsRuleNames(); - ctx.getSessionVariable().setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); - statementContext.invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - Plan plan; - try { - // must disable constant folding by be, because be constant folding may return wrong type - ctx.getSessionVariable().setVarOnce(SessionVariable.ENABLE_FOLD_CONSTANT_BY_BE, "false"); - plan = planner.planWithLock(logicalSink, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); - } finally { - // after operate, roll back the disable rules - ctx.getSessionVariable().setDisableNereidsRules(String.join(",", tempDisableRules)); - statementContext.invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - } - // can not contain VIEW or MTMV - analyzeBaseTables(planner.getAnalyzedPlan()); - // can not contain Random function - analyzeExpressions(planner.getAnalyzedPlan(), mvProperties); - // can not contain partition or tablets - boolean containTableQueryOperator = MaterializedViewUtils.containTableQueryOperator( - planner.getAnalyzedPlan()); - if (containTableQueryOperator) { - throw new AnalysisException("can not contain invalid expression"); - } - - Set baseTables = Sets.newHashSet(statementContext.getTables().values()); - for (TableIf table : baseTables) { - if (table.isTemporary()) { - throw new AnalysisException("do not support create materialized view on temporary table (" - + Util.getTempTableDisplayName(table.getName()) + ")"); - } - } - getMTMVRelation(baseTables, ctx); - this.mvPartitionInfo = mvPartitionDefinition.analyzeAndTransferToMTMVPartitionInfo(planner); - this.partitionDesc = generatePartitionDesc(ctx); - columns = MTMVPlanUtil.generateColumns(plan, ctx, mvPartitionInfo.getPartitionCol(), - (distribution == null || CollectionUtils.isEmpty(distribution.getCols())) ? Sets.newHashSet() - : Sets.newHashSet(distribution.getCols()), - simpleColumnDefinitions, properties); - analyzeKeys(); - } - } - - private void analyzeKeys() { - boolean enableDuplicateWithoutKeysByDefault = false; - try { - if (properties != null) { - enableDuplicateWithoutKeysByDefault = - PropertyAnalyzer.analyzeEnableDuplicateWithoutKeysByDefault(properties); - } - } catch (Exception e) { - throw new AnalysisException(e.getMessage(), e.getCause()); - } - if (keys.isEmpty() && !enableDuplicateWithoutKeysByDefault) { - keys = Lists.newArrayList(); - int keyLength = 0; - for (ColumnDefinition column : columns) { - DataType type = column.getType(); - Type catalogType = column.getType().toCatalogDataType(); - keyLength += catalogType.getIndexSize(); - if (keys.size() >= FeConstants.shortkey_max_column_count - || keyLength > FeConstants.shortkey_maxsize_bytes) { - if (keys.isEmpty() && type.isStringLikeType()) { - keys.add(column.getName()); - column.setIsKey(true); - } - break; - } - if (column.getAggType() != null) { - break; - } - if (!catalogType.couldBeShortKey()) { - break; - } - keys.add(column.getName()); - column.setIsKey(true); - if (type.isVarcharType()) { - break; - } - } - } - } - - // Should use analyzed plan for collect views and tables - private void getMTMVRelation(Set tables, ConnectContext ctx) { - this.relation = MTMVPlanUtil.generateMTMVRelation(tables, ctx); + public void analyzeQuery(ConnectContext ctx) throws UserException { + MTMVAnalyzeQueryInfo mtmvAnalyzeQueryInfo = MTMVPlanUtil.analyzeQuery(ctx, this.mvProperties, this.querySql, + this.mvPartitionDefinition, this.distribution, this.simpleColumnDefinitions, this.properties, this.keys, + this.logicalQuery); + this.mvPartitionInfo = mtmvAnalyzeQueryInfo.getMvPartitionInfo(); + this.columns = mtmvAnalyzeQueryInfo.getColumnDefinitions(); + this.relation = mtmvAnalyzeQueryInfo.getRelation(); } private PartitionDesc generatePartitionDesc(ConnectContext ctx) { @@ -370,41 +251,6 @@ private PartitionDesc generatePartitionDesc(ConnectContext ctx) { } } - private void analyzeBaseTables(Plan plan) { - List subQuerys = plan.collectToList(node -> node instanceof LogicalSubQueryAlias); - for (Object subquery : subQuerys) { - List qualifier = ((LogicalSubQueryAlias) subquery).getQualifier(); - if (!CollectionUtils.isEmpty(qualifier) && qualifier.size() == 3) { - try { - TableIf table = Env.getCurrentEnv().getCatalogMgr() - .getCatalogOrAnalysisException(qualifier.get(0)) - .getDbOrAnalysisException(qualifier.get(1)).getTableOrAnalysisException(qualifier.get(2)); - if (table instanceof View) { - throw new AnalysisException("can not contain VIEW"); - } - } catch (org.apache.doris.common.AnalysisException e) { - LOG.warn(e.getMessage(), e); - } - } - } - } - - private void analyzeExpressions(Plan plan, Map mvProperties) { - boolean enableNondeterministicFunction = Boolean.parseBoolean( - mvProperties.get(PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION)); - if (enableNondeterministicFunction) { - return; - } - List functionCollectResult = MaterializedViewUtils.extractNondeterministicFunction(plan); - if (!CollectionUtils.isEmpty(functionCollectResult)) { - throw new AnalysisException(String.format( - "can not contain nonDeterministic expression, the expression is %s. " - + "Should add 'enable_nondeterministic_function' = 'true' property " - + "when create materialized view if you know the property real meaning entirely", - functionCollectResult.stream().map(Expression::toString).collect(Collectors.joining(",")))); - } - } - /** * set CreateTableInfo Information */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java index 750e36d41a61af..9e7e7bc19cba19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalCheckPolicy.java @@ -53,7 +53,9 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; /** * Logical Check Policy @@ -140,6 +142,11 @@ public RelatedPolicy findPolicy(LogicalPlan logicalPlan, CascadesContext cascade if (!(logicalPlan instanceof CatalogRelation || logicalPlan instanceof LogicalView)) { return RelatedPolicy.NO_POLICY; } + Optional>> mvRefreshPredicates = cascadesContext.getStatementContext() + .getMvRefreshPredicates(); + if (mvRefreshPredicates.isPresent()) { + return findPolicyByMvRefresh(mvRefreshPredicates.get(), logicalPlan); + } ConnectContext connectContext = cascadesContext.getConnectContext(); AccessControllerManager accessManager = connectContext.getEnv().getAccessManager(); UserIdentity currentUserIdentity = connectContext.getCurrentUserIdentity(); @@ -202,6 +209,16 @@ public RelatedPolicy findPolicy(LogicalPlan logicalPlan, CascadesContext cascade ); } + private RelatedPolicy findPolicyByMvRefresh(Map> mvRefreshPredicates, + LogicalPlan logicalPlan) { + TableIf table = logicalPlan instanceof CatalogRelation ? ((CatalogRelation) logicalPlan).getTable() + : ((LogicalView) logicalPlan).getView(); + if (mvRefreshPredicates.containsKey(table)) { + return new RelatedPolicy(Optional.of(ExpressionUtils.or(mvRefreshPredicates.get(table))), Optional.empty()); + } + return RelatedPolicy.NO_POLICY; + } + private Expression mergeRowPolicy(List policies) { List orList = new ArrayList<>(); List andList = new ArrayList<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index c40dd0a03f2a0c..c22147ffdd6509 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -71,7 +71,6 @@ import org.apache.doris.job.extensions.insert.streaming.StreamingInsertTask; import org.apache.doris.job.extensions.mtmv.MTMVJob; import org.apache.doris.job.task.AbstractTask; -import org.apache.doris.mtmv.BaseTableInfo; import org.apache.doris.mtmv.MTMVPartitionUtil; import org.apache.doris.mtmv.MTMVRelation; import org.apache.doris.mtmv.MTMVStatus; @@ -131,8 +130,8 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; public class MetadataGenerator { private static final Logger LOG = LogManager.getLogger(MetadataGenerator.class); @@ -712,19 +711,19 @@ private static TFetchSchemaTableDataResult viewDependencyMetadataResult(TSchemaT continue; } MTMVRelation relation = ((MTMV) table).getRelation(); - Set tablesOneLevel = relation.getBaseTablesOneLevel(); - for (BaseTableInfo info : tablesOneLevel) { - TRow trow = new TRow(); - trow.addToColumnValue(new TCell().setStringVal(InternalCatalog.INTERNAL_CATALOG_NAME)); - trow.addToColumnValue(new TCell().setStringVal(dbName)); - trow.addToColumnValue(new TCell().setStringVal(tableName)); - trow.addToColumnValue(new TCell().setStringVal(table.getType().name())); - trow.addToColumnValue(new TCell().setStringVal(info.getCtlName())); - trow.addToColumnValue(new TCell().setStringVal(info.getDbName())); - trow.addToColumnValue(new TCell().setStringVal(info.getTableName())); - trow.addToColumnValue(new TCell().setStringVal(info.getType())); - dataBatch.add(trow); - } + Stream.concat(relation.getBaseTablesOneLevel().stream(), relation.getBaseViewsOneLevel().stream()) + .forEach(info -> { + TRow trow = new TRow(); + trow.addToColumnValue(new TCell().setStringVal(InternalCatalog.INTERNAL_CATALOG_NAME)); + trow.addToColumnValue(new TCell().setStringVal(dbName)); + trow.addToColumnValue(new TCell().setStringVal(tableName)); + trow.addToColumnValue(new TCell().setStringVal(table.getType().name())); + trow.addToColumnValue(new TCell().setStringVal(info.getCtlName())); + trow.addToColumnValue(new TCell().setStringVal(info.getDbName())); + trow.addToColumnValue(new TCell().setStringVal(info.getTableName())); + trow.addToColumnValue(new TCell().setStringVal(info.getType())); + dataBatch.add(trow); + }); } else if (table instanceof View) { String tableName = table.getName(); if (FrontendConjunctsUtils.isFiltered(viewNameConjuncts, "VIEW_NAME", tableName)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java index d90b43482eba1a..fe69414a34e229 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java @@ -137,7 +137,7 @@ public void setUp() throws NoSuchMethodException, SecurityException, AnalysisExc minTimes = 0; result = true; - relation.getBaseTablesOneLevel(); + relation.getBaseTablesOneLevelAndFromView(); minTimes = 0; result = baseTables; diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java index 25e73b37506e29..ca33a4d69973d6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java @@ -17,15 +17,27 @@ package org.apache.doris.mtmv; +import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Config; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.job.exception.JobException; +import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.glue.LogicalPlanAdapter; +import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.sqltest.SqlTestBase; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; +import org.apache.doris.nereids.trees.plans.commands.info.DistributionDescriptor; +import org.apache.doris.nereids.trees.plans.commands.info.MTMVPartitionDefinition; import org.apache.doris.nereids.trees.plans.commands.info.SimpleColumnDefinition; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DecimalV2Type; @@ -66,7 +78,7 @@ public void testGenerateColumnsBySql() throws Exception { + ")\n" ); - String querySql = "select * from T1"; + String querySql = "select * from test.T1"; List actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null, Sets.newHashSet(), Lists.newArrayList(), Maps.newHashMap()); @@ -83,14 +95,14 @@ public void testGenerateColumnsBySql() throws Exception { new ColumnDefinition(Column.ROW_STORE_COL, StringType.INSTANCE, false)); checkRes(expect, actual); - querySql = "select T1.id from T1 inner join T2 on T1.id = T2.id"; + querySql = "select T1.id from test.T1 inner join test.T2 on T1.id = T2.id"; actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null, Sets.newHashSet(), Lists.newArrayList(), Maps.newHashMap()); expect = Lists.newArrayList(new ColumnDefinition("id", BigIntType.INSTANCE, true)); checkRes(expect, actual); - querySql = "select id,sum(score) from T1 group by id"; + querySql = "select id,sum(score) from test.T1 group by id"; actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null, Sets.newHashSet(), Lists.newArrayList(), Maps.newHashMap()); @@ -98,7 +110,7 @@ public void testGenerateColumnsBySql() throws Exception { new ColumnDefinition("__sum_1", BigIntType.INSTANCE, true)); checkRes(expect, actual); - querySql = "select id,sum(score) from T1 group by id"; + querySql = "select id,sum(score) from test.T1 group by id"; actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null, Sets.newHashSet(), Lists.newArrayList(new SimpleColumnDefinition("id", null), new SimpleColumnDefinition("sum_score", null)), @@ -107,7 +119,7 @@ public void testGenerateColumnsBySql() throws Exception { new ColumnDefinition("sum_score", BigIntType.INSTANCE, true)); checkRes(expect, actual); - querySql = "select * from MTMVPlanUtilTestT1"; + querySql = "select * from test.MTMVPlanUtilTestT1"; actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null, Sets.newHashSet(), Lists.newArrayList(), Maps.newHashMap()); @@ -115,7 +127,7 @@ public void testGenerateColumnsBySql() throws Exception { new ColumnDefinition("score", StringType.INSTANCE, true)); checkRes(expect, actual); - querySql = "select score from MTMVPlanUtilTestT1"; + querySql = "select score from test.MTMVPlanUtilTestT1"; actual = MTMVPlanUtil.generateColumnsBySql(querySql, connectContext, null, Sets.newHashSet(), Lists.newArrayList(), Maps.newHashMap()); @@ -279,4 +291,209 @@ public void testGenerateColumns(@Mocked SlotReference slot, @Mocked Plan plan) { Lists.newArrayList(new SimpleColumnDefinition("col1", "c1"), new SimpleColumnDefinition("col1", "c2")), null)); } + + @Test + public void testAnalyzeQuerynNonDeterministic() throws Exception { + String querySql = "select *,now() from test.T1"; + MTMVPartitionDefinition mtmvPartitionDefinition = new MTMVPartitionDefinition(); + mtmvPartitionDefinition.setPartitionType(MTMVPartitionType.SELF_MANAGE); + DistributionDescriptor distributionDescriptor = new DistributionDescriptor(false, true, 10, + Lists.newArrayList("id")); + StatementBase parsedStmt = new NereidsParser().parseSQL(querySql).get(0); + LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + + AnalysisException exception = Assertions.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, () -> { + MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), querySql, mtmvPartitionDefinition, + distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan); + }); + Assertions.assertTrue(exception.getMessage().contains("nonDeterministic")); + } + + @Test + public void testAnalyzeQueryFromTablet() throws Exception { + String querySql = "select * from test.T1 tablet (20001)"; + MTMVPartitionDefinition mtmvPartitionDefinition = new MTMVPartitionDefinition(); + mtmvPartitionDefinition.setPartitionType(MTMVPartitionType.SELF_MANAGE); + DistributionDescriptor distributionDescriptor = new DistributionDescriptor(false, true, 10, + Lists.newArrayList("id")); + StatementBase parsedStmt = new NereidsParser().parseSQL(querySql).get(0); + LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + + AnalysisException exception = Assertions.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, () -> { + MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), querySql, mtmvPartitionDefinition, + distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan); + }); + Assertions.assertTrue(exception.getMessage().contains("invalid expression")); + } + + @Test + public void testAnalyzeQueryFromTempTable() throws Exception { + createTables( + "CREATE TEMPORARY TABLE IF NOT EXISTS temp_t1 (\n" + + " id varchar(10),\n" + + " score String\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "AUTO PARTITION BY LIST(`id`)\n" + + "(\n" + + ")\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")\n" + ); + String querySql = "select * from test.temp_t1"; + MTMVPartitionDefinition mtmvPartitionDefinition = new MTMVPartitionDefinition(); + mtmvPartitionDefinition.setPartitionType(MTMVPartitionType.SELF_MANAGE); + DistributionDescriptor distributionDescriptor = new DistributionDescriptor(false, true, 10, + Lists.newArrayList("id")); + StatementBase parsedStmt = new NereidsParser().parseSQL(querySql).get(0); + LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + + AnalysisException exception = Assertions.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, () -> { + MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), querySql, mtmvPartitionDefinition, + distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan); + }); + Assertions.assertTrue(exception.getMessage().contains("temporary")); + } + + @Test + public void testAnalyzeQueryFollowBaseTableFailed() throws Exception { + String querySql = "select * from test.T4"; + MTMVPartitionDefinition mtmvPartitionDefinition = new MTMVPartitionDefinition(); + mtmvPartitionDefinition.setPartitionType(MTMVPartitionType.FOLLOW_BASE_TABLE); + mtmvPartitionDefinition.setPartitionCol("score"); + DistributionDescriptor distributionDescriptor = new DistributionDescriptor(false, true, 10, + Lists.newArrayList("id")); + StatementBase parsedStmt = new NereidsParser().parseSQL(querySql).get(0); + LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + + AnalysisException exception = Assertions.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, () -> { + MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), querySql, mtmvPartitionDefinition, + distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan); + }); + Assertions.assertTrue(exception.getMessage().contains("suitable")); + } + + @Test + public void testAnalyzeQueryNormal() throws Exception { + String querySql = "select * from test.T4"; + MTMVPartitionDefinition mtmvPartitionDefinition = new MTMVPartitionDefinition(); + mtmvPartitionDefinition.setPartitionType(MTMVPartitionType.FOLLOW_BASE_TABLE); + mtmvPartitionDefinition.setPartitionCol("id"); + DistributionDescriptor distributionDescriptor = new DistributionDescriptor(false, true, 10, + Lists.newArrayList("id")); + StatementBase parsedStmt = new NereidsParser().parseSQL(querySql).get(0); + LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + MTMVAnalyzeQueryInfo mtmvAnalyzeQueryInfo = MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), + querySql, mtmvPartitionDefinition, + distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan); + Assertions.assertTrue(mtmvAnalyzeQueryInfo.getRelation().getBaseTables().size() == 1); + Assertions.assertTrue(mtmvAnalyzeQueryInfo.getMvPartitionInfo().getRelatedCol().equals("id")); + Assertions.assertTrue(mtmvAnalyzeQueryInfo.getColumnDefinitions().size() == 2); + } + + @Test + public void testEnsureMTMVQueryUsable() throws Exception { + createMvByNereids("create materialized view mv1 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select * from test.T4;"); + Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test"); + MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv1"); + Assertions.assertDoesNotThrow( + () -> MTMVPlanUtil.ensureMTMVQueryUsable(mtmv, + MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_GENERATE_MTMV_CACHE))); + } + + @Test + public void testEnsureMTMVQueryAnalyzeFailed() throws Exception { + createTable("CREATE TABLE IF NOT EXISTS analyze_faild_t_partition (\n" + + " id bigint not null,\n" + + " score bigint\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "AUTO PARTITION BY LIST(`score`)\n" + + "(\n" + + ")\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")\n"); + createTable("CREATE TABLE IF NOT EXISTS analyze_faild_t_not_partition (\n" + + " id bigint not null,\n" + + " score bigint\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")\n"); + createView("create view analyze_faild_v1 as select * from test.analyze_faild_t_partition"); + + createMvByNereids("create materialized view analyze_faild_mv1 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n" + + " PARTITION BY (score)\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select * from test.analyze_faild_v1;"); + dropView("drop view analyze_faild_v1"); + createView("create view analyze_faild_v1 as select * from test.analyze_faild_t_not_partition"); + Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test"); + MTMV mtmv = (MTMV) db.getTableOrAnalysisException("analyze_faild_mv1"); + JobException exception = Assertions.assertThrows( + org.apache.doris.job.exception.JobException.class, () -> { + MTMVPlanUtil.ensureMTMVQueryUsable(mtmv, + MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_GENERATE_MTMV_CACHE)); + }); + Assertions.assertTrue(exception.getMessage().contains("suitable")); + } + + @Test + public void testEnsureMTMVQueryNotEqual() throws Exception { + createTable("CREATE TABLE IF NOT EXISTS t_partition1 (\n" + + " id bigint not null,\n" + + " score bigint\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "AUTO PARTITION BY LIST(`score`)\n" + + "(\n" + + ")\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")\n"); + createTable("CREATE TABLE IF NOT EXISTS t_partition2 (\n" + + " id bigint not null,\n" + + " score bigint\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "AUTO PARTITION BY LIST(`score`)\n" + + "(\n" + + ")\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")\n"); + createView("create view t_partition1_v1 as select * from test.t_partition1"); + + createMvByNereids("create materialized view t_partition1_mv1 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n" + + " PARTITION BY (score)\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select * from test.t_partition1_v1;"); + dropView("drop view t_partition1_v1"); + createView("create view t_partition1_v1 as select * from test.t_partition2"); + Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test"); + MTMV mtmv = (MTMV) db.getTableOrAnalysisException("t_partition1_mv1"); + JobException exception = Assertions.assertThrows( + org.apache.doris.job.exception.JobException.class, () -> { + MTMVPlanUtil.ensureMTMVQueryUsable(mtmv, + MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_GENERATE_MTMV_CACHE)); + }); + Assertions.assertTrue(exception.getMessage().contains("changed")); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationManagerTest.java index 40263705c43f99..d3c2652208e37e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationManagerTest.java @@ -95,37 +95,38 @@ public void setUp() throws NoSuchMethodException, SecurityException, AnalysisExc } @Test - public void testGetMtmvsByBaseTableOneLevel() { + public void testGetMtmvsByBaseTableOneLevelAndFromView() { // mock mv2==>mv1,t3; mv1==>t4 MTMVRelationManager manager = new MTMVRelationManager(); MTMVRelation mv2Relation = new MTMVRelation(Sets.newHashSet(mv1, t3, t4), Sets.newHashSet(mv1, t3), - Sets.newHashSet()); - MTMVRelation mv1Relation = new MTMVRelation(Sets.newHashSet(t4), Sets.newHashSet(t4), - Sets.newHashSet()); + Sets.newHashSet(mv1, t3), + Sets.newHashSet(), Sets.newHashSet()); + MTMVRelation mv1Relation = new MTMVRelation(Sets.newHashSet(t4), Sets.newHashSet(t4), Sets.newHashSet(t4), + Sets.newHashSet(), Sets.newHashSet()); manager.refreshMTMVCache(mv2Relation, mv2); manager.refreshMTMVCache(mv1Relation, mv1); // should return mv2 - Set mv1OneLevel = manager.getMtmvsByBaseTableOneLevel(mv1); + Set mv1OneLevel = manager.getMtmvsByBaseTableOneLevelAndFromView(mv1); Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv2), mv1OneLevel)); // should return mv2 - Set t3OneLevel = manager.getMtmvsByBaseTableOneLevel(t3); + Set t3OneLevel = manager.getMtmvsByBaseTableOneLevelAndFromView(t3); Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv2), t3OneLevel)); // should return mv1 - Set t4OneLevel = manager.getMtmvsByBaseTableOneLevel(t4); + Set t4OneLevel = manager.getMtmvsByBaseTableOneLevelAndFromView(t4); Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv1), t4OneLevel)); // update mv2 only use t3,remove mv1 - mv2Relation = new MTMVRelation(Sets.newHashSet(t3), Sets.newHashSet(t3), - Sets.newHashSet()); + mv2Relation = new MTMVRelation(Sets.newHashSet(t3), Sets.newHashSet(t3), Sets.newHashSet(t3), + Sets.newHashSet(), Sets.newHashSet()); manager.refreshMTMVCache(mv2Relation, mv2); // should return empty - mv1OneLevel = manager.getMtmvsByBaseTableOneLevel(mv1); + mv1OneLevel = manager.getMtmvsByBaseTableOneLevelAndFromView(mv1); Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(), mv1OneLevel)); // should return mv2 - t3OneLevel = manager.getMtmvsByBaseTableOneLevel(t3); + t3OneLevel = manager.getMtmvsByBaseTableOneLevelAndFromView(t3); Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv2), t3OneLevel)); // should return mv1 - t4OneLevel = manager.getMtmvsByBaseTableOneLevel(t4); + t4OneLevel = manager.getMtmvsByBaseTableOneLevelAndFromView(t4); Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv1), t4OneLevel)); } @@ -134,9 +135,10 @@ public void testGetMtmvsByBaseTable() { // mock mv2==>mv1,t3; mv1==>t4 MTMVRelationManager manager = new MTMVRelationManager(); MTMVRelation mv2Relation = new MTMVRelation(Sets.newHashSet(mv1, t3, t4), Sets.newHashSet(mv1, t3), - Sets.newHashSet()); - MTMVRelation mv1Relation = new MTMVRelation(Sets.newHashSet(t4), Sets.newHashSet(t4), - Sets.newHashSet()); + Sets.newHashSet(mv1, t3), + Sets.newHashSet(), Sets.newHashSet()); + MTMVRelation mv1Relation = new MTMVRelation(Sets.newHashSet(t4), Sets.newHashSet(t4), Sets.newHashSet(t4), + Sets.newHashSet(), Sets.newHashSet()); manager.refreshMTMVCache(mv2Relation, mv2); manager.refreshMTMVCache(mv1Relation, mv1); // should return mv2 @@ -150,17 +152,17 @@ public void testGetMtmvsByBaseTable() { Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv1, mv2), t4All)); // update mv2 only use t3,remove mv1 - mv2Relation = new MTMVRelation(Sets.newHashSet(t3), Sets.newHashSet(t3), - Sets.newHashSet()); + mv2Relation = new MTMVRelation(Sets.newHashSet(t3), Sets.newHashSet(t3), Sets.newHashSet(t3), + Sets.newHashSet(), Sets.newHashSet()); manager.refreshMTMVCache(mv2Relation, mv2); // should return empty - mv1All = manager.getMtmvsByBaseTableOneLevel(mv1); + mv1All = manager.getMtmvsByBaseTableOneLevelAndFromView(mv1); Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(), mv1All)); // should return mv2 - t3All = manager.getMtmvsByBaseTableOneLevel(t3); + t3All = manager.getMtmvsByBaseTableOneLevelAndFromView(t3); Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv2), t3All)); // should return mv1 - t4All = manager.getMtmvsByBaseTableOneLevel(t4); + t4All = manager.getMtmvsByBaseTableOneLevelAndFromView(t4); Assert.assertTrue(CollectionUtils.isEqualCollection(Sets.newHashSet(mv1), t4All)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationTest.java new file mode 100644 index 00000000000000..3b93daca9b7453 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelationTest.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Sets; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class MTMVRelationTest extends TestWithFeService { + // t1 => v1 => v2 + // t2 => mv1 + // mv1 join v2 => mv2 + @Test + public void testMTMVRelation() throws Exception { + createDatabaseAndUse("db1"); + createTables( + "CREATE TABLE IF NOT EXISTS t1 (\n" + + " id varchar(10),\n" + + " score String\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")\n", + "CREATE TABLE IF NOT EXISTS t2 (\n" + + " id varchar(10),\n" + + " score String\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")\n" + ); + createView("create view v1 as select * from t1"); + createView("create view v2 as select * from v1"); + createMvByNereids("create materialized view mv1 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select * from t2;"); + createMvByNereids("create materialized view mv2 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select mv1.* from mv1, v2 ;"); + Database db1 = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("db1"); + MTMV mtmv = (MTMV) db1.getTableOrAnalysisException("mv2"); + MTMVRelation relation = mtmv.getRelation(); + BaseTableInfo t1 = new BaseTableInfo(db1.getTableOrAnalysisException("t1")); + BaseTableInfo t2 = new BaseTableInfo(db1.getTableOrAnalysisException("t2")); + BaseTableInfo v1 = new BaseTableInfo(db1.getTableOrAnalysisException("v1")); + BaseTableInfo v2 = new BaseTableInfo(db1.getTableOrAnalysisException("v2")); + BaseTableInfo mv1 = new BaseTableInfo(db1.getTableOrAnalysisException("mv1")); + BaseTableInfo mv2 = new BaseTableInfo(db1.getTableOrAnalysisException("mv2")); + // test forward index + Assertions.assertEquals(Sets.newHashSet(t1, t2, mv1), relation.getBaseTables()); + Assertions.assertEquals(Sets.newHashSet(v1, v2), relation.getBaseViews()); + Assertions.assertEquals(Sets.newHashSet(mv1), relation.getBaseTablesOneLevel()); + Assertions.assertEquals(Sets.newHashSet(v2), relation.getBaseViewsOneLevel()); + Assertions.assertEquals(Sets.newHashSet(mv1, t1), relation.getBaseTablesOneLevelAndFromView()); + + // test inverted index + MTMVRelationManager relationManager = Env.getCurrentEnv().getMtmvService().getRelationManager(); + Assertions.assertEquals(Sets.newHashSet(mv2), relationManager.getMtmvsByBaseTable(t1)); + Assertions.assertEquals(Sets.newHashSet(mv2, mv1), relationManager.getMtmvsByBaseTable(t2)); + Assertions.assertEquals(Sets.newHashSet(mv2), relationManager.getMtmvsByBaseTable(mv1)); + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTable(v1)); + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTable(v2)); + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTable(mv2)); + + Assertions.assertEquals(Sets.newHashSet(mv2), relationManager.getMtmvsByBaseTableOneLevelAndFromView(t1)); + Assertions.assertEquals(Sets.newHashSet(mv1), relationManager.getMtmvsByBaseTableOneLevelAndFromView(t2)); + Assertions.assertEquals(Sets.newHashSet(mv2), relationManager.getMtmvsByBaseTableOneLevelAndFromView(mv1)); + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTableOneLevelAndFromView(v1)); + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTableOneLevelAndFromView(v2)); + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTableOneLevelAndFromView(mv2)); + + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseView(t1)); + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseView(t2)); + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseView(mv1)); + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseView(mv2)); + Assertions.assertEquals(Sets.newHashSet(mv2), relationManager.getMtmvsByBaseView(v1)); + Assertions.assertEquals(Sets.newHashSet(mv2), relationManager.getMtmvsByBaseView(v2)); + + dropMvByNereids("drop materialized view mv2"); + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTable(t1)); + Assertions.assertEquals(Sets.newHashSet(mv1), relationManager.getMtmvsByBaseTable(t2)); + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTable(mv1)); + + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTableOneLevelAndFromView(t1)); + Assertions.assertEquals(Sets.newHashSet(mv1), relationManager.getMtmvsByBaseTableOneLevelAndFromView(t2)); + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseTableOneLevelAndFromView(mv1)); + + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseView(v1)); + Assertions.assertEquals(Sets.newHashSet(), relationManager.getMtmvsByBaseView(v2)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVSchemaChangeTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVSchemaChangeTest.java new file mode 100644 index 00000000000000..e911bd2287f9e7 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVSchemaChangeTest.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class MTMVSchemaChangeTest extends TestWithFeService { + // t1 => v1 => mv1 + @Test + public void testDropBaseView() throws Exception { + createDatabaseAndUse("db1"); + createTables( + "CREATE TABLE IF NOT EXISTS t1 (\n" + + " id varchar(10),\n" + + " score String\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")\n" + ); + createView("create view v1 as select * from t1"); + createMvByNereids("create materialized view mv1 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select * from v1 ;"); + Database db1 = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("db1"); + MTMV mtmv = (MTMV) db1.getTableOrAnalysisException("mv1"); + MTMVRefreshSnapshot mtmvRefreshSnapshot = new MTMVRefreshSnapshot(); + mtmv.setRefreshSnapshot(mtmvRefreshSnapshot); + Assertions.assertTrue(mtmv.getRefreshSnapshot().equals(mtmvRefreshSnapshot)); + Assertions.assertTrue(mtmv.getSchemaChangeVersion() == 0); + Assertions.assertTrue(mtmv.getStatus().getState().equals(MTMVState.INIT)); + dropView("drop view v1"); + Assertions.assertFalse(mtmv.getRefreshSnapshot().equals(mtmvRefreshSnapshot)); + Assertions.assertTrue(mtmv.getSchemaChangeVersion() == 1); + Assertions.assertTrue(mtmv.getStatus().getState().equals(MTMVState.SCHEMA_CHANGE)); + } + + @Test + public void testDropBaseTable() throws Exception { + createDatabaseAndUse("db2"); + createTables( + "CREATE TABLE IF NOT EXISTS t1 (\n" + + " id varchar(10),\n" + + " score String\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ")\n" + ); + createView("create view v1 as select * from t1"); + createMvByNereids("create materialized view mv1 BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select * from v1 ;"); + Database db2 = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("db2"); + MTMV mtmv = (MTMV) db2.getTableOrAnalysisException("mv1"); + MTMVRefreshSnapshot mtmvRefreshSnapshot = new MTMVRefreshSnapshot(); + mtmv.setRefreshSnapshot(mtmvRefreshSnapshot); + Assertions.assertTrue(mtmv.getRefreshSnapshot().equals(mtmvRefreshSnapshot)); + Assertions.assertTrue(mtmv.getSchemaChangeVersion() == 0); + Assertions.assertTrue(mtmv.getStatus().getState().equals(MTMVState.INIT)); + dropTable("t1", true); + Assertions.assertTrue(mtmv.getRefreshSnapshot().equals(mtmvRefreshSnapshot)); + Assertions.assertTrue(mtmv.getSchemaChangeVersion() == 1); + Assertions.assertTrue(mtmv.getStatus().getState().equals(MTMVState.SCHEMA_CHANGE)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java index da9315d8ebc576..d2008627b731bb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java @@ -44,7 +44,8 @@ public class MTMVTaskTest { private String poneName = "p1"; private String ptwoName = "p2"; private List allPartitionNames = Lists.newArrayList(poneName, ptwoName); - private MTMVRelation relation = new MTMVRelation(Sets.newHashSet(), Sets.newHashSet(), Sets.newHashSet()); + private MTMVRelation relation = new MTMVRelation(Sets.newHashSet(), Sets.newHashSet(), Sets.newHashSet(), + Sets.newHashSet(), Sets.newHashSet()); @Mocked private MTMV mtmv; diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java index b8c438348ac02b..5da3a045231fe9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java @@ -63,7 +63,7 @@ public void testToInfoString() { + "finishTimeMs=null, taskType=null, errMsg='null'}]}, mvProperties={}, " + "relation=MTMVRelation{baseTables=[], baseTablesOneLevel=[], baseViews=[]}, " + "mvPartitionInfo=MTMVPartitionInfo{partitionType=null, relatedTable=null, " - + "relatedCol='null', partitionCol='null'}, " + + "relatedCol='null', partitionCol='null', expr='null'}, " + "refreshSnapshot=MTMVRefreshSnapshot{partitionSnapshots={}}, id=1, name='null', " + "qualifiedDbName='db1', comment='comment1'}"; MTMV mtmv = new MTMV(); @@ -75,7 +75,8 @@ public void testToInfoString() { mtmv.setStatus(new MTMVStatus()); mtmv.setJobInfo(buildMTMVJobInfo(mtmv)); mtmv.setMvProperties(new HashMap<>()); - mtmv.setRelation(new MTMVRelation(Sets.newHashSet(), Sets.newHashSet(), Sets.newHashSet())); + mtmv.setRelation(new MTMVRelation(Sets.newHashSet(), Sets.newHashSet(), Sets.newHashSet(), Sets.newHashSet(), + Sets.newHashSet())); mtmv.setMvPartitionInfo(new MTMVPartitionInfo()); mtmv.setRefreshSnapshot(new MTMVRefreshSnapshot()); Assert.assertEquals(expect, mtmv.toInfoString()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index 2e4361000c92da..8e997062f50bfd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -148,7 +148,7 @@ public PlanChecker setIsQuery() { } public PlanChecker analyze() { - this.cascadesContext.newTableCollector().collect(); + this.cascadesContext.newTableCollector(true).collect(); this.cascadesContext.newAnalyzer().analyze(); this.cascadesContext.setCteContext(new CTEContext()); MemoTestUtils.initMemoAndValidState(cascadesContext); @@ -157,7 +157,7 @@ public PlanChecker analyze() { public PlanChecker analyze(Plan plan) { this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, plan); - this.cascadesContext.newTableCollector().collect(); + this.cascadesContext.newTableCollector(true).collect(); this.cascadesContext.setCteContext(new CTEContext()); Set originDisableRules = connectContext.getSessionVariable().getDisableNereidsRuleNames(); Set disableRuleWithAuth = Sets.newHashSet(originDisableRules); @@ -171,7 +171,7 @@ public PlanChecker analyze(Plan plan) { public PlanChecker analyze(String sql) { this.cascadesContext = MemoTestUtils.createCascadesContext(connectContext, sql); - this.cascadesContext.newTableCollector().collect(); + this.cascadesContext.newTableCollector(true).collect(); this.cascadesContext.newAnalyzer().analyze(); this.cascadesContext.setCteContext(new CTEContext()); MemoTestUtils.initMemoAndValidState(cascadesContext); diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index abb50a7a0747f7..972b18fd9467d5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -63,6 +63,7 @@ import org.apache.doris.nereids.trees.plans.commands.DropRowPolicyCommand; import org.apache.doris.nereids.trees.plans.commands.DropSqlBlockRuleCommand; import org.apache.doris.nereids.trees.plans.commands.DropTableCommand; +import org.apache.doris.nereids.trees.plans.commands.DropViewCommand; import org.apache.doris.nereids.trees.plans.commands.GrantRoleCommand; import org.apache.doris.nereids.trees.plans.commands.GrantTablePrivilegeCommand; import org.apache.doris.nereids.trees.plans.commands.RecoverTableCommand; @@ -712,6 +713,12 @@ public void createView(String sql) throws Exception { command.run(connectContext, new StmtExecutor(connectContext, sql)); } + public void dropView(String sql) throws Exception { + NereidsParser nereidsParser = new NereidsParser(); + DropViewCommand command = (DropViewCommand) nereidsParser.parseSingle(sql); + command.run(connectContext, new StmtExecutor(connectContext, sql)); + } + protected void createPolicy(String sql) throws Exception { NereidsParser nereidsParser = new NereidsParser(); CreatePolicyCommand command = (CreatePolicyCommand) nereidsParser.parseSingle(sql); diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view.out new file mode 100644 index 00000000000000..90c7a97c096ebb --- /dev/null +++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view.out @@ -0,0 +1,30 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !is_sync_init -- +false + +-- !is_sync_after_refresh -- +true + +-- !after_refresh -- +1 1 + +-- !not_need_refresh -- +NOT_REFRESH + +-- !is_sync_data_change -- +false + +-- !trigger_view_need_refresh -- +COMPLETE + +-- !after_trigger_view -- +1 1 +2 2 + +-- !trigger_table_not_need_refresh -- +NOT_REFRESH + +-- !after_trigger_table -- +1 1 +2 2 + diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter.out new file mode 100644 index 00000000000000..5ad3271eba9d50 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter.out @@ -0,0 +1,19 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_1 -- +1 2 + +-- !state_alter_view -- +SCHEMA_CHANGE + +-- !state_after_refresh -- +NORMAL + +-- !need_refresh_complete -- +COMPLETE + +-- !select_2 -- +1 3 + +-- !need_refresh_failed -- +FAILED + diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter_table.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter_table.out new file mode 100644 index 00000000000000..f4dcc52e9d2a54 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_alter_table.out @@ -0,0 +1,19 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_1 -- +1 1 + +-- !state_after_drop -- +SCHEMA_CHANGE + +-- !need_refresh_failed -- +FAILED + +-- !need_refresh_complete -- +COMPLETE + +-- !select_2 -- +2 2 + +-- !state_after_refresh -- +NORMAL + diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view_commit.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_commit.out new file mode 100644 index 00000000000000..0f44c5dce4f0fc --- /dev/null +++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_commit.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !is_sync_init -- +false + +-- !after_refresh -- +1 1 + diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view_cte.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_cte.out new file mode 100644 index 00000000000000..ebad07a8c2c16b --- /dev/null +++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_cte.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !is_sync_init -- +false + +-- !after_refresh -- +1 2 2 3 + diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view_mtmv.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_mtmv.out new file mode 100644 index 00000000000000..2cb90afa89ee29 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_mtmv.out @@ -0,0 +1,20 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_mv1 -- +1 2017-01-15 1 +1 2017-02-15 2 +1 2017-03-15 3 + +-- !is_sync_mv2 -- +true + +-- !select_mv2 -- +1 2017-01-15 1 +1 2017-02-15 2 +1 2017-03-15 3 + +-- !is_sync_data_change -- +false + +-- !pct_refresh -- +PARTIAL + diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view_pct.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_pct.out new file mode 100644 index 00000000000000..2281daaaadb6ed --- /dev/null +++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_pct.out @@ -0,0 +1,21 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !is_sync_init -- +false + +-- !is_sync_after_refresh -- +true + +-- !after_refresh -- +1 2017-01-15 1 +1 2017-02-15 2 +1 2017-03-15 3 + +-- !not_need_refresh -- +NOT_REFRESH + +-- !is_sync_data_change -- +false + +-- !pct_refresh -- +PARTIAL + diff --git a/regression-test/data/mtmv_p0/test_create_mtmv_with_view_rollup.out b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_rollup.out new file mode 100644 index 00000000000000..4c30d8bd01a3d5 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_create_mtmv_with_view_rollup.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !is_sync_after_refresh -- +true + +-- !after_refresh -- +1 2017-01-15 1 +1 2017-02-15 2 +1 2017-03-15 3 + diff --git a/regression-test/data/mtmv_p0/test_limit_partition_view_mtmv.out b/regression-test/data/mtmv_p0/test_limit_partition_view_mtmv.out new file mode 100644 index 00000000000000..a655b5fdeffecc --- /dev/null +++ b/regression-test/data/mtmv_p0/test_limit_partition_view_mtmv.out @@ -0,0 +1,17 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !date_list -- +1 2038-01-01 + +-- !varchar_list -- +1 20380101 + +-- !varchar_list -- +1 20380101 + +-- !date_range -- +1 2038-01-02 + +-- !date_range_all -- +1 2038-01-02 +2 2020-01-02 + diff --git a/regression-test/data/mtmv_p0/test_view_hive_mtmv.out b/regression-test/data/mtmv_p0/test_view_hive_mtmv.out new file mode 100644 index 00000000000000..165e9809af8be5 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_view_hive_mtmv.out @@ -0,0 +1,28 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !refresh_one_partition -- +1 A 20230101 +2 B 20230101 +3 C 20230101 + +-- !refresh_complete -- +1 A 20230101 +2 B 20230101 +3 C 20230101 +4 D 20230102 +5 E 20230102 +6 F 20230102 + +-- !is_sync_before_rebuild -- +true + +-- !is_sync_after_rebuild -- +true + +-- !refresh_complete_rebuild -- +1 A 20230101 +2 B 20230101 +3 C 20230101 +4 D 20230102 +5 E 20230102 +6 F 20230102 + diff --git a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy index 0c8456fd0a2f58..32656fc753754d 100644 --- a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy @@ -146,21 +146,6 @@ suite("test_build_mtmv") { log.info(e.getMessage()) } - // not allow create mv use view - try { - sql """ - CREATE MATERIALIZED VIEW ${mvNameRenamed} - BUILD DEFERRED REFRESH COMPLETE ON MANUAL - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ('replication_num' = '1') - AS - SELECT * from ${viewName}; - """ - Assert.fail(); - } catch (Exception e) { - log.info(e.getMessage()) - } - sql """ DROP MATERIALIZED VIEW ${mvName} """ diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view.groovy new file mode 100644 index 00000000000000..aab474454d3e0a --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view.groovy @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_create_mtmv_with_view","mtmv") { + String suiteName = "test_create_mtmv_with_view" + String tableName = "${suiteName}_table" + String mvName = "${suiteName}_mv" + String viewName = "${suiteName}_view" + String dbName = context.config.getDbNameByFile(context.file) + sql """drop table if exists `${tableName}`""" + sql """drop view if exists `${viewName}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE ${tableName} + ( + k2 TINYINT, + k3 INT not null + ) + DISTRIBUTED BY HASH(k2) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql""" + create view ${viewName} as select * from ${tableName}; + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${viewName}; + """ + + sql """ + insert into ${tableName} values(1,1); + """ + order_qt_is_sync_init "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'" + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_is_sync_after_refresh "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'" + order_qt_after_refresh "SELECT * FROM ${mvName}" + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_not_need_refresh "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;" + + sql """ + insert into ${tableName} values(2,2); + """ + order_qt_is_sync_data_change "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'" + + // test excluded_trigger_tables + // view should not useful + sql """ + alter Materialized View ${mvName} set("excluded_trigger_tables"="internal.${dbName}.${viewName}"); + """ + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + + order_qt_trigger_view_need_refresh "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;" + order_qt_after_trigger_view "SELECT * FROM ${mvName}" + + // table should useful + sql """ + alter Materialized View ${mvName} set("excluded_trigger_tables"="internal.${dbName}.${tableName}"); + """ + sql """ + insert into ${tableName} values(3,3); + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + + order_qt_trigger_table_not_need_refresh "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;" + order_qt_after_trigger_table "SELECT * FROM ${mvName}" + + + sql """drop view if exists `${viewName}`""" + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" +} diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter.groovy new file mode 100644 index 00000000000000..3973fde6035ed6 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter.groovy @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_create_mtmv_with_view_alter","mtmv") { + String suiteName = "test_create_mtmv_with_view_alter" + String tableName = "${suiteName}_table" + String mvName = "${suiteName}_mv" + String viewName = "${suiteName}_view" + String dbName = context.config.getDbNameByFile(context.file) + sql """drop table if exists `${tableName}`""" + sql """drop view if exists `${viewName}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE ${tableName} + ( + k2 TINYINT, + k3 INT not null + ) + DISTRIBUTED BY HASH(k2) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql""" + create view ${viewName} as select k2,k3+1 as k4 from ${tableName}; + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${viewName}; + """ + + sql """ + insert into ${tableName} values(1,1); + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_select_1 "SELECT * FROM ${mvName}" + // should refresh all + sql""" + alter view ${viewName} as select k2,k3+2 as k4 from ${tableName}; + """ + + order_qt_state_alter_view "select State from mv_infos('database'='${dbName}') where Name='${mvName}'" + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_state_after_refresh "select State from mv_infos('database'='${dbName}') where Name='${mvName}'" + order_qt_need_refresh_complete "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;" + order_qt_select_2 "SELECT * FROM ${mvName}" + + // column k4 not exist, should refresh failed + sql""" + alter view ${viewName} as select k2,k3+3 as k5 from ${tableName}; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + def jobName = getJobName(dbName, mvName); + waitingMTMVTaskFinishedNotNeedSuccess(jobName) + order_qt_need_refresh_failed "select Status from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;" + + sql """drop view if exists `${viewName}`""" + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" +} diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter_table.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter_table.groovy new file mode 100644 index 00000000000000..b8802bf9f9fc54 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_alter_table.groovy @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_create_mtmv_with_view_alter_table","mtmv") { + String suiteName = "test_create_mtmv_with_view_alter_table" + String tableName = "${suiteName}_table" + String mvName = "${suiteName}_mv" + String viewName = "${suiteName}_view" + String dbName = context.config.getDbNameByFile(context.file) + sql """drop table if exists `${tableName}`""" + sql """drop view if exists `${viewName}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE ${tableName} + ( + k2 TINYINT, + k3 INT not null + ) + DISTRIBUTED BY HASH(k2) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql""" + create view ${viewName} as select * from ${tableName}; + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${viewName}; + """ + + sql """ + insert into ${tableName} values(1,1); + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_select_1 "SELECT * FROM ${mvName}" + + sql """ + drop table ${tableName}; + """ + order_qt_state_after_drop "select State from mv_infos('database'='${dbName}') where Name='${mvName}'" + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + def jobName = getJobName(dbName, mvName); + waitingMTMVTaskFinishedNotNeedSuccess(jobName) + order_qt_need_refresh_failed "select Status from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;" + + sql """ + CREATE TABLE ${tableName} + ( + k2 TINYINT, + k3 INT not null + ) + DISTRIBUTED BY HASH(k2) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + insert into ${tableName} values(2,2); + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_need_refresh_complete "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;" + order_qt_select_2 "SELECT * FROM ${mvName}" + order_qt_state_after_refresh "select State from mv_infos('database'='${dbName}') where Name='${mvName}'" + + sql """drop view if exists `${viewName}`""" + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" +} diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_commit.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_commit.groovy new file mode 100644 index 00000000000000..e977e05e33e690 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_commit.groovy @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_create_mtmv_with_view_commit","mtmv") { + String suiteName = "test_create_mtmv_with_view_commit" + String tableName = "${suiteName}_table" + String mvName = "${suiteName}_mv" + String viewName = "${suiteName}_view" + String dbName = context.config.getDbNameByFile(context.file) + sql """drop table if exists `${tableName}`""" + sql """drop view if exists `${viewName}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE ${tableName} + ( + k2 TINYINT, + k3 INT not null + ) + DISTRIBUTED BY HASH(k2) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql""" + create view ${viewName} as select * from ${tableName}; + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON COMMIT + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${viewName}; + """ + + order_qt_is_sync_init "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'" + + sql """ + insert into ${tableName} values(1,1); + """ + + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_after_refresh "SELECT * FROM ${mvName}" + + sql """drop view if exists `${viewName}`""" + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" +} diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_cte.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_cte.groovy new file mode 100644 index 00000000000000..3bc03bd2d2d8e9 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_cte.groovy @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_create_mtmv_with_view_cte","mtmv") { + String suiteName = "test_create_mtmv_with_view_cte" + String tableName1 = "${suiteName}_table1" + String tableName2 = "${suiteName}_table2" + String mvName = "${suiteName}_mv" + String viewName = "${suiteName}_view" + String dbName = context.config.getDbNameByFile(context.file) + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop view if exists `${viewName}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE ${tableName1} + ( + k1 INT, + k2 INT not null + ) + DISTRIBUTED BY HASH(k2) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + CREATE TABLE ${tableName2} + ( + k3 INT, + k4 INT not null + ) + DISTRIBUTED BY HASH(k4) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql""" + create view ${viewName} as select * from ${tableName1}; + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON COMMIT + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + with cte1 as ( + select * from ${tableName2} + ) + SELECT * from ${viewName} a inner join cte1 b on a.k2=b.k3; + """ + + order_qt_is_sync_init "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'" + + sql """ + insert into ${tableName1} values(1,2); + """ + sql """ + insert into ${tableName2} values(2,3); + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_after_refresh "SELECT * FROM ${mvName}" + + sql """drop view if exists `${viewName}`""" + sql """drop table if exists `${tableName1}`""" + sql """drop table if exists `${tableName2}`""" + sql """drop materialized view if exists ${mvName};""" +} diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_mtmv.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_mtmv.groovy new file mode 100644 index 00000000000000..63e83b7890ddb1 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_mtmv.groovy @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_create_mtmv_with_view_mtmv","mtmv") { + String suiteName = "test_create_mtmv_with_view_mtmv" + String tableName = "${suiteName}_table" + String mvName1 = "${suiteName}_mv1" + String mvName2 = "${suiteName}_mv2" + String viewName = "${suiteName}_view" + String dbName = context.config.getDbNameByFile(context.file) + sql """drop table if exists `${tableName}`""" + sql """drop view if exists `${viewName}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName2};""" + + sql """ + CREATE TABLE `${tableName}` ( + `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"', + `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"', + `num` SMALLINT NOT NULL COMMENT '\"数量\"' + ) ENGINE=OLAP + DUPLICATE KEY(`user_id`, `date`, `num`) + COMMENT 'OLAP' + PARTITION BY RANGE(`date`) + (PARTITION p201701 VALUES [('0000-01-01'), ('2017-02-01')), + PARTITION p201702 VALUES [('2017-02-01'), ('2017-03-01')), + PARTITION p201703 VALUES [('2017-03-01'), ('2017-04-01'))) + DISTRIBUTED BY HASH(`user_id`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + + sql """ + insert into ${tableName} values(1,"2017-01-15",1),(1,"2017-02-15",2),(1,"2017-03-15",3); + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName1} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(`date`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${tableName}; + """ + + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName1) + + order_qt_select_mv1 "SELECT * FROM ${mvName1}" + + sql""" + create view ${viewName} as select * from ${mvName1}; + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName2} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(`date`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${viewName}; + """ + + + def showPartitionsResult = sql """show partitions from ${mvName2}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertTrue(showPartitionsResult.toString().contains("p_00000101_20170201")) + assertTrue(showPartitionsResult.toString().contains("p_20170201_20170301")) + assertTrue(showPartitionsResult.toString().contains("p_20170301_20170401")) + + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName2) + order_qt_is_sync_mv2 "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName2}'" + order_qt_select_mv2 "SELECT * FROM ${mvName2}" + + sql """ + insert into ${tableName} values(1,"2017-01-15",4); + """ + order_qt_is_sync_data_change "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName1}'" + + sql """ + REFRESH MATERIALIZED VIEW ${mvName1} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName1) + sql """ + REFRESH MATERIALIZED VIEW ${mvName2} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName2) + order_qt_pct_refresh "select RefreshMode from tasks('type'='mv') where MvName='${mvName2}' order by CreateTime desc limit 1;" + + mv_rewrite_success_without_check_chosen("select * from ${mvName1}", "${mvName2}") + mv_rewrite_success_without_check_chosen("select * from ${tableName}", "${mvName1}") + mv_rewrite_success_without_check_chosen("select * from ${viewName}", "${mvName2}") + // when nest mv is partitioned, rewrite has bug, after fixed, open it + // mv_rewrite_success_without_check_chosen("select * from ${tableName}", "${mvName2}") + + sql """drop view if exists `${viewName}`""" + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName1};""" + sql """drop materialized view if exists ${mvName2};""" +} diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_pct.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_pct.groovy new file mode 100644 index 00000000000000..ac4157b91f1bcb --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_pct.groovy @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_create_mtmv_with_view_pct","mtmv") { + String suiteName = "test_create_mtmv_with_view_pct" + String tableName = "${suiteName}_table" + String mvName = "${suiteName}_mv" + String viewName = "${suiteName}_view" + String dbName = context.config.getDbNameByFile(context.file) + sql """drop table if exists `${tableName}`""" + sql """drop view if exists `${viewName}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE `${tableName}` ( + `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"', + `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"', + `num` SMALLINT NOT NULL COMMENT '\"数量\"' + ) ENGINE=OLAP + DUPLICATE KEY(`user_id`, `date`, `num`) + COMMENT 'OLAP' + PARTITION BY RANGE(`date`) + (PARTITION p201701 VALUES [('0000-01-01'), ('2017-02-01')), + PARTITION p201702 VALUES [('2017-02-01'), ('2017-03-01')), + PARTITION p201703 VALUES [('2017-03-01'), ('2017-04-01'))) + DISTRIBUTED BY HASH(`user_id`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + + sql """ + insert into ${tableName} values(1,"2017-01-15",1),(1,"2017-02-15",2),(1,"2017-03-15",3); + """ + + sql""" + create view ${viewName} as select * from ${tableName}; + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(`date`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${viewName}; + """ + + def showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertTrue(showPartitionsResult.toString().contains("p_00000101_20170201")) + assertTrue(showPartitionsResult.toString().contains("p_20170201_20170301")) + assertTrue(showPartitionsResult.toString().contains("p_20170301_20170401")) + + order_qt_is_sync_init "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'" + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_is_sync_after_refresh "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'" + order_qt_after_refresh "SELECT * FROM ${mvName}" + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_not_need_refresh "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;" + + sql """ + insert into ${tableName} values(1,"2017-01-15",4); + """ + order_qt_is_sync_data_change "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'" + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_pct_refresh "select RefreshMode from tasks('type'='mv') where MvName='${mvName}' order by CreateTime desc limit 1;" + + mv_rewrite_success_without_check_chosen("select * from ${viewName}", "${mvName}") + mv_rewrite_success_without_check_chosen("select * from ${tableName}", "${mvName}") + + sql """drop view if exists `${viewName}`""" + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" +} diff --git a/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_rollup.groovy b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_rollup.groovy new file mode 100644 index 00000000000000..0d65be6382c64d --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_create_mtmv_with_view_rollup.groovy @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_create_mtmv_with_view_rollup","mtmv") { + String suiteName = "test_create_mtmv_with_view_rollup" + String tableName = "${suiteName}_table" + String mvName = "${suiteName}_mv" + String viewName = "${suiteName}_view" + String dbName = context.config.getDbNameByFile(context.file) + sql """drop table if exists `${tableName}`""" + sql """drop view if exists `${viewName}`""" + sql """drop materialized view if exists ${mvName};""" + + sql """ + CREATE TABLE `${tableName}` ( + `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"', + `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"', + `num` SMALLINT NOT NULL COMMENT '\"数量\"' + ) ENGINE=OLAP + DUPLICATE KEY(`user_id`, `date`, `num`) + COMMENT 'OLAP' + PARTITION BY RANGE(`date`) + (PARTITION p201701 VALUES [('2017-01-01'), ('2017-02-01')), + PARTITION p201702 VALUES [('2017-02-01'), ('2017-03-01')), + PARTITION p201703 VALUES [('2017-03-01'), ('2017-04-01'))) + DISTRIBUTED BY HASH(`user_id`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + + sql """ + insert into ${tableName} values(1,"2017-01-15",1),(1,"2017-02-15",2),(1,"2017-03-15",3); + """ + + sql""" + create view ${viewName} as select * from ${tableName}; + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (date_trunc(`date`,'year')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * from ${viewName}; + """ + + def showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertEquals(1, showPartitionsResult.size()) + assertTrue(showPartitionsResult.toString().contains("p_20170101_20180101")) + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_is_sync_after_refresh "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'" + order_qt_after_refresh "SELECT * FROM ${mvName}" + + sql """drop view if exists `${viewName}`""" + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" +} diff --git a/regression-test/suites/mtmv_p0/test_limit_partition_view_mtmv.groovy b/regression-test/suites/mtmv_p0/test_limit_partition_view_mtmv.groovy new file mode 100644 index 00000000000000..9d399660b6e667 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_limit_partition_view_mtmv.groovy @@ -0,0 +1,254 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_limit_partition_view_mtmv","mtmv") { + String suiteName = "test_limit_partition_view_mtmv" + String tableName = "${suiteName}_table" + String mvName = "${suiteName}_mv" + String viewName = "${suiteName}_view" + String dbName = context.config.getDbNameByFile(context.file) + + sql """drop table if exists `${tableName}`""" + sql """drop view if exists `${viewName}`""" + sql """drop materialized view if exists ${mvName};""" + + // list partition date type + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + sql """ + CREATE TABLE `${tableName}` ( + `k1` LARGEINT NOT NULL COMMENT '\"用户id\"', + `k2` DATE NOT NULL COMMENT '\"数据灌入日期时间\"' + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + PARTITION BY list(`k2`) + ( + PARTITION p_20380101 VALUES IN ("2038-01-01"), + PARTITION p_20200101 VALUES IN ("2020-01-01") + ) + DISTRIBUTED BY HASH(`k1`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + sql """ + insert into ${tableName} values(1,"2038-01-01"),(2,"2020-01-01"); + """ + + sql""" + create view ${viewName} as select * from ${tableName}; + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(`k2`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1', + 'partition_sync_limit'='2', + 'partition_sync_time_unit'='YEAR' + ) + AS + SELECT * FROM ${viewName}; + """ + def showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertEquals(1, showPartitionsResult.size()) + assertTrue(showPartitionsResult.toString().contains("p_20380101")) + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_date_list "SELECT * FROM ${mvName} order by k1,k2" + + + + // list partition string type + sql """drop table if exists `${tableName}`""" + sql """drop view if exists `${viewName}`""" + sql """drop materialized view if exists ${mvName};""" + sql """ + CREATE TABLE `${tableName}` ( + `k1` LARGEINT NOT NULL COMMENT '\"用户id\"', + `k2` VARCHAR(100) NOT NULL COMMENT '\"数据灌入日期时间\"' + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + PARTITION BY list(`k2`) + ( + PARTITION p_20380101 VALUES IN ("20380101"), + PARTITION p_20200101 VALUES IN ("20200101") + ) + DISTRIBUTED BY HASH(`k1`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + sql """ + insert into ${tableName} values(1,"20380101"),(2,"20200101"); + """ + sql""" + create view ${viewName} as select * from ${tableName}; + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(`k2`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1', + 'partition_sync_limit'='2', + 'partition_sync_time_unit'='MONTH', + 'partition_date_format'='%Y%m%d' + ) + AS + SELECT * FROM ${viewName}; + """ + showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertEquals(1, showPartitionsResult.size()) + assertTrue(showPartitionsResult.toString().contains("p_20380101")) + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_varchar_list "SELECT * FROM ${mvName} order by k1,k2" + + + // list partition int type + sql """drop table if exists `${tableName}`""" + sql """drop view if exists `${viewName}`""" + sql """drop materialized view if exists ${mvName};""" + sql """ + CREATE TABLE `${tableName}` ( + `k1` LARGEINT NOT NULL COMMENT '\"用户id\"', + `k2` int NOT NULL COMMENT '\"数据灌入日期时间\"' + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + PARTITION BY list(`k2`) + ( + PARTITION p_20380101 VALUES IN ("20380101"), + PARTITION p_20200101 VALUES IN ("20200101") + ) + DISTRIBUTED BY HASH(`k1`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + sql """ + insert into ${tableName} values(1,20380101),(2,20200101); + """ + sql""" + create view ${viewName} as select * from ${tableName}; + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(`k2`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1', + 'partition_sync_limit'='2', + 'partition_sync_time_unit'='DAY', + 'partition_date_format'='%Y%m%d' + ) + AS + SELECT * FROM ${viewName}; + """ + showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertEquals(1, showPartitionsResult.size()) + assertTrue(showPartitionsResult.toString().contains("p_20380101")) + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_varchar_list "SELECT * FROM ${mvName} order by k1,k2" + + + // range partition date type + sql """drop table if exists `${tableName}`""" + sql """drop view if exists `${viewName}`""" + sql """drop materialized view if exists ${mvName};""" + sql """ + CREATE TABLE `${tableName}` ( + `k1` LARGEINT NOT NULL COMMENT '\"用户id\"', + `k2` DATE NOT NULL COMMENT '\"数据灌入日期时间\"' + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + PARTITION BY range(`k2`) + ( + PARTITION p2038 VALUES [("2038-01-01"),("2038-01-03")), + PARTITION p2020 VALUES [("2020-01-01"),("2020-01-03")) + ) + DISTRIBUTED BY HASH(`k1`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + sql """ + insert into ${tableName} values(1,"2038-01-02"),(2,"2020-01-02"); + """ + sql""" + create view ${viewName} as select * from ${tableName}; + """ + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(`k2`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1', + 'partition_sync_limit'='2', + 'partition_sync_time_unit'='YEAR' + ) + AS + SELECT * FROM ${viewName}; + """ + showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertEquals(1, showPartitionsResult.size()) + assertTrue(showPartitionsResult.toString().contains("p_20380101_20380103")) + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_date_range "SELECT * FROM ${mvName} order by k1,k2" + + + // alter + sql """ + alter Materialized View ${mvName} set("partition_sync_limit"=""); + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + waitingMTMVTaskFinishedByMvName(mvName) + showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertEquals(2, showPartitionsResult.size()) + assertTrue(showPartitionsResult.toString().contains("p_20380101_20380103")) + assertTrue(showPartitionsResult.toString().contains("p_20200101_20200103")) + order_qt_date_range_all "SELECT * FROM ${mvName} order by k1,k2" + + sql """drop table if exists `${tableName}`""" + sql """drop view if exists `${viewName}`""" + sql """drop materialized view if exists ${mvName};""" +} diff --git a/regression-test/suites/mtmv_p0/test_view_hive_mtmv.groovy b/regression-test/suites/mtmv_p0/test_view_hive_mtmv.groovy new file mode 100644 index 00000000000000..98472d80746d73 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_view_hive_mtmv.groovy @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_view_hive_mtmv", "p0,external,hive,external_docker,external_docker_hive") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("diable Hive test.") + return; + } + String suiteName = "test_create_mtmv_with_view_rollup" + String mvName = "${suiteName}_mv" + String viewName = "${suiteName}_view" + String dbName = context.config.getDbNameByFile(context.file) + for (String hivePrefix : ["hive3"]) { + try { + String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") + String catalog_name = "${suiteName}_${hivePrefix}_test_catalog" + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + sql """drop catalog if exists ${catalog_name}""" + sql """drop view if exists `${viewName}`""" + sql """drop materialized view if exists ${mvName};""" + sql """create catalog if not exists ${catalog_name} properties ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + );""" + + sql""" + create view ${viewName} as select * from ${catalog_name}.`default`.mtmv_base1; + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(`part_col`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT * FROM ${viewName}; + """ + def showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertTrue(showPartitionsResult.toString().contains("p_20230101")) + assertTrue(showPartitionsResult.toString().contains("p_20230102")) + + // refresh one partitions + sql """ + REFRESH MATERIALIZED VIEW ${mvName} partitions(p_20230101); + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_refresh_one_partition "SELECT * FROM ${mvName} order by id" + + //refresh complete + sql """ + REFRESH MATERIALIZED VIEW ${mvName} auto + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_refresh_complete "SELECT * FROM ${mvName} order by id" + + order_qt_is_sync_before_rebuild "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'" + // rebuild catalog, should not Affects MTMV + sql """drop catalog if exists ${catalog_name}""" + sql """create catalog if not exists ${catalog_name} properties ( + "type"="hms", + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}' + );""" + + order_qt_is_sync_after_rebuild "select SyncWithBaseTables from mv_infos('database'='${dbName}') where Name='${mvName}'" + + // should refresh normal after catalog rebuild + sql """ + REFRESH MATERIALIZED VIEW ${mvName} complete + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_refresh_complete_rebuild "SELECT * FROM ${mvName} order by id" + + sql """drop materialized view if exists ${mvName};""" + sql """drop view if exists `${viewName}`""" + sql """drop catalog if exists ${catalog_name}""" + } finally { + } + } +} +