From c174ffed02f4f734deb13c5134e83c24381dee7a Mon Sep 17 00:00:00 2001 From: daidai Date: Fri, 15 Aug 2025 11:04:47 +0800 Subject: [PATCH 1/4] [enchement](tvf)support tvf topn lazy materialize. --- .../translator/PhysicalPlanTranslator.java | 6 +- .../post/materialize/LazyMaterializeTopN.java | 48 ++++++++---- .../post/materialize/LazySlotPruning.java | 22 +++++- .../materialize/MaterializeProbeVisitor.java | 30 ++++++++ .../post/materialize/MaterializeSource.java | 6 +- .../rules/analysis/BindExpression.java | 3 +- .../nereids/rules/analysis/BindRelation.java | 2 +- ...gicalTVFRelationToPhysicalTVFRelation.java | 2 +- .../rules/rewrite/OperativeColumnDerive.java | 12 +++ .../plans/logical/LogicalTVFRelation.java | 29 ++++++-- .../physical/PhysicalLazyMaterialize.java | 31 +++++--- .../PhysicalLazyMaterializeTVFScan.java | 73 +++++++++++++++++++ .../plans/physical/PhysicalTVFRelation.java | 22 ++++-- .../trees/plans/visitor/PlanVisitor.java | 5 ++ 14 files changed, 241 insertions(+), 50 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeTVFScan.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 215bd0221cb3a7..d56f8e8a8834c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -115,7 +115,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PreAggStatus; import org.apache.doris.nereids.trees.plans.algebra.Aggregate; -import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.algebra.Relation; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalJoin; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort; import org.apache.doris.nereids.trees.plans.physical.PhysicalAssertNumRows; @@ -2663,7 +2663,7 @@ public PlanFragment visitPhysicalLazyMaterialize(PhysicalLazyMaterialize rowStoreFlags = new ArrayList<>(); - for (CatalogRelation relation : materialize.getRelations()) { + for (Relation relation : materialize.getRelations()) { rowStoreFlags.add(shouldUseRowStore(relation)); } materializeNode.setRowStoreFlags(rowStoreFlags); @@ -2677,7 +2677,7 @@ public PlanFragment visitPhysicalLazyMaterialize(PhysicalLazyMaterialize> relationToLazySlotMap = new HashMap<>(); + Map> relationToLazySlotMap = new HashMap<>(); for (Slot slot : lazyMaterializeSlots) { MaterializeSource source = materializeMap.get(slot); relationToLazySlotMap.computeIfAbsent(source.relation, relation -> new ArrayList<>()).add(slot); @@ -102,23 +105,42 @@ public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) { Plan result = topN; List originOutput = topN.getOutput(); - BiMap relationToRowId = HashBiMap.create(relationToLazySlotMap.size()); + BiMap relationToRowId = HashBiMap.create(relationToLazySlotMap.size()); HashSet rowIdSet = new HashSet<>(); // we should use threadStatementContext, not ctx.getStatementContext(), because the StatisticsCleaner // will generate two statementContext, and reuse the plan which generated by outer StatementContext, // so we should generate exprId by the outer StatementContext, or else generate conflict expr id StatementContext threadStatementContext = StatementScopeIdGenerator.getStatementContext(); - for (CatalogRelation relation : relationToLazySlotMap.keySet()) { - Column rowIdCol = new Column(Column.GLOBAL_ROWID_COL + relation.getTable().getName(), - Type.STRING, false, null, false, - "", relation.getTable().getName() + ".global_row_id"); - SlotReference rowIdSlot = SlotReference.fromColumn( - threadStatementContext.getNextExprId(), relation.getTable(), rowIdCol, relation.getQualifier()); - result = result.accept(new LazySlotPruning(), - new LazySlotPruning.Context((PhysicalCatalogRelation) relation, - rowIdSlot, relationToLazySlotMap.get(relation))); - relationToRowId.put(relation, rowIdSlot); - rowIdSet.add(rowIdSlot); + for (Relation relation : relationToLazySlotMap.keySet()) { + if (relation instanceof CatalogRelation) { + CatalogRelation catalogRelation = (CatalogRelation) relation; + Column rowIdCol = new Column(Column.GLOBAL_ROWID_COL + catalogRelation.getTable().getName(), + Type.STRING, false, null, false, + "", catalogRelation.getTable().getName() + ".global_row_id"); + SlotReference rowIdSlot = SlotReference.fromColumn(threadStatementContext.getNextExprId(), + catalogRelation.getTable(), rowIdCol, catalogRelation.getQualifier()); + result = result.accept(new LazySlotPruning(), + new LazySlotPruning.Context((PhysicalCatalogRelation) relation, + rowIdSlot, relationToLazySlotMap.get(relation))); + relationToRowId.put(catalogRelation, rowIdSlot); + rowIdSet.add(rowIdSlot); + } else if (relation instanceof PhysicalTVFRelation) { + PhysicalTVFRelation tvfRelation = (PhysicalTVFRelation) relation; + + // Column rowIdCol = new Column(Column.GLOBAL_ROWID_COL + tvfRelation.getFunction().getName(), + // Type.STRING, false, null, false, + // "", tvfRelation.getFunction().getName() + ".global_row_id"); + SlotReference rowIdSlot = new SlotReference( + Column.GLOBAL_ROWID_COL + tvfRelation.getFunction().getName(), StringType.INSTANCE); + + // SlotReference rowIdSlot = SlotReference.fromColumn(threadStatementContext.getNextExprId(), + // catalogRelation.getTable(), rowIdCol, catalogRelation.getQualifier()); + result = result.accept(new LazySlotPruning(), + new LazySlotPruning.Context((PhysicalTVFRelation) relation, + rowIdSlot, relationToLazySlotMap.get(relation))); + relationToRowId.put(tvfRelation, rowIdSlot); + rowIdSet.add(rowIdSlot); + } } // materialize.child.output requires diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java index f9858538d084ce..8c53156e93618e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazySlotPruning.java @@ -26,16 +26,18 @@ import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer; import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEProducer; -import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeFileScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeTVFScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalRepeat; import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import com.google.common.collect.ImmutableList; @@ -53,17 +55,17 @@ public class LazySlotPruning extends DefaultPlanRewriter lazySlots; private SlotReference rowIdSlot; - public Context(PhysicalCatalogRelation scan, SlotReference rowIdSlot, List lazySlots) { + public Context(PhysicalRelation scan, SlotReference rowIdSlot, List lazySlots) { this.scan = scan; this.lazySlots = lazySlots; this.rowIdSlot = rowIdSlot; } - private Context(PhysicalCatalogRelation scan, List lazySlots, SlotReference rowIdSlot) { + private Context(PhysicalRelation scan, List lazySlots, SlotReference rowIdSlot) { this.scan = scan; this.lazySlots = lazySlots; this.rowIdSlot = rowIdSlot; @@ -130,6 +132,18 @@ public Plan visitPhysicalFileScan(PhysicalFileScan scan, Context context) { } } + @Override + public Plan visitPhysicalTVFRelation(PhysicalTVFRelation tvfRelation, Context context) { + if (tvfRelation.getOutput().containsAll(context.lazySlots)) { + PhysicalLazyMaterializeTVFScan lazyScan = new PhysicalLazyMaterializeTVFScan(tvfRelation, + context.rowIdSlot, context.lazySlots); + return lazyScan; + } else { + // should not hit here + throw new RuntimeException("Lazy materialize fault"); + } + } + @Override public Plan visitPhysicalLazyMaterializeOlapScan(PhysicalLazyMaterializeOlapScan scan, Context context) { // should not come here diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java index e42a0b4c1e1b8f..1c6fa677e5c6cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java @@ -26,18 +26,21 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.table.Local; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterialize; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import com.google.common.collect.ImmutableSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.Map; import java.util.Optional; import java.util.Set; @@ -101,6 +104,17 @@ boolean checkRelationTableSupportedType(PhysicalCatalogRelation relation) { return true; } + boolean checkTvfRelationTableSupportedType(PhysicalTVFRelation tvfRelation) { + Map properties = tvfRelation.getFunction().getTVFProperties().getMap(); + + if (tvfRelation.getFunction() instanceof Local) { + if (properties.containsKey("format") && properties.get("format").equals("parquet")) { + return true; + } + } + return false; + } + @Override public Optional visitPhysicalOlapScan(PhysicalOlapScan scan, ProbeContext context) { if (scan.getSelectedIndexId() == scan.getTable().getBaseIndexId()) { @@ -125,6 +139,22 @@ public Optional visitPhysicalCatalogRelation( return Optional.empty(); } + @Override + public Optional visitPhysicalTVFRelation( + PhysicalTVFRelation tvfRelation, ProbeContext context) { + if (checkTvfRelationTableSupportedType(tvfRelation) && tvfRelation.getOutput().contains(context.slot) + && !tvfRelation.getOperativeSlots().contains(context.slot)) { + // lazy materialize slot must be a passive slot + if (context.slot.getOriginalColumn().isPresent()) { + return Optional.of(new MaterializeSource(tvfRelation, context.slot)); + } else { + LOG.info("lazy materialize {} failed, because its column is empty", context.slot); + } + } + + return Optional.empty(); + } + @Override public Optional visitPhysicalLazyMaterialize( PhysicalLazyMaterialize materialize, ProbeContext context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeSource.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeSource.java index d367dd7957d323..30012f44b4d5df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeSource.java @@ -18,19 +18,19 @@ package org.apache.doris.nereids.processor.post.materialize; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.algebra.Relation; /** the table and slot used to do lazy materialize */ public class MaterializeSource { - public final CatalogRelation relation; + public final Relation relation; public final SlotReference baseSlot; /* constructor */ - public MaterializeSource(CatalogRelation relation, SlotReference baseSlot) { + public MaterializeSource(Relation relation, SlotReference baseSlot) { this.relation = relation; this.baseSlot = baseSlot; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 90614b29135e24..bc85a6f2e42849 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -1336,7 +1336,8 @@ private LogicalTVFRelation bindTableValuedFunction(MatchingContext expressions, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index f8512e4fc76a8d..253dac3db5f644 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -371,7 +371,7 @@ private Optional handleMetaTable(TableIf table, UnboundRelation unb Optional tvf = table.getSysTableFunction( qualifiedTableName.get(0), qualifiedTableName.get(1), qualifiedTableName.get(2)); if (tvf.isPresent()) { - return Optional.of(new LogicalTVFRelation(unboundRelation.getRelationId(), tvf.get())); + return Optional.of(new LogicalTVFRelation(unboundRelation.getRelationId(), tvf.get(), ImmutableList.of())); } return Optional.empty(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalTVFRelationToPhysicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalTVFRelationToPhysicalTVFRelation.java index 4828f77d5eee9b..c5eebaaab0c7a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalTVFRelationToPhysicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalTVFRelationToPhysicalTVFRelation.java @@ -29,7 +29,7 @@ public class LogicalTVFRelationToPhysicalTVFRelation extends OneImplementationRu public Rule build() { return logicalTVFRelation() .then(relation -> new PhysicalTVFRelation(relation.getRelationId(), - relation.getFunction(), relation.getLogicalProperties())) + relation.getFunction(), relation.getOperativeSlots(), relation.getLogicalProperties())) .toRule(RuleType.LOGICAL_TVF_RELATION_TO_PHYSICAL_TVF_RELATION); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OperativeColumnDerive.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OperativeColumnDerive.java index 82523e76185923..da2127af8d6e49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OperativeColumnDerive.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/OperativeColumnDerive.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; @@ -174,6 +175,17 @@ public Plan visitLogicalCatalogRelation(LogicalCatalogRelation relation, DeriveC return relation.withOperativeSlots(operandSlots.build()); } + @Override + public Plan visitLogicalTVFRelation(LogicalTVFRelation relation, DeriveContext context) { + ImmutableSet.Builder operandSlots = ImmutableSet.builder(); + for (Slot slot : relation.getOutput()) { + if (context.operativeSlotIds.contains(slot.getExprId().asInt())) { + operandSlots.add(slot); + } + } + return relation.withOperativeSlots(operandSlots.build()); + } + /** * DeriveContext */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java index 76152fedb923e9..a434f81b2313ba 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTVFRelation.java @@ -36,6 +36,7 @@ import com.google.common.collect.ImmutableList; +import java.util.Collection; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -45,34 +46,48 @@ public class LogicalTVFRelation extends LogicalRelation implements TVFRelation, private final TableValuedFunction function; private final ImmutableList qualifier; + private final ImmutableList operativeSlots; - public LogicalTVFRelation(RelationId id, TableValuedFunction function) { + public LogicalTVFRelation(RelationId id, TableValuedFunction function, ImmutableList operativeSlots) { super(id, PlanType.LOGICAL_TVF_RELATION); + this.operativeSlots = operativeSlots; this.function = function; qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName()); } - public LogicalTVFRelation(RelationId id, TableValuedFunction function, Optional groupExpression, - Optional logicalProperties) { + public LogicalTVFRelation(RelationId id, TableValuedFunction function, ImmutableList operativeSlots, + Optional groupExpression, Optional logicalProperties) { super(id, PlanType.LOGICAL_TVF_RELATION, groupExpression, logicalProperties); + this.operativeSlots = operativeSlots; this.function = function; qualifier = ImmutableList.of(TableValuedFunctionIf.TVF_TABLE_PREFIX + function.getName()); } @Override public LogicalTVFRelation withGroupExpression(Optional groupExpression) { - return new LogicalTVFRelation(relationId, function, groupExpression, Optional.of(getLogicalProperties())); + return new LogicalTVFRelation(relationId, function, operativeSlots, + groupExpression, Optional.of(getLogicalProperties())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalTVFRelation(relationId, function, groupExpression, logicalProperties); + return new LogicalTVFRelation(relationId, function, operativeSlots, groupExpression, logicalProperties); } @Override public LogicalTVFRelation withRelationId(RelationId relationId) { - return new LogicalTVFRelation(relationId, function, Optional.empty(), Optional.empty()); + return new LogicalTVFRelation(relationId, function, operativeSlots, Optional.empty(), + Optional.of(getLogicalProperties())); + } + + public LogicalTVFRelation withOperativeSlots(Collection operativeSlots) { + return new LogicalTVFRelation(relationId, function, Utils.fastToImmutableList(operativeSlots), + Optional.empty(), Optional.of(getLogicalProperties())); + } + + public List getOperativeSlots() { + return operativeSlots; } @Override @@ -87,7 +102,7 @@ public boolean equals(Object o) { return false; } LogicalTVFRelation that = (LogicalTVFRelation) o; - return Objects.equals(function, that.function); + return Objects.equals(function, that.function) && Objects.equals(operativeSlots, that.operativeSlots); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java index 546c440c3c2389..f3106e068be7f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterialize.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.TableIf; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.processor.post.materialize.MaterializeSource; import org.apache.doris.nereids.properties.DataTrait.Builder; @@ -30,6 +31,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import org.apache.doris.nereids.trees.plans.algebra.Relation; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.statistics.Statistics; @@ -49,9 +51,9 @@ */ public class PhysicalLazyMaterialize extends PhysicalUnary { - private final Map> relationToLazySlotMap; + private final Map> relationToLazySlotMap; - private final BiMap relationToRowId; + private final BiMap relationToRowId; private final Map materializeMap; @@ -65,7 +67,7 @@ public class PhysicalLazyMaterialize extends PhysicalUn private List> lazySlotLocations = new ArrayList<>(); private List> lazyTableIdxs = new ArrayList<>(); - private final List relations; + private final List relations; /** * constructor @@ -73,8 +75,8 @@ public class PhysicalLazyMaterialize extends PhysicalUn public PhysicalLazyMaterialize(CHILD_TYPE child, List materializeInput, List materializedSlots, - Map> relationToLazySlotMap, - BiMap relationToRowId, + Map> relationToLazySlotMap, + BiMap relationToRowId, Map materializeMap) { this(child, materializeInput, materializedSlots, relationToLazySlotMap, relationToRowId, materializeMap, null, null); @@ -86,8 +88,8 @@ public PhysicalLazyMaterialize(CHILD_TYPE child, public PhysicalLazyMaterialize(CHILD_TYPE child, List materializeInput, List materializedSlots, - Map> relationToLazySlotMap, - BiMap relationToRowId, + Map> relationToLazySlotMap, + BiMap relationToRowId, Map materializeMap, PhysicalProperties physicalProperties, Statistics statistics) { super(PlanType.PHYSICAL_MATERIALIZE, Optional.empty(), @@ -111,8 +113,15 @@ public PhysicalLazyMaterialize(CHILD_TYPE child, for (; idx < materializeInput.size(); idx++) { Slot rowId = materializeInput.get(idx); rowIdListBuilder.add(rowId); - CatalogRelation rel = relationToRowId.inverse().get(rowId); - TableIf relationTable = rel.getTable(); + Relation rel = relationToRowId.inverse().get(rowId); + TableIf relationTable; + if (rel instanceof CatalogRelation) { + relationTable = ((CatalogRelation) rel).getTable(); + } else if (rel instanceof PhysicalTVFRelation) { + relationTable = ((PhysicalTVFRelation) rel).getFunction().getTable(); + } else { + throw new AnalysisException("Unsupported relation type: " + rel); + } List lazyColumnForRel = new ArrayList<>(); lazyColumns.add(lazyColumnForRel); @@ -191,7 +200,7 @@ public String toString() { StringBuilder builder = new StringBuilder(); builder.append("PhysicalLazyMaterialize [Output= (") .append(getOutput()).append("), lazySlots= ("); - for (Map.Entry> entry : relationToLazySlotMap.entrySet()) { + for (Map.Entry> entry : relationToLazySlotMap.entrySet()) { builder.append(entry.getValue()); } builder.append(")]"); @@ -226,7 +235,7 @@ public int compare(Slot slot, Slot t1) { return shapeBuilder.toString(); } - public List getRelations() { + public List getRelations() { return relations; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeTVFScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeTVFScan.java new file mode 100644 index 00000000000000..46e4a2b5af4f71 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeTVFScan.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.physical; + +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +/** + wrapper for FileScan used for lazy materialization + */ +public class PhysicalLazyMaterializeTVFScan extends PhysicalTVFRelation { + private PhysicalTVFRelation scan; + private SlotReference rowId; + private final List lazySlots; + private List output; + + /** + * PhysicalLazyMaterializeTVFScan + */ + public PhysicalLazyMaterializeTVFScan(PhysicalTVFRelation scan, SlotReference rowId, List lazySlots) { + super(scan.getRelationId(), scan.getFunction(), scan.getOperativeSlots(), scan.getLogicalProperties()); + this.scan = scan; + this.rowId = rowId; + this.lazySlots = lazySlots; + } + + @Override + public List computeOutput() { + if (output == null) { + output = ImmutableList.builder() + .addAll(scan.getOperativeSlots()) + .add(rowId).build(); + } + return output; + } + + @Override + public List getOutput() { + return computeOutput(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("PhysicalLazyMaterializeTVFScan[") + .append(scan.toString()); + + if (!getAppliedRuntimeFilters().isEmpty()) { + getAppliedRuntimeFilters().forEach(rf -> sb.append(" RF").append(rf.getId().asInt())); + } + sb.append("]"); + return sb.toString(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java index f515adffed37fb..2030afce3bb6d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java @@ -37,6 +37,7 @@ import com.google.common.collect.ImmutableList; +import java.util.Collection; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -45,39 +46,48 @@ public class PhysicalTVFRelation extends PhysicalRelation implements TVFRelation, BlockFuncDepsPropagation { private final TableValuedFunction function; + private final ImmutableList operativeSlots; - public PhysicalTVFRelation(RelationId id, TableValuedFunction function, LogicalProperties logicalProperties) { + public PhysicalTVFRelation(RelationId id, TableValuedFunction function, Collection operativeSlots, + LogicalProperties logicalProperties) { super(id, PlanType.PHYSICAL_TVF_RELATION, Optional.empty(), logicalProperties); + this.operativeSlots = ImmutableList.copyOf(operativeSlots); this.function = Objects.requireNonNull(function, "function can not be null"); } - public PhysicalTVFRelation(RelationId id, TableValuedFunction function, Optional groupExpression, + public PhysicalTVFRelation(RelationId id, TableValuedFunction function, Collection operativeSlots, + Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { super(id, PlanType.PHYSICAL_TVF_RELATION, groupExpression, logicalProperties, physicalProperties, statistics); + this.operativeSlots = ImmutableList.copyOf(operativeSlots); this.function = Objects.requireNonNull(function, "function can not be null"); } @Override public PhysicalTVFRelation withGroupExpression(Optional groupExpression) { - return new PhysicalTVFRelation(relationId, function, groupExpression, getLogicalProperties(), + return new PhysicalTVFRelation(relationId, function, operativeSlots, groupExpression, getLogicalProperties(), physicalProperties, statistics); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalTVFRelation(relationId, function, groupExpression, + return new PhysicalTVFRelation(relationId, function, operativeSlots, groupExpression, logicalProperties.get(), physicalProperties, statistics); } @Override public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalTVFRelation(relationId, function, Optional.empty(), + return new PhysicalTVFRelation(relationId, function, operativeSlots, Optional.empty(), getLogicalProperties(), physicalProperties, statistics); } + public List getOperativeSlots() { + return operativeSlots; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -90,7 +100,7 @@ public boolean equals(Object o) { return false; } PhysicalTVFRelation that = (PhysicalTVFRelation) o; - return Objects.equals(function, that.function); + return Objects.equals(function, that.function) && Objects.equals(operativeSlots, that.operativeSlots); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java index ebb84f692a4da0..1c5b46c147929a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/PlanVisitor.java @@ -72,6 +72,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterialize; import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeFileScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeTVFScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit; import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN; @@ -282,6 +283,10 @@ public R visitPhysicalLazyMaterializeFileScan(PhysicalLazyMaterializeFileScan sc return visit(scan, context); } + public R visitPhysicalLazyMaterializeTVFScan(PhysicalLazyMaterializeTVFScan scan, C context) { + return visit(scan, context); + } + public R visitPhysicalLazyMaterializeOlapScan(PhysicalLazyMaterializeOlapScan scan, C context) { return visit(scan, context); } From e1e9592f931abc18a30118f8985e08a3a1296b62 Mon Sep 17 00:00:00 2001 From: daidai Date: Sun, 17 Aug 2025 17:56:41 +0800 Subject: [PATCH 2/4] fix plan --- .../translator/PhysicalPlanTranslator.java | 36 +++++++++++++++++- .../post/materialize/LazyMaterializeTopN.java | 17 +++++---- .../materialize/MaterializeProbeVisitor.java | 12 +++--- .../PhysicalLazyMaterializeTVFScan.java | 37 ++++++++++++++++++- .../plans/physical/PhysicalTVFRelation.java | 7 ++-- 5 files changed, 91 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index d56f8e8a8834c9..6ef0798475ef0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -144,6 +144,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterialize; import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeTVFScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit; import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan; @@ -1071,7 +1072,7 @@ public PlanFragment visitPhysicalSchemaScan(PhysicalSchemaScan schemaScan, PlanT @Override public PlanFragment visitPhysicalTVFRelation(PhysicalTVFRelation tvfRelation, PlanTranslatorContext context) { - List slots = tvfRelation.getLogicalProperties().getOutput(); + List slots = tvfRelation.getOutput(); TupleDescriptor tupleDescriptor = generateTupleDesc(slots, tvfRelation.getFunction().getTable(), context); TableValuedFunctionIf catalogFunction = tvfRelation.getFunction().getCatalogFunction(); @@ -2687,6 +2688,39 @@ private boolean shouldUseRowStore(Relation rel) { return useRowStore; } + @Override + public PlanFragment visitPhysicalLazyMaterializeTVFScan(PhysicalLazyMaterializeTVFScan tvfRelation, + PlanTranslatorContext context) { + List slots = tvfRelation.getOutput(); + TupleDescriptor tupleDescriptor = generateTupleDesc(slots, tvfRelation.getFunction().getTable(), context); + + TableValuedFunctionIf catalogFunction = tvfRelation.getFunction().getCatalogFunction(); + SessionVariable sv = ConnectContext.get().getSessionVariable(); + ScanNode scanNode = catalogFunction.getScanNode(context.nextPlanNodeId(), tupleDescriptor, sv); + scanNode.setNereidsId(tvfRelation.getId()); + context.getNereidsIdToPlanNodeIdMap().put(tvfRelation.getId(), scanNode.getId()); + Utils.execWithUncheckedException(scanNode::init); + context.getRuntimeTranslator().ifPresent( + runtimeFilterGenerator -> runtimeFilterGenerator.getContext().getTargetListByScan(tvfRelation) + .forEach(expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, scanNode, context) + ) + ); + context.addScanNode(scanNode, tvfRelation); + + // TODO: it is weird update label in this way + // set label for explain + for (Slot slot : slots) { + String tableColumnName = TableValuedFunctionIf.TVF_TABLE_PREFIX + tvfRelation.getFunction().getName() + + "." + slot.getName(); + context.findSlotRef(slot.getExprId()).setLabel(tableColumnName); + } + + PlanFragment planFragment = createPlanFragment(scanNode, DataPartition.RANDOM, tvfRelation); + context.addPlanFragment(planFragment); + updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), tvfRelation); + return planFragment; + } + @Override public PlanFragment visitPhysicalLazyMaterializeOlapScan(PhysicalLazyMaterializeOlapScan lazyScan, PlanTranslatorContext context) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java index ab9152c9b8ee2a..f1f02d770c5c1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/LazyMaterializeTopN.java @@ -34,10 +34,10 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN; -import org.apache.doris.nereids.types.StringType; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.HashMap; @@ -127,19 +127,20 @@ public Plan visitPhysicalTopN(PhysicalTopN topN, CascadesContext ctx) { } else if (relation instanceof PhysicalTVFRelation) { PhysicalTVFRelation tvfRelation = (PhysicalTVFRelation) relation; - // Column rowIdCol = new Column(Column.GLOBAL_ROWID_COL + tvfRelation.getFunction().getName(), - // Type.STRING, false, null, false, - // "", tvfRelation.getFunction().getName() + ".global_row_id"); - SlotReference rowIdSlot = new SlotReference( - Column.GLOBAL_ROWID_COL + tvfRelation.getFunction().getName(), StringType.INSTANCE); + Column rowIdCol = new Column(Column.GLOBAL_ROWID_COL + tvfRelation.getFunction().getName(), + Type.STRING, false, null, false, + "", tvfRelation.getFunction().getName() + ".global_row_id"); - // SlotReference rowIdSlot = SlotReference.fromColumn(threadStatementContext.getNextExprId(), - // catalogRelation.getTable(), rowIdCol, catalogRelation.getQualifier()); + SlotReference rowIdSlot = SlotReference.fromColumn(threadStatementContext.getNextExprId(), + tvfRelation.getFunction().getTable(), rowIdCol, ImmutableList.of()); result = result.accept(new LazySlotPruning(), new LazySlotPruning.Context((PhysicalTVFRelation) relation, rowIdSlot, relationToLazySlotMap.get(relation))); relationToRowId.put(tvfRelation, rowIdSlot); rowIdSet.add(rowIdSlot); + } else { + // should not reach here. + throw new RuntimeException("LazyMaterializeTopN not support this relation." + relation); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java index 1c6fa677e5c6cc..1e18fddf6699c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/materialize/MaterializeProbeVisitor.java @@ -26,7 +26,6 @@ import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.functions.table.Local; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterialize; @@ -104,11 +103,14 @@ boolean checkRelationTableSupportedType(PhysicalCatalogRelation relation) { return true; } - boolean checkTvfRelationTableSupportedType(PhysicalTVFRelation tvfRelation) { + boolean checkTVFRelationTableSupportedType(PhysicalTVFRelation tvfRelation) { Map properties = tvfRelation.getFunction().getTVFProperties().getMap(); + String functionName = tvfRelation.getFunction().getName(); - if (tvfRelation.getFunction() instanceof Local) { - if (properties.containsKey("format") && properties.get("format").equals("parquet")) { + if (functionName.equals("local") || functionName.equals("s3") || functionName.equals("hdfs")) { + if (properties.containsKey("format") + && (properties.get("format").equalsIgnoreCase("parquet") + || properties.get("format").equalsIgnoreCase("orc"))) { return true; } } @@ -142,7 +144,7 @@ public Optional visitPhysicalCatalogRelation( @Override public Optional visitPhysicalTVFRelation( PhysicalTVFRelation tvfRelation, ProbeContext context) { - if (checkTvfRelationTableSupportedType(tvfRelation) && tvfRelation.getOutput().contains(context.slot) + if (checkTVFRelationTableSupportedType(tvfRelation) && tvfRelation.getOutput().contains(context.slot) && !tvfRelation.getOperativeSlots().contains(context.slot)) { // lazy materialize slot must be a passive slot if (context.slot.getOriginalColumn().isPresent()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeTVFScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeTVFScan.java index 46e4a2b5af4f71..03787887563177 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeTVFScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeTVFScan.java @@ -17,15 +17,22 @@ package org.apache.doris.nereids.trees.plans.physical; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.statistics.Statistics; import com.google.common.collect.ImmutableList; import java.util.List; +import java.util.Optional; /** - wrapper for FileScan used for lazy materialization + wrapper for PhysicalTVFRelation used for lazy materialization */ public class PhysicalLazyMaterializeTVFScan extends PhysicalTVFRelation { private PhysicalTVFRelation scan; @@ -70,4 +77,32 @@ public String toString() { sb.append("]"); return sb.toString(); } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitPhysicalTVFRelation(this, context); + } + + @Override + public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, + Statistics statistics) { + PhysicalTVFRelation tvfRelation = new PhysicalTVFRelation(relationId, function, operativeSlots, + Optional.empty(), getLogicalProperties(), physicalProperties, statistics); + return new PhysicalLazyMaterializeTVFScan(tvfRelation, rowId, lazySlots); + } + + @Override + public PhysicalLazyMaterializeTVFScan withGroupExpression(Optional groupExpression) { + PhysicalTVFRelation tvfRelation = new PhysicalTVFRelation(relationId, function, operativeSlots, + groupExpression, getLogicalProperties(), physicalProperties, statistics); + return new PhysicalLazyMaterializeTVFScan(tvfRelation, rowId, lazySlots); + } + + @Override + public Plan withGroupExprLogicalPropChildren(Optional groupExpression, + Optional logicalProperties, List children) { + PhysicalTVFRelation tvfRelation = new PhysicalTVFRelation(relationId, function, operativeSlots, + groupExpression, logicalProperties.get(), physicalProperties, statistics); + return new PhysicalLazyMaterializeTVFScan(tvfRelation, rowId, lazySlots); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java index 2030afce3bb6d9..84bf5aa4bf75f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTVFRelation.java @@ -45,8 +45,8 @@ /** PhysicalTableValuedFunctionRelation */ public class PhysicalTVFRelation extends PhysicalRelation implements TVFRelation, BlockFuncDepsPropagation { - private final TableValuedFunction function; - private final ImmutableList operativeSlots; + protected final TableValuedFunction function; + protected final ImmutableList operativeSlots; public PhysicalTVFRelation(RelationId id, TableValuedFunction function, Collection operativeSlots, LogicalProperties logicalProperties) { @@ -113,7 +113,8 @@ public String toString() { return Utils.toSqlString("PhysicalTVFRelation", "qualified", Utils.qualifiedName(ImmutableList.of(), function.getTable().getName()), "output", getOutput(), - "function", function.toSql() + "function", function.toSql(), + "operativeSlots", operativeSlots ); } From 89298a28857793a6fbd1aaa689a5dc4c7ff436f4 Mon Sep 17 00:00:00 2001 From: daidai Date: Mon, 18 Aug 2025 15:26:55 +0800 Subject: [PATCH 3/4] add case. --- .../tvf/test_tvf_topn_lazy_mat.out | 429 ++++++++++++++++++ .../tvf/test_tvf_topn_lazy_mat.groovy | 216 +++++++++ 2 files changed, 645 insertions(+) create mode 100644 regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out create mode 100644 regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy diff --git a/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out b/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out new file mode 100644 index 00000000000000..b965c865606dd4 --- /dev/null +++ b/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out @@ -0,0 +1,429 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !1 -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 + +-- !2 -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 +5 user5 5.0 false 2.5 + +-- !3 -- +0.5 1.0 false user1 +1.0 2.0 true user2 +1.5 3.0 false user3 +2.0 4.0 true user4 + +-- !4 -- +1.0 user1 1 +2.0 user2 2 + +-- !test_basic -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 + +-- !test_partial -- +2 user2 1.0 +4 user4 2.0 +6 user6 3.0 + +-- !test_multi_sort -- +1 user1 1.0 0.5 +2 user2 2.0 1.0 +3 user3 3.0 1.5 + +-- !test_filter -- + +-- !test_subquery -- +2 user2 3.0 +4 user4 6.0 +6 user6 9.0 + +-- !test_agg -- +1 0.5 1.0 +2 1.0 2.0 +3 1.5 3.0 + +-- !test_basic -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 +5 user5 5.0 false 2.5 +6 user6 6.0 true 3.0 +7 user7 7.0 false 3.5 + +-- !test_partial -- +2 user2 1.0 +4 user4 2.0 +6 user6 3.0 +8 user8 4.0 + +-- !test_multi_sort -- +1 user1 1.0 0.5 +2 user2 2.0 1.0 +3 user3 3.0 1.5 +4 user4 4.0 2.0 +5 user5 5.0 2.5 +6 user6 6.0 3.0 +7 user7 7.0 3.5 + +-- !test_filter -- + +-- !test_subquery -- +2 user2 3.0 +4 user4 6.0 +6 user6 9.0 +8 user8 12.0 + +-- !test_agg -- +1 0.5 1.0 +2 1.0 2.0 +3 1.5 3.0 +4 2.0 4.0 +5 2.5 5.0 +6 3.0 6.0 +7 3.5 7.0 + +-- !1 -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 + +-- !2 -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 +5 user5 5.0 false 2.5 + +-- !3 -- +0.5 1.0 false user1 +1.0 2.0 true user2 +1.5 3.0 false user3 +2.0 4.0 true user4 + +-- !4 -- +1.0 user1 1 +2.0 user2 2 + +-- !test_basic -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 + +-- !test_partial -- +2 user2 1.0 +4 user4 2.0 +6 user6 3.0 + +-- !test_multi_sort -- +1 user1 1.0 0.5 +2 user2 2.0 1.0 +3 user3 3.0 1.5 + +-- !test_filter -- + +-- !test_subquery -- +2 user2 3.0 +4 user4 6.0 +6 user6 9.0 + +-- !test_agg -- +1 0.5 1.0 +2 1.0 2.0 +3 1.5 3.0 + +-- !test_basic -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 +5 user5 5.0 false 2.5 +6 user6 6.0 true 3.0 +7 user7 7.0 false 3.5 + +-- !test_partial -- +2 user2 1.0 +4 user4 2.0 +6 user6 3.0 +8 user8 4.0 + +-- !test_multi_sort -- +1 user1 1.0 0.5 +2 user2 2.0 1.0 +3 user3 3.0 1.5 +4 user4 4.0 2.0 +5 user5 5.0 2.5 +6 user6 6.0 3.0 +7 user7 7.0 3.5 + +-- !test_filter -- + +-- !test_subquery -- +2 user2 3.0 +4 user4 6.0 +6 user6 9.0 +8 user8 12.0 + +-- !test_agg -- +1 0.5 1.0 +2 1.0 2.0 +3 1.5 3.0 +4 2.0 4.0 +5 2.5 5.0 +6 3.0 6.0 +7 3.5 7.0 + +-- !join_1 -- +1 user1 1.0 false 0.5 1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 4 user4 4.0 true 2.0 +5 user5 5.0 false 2.5 5 user5 5.0 false 2.5 + +-- !join_2 -- +1 user10 +2 user11 +3 user12 +4 user13 +5 user14 + +-- !join_3 -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 +5 user5 5.0 false 2.5 + +-- !join_4 -- +1 user1 1.0 false 0.5 2 user2 2.0 true 1.0 +1 user1 1.0 false 0.5 4 user4 4.0 true 2.0 +1 user1 1.0 false 0.5 8 user8 8.0 true 4.0 +1 user1 1.0 false 0.5 9 user9 9.0 false 4.5 + +-- !join_5 -- +1 user1 1.0 false 0.5 10 user10 10.0 true 5.0 1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 11 user11 11.0 false 5.5 2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 12 user12 12.0 true 6.0 3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 13 user13 13.0 false 6.5 4 user4 4.0 true 2.0 +5 user5 5.0 false 2.5 14 user14 14.0 true 7.0 5 user5 5.0 false 2.5 + +-- !1 -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 + +-- !2 -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 +5 user5 5.0 false 2.5 + +-- !3 -- +0.5 1.0 false user1 +1.0 2.0 true user2 +1.5 3.0 false user3 +2.0 4.0 true user4 + +-- !4 -- +1.0 user1 1 +2.0 user2 2 + +-- !test_basic -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 + +-- !test_partial -- +2 user2 1.0 +4 user4 2.0 +6 user6 3.0 + +-- !test_multi_sort -- +1 user1 1.0 0.5 +2 user2 2.0 1.0 +3 user3 3.0 1.5 + +-- !test_filter -- + +-- !test_subquery -- +2 user2 3.0 +4 user4 6.0 +6 user6 9.0 + +-- !test_agg -- +1 0.5 1.0 +2 1.0 2.0 +3 1.5 3.0 + +-- !test_basic -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 +5 user5 5.0 false 2.5 +6 user6 6.0 true 3.0 +7 user7 7.0 false 3.5 + +-- !test_partial -- +2 user2 1.0 +4 user4 2.0 +6 user6 3.0 +8 user8 4.0 + +-- !test_multi_sort -- +1 user1 1.0 0.5 +2 user2 2.0 1.0 +3 user3 3.0 1.5 +4 user4 4.0 2.0 +5 user5 5.0 2.5 +6 user6 6.0 3.0 +7 user7 7.0 3.5 + +-- !test_filter -- + +-- !test_subquery -- +2 user2 3.0 +4 user4 6.0 +6 user6 9.0 +8 user8 12.0 + +-- !test_agg -- +1 0.5 1.0 +2 1.0 2.0 +3 1.5 3.0 +4 2.0 4.0 +5 2.5 5.0 +6 3.0 6.0 +7 3.5 7.0 + +-- !1 -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 + +-- !2 -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 +5 user5 5.0 false 2.5 + +-- !3 -- +0.5 1.0 false user1 +1.0 2.0 true user2 +1.5 3.0 false user3 +2.0 4.0 true user4 + +-- !4 -- +1.0 user1 1 +2.0 user2 2 + +-- !test_basic -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 + +-- !test_partial -- +2 user2 1.0 +4 user4 2.0 +6 user6 3.0 + +-- !test_multi_sort -- +1 user1 1.0 0.5 +2 user2 2.0 1.0 +3 user3 3.0 1.5 + +-- !test_filter -- + +-- !test_subquery -- +2 user2 3.0 +4 user4 6.0 +6 user6 9.0 + +-- !test_agg -- +1 0.5 1.0 +2 1.0 2.0 +3 1.5 3.0 + +-- !test_basic -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 +5 user5 5.0 false 2.5 +6 user6 6.0 true 3.0 +7 user7 7.0 false 3.5 + +-- !test_partial -- +2 user2 1.0 +4 user4 2.0 +6 user6 3.0 +8 user8 4.0 + +-- !test_multi_sort -- +1 user1 1.0 0.5 +2 user2 2.0 1.0 +3 user3 3.0 1.5 +4 user4 4.0 2.0 +5 user5 5.0 2.5 +6 user6 6.0 3.0 +7 user7 7.0 3.5 + +-- !test_filter -- + +-- !test_subquery -- +2 user2 3.0 +4 user4 6.0 +6 user6 9.0 +8 user8 12.0 + +-- !test_agg -- +1 0.5 1.0 +2 1.0 2.0 +3 1.5 3.0 +4 2.0 4.0 +5 2.5 5.0 +6 3.0 6.0 +7 3.5 7.0 + +-- !join_1 -- +1 user1 1.0 false 0.5 1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 4 user4 4.0 true 2.0 +5 user5 5.0 false 2.5 5 user5 5.0 false 2.5 + +-- !join_2 -- +1 user10 +2 user11 +3 user12 +4 user13 +5 user14 + +-- !join_3 -- +1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 +5 user5 5.0 false 2.5 + +-- !join_4 -- +1 user1 1.0 false 0.5 2 user2 2.0 true 1.0 +1 user1 1.0 false 0.5 4 user4 4.0 true 2.0 +1 user1 1.0 false 0.5 8 user8 8.0 true 4.0 +1 user1 1.0 false 0.5 9 user9 9.0 false 4.5 + +-- !join_5 -- +1 user1 1.0 false 0.5 10 user10 10.0 true 5.0 1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 11 user11 11.0 false 5.5 2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 12 user12 12.0 true 6.0 3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 13 user13 13.0 false 6.5 4 user4 4.0 true 2.0 +5 user5 5.0 false 2.5 14 user14 14.0 true 7.0 5 user5 5.0 false 2.5 + diff --git a/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy new file mode 100644 index 00000000000000..30bda05dbcfbc3 --- /dev/null +++ b/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy @@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_tvf_topn_lazy_mat","external,hive,tvf,external_docker") { + String hdfs_port = context.config.otherConfigs.get("hive2HdfsPort") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + + + String enabled = context.config.otherConfigs.get("enableHiveTest") + + + + def hdfsUserName = "doris" + def defaultFS = "hdfs://${externalEnvIp}:${hdfs_port}" + def parquet_uri = "${defaultFS}" + "/user/doris/preinstalled_data/parquet_table/parquet_global_lazy_mat_table/file_id=1/example_1.parquet"; + def orc_uri1 = "${defaultFS}" + "/user/doris/preinstalled_data/orc_table/orc_global_lazy_mat_table/file_id=1/example_1.orc" + def orc_uri2 = "${defaultFS}" + "/user/doris/preinstalled_data/orc_table/orc_global_lazy_mat_table/file_id=2/example_2.orc" + + def tvf_parquet_1 = """ HDFS( + "uri" = "${parquet_uri}", + "hadoop.username" = "doris", + "format" = "parquet") """ + + def tvf_orc_1 = """ HDFS("uri" = "${orc_uri1}","hadoop.username" = "doris","format" = "ORC") """ + def tvf_orc_2 = """ HDFS("uri" = "${orc_uri2}","hadoop.username" = "doris", "format" = "orc") """ + + + def runTopNLazyMatTests = { + def limitValues = [3, 7] + + // Single table query tests, using nested loops to iterate through tables and limit values + for (String table : ["${tvf_orc_1}", "${tvf_parquet_1}"]) { + qt_1 """ select * from ${table} order by id limit 3; """ + qt_2 """ select * from ${table} order by score limit 5; """ + qt_3 """ select score, value, active,name from ${table} order by value limit 4; """ + qt_4 """ select value,name,id from ${table} order by name,score limit 2; """ + + for (int limit : limitValues) { + // Basic query + qt_test_basic """ + select * from ${table} + where value > 0 + order by id, score desc + limit ${limit}; + """ + + // Partial columns query + qt_test_partial """ + select id, name, score + from ${table} + where active = true + order by id, value desc + limit ${limit}; + """ + + // Multi-field sorting + qt_test_multi_sort """ + select id, name, value, score + from ${table} + order by id, score desc, value asc + limit ${limit}; + """ + + // Filter condition query + qt_test_filter """ + select id, name, value + from ${table} + where active = true and value > 100 + order by id, value desc + limit ${limit}; + """ + + // Subquery + qt_test_subquery """ + select t.id, t.name, t.total_score + from ( + select id, name, (value + score) as total_score + from ${table} + where active = true + ) t + order by t.id, t.total_score desc + limit ${limit}; + """ + + // Aggregation query + qt_test_agg """ + select + id, + max(score) as max_score, + avg(value) as avg_value + from ${table} + group by id + order by id, max_score desc + limit ${limit}; + """ + } + } + } + + def runjointest = { + qt_join_1 """ + select * from ${tvf_parquet_1} as a join ${tvf_orc_1} as b on a.id = b.id order by a.name limit 5; + """ + + qt_join_2 """ + select a.id,b.name from ${tvf_parquet_1} as a join ${tvf_orc_2} as b on a.id + 9 = b.id order by a.id limit 5; + """ + + qt_join_3 """ + select a.* from ${tvf_orc_1} as a join ${tvf_orc_2} as b on a.id + 9 = b.id order by b.name limit 5; + """ + + qt_join_4 """ + select * from ${tvf_orc_1} as a join ${tvf_orc_1} as b on a.id = a.value order by a.id limit 4; + """ + + qt_join_5 """ + select * from ${tvf_orc_1} as a join ${tvf_orc_2} as b join ${tvf_parquet_1} as c + on a.value + 9 = b.id and c.name = a.name order by a.score limit 5; + """ + + + } + + + if (enabled != null && enabled.equalsIgnoreCase("true")) { + //id | name | value | active | score + sql """ set enable_topn_lazy_materialization=true; """ + explain { + sql """ verbose select * from ${tvf_parquet_1} order by id limit 5; """ + contains("VMaterializeNode") + contains("projectList:[id, name, value, active, score]") + + contains("column_descs_lists[[`name` text NULL, `value` double NULL, `active` boolean NULL, `score` double NULL]]") + contains("locations: [[1, 2, 3, 4]]") + contains("table_idxs: [[1, 2, 3, 4]]") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__hdfs]") + contains("isTopMaterializeNode: true") + contains("SlotDescriptor{id=0, col=id, colUniqueId=-1, type=bigint, nullable=true") + contains("SlotDescriptor{id=1, col=__DORIS_GLOBAL_ROWID_COL__hdfs, colUniqueId=-1, type=text, nullable=false,") + } + + + explain { + sql """ verbose select name,value,score from ${tvf_orc_1} order by id limit 5; """ + contains("VMaterializeNode") + + contains("projectList:[name, value, score]") + contains("column_descs_lists[[`name` text NULL, `value` double NULL, `score` double NULL]]") + contains("locations: [[1, 2, 3]]") + contains("table_idxs: [[1, 2, 4]]") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__hdfs]") + contains("isTopMaterializeNode: true") + + + contains("SlotDescriptor{id=0, col=id, colUniqueId=-1, type=int, nullable=true") + contains("SlotDescriptor{id=1, col=__DORIS_GLOBAL_ROWID_COL__hdfs, colUniqueId=-1, type=text, nullable=false,") + } + + + explain { + sql """verbose select * from ${tvf_parquet_1} as a join ${tvf_orc_1} as b on a.id = b.id order by a.name limit 5; """ + contains("VMaterializeNode") + contains("projectList:[id, name, value, active, score, id, name, value, active, score]") + contains("column_descs_lists[[`value` double NULL, `active` boolean NULL, `score` double NULL], [`name` text NULL, `value` double NULL, `active` boolean NULL, `score` double NULL]]") + contains("locations: [[3, 4, 5], [6, 7, 8, 9]]") + contains("table_idxs: [[2, 3, 4], [1, 2, 3, 4]]") + contains("row_ids: [__DORIS_GLOBAL_ROWID_COL__hdfs, __DORIS_GLOBAL_ROWID_COL__hdfs]") + contains("isTopMaterializeNode: true") + } + + runTopNLazyMatTests() + runjointest() + + + + + sql """ set enable_topn_lazy_materialization=false; """ + explain { + sql """ verbose select * from ${tvf_parquet_1} order by id limit 5; """ + notContains ("VMaterializeNode") + } + + + explain { + sql """ verbose select name,value,score from ${tvf_orc_1} order by id limit 5; """ + notContains ("VMaterializeNode") + } + + explain { + sql """verbose select * from ${tvf_parquet_1} as a join ${tvf_orc_1} as b on a.id = b.id order by a.name limit 5; """ + notContains ("VMaterializeNode") + } + + + + runTopNLazyMatTests() + runjointest() + } + +} + From 53227b2fbbe998d8d3acc01f44f8e841669238dc Mon Sep 17 00:00:00 2001 From: daidai Date: Tue, 19 Aug 2025 11:35:41 +0800 Subject: [PATCH 4/4] fix case. --- .../tvf/test_tvf_topn_lazy_mat.out | 36 +++++++++---------- .../tvf/test_tvf_topn_lazy_mat.groovy | 14 ++++---- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out b/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out index b965c865606dd4..a104af3c757d89 100644 --- a/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out +++ b/regression-test/data/external_table_p0/tvf/test_tvf_topn_lazy_mat.out @@ -179,34 +179,34 @@ 6 3.0 6.0 7 3.5 7.0 --- !join_1 -- +-- !test_join_1 -- 1 user1 1.0 false 0.5 1 user1 1.0 false 0.5 2 user2 2.0 true 1.0 2 user2 2.0 true 1.0 3 user3 3.0 false 1.5 3 user3 3.0 false 1.5 4 user4 4.0 true 2.0 4 user4 4.0 true 2.0 5 user5 5.0 false 2.5 5 user5 5.0 false 2.5 --- !join_2 -- +-- !test_join_2 -- 1 user10 2 user11 3 user12 4 user13 5 user14 --- !join_3 -- +-- !test_join_3 -- 1 user1 1.0 false 0.5 2 user2 2.0 true 1.0 3 user3 3.0 false 1.5 4 user4 4.0 true 2.0 5 user5 5.0 false 2.5 --- !join_4 -- -1 user1 1.0 false 0.5 2 user2 2.0 true 1.0 -1 user1 1.0 false 0.5 4 user4 4.0 true 2.0 -1 user1 1.0 false 0.5 8 user8 8.0 true 4.0 -1 user1 1.0 false 0.5 9 user9 9.0 false 4.5 +-- !test_join_4 -- +1 user1 1.0 false 0.5 1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 4 user4 4.0 true 2.0 --- !join_5 -- +-- !test_join_5 -- 1 user1 1.0 false 0.5 10 user10 10.0 true 5.0 1 user1 1.0 false 0.5 2 user2 2.0 true 1.0 11 user11 11.0 false 5.5 2 user2 2.0 true 1.0 3 user3 3.0 false 1.5 12 user12 12.0 true 6.0 3 user3 3.0 false 1.5 @@ -393,34 +393,34 @@ 6 3.0 6.0 7 3.5 7.0 --- !join_1 -- +-- !test_join_1 -- 1 user1 1.0 false 0.5 1 user1 1.0 false 0.5 2 user2 2.0 true 1.0 2 user2 2.0 true 1.0 3 user3 3.0 false 1.5 3 user3 3.0 false 1.5 4 user4 4.0 true 2.0 4 user4 4.0 true 2.0 5 user5 5.0 false 2.5 5 user5 5.0 false 2.5 --- !join_2 -- +-- !test_join_2 -- 1 user10 2 user11 3 user12 4 user13 5 user14 --- !join_3 -- +-- !test_join_3 -- 1 user1 1.0 false 0.5 2 user2 2.0 true 1.0 3 user3 3.0 false 1.5 4 user4 4.0 true 2.0 5 user5 5.0 false 2.5 --- !join_4 -- -1 user1 1.0 false 0.5 2 user2 2.0 true 1.0 -1 user1 1.0 false 0.5 4 user4 4.0 true 2.0 -1 user1 1.0 false 0.5 8 user8 8.0 true 4.0 -1 user1 1.0 false 0.5 9 user9 9.0 false 4.5 +-- !test_join_4 -- +1 user1 1.0 false 0.5 1 user1 1.0 false 0.5 +2 user2 2.0 true 1.0 2 user2 2.0 true 1.0 +3 user3 3.0 false 1.5 3 user3 3.0 false 1.5 +4 user4 4.0 true 2.0 4 user4 4.0 true 2.0 --- !join_5 -- +-- !test_join_5 -- 1 user1 1.0 false 0.5 10 user10 10.0 true 5.0 1 user1 1.0 false 0.5 2 user2 2.0 true 1.0 11 user11 11.0 false 5.5 2 user2 2.0 true 1.0 3 user3 3.0 false 1.5 12 user12 12.0 true 6.0 3 user3 3.0 false 1.5 diff --git a/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy index 30bda05dbcfbc3..24acb8648a6340 100644 --- a/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_tvf_topn_lazy_mat.groovy @@ -112,30 +112,30 @@ suite("test_tvf_topn_lazy_mat","external,hive,tvf,external_docker") { } def runjointest = { - qt_join_1 """ + qt_test_join_1 """ select * from ${tvf_parquet_1} as a join ${tvf_orc_1} as b on a.id = b.id order by a.name limit 5; """ - qt_join_2 """ + qt_test_join_2 """ select a.id,b.name from ${tvf_parquet_1} as a join ${tvf_orc_2} as b on a.id + 9 = b.id order by a.id limit 5; """ - qt_join_3 """ + qt_test_join_3 """ select a.* from ${tvf_orc_1} as a join ${tvf_orc_2} as b on a.id + 9 = b.id order by b.name limit 5; """ - qt_join_4 """ - select * from ${tvf_orc_1} as a join ${tvf_orc_1} as b on a.id = a.value order by a.id limit 4; + qt_test_join_4 """ + select * from ${tvf_orc_1} as a join ${tvf_orc_1} as b on a.id = b.value order by a.id limit 4; """ - qt_join_5 """ + qt_test_join_5 """ select * from ${tvf_orc_1} as a join ${tvf_orc_2} as b join ${tvf_parquet_1} as c on a.value + 9 = b.id and c.name = a.name order by a.score limit 5; """ } - + sql """ set disable_join_reorder=true; """ if (enabled != null && enabled.equalsIgnoreCase("true")) { //id | name | value | active | score