Skip to content

[opt](complex type) support prune nested column through lateral view #58776

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
924060929 merged 2 commits into from
Dec 26, 2025
Merged

[opt](complex type) support prune nested column through lateral view #58776

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6347bb1
prune nested column through lateral view
924060929 Dec 5, 2025
19f685c
fix
924060929 Dec 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion ...e/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -449,9 +449,14 @@ public TAccessPathType getType() {
public void setType(TAccessPathType type) {
this.type = type;
}

public AccessPathBuilder getAccessPathBuilder() {
return accessPathBuilder;
}
}

private static class AccessPathBuilder {
/** AccessPathBuilder */
public static class AccessPathBuilder {
private LinkedList<String> accessPath;

public AccessPathBuilder() {
Expand All @@ -463,6 +468,16 @@ public AccessPathBuilder addPrefix(String prefix) {
return this;
}

public AccessPathBuilder addSuffix(String suffix) {
accessPath.addLast(suffix);
return this;
}

public AccessPathBuilder addSuffix(List<String> suffix) {
accessPath.addAll(suffix);
return this;
}

public AccessPathBuilder removePrefix() {
accessPath.removeFirst();
return this;
Expand Down
152 changes: 152 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathPlanCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,29 @@

package org.apache.doris.nereids.rules.rewrite;

import org.apache.doris.analysis.AccessPathInfo;
import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectAccessPathResult;
import org.apache.doris.nereids.rules.rewrite.AccessPathExpressionCollector.CollectorContext;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.expressions.functions.generator.Explode;
import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeMap;
import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeMapOuter;
import org.apache.doris.nereids.trees.expressions.functions.generator.ExplodeOuter;
import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplode;
import org.apache.doris.nereids.trees.expressions.functions.generator.PosExplodeOuter;
import org.apache.doris.nereids.trees.expressions.literal.StructLiteral;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalTVFRelation;
Expand All @@ -45,6 +56,8 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;

/** AccessPathPlanCollector */
public class AccessPathPlanCollector extends DefaultPlanVisitor<Void, StatementContext> {
Expand All @@ -56,6 +69,145 @@ public Map<Slot, List<CollectAccessPathResult>> collect(Plan root, StatementCont
return scanSlotToAccessPaths;
}

@Override
public Void visitLogicalGenerate(LogicalGenerate<? extends Plan> generate, StatementContext context) {
List<Function> generators = generate.getGenerators();
List<Slot> output = generate.getGeneratorOutput();

AccessPathExpressionCollector exprCollector
= new AccessPathExpressionCollector(context, allSlotToAccessPaths, false);
for (int i = 0; i < output.size(); i++) {
Slot generatorOutput = output.get(i);
Function function = generators.get(i);
Collection<CollectAccessPathResult> accessPaths = allSlotToAccessPaths.get(
generatorOutput.getExprId().asInt());
if (function instanceof Explode || function instanceof ExplodeOuter) {
if (accessPaths.isEmpty()) {
// use the whole column
for (Expression child : function.children()) {
exprCollector.collect(child);
}
} else {
for (CollectAccessPathResult accessPath : accessPaths) {
List<String> path = accessPath.getPath();
if (function.arity() == 1) {
// $c$1.VALUES.b
CollectorContext argumentContext = new CollectorContext(context, false);
argumentContext.setType(accessPath.getType());
argumentContext.getAccessPathBuilder()
.addSuffix(AccessPathInfo.ACCESS_ALL)
.addSuffix(path.subList(1, path.size()));
function.child(0).accept(exprCollector, argumentContext);
continue;
} else if (path.size() >= 2) {
// $c$1.col1.VALUES.b will be extract 'col1'
String colName = path.get(1);
// extract '1' in 'col1'
int colIndex = Integer.parseInt(colName.substring(StructLiteral.COL_PREFIX.length())) - 1;
CollectorContext argumentContext = new CollectorContext(context, false);
argumentContext.setType(accessPath.getType());
argumentContext.getAccessPathBuilder()
.addSuffix(AccessPathInfo.ACCESS_ALL)
.addSuffix(path.subList(2, path.size()));
function.child(colIndex).accept(exprCollector, argumentContext);
continue;
}
// use the whole column
for (Expression child : function.children()) {
exprCollector.collect(child);
}
}
}
} else if (function instanceof ExplodeMap || function instanceof ExplodeMapOuter) {
if (accessPaths.isEmpty()) {
// use the whole column
for (Expression child : function.children()) {
exprCollector.collect(child);
}
} else {
for (CollectAccessPathResult accessPath : accessPaths) {
List<String> path = accessPath.getPath();
if (path.size() >= 2) {
if (path.get(1).equalsIgnoreCase(StructLiteral.COL_PREFIX + "1")) {
// key
for (Expression child : function.children()) {
CollectorContext argumentContext = new CollectorContext(context, false);
argumentContext.setType(accessPath.getType());
argumentContext.getAccessPathBuilder()
.addSuffix(AccessPathInfo.ACCESS_MAP_KEYS)
.addSuffix(path.subList(2, path.size()));
child.accept(exprCollector, argumentContext);
}
continue;
} else if (path.get(1).equalsIgnoreCase(StructLiteral.COL_PREFIX + "2")) {
// value
for (Expression child : function.children()) {
CollectorContext argumentContext = new CollectorContext(context, false);
argumentContext.setType(accessPath.getType());
argumentContext.getAccessPathBuilder()
.addSuffix(AccessPathInfo.ACCESS_MAP_VALUES)
.addSuffix(path.subList(2, path.size()));
child.accept(exprCollector, argumentContext);
}
continue;
}
}
// use the whole column
exprCollector.collect(function.child(0));
}
}
} else if (function instanceof PosExplode || function instanceof PosExplodeOuter) {
if (accessPaths.isEmpty()) {
// use the whole column
for (Expression child : function.children()) {
exprCollector.collect(child);
}
} else {
boolean useWholeItem = false;
Set<Integer> prunedChildIndex = new TreeSet<>();
for (CollectAccessPathResult accessPath : accessPaths) {
List<String> path = accessPath.getPath();
if (path.size() >= 2) {
// $c$1.col1.VALUES.b will be extract 'col1'
String colName = path.get(1);
if (colName.startsWith(StructLiteral.COL_PREFIX)) {
// $c$1.col1.VALUES.b will be extract 'col1'
// extract '1' in 'col1'
int colIndex
= Integer.parseInt(colName.substring(StructLiteral.COL_PREFIX.length())) - 1;
CollectorContext argumentContext = new CollectorContext(context, false);
argumentContext.setType(accessPath.getType());
argumentContext.getAccessPathBuilder()
.addSuffix(AccessPathInfo.ACCESS_ALL)
.addSuffix(path.subList(2, path.size()));
function.child(colIndex).accept(exprCollector, argumentContext);
prunedChildIndex.add(colIndex);
}
} else {
useWholeItem = true;
break;
}
}
if (useWholeItem) {
// use the whole column
for (Expression child : function.children()) {
exprCollector.collect(child);
}
} else {
for (int j = 0; j < function.arity(); j++) {
if (!prunedChildIndex.contains(j)) {
exprCollector.collect(function.child(j));
}
}
}
}
} else {
exprCollector.collect(function);
}
}
return generate.child().accept(this, context);
}

@Override
public Void visitLogicalProject(LogicalProject<? extends Plan> project, StatementContext context) {
AccessPathExpressionCollector exprCollector
Expand Down
6 changes: 6 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SlotTypeReplacer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,12 @@ public Plan visitLogicalGenerate(LogicalGenerate<? extends Plan> generate, Void

Pair<Boolean, List<Function>> replacedGenerators
= replaceExpressions(generate.getGenerators(), false, false);
for (int i = 0; i < replacedGenerators.second.size(); i++) {
DataType dataType = replacedGenerators.second.get(i).getDataType();
replacedDataTypes.put(generate.getGeneratorOutput().get(i).getExprId().asInt(),
new AccessPathInfo(dataType, null, null)
);
}
Pair<Boolean, List<Slot>> replacedGeneratorOutput
= replaceExpressions(generate.getGeneratorOutput(), false, false);
if (replacedGenerators.first || replacedGeneratorOutput.first) {
Expand Down
11 changes: 6 additions & 5 deletions ...src/main/java/org/apache/doris/nereids/trees/expressions/functions/generator/Explode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,14 @@
import org.apache.doris.nereids.trees.expressions.functions.ComputePrecision;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.functions.SearchSignature;
import org.apache.doris.nereids.trees.expressions.literal.StructLiteral;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.NullType;
import org.apache.doris.nereids.types.StructField;
import org.apache.doris.nereids.types.StructType;
import org.apache.doris.nereids.util.ExpressionUtils;

import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
Expand All @@ -42,12 +44,11 @@
* where the first column contains 1, 2, 3, and the second column contains 4, 5, 6.
*/
public class Explode extends TableGeneratingFunction implements CustomSignature, ComputePrecision, AlwaysNullable {

/**
* constructor with one or more argument.
*/
public Explode(Expression[] args) {
super("explode", args);
public Explode(Expression arg, Expression... others) {
super("explode", ExpressionUtils.mergeArguments(arg, others));
}

/** constructor for withChildren and reuse signature */
Expand Down Expand Up @@ -77,10 +78,10 @@ public FunctionSignature customSignature() {
if (children.get(i).getDataType().isNullType()) {
arguments.add(ArrayType.of(NullType.INSTANCE));
structFields.add(
new StructField("col" + (i + 1), NullType.INSTANCE, true, ""));
new StructField(StructLiteral.COL_PREFIX + (i + 1), NullType.INSTANCE, true, ""));
} else if (children.get(i).getDataType().isArrayType()) {
structFields.add(
new StructField("col" + (i + 1),
new StructField(StructLiteral.COL_PREFIX + (i + 1),
((ArrayType) (children.get(i)).getDataType()).getItemType(), true, ""));
arguments.add(children.get(i).getDataType());
} else {
Expand Down
3 changes: 1 addition & 2 deletions ...rg/apache/doris/nereids/trees/expressions/functions/generator/ExplodeJsonArrayDouble.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {

@Override
public Expression rewriteWhenAnalyze() {
Expression[] args = {new Cast(children.get(0), ArrayType.of(DoubleType.INSTANCE))};
return new Explode(args);
return new Explode(new Cast(children.get(0), ArrayType.of(DoubleType.INSTANCE)));
}
}
3 changes: 1 addition & 2 deletions ...ache/doris/nereids/trees/expressions/functions/generator/ExplodeJsonArrayDoubleOuter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {

@Override
public Expression rewriteWhenAnalyze() {
Expression[] args = {new Cast(children.get(0), ArrayType.of(DoubleType.INSTANCE))};
return new ExplodeOuter(args);
return new ExplodeOuter(new Cast(children.get(0), ArrayType.of(DoubleType.INSTANCE)));
}
}
3 changes: 1 addition & 2 deletions ...a/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeJsonArrayInt.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {

@Override
public Expression rewriteWhenAnalyze() {
Expression[] args = {new Cast(children.get(0), ArrayType.of(BigIntType.INSTANCE))};
return new Explode(args);
return new Explode(new Cast(children.get(0), ArrayType.of(BigIntType.INSTANCE)));
}
}
3 changes: 1 addition & 2 deletions .../apache/doris/nereids/trees/expressions/functions/generator/ExplodeJsonArrayIntOuter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {

@Override
public Expression rewriteWhenAnalyze() {
Expression[] args = {new Cast(children.get(0), ArrayType.of(BigIntType.INSTANCE))};
return new ExplodeOuter(args);
return new ExplodeOuter(new Cast(children.get(0), ArrayType.of(BigIntType.INSTANCE)));
}
}
3 changes: 1 addition & 2 deletions .../org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeJsonArrayJson.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {

@Override
public Expression rewriteWhenAnalyze() {
Expression[] args = {new Cast(children.get(0), ArrayType.of(JsonType.INSTANCE))};
return new Explode(args);
return new Explode(new Cast(children.get(0), ArrayType.of(JsonType.INSTANCE)));
}
}
3 changes: 1 addition & 2 deletions ...apache/doris/nereids/trees/expressions/functions/generator/ExplodeJsonArrayJsonOuter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {

@Override
public Expression rewriteWhenAnalyze() {
Expression[] args = {new Cast(children.get(0), ArrayType.of(JsonType.INSTANCE))};
return new ExplodeOuter(args);
return new ExplodeOuter(new Cast(children.get(0), ArrayType.of(JsonType.INSTANCE)));
}
}
3 changes: 1 addition & 2 deletions ...rg/apache/doris/nereids/trees/expressions/functions/generator/ExplodeJsonArrayString.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {

@Override
public Expression rewriteWhenAnalyze() {
Expression[] args = {new Cast(children.get(0), ArrayType.of(VarcharType.SYSTEM_DEFAULT))};
return new Explode(args);
return new Explode(new Cast(children.get(0), ArrayType.of(VarcharType.SYSTEM_DEFAULT)));
}
}
3 changes: 1 addition & 2 deletions ...ache/doris/nereids/trees/expressions/functions/generator/ExplodeJsonArrayStringOuter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {

@Override
public Expression rewriteWhenAnalyze() {
Expression[] args = {new Cast(children.get(0), ArrayType.of(VarcharType.SYSTEM_DEFAULT))};
return new ExplodeOuter(args);
return new ExplodeOuter(new Cast(children.get(0), ArrayType.of(VarcharType.SYSTEM_DEFAULT)));
}
}
6 changes: 4 additions & 2 deletions .../main/java/org/apache/doris/nereids/trees/expressions/functions/generator/ExplodeMap.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
import org.apache.doris.nereids.trees.expressions.literal.StructLiteral;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.MapType;
Expand Down Expand Up @@ -70,10 +71,11 @@ public void checkLegalityBeforeTypeCoercion() {

@Override
public List<FunctionSignature> getSignatures() {
MapType dataType = (MapType) child().getDataType();
return ImmutableList.of(
FunctionSignature.ret(new StructType(ImmutableList.of(
new StructField("col1", ((MapType) child().getDataType()).getKeyType(), true, ""),
new StructField("col2", ((MapType) child().getDataType()).getValueType(), true, ""))))
new StructField(StructLiteral.COL_PREFIX + "1", dataType.getKeyType(), true, ""),
new StructField(StructLiteral.COL_PREFIX + "2", dataType.getValueType(), true, ""))))
.args(child().getDataType())
);
}
Expand Down
Loading
Loading