diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index c6e5a5f1b427d8..171d6ed137bf6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -263,6 +263,7 @@ import org.apache.doris.nereids.trees.expressions.CaseWhen; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.DefaultValueSlot; +import org.apache.doris.nereids.trees.expressions.DereferenceExpression; import org.apache.doris.nereids.trees.expressions.Divide; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Exists; @@ -2257,8 +2258,7 @@ public Expression visitDereference(DereferenceContext ctx) { UnboundSlot slot = new UnboundSlot(nameParts, Optional.empty()); return slot; } else { - // todo: base is an expression, may be not a table name. - throw new ParseException("Unsupported dereference expression: " + ctx.getText(), ctx); + return new DereferenceExpression(e, new StringLiteral(ctx.identifier().getText())); } }); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java index 8ce5cc270965a0..890adf9babf725 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java @@ -440,12 +440,12 @@ private LogicalHaving bindHavingAggregate( Scope groupBySlotsScope = toScope(cascadesContext, groupBySlots.build()); return (analyzer, unboundSlot) -> { - List boundInGroupBy = analyzer.bindSlotByScope(unboundSlot, groupBySlotsScope); + List boundInGroupBy = analyzer.bindSlotByScope(unboundSlot, groupBySlotsScope); if (!boundInGroupBy.isEmpty()) { return ImmutableList.of(boundInGroupBy.get(0)); } - List boundInAggOutput = analyzer.bindSlotByScope(unboundSlot, aggOutputScope); + List boundInAggOutput = analyzer.bindSlotByScope(unboundSlot, aggOutputScope); if (!boundInAggOutput.isEmpty()) { return ImmutableList.of(boundInAggOutput.get(0)); } @@ -515,7 +515,7 @@ private LogicalHaving bindHavingByScopes( SimpleExprAnalyzer analyzer = buildCustomSlotBinderAnalyzer( having, cascadesContext, defaultScope, false, true, (self, unboundSlot) -> { - List slots = self.bindSlotByScope(unboundSlot, defaultScope); + List slots = self.bindSlotByScope(unboundSlot, defaultScope); if (!slots.isEmpty()) { return slots; } @@ -837,7 +837,7 @@ private List bindGroupBy( // see: https://github.com/apache/doris/pull/15240 // // first, try to bind by agg.child.output - List slotsInChildren = self.bindExactSlotsByThisScope(unboundSlot, childOutputScope); + List slotsInChildren = self.bindExactSlotsByThisScope(unboundSlot, childOutputScope); if (slotsInChildren.size() == 1) { // bind succeed return slotsInChildren; @@ -845,7 +845,7 @@ private List bindGroupBy( // second, bind failed: // if the slot not found, or more than one candidate slots found in agg.child.output, // then try to bind by agg.output - List slotsInOutput = self.bindExactSlotsByThisScope( + List slotsInOutput = self.bindExactSlotsByThisScope( unboundSlot, aggOutputScopeWithoutAggFun.get()); if (slotsInOutput.isEmpty()) { // if slotsInChildren.size() > 1 && slotsInOutput.isEmpty(), @@ -854,7 +854,7 @@ private List bindGroupBy( } Builder useOutputExpr = ImmutableList.builderWithExpectedSize(slotsInOutput.size()); - for (Slot slotInOutput : slotsInOutput) { + for (Expression slotInOutput : slotsInOutput) { // mappingSlot is provided by aggOutputScopeWithoutAggFun // and no non-MappingSlot slot exist in the Scope, so we // can direct cast it safely @@ -945,7 +945,7 @@ private Plan bindSortWithoutSetOperation(MatchingContext> ctx) sort, cascadesContext, inputScope, true, false, (self, unboundSlot) -> { // first, try to bind slot in Scope(input.output) - List slotsInInput = self.bindExactSlotsByThisScope(unboundSlot, inputScope); + List slotsInInput = self.bindExactSlotsByThisScope(unboundSlot, inputScope); if (!slotsInInput.isEmpty()) { // bind succeed return ImmutableList.of(slotsInInput.get(0)); @@ -1160,7 +1160,7 @@ private SimpleExprAnalyzer getAnalyzerForOrderByAggFunc(Plan finalInput, Cascade sort, cascadesContext, inputScope, true, false, (analyzer, unboundSlot) -> { if (finalInput instanceof LogicalAggregate) { - List boundInOutputWithoutAggFunc = analyzer.bindSlotByScope(unboundSlot, + List boundInOutputWithoutAggFunc = analyzer.bindSlotByScope(unboundSlot, outputWithoutAggFunc); if (!boundInOutputWithoutAggFunc.isEmpty()) { return ImmutableList.of(boundInOutputWithoutAggFunc.get(0)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java index c2dce7f591ea79..c717e22f8baf57 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ExpressionAnalyzer.java @@ -48,6 +48,7 @@ import org.apache.doris.nereids.trees.expressions.CaseWhen; import org.apache.doris.nereids.trees.expressions.Cast; import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.DereferenceExpression; import org.apache.doris.nereids.trees.expressions.Divide; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -71,6 +72,7 @@ import org.apache.doris.nereids.trees.expressions.functions.agg.NullableAggregateFunction; import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt; import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda; +import org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement; import org.apache.doris.nereids.trees.expressions.functions.udf.AliasUdfBuilder; import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdaf; import org.apache.doris.nereids.trees.expressions.functions.udf.JavaUdf; @@ -88,6 +90,8 @@ import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.StructField; +import org.apache.doris.nereids.types.StructType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.util.ExpressionUtils; import org.apache.doris.nereids.util.TypeCoercionUtils; @@ -277,6 +281,25 @@ public Expression visitUnboundAlias(UnboundAlias unboundAlias, ExpressionRewrite } } + @Override + public Expression visitDereferenceExpression(DereferenceExpression dereferenceExpression, + ExpressionRewriteContext context) { + Expression expression = dereferenceExpression.child(0).accept(this, context); + DataType dataType = expression.getDataType(); + if (dataType.isStructType()) { + StructType structType = (StructType) dataType; + StructField field = structType.getField(dereferenceExpression.fieldName); + if (field != null) { + return new StructElement(expression, dereferenceExpression.child(1)); + } + } else if (dataType.isMapType()) { + return new ElementAt(expression, dereferenceExpression.child(1)); + } else if (dataType.isVariantType()) { + return new ElementAt(expression, dereferenceExpression.child(1)); + } + throw new AnalysisException("Can not dereference field: " + dereferenceExpression.fieldName); + } + @Override public Expression visitUnboundSlot(UnboundSlot unboundSlot, ExpressionRewriteContext context) { Optional outerScope = getScope().getOuterScope(); @@ -887,13 +910,13 @@ protected List bindSlotByThisScope(UnboundSlot unboundSlot return bindSlotByScope(unboundSlot, getScope()); } - protected List bindExactSlotsByThisScope(UnboundSlot unboundSlot, Scope scope) { - List candidates = bindSlotByScope(unboundSlot, scope); + protected List bindExactSlotsByThisScope(UnboundSlot unboundSlot, Scope scope) { + List candidates = bindSlotByScope(unboundSlot, scope); if (candidates.size() == 1) { return candidates; } - List extractSlots = Utils.filterImmutableList(candidates, bound -> - unboundSlot.getNameParts().size() == bound.getQualifier().size() + 1 + List extractSlots = Utils.filterImmutableList(candidates, bound -> + bound instanceof Slot && unboundSlot.getNameParts().size() == ((Slot) bound).getQualifier().size() + 1 ); // we should return origin candidates slots if extract slots is empty, // and then throw an ambiguous exception @@ -912,33 +935,137 @@ private List addSqlIndexInfo(List slots, Optional bindSlotByScope(UnboundSlot unboundSlot, Scope scope) { + public List bindSlotByScope(UnboundSlot unboundSlot, Scope scope) { List nameParts = unboundSlot.getNameParts(); Optional> idxInSql = unboundSlot.getIndexInSqlString(); int namePartSize = nameParts.size(); switch (namePartSize) { // column case 1: { - return addSqlIndexInfo(bindSingleSlotByName(nameParts.get(0), scope), idxInSql); + return (List) bindExpressionByColumn(unboundSlot, nameParts, idxInSql, scope); } // table.column case 2: { - return addSqlIndexInfo(bindSingleSlotByTable(nameParts.get(0), nameParts.get(1), scope), idxInSql); + return (List) bindExpressionByTableColumn(unboundSlot, nameParts, idxInSql, scope); } // db.table.column case 3: { - return addSqlIndexInfo(bindSingleSlotByDb(nameParts.get(0), nameParts.get(1), nameParts.get(2), scope), - idxInSql); + return (List) bindExpressionByDbTableColumn(unboundSlot, nameParts, idxInSql, scope); } // catalog.db.table.column - case 4: { - return addSqlIndexInfo(bindSingleSlotByCatalog( - nameParts.get(0), nameParts.get(1), nameParts.get(2), nameParts.get(3), scope), idxInSql); - } default: { - throw new AnalysisException("Not supported name: " + StringUtils.join(nameParts, ".")); + return (List) bindExpressionByCatalogDbTableColumn(unboundSlot, nameParts, idxInSql, scope); + } + } + } + + private List bindExpressionByCatalogDbTableColumn( + UnboundSlot unboundSlot, List nameParts, Optional> idxInSql, Scope scope) { + List slots = addSqlIndexInfo(bindSingleSlotByCatalog( + nameParts.get(0), nameParts.get(1), nameParts.get(2), nameParts.get(3), scope), idxInSql); + if (slots.isEmpty()) { + return bindExpressionByDbTableColumn(unboundSlot, nameParts, idxInSql, scope); + } else if (slots.size() > 1) { + return slots; + } + if (nameParts.size() == 4) { + return slots; + } + + Optional expression = bindNestedFields( + unboundSlot, slots.get(0), nameParts.subList(4, nameParts.size()) + ); + if (!expression.isPresent()) { + return slots; + } + return ImmutableList.of(expression.get()); + } + + private List bindExpressionByDbTableColumn( + UnboundSlot unboundSlot, List nameParts, Optional> idxInSql, Scope scope) { + List slots = addSqlIndexInfo( + bindSingleSlotByDb(nameParts.get(0), nameParts.get(1), nameParts.get(2), scope), idxInSql); + if (slots.isEmpty()) { + return bindExpressionByTableColumn(unboundSlot, nameParts, idxInSql, scope); + } else if (slots.size() > 1) { + return slots; + } + if (nameParts.size() == 3) { + return slots; + } + + Optional expression = bindNestedFields( + unboundSlot, slots.get(0), nameParts.subList(3, nameParts.size()) + ); + if (!expression.isPresent()) { + return slots; + } + return ImmutableList.of(expression.get()); + } + + private List bindExpressionByTableColumn( + UnboundSlot unboundSlot, List nameParts, Optional> idxInSql, Scope scope) { + List slots = addSqlIndexInfo(bindSingleSlotByTable(nameParts.get(0), nameParts.get(1), scope), idxInSql); + if (slots.isEmpty()) { + return bindExpressionByColumn(unboundSlot, nameParts, idxInSql, scope); + } else if (slots.size() > 1) { + return slots; + } + if (nameParts.size() == 2) { + return slots; + } + + Optional expression = bindNestedFields( + unboundSlot, slots.get(0), nameParts.subList(2, nameParts.size()) + ); + if (!expression.isPresent()) { + return slots; + } + return ImmutableList.of(expression.get()); + } + + private List bindExpressionByColumn( + UnboundSlot unboundSlot, List nameParts, Optional> idxInSql, Scope scope) { + List slots = addSqlIndexInfo(bindSingleSlotByName(nameParts.get(0), scope), idxInSql); + if (slots.size() != 1) { + return slots; + } + if (nameParts.size() == 1) { + return slots; + } + Optional expression = bindNestedFields( + unboundSlot, slots.get(0), nameParts.subList(1, nameParts.size()) + ); + if (!expression.isPresent()) { + return slots; + } + return ImmutableList.of(expression.get()); + } + + private Optional bindNestedFields(UnboundSlot unboundSlot, Slot slot, List fieldNames) { + Expression expression = slot; + String lastFieldName = slot.getName(); + for (String fieldName : fieldNames) { + DataType dataType = expression.getDataType(); + if (dataType.isStructType()) { + StructType structType = (StructType) dataType; + StructField field = structType.getField(fieldName); + if (field == null) { + throw new AnalysisException("No such struct field '" + fieldName + "' in '" + lastFieldName + "'"); + } + lastFieldName = fieldName; + expression = new StructElement(expression, new StringLiteral(fieldName)); + continue; + } else if (dataType.isMapType()) { + expression = new ElementAt(expression, new StringLiteral(fieldName)); + continue; + } else if (dataType.isVariantType()) { + expression = new ElementAt(expression, new StringLiteral(fieldName)); + continue; } + throw new AnalysisException("No such field '" + fieldName + "' in '" + lastFieldName + "'"); } + return Optional.of(new Alias(expression)); } public static boolean compareDbName(String boundedDbName, String unBoundDbName) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/DereferenceExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/DereferenceExpression.java new file mode 100644 index 00000000000000..41f6048cf30c4b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/DereferenceExpression.java @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions; + +import org.apache.doris.nereids.analyzer.Unbound; +import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable; +import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; + +import com.google.common.collect.ImmutableList; + +/** DereferenceExpression*/ +public class DereferenceExpression extends Expression implements BinaryExpression, PropagateNullable, Unbound { + public final String fieldName; + + public DereferenceExpression(Expression expression, StringLiteral fieldName) { + super(ImmutableList.of(expression, fieldName)); + this.fieldName = fieldName.getValue(); + } + + @Override + public R accept(ExpressionVisitor visitor, C context) { + return visitor.visitDereferenceExpression(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java index c159589e6db690..09284662cb5fa4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java @@ -40,6 +40,7 @@ import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; import org.apache.doris.nereids.trees.expressions.CompoundPredicate; import org.apache.doris.nereids.trees.expressions.DefaultValueSlot; +import org.apache.doris.nereids.trees.expressions.DereferenceExpression; import org.apache.doris.nereids.trees.expressions.Divide; import org.apache.doris.nereids.trees.expressions.EqualTo; import org.apache.doris.nereids.trees.expressions.Exists; @@ -531,4 +532,8 @@ public R visitUnboundStar(UnboundStar unboundStar, C context) { public R visitUnboundVariable(UnboundVariable unboundVariable, C context) { return visit(unboundVariable, context); } + + public R visitDereferenceExpression(DereferenceExpression dereferenceExpression, C context) { + return visit(dereferenceExpression, context); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java index 20f0947b321572..0374c576d3e1ac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/StructType.java @@ -71,6 +71,10 @@ public Map getNameToFields() { return nameToFields.get(); } + public StructField getField(String name) { + return nameToFields.get().get(name.toLowerCase()); + } + @Override public DataType conversion() { return new StructType(fields.stream().map(StructField::conversion).collect(Collectors.toList())); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/TestDereference.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/TestDereference.java new file mode 100644 index 00000000000000..4731a4c988b513 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/TestDereference.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.analysis; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.VariantType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.test.TestExternalCatalog.TestCatalogProvider; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class TestDereference extends TestWithFeService { + + private static final Map>> CATALOG_META = ImmutableMap.of( + "t", ImmutableMap.of( + "t", ImmutableList.of( + new Column("id", PrimitiveType.INT), + new Column("t", new VariantType()) + ) + ) + ); + + @Override + protected void runBeforeAll() throws Exception { + FeConstants.runningUnitTest = true; + createCatalog("create catalog t properties(" + + " \"type\"=\"test\"," + + " \"catalog_provider.class\"=\"org.apache.doris.nereids.rules.analysis.TestDereference$CustomCatalogProvider\"" + + ")"); + connectContext.changeDefaultCatalog("t"); + useDatabase("t"); + } + + @Test + public void testBindPriority() { + // column + testBind("select t from t"); + // table.column + testBind("select t.t from t"); + // db.table.column + testBind("select t.t.t from t"); + // catalog.db.table.column + testBind("select t.t.t.t from t"); + // catalog.db.table.column.subColumn + testBind("select t.t.t.t.t from t"); + // catalog.db.table.column.subColumn.subColumn2 + testBind("select t.t.t.t.t.t from t"); + } + + private void testBind(String sql) { + PlanChecker.from(connectContext) + .analyze(sql) + .rewrite(); + } + + public static class CustomCatalogProvider implements TestCatalogProvider { + + @Override + public Map>> getMetadata() { + return CATALOG_META; + } + } +} diff --git a/regression-test/suites/query_p0/test_dereference.groovy b/regression-test/suites/query_p0/test_dereference.groovy new file mode 100644 index 00000000000000..30c123e3c37e49 --- /dev/null +++ b/regression-test/suites/query_p0/test_dereference.groovy @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import com.google.common.collect.Lists + +suite("test_dereference") { + multi_sql """ + drop table if exists test_dereference; + create table test_dereference( + id int, + a array, + m map, + s struct, + v variant + ) + distributed by hash(id) + properties( + 'replication_num'='1' + ); + + insert into test_dereference + values (1, array(1, 2, 3, 4, 5), map('a', 1, 'b', 2, 'c', 3), struct(1, 2), '{"v": {"v":200}}') + """ + + test { + sql "select cardinality(a), map_size(m), map_keys(m), map_values(m), m.a, m.b, m.c, s.a, s.b, v.v.v from test_dereference" + result([[5L, 3L, '["a", "b", "c"]', '[1, 2, 3]', 1, 2, 3, 1, 2d, "200"]]) + } + + multi_sql """ + drop table if exists test_dereference2; + create table test_dereference2( + id int, + s struct>>, + v variant + ) + distributed by hash(id) + properties( + 'replication_num'='1' + ); + + insert into test_dereference2 + values (1, struct(struct(struct(100))), '{"v": {"v": 200}}') + """ + + test { + sql "select s.s.s.s, v.v.v from test_dereference2" + result([[100, "200"]]) + } + + test { + sql "select s.a from test_dereference2" + exception "No such struct field 'a' in 's'" + } +} \ No newline at end of file