Skip to content

separate projection and alias-projection #4232

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jackwener wants to merge 1 commit into from
Closed

separate projection and alias-projection #4232

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions datafusion/core/tests/sql/explain_analyze.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -744,9 +744,10 @@ async fn test_physical_plan_display_indent_multi_children() {
" CoalesceBatchesExec: target_batch_size=4096",
" RepartitionExec: partitioning=Hash([Column { name: \"c2\", index: 0 }], 9000)",
" ProjectionExec: expr=[c2@0 as c2]",
" ProjectionExec: expr=[c1@0 as c2]",
" RepartitionExec: partitioning=RoundRobinBatch(9000)",
" CsvExec: files=[ARROW_TEST_DATA/csv/aggregate_test_100.csv], has_header=true, limit=None, projection=[c1, c2]",
" ProjectionExec: expr=[c2@0 as c2]",
" ProjectionExec: expr=[c1@0 as c2]",
" RepartitionExec: partitioning=RoundRobinBatch(9000)",
" CsvExec: files=[ARROW_TEST_DATA/csv/aggregate_test_100.csv], has_header=true, limit=None, projection=[c1, c2]",
];

let normalizer = ExplainNormalizer::new();
Expand Down
65 changes: 35 additions & 30 deletions datafusion/core/tests/sql/subqueries.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,15 @@ where c_acctbal < (
Inner Join: customer.c_custkey = __sq_2.o_custkey
TableScan: customer projection=[c_custkey, c_acctbal]
Projection: orders.o_custkey, SUM(orders.o_totalprice) AS __value, alias=__sq_2
Aggregate: groupBy=[[orders.o_custkey]], aggr=[[SUM(orders.o_totalprice)]]
Filter: CAST(orders.o_totalprice AS Decimal128(25, 2)) < __sq_1.__value
Inner Join: orders.o_orderkey = __sq_1.l_orderkey
TableScan: orders projection=[o_orderkey, o_custkey, o_totalprice]
Projection: lineitem.l_orderkey, SUM(lineitem.l_extendedprice) AS price AS __value, alias=__sq_1
Aggregate: groupBy=[[lineitem.l_orderkey]], aggr=[[SUM(lineitem.l_extendedprice)]]
TableScan: lineitem projection=[l_orderkey, l_extendedprice]"#
Projection: orders.o_custkey
Aggregate: groupBy=[[orders.o_custkey]], aggr=[[SUM(orders.o_totalprice)]]
Filter: CAST(orders.o_totalprice AS Decimal128(25, 2)) < __sq_1.__value
Inner Join: orders.o_orderkey = __sq_1.l_orderkey
TableScan: orders projection=[o_orderkey, o_custkey, o_totalprice]
Projection: lineitem.l_orderkey, SUM(lineitem.l_extendedprice) AS price AS __value, alias=__sq_1
Projection: lineitem.l_orderkey
Comment on lines +64 to +65
Copy link
Member Author

@jackwener jackwener Nov 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Problems like this indicate pushdown rule exist some bug.

Aggregate: groupBy=[[lineitem.l_orderkey]], aggr=[[SUM(lineitem.l_extendedprice)]]
TableScan: lineitem projection=[l_orderkey, l_extendedprice]"#
.to_string();
assert_eq!(actual, expected);

Expand Down Expand Up @@ -154,15 +156,16 @@ order by s_acctbal desc, n_name, s_name, p_partkey;"#;
Filter: region.r_name = Utf8("EUROPE")
TableScan: region projection=[r_regionkey, r_name], partial_filters=[region.r_name = Utf8("EUROPE")]
Projection: partsupp.ps_partkey, MIN(partsupp.ps_supplycost) AS __value, alias=__sq_1
Aggregate: groupBy=[[partsupp.ps_partkey]], aggr=[[MIN(partsupp.ps_supplycost)]]
Inner Join: nation.n_regionkey = region.r_regionkey
Inner Join: supplier.s_nationkey = nation.n_nationkey
Inner Join: partsupp.ps_suppkey = supplier.s_suppkey
TableScan: partsupp projection=[ps_partkey, ps_suppkey, ps_supplycost]
TableScan: supplier projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment]
TableScan: nation projection=[n_nationkey, n_name, n_regionkey]
Filter: region.r_name = Utf8("EUROPE")
TableScan: region projection=[r_regionkey, r_name], partial_filters=[region.r_name = Utf8("EUROPE")]"#
Projection: partsupp.ps_partkey
Aggregate: groupBy=[[partsupp.ps_partkey]], aggr=[[MIN(partsupp.ps_supplycost)]]
Inner Join: nation.n_regionkey = region.r_regionkey
Inner Join: supplier.s_nationkey = nation.n_nationkey
Inner Join: partsupp.ps_suppkey = supplier.s_suppkey
TableScan: partsupp projection=[ps_partkey, ps_suppkey, ps_supplycost]
TableScan: supplier projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment]
TableScan: nation projection=[n_nationkey, n_name, n_regionkey]
Filter: region.r_name = Utf8("EUROPE")
TableScan: region projection=[r_regionkey, r_name], partial_filters=[region.r_name = Utf8("EUROPE")]"#
.to_string();
assert_eq!(actual, expected);

Expand Down Expand Up @@ -336,9 +339,10 @@ order by s_name;
Filter: part.p_name LIKE Utf8("forest%")
TableScan: part projection=[p_partkey, p_name], partial_filters=[part.p_name LIKE Utf8("forest%")]
Projection: lineitem.l_partkey, lineitem.l_suppkey, Float64(0.5) * CAST(SUM(lineitem.l_quantity) AS Float64) AS __value, alias=__sq_3
Aggregate: groupBy=[[lineitem.l_partkey, lineitem.l_suppkey]], aggr=[[SUM(lineitem.l_quantity)]]
Filter: lineitem.l_shipdate >= Date32("8766")
TableScan: lineitem projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], partial_filters=[lineitem.l_shipdate >= Date32("8766")]"#
Projection: lineitem.l_partkey, lineitem.l_suppkey
Aggregate: groupBy=[[lineitem.l_partkey, lineitem.l_suppkey]], aggr=[[SUM(lineitem.l_quantity)]]
Filter: lineitem.l_shipdate >= Date32("8766")
TableScan: lineitem projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], partial_filters=[lineitem.l_shipdate >= Date32("8766")]"#
.to_string();
assert_eq!(actual, expected);

Expand Down Expand Up @@ -383,17 +387,18 @@ order by cntrycode;"#;
Projection: custsale.cntrycode, COUNT(UInt8(1)) AS numcust, SUM(custsale.c_acctbal) AS totacctbal
Aggregate: groupBy=[[custsale.cntrycode]], aggr=[[COUNT(UInt8(1)), SUM(custsale.c_acctbal)]]
Projection: cntrycode, customer.c_acctbal, alias=custsale
Projection: substr(customer.c_phone, Int64(1), Int64(2)) AS cntrycode, customer.c_acctbal
Filter: CAST(customer.c_acctbal AS Decimal128(19, 6)) > __sq_1.__value
CrossJoin:
LeftAnti Join: customer.c_custkey = orders.o_custkey
Filter: substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8("13"), Utf8("31"), Utf8("23"), Utf8("29"), Utf8("30"), Utf8("18"), Utf8("17")])
TableScan: customer projection=[c_custkey, c_phone, c_acctbal], partial_filters=[substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8("13"), Utf8("31"), Utf8("23"), Utf8("29"), Utf8("30"), Utf8("18"), Utf8("17")])]
TableScan: orders projection=[o_custkey]
Projection: AVG(customer.c_acctbal) AS __value, alias=__sq_1
Aggregate: groupBy=[[]], aggr=[[AVG(customer.c_acctbal)]]
Filter: customer.c_acctbal > Decimal128(Some(0),15,2) AND substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8("13"), Utf8("31"), Utf8("23"), Utf8("29"), Utf8("30"), Utf8("18"), Utf8("17")])
TableScan: customer projection=[c_phone, c_acctbal], partial_filters=[CAST(customer.c_acctbal AS Decimal128(30, 15)) > Decimal128(Some(0),30,15), substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8("13"), Utf8("31"), Utf8("23"), Utf8("29"), Utf8("30"), Utf8("18"), Utf8("17")]), customer.c_acctbal > Decimal128(Some(0),15,2)]"#
Projection: cntrycode, customer.c_acctbal
Projection: substr(customer.c_phone, Int64(1), Int64(2)) AS cntrycode, customer.c_acctbal
Filter: CAST(customer.c_acctbal AS Decimal128(19, 6)) > __sq_1.__value
CrossJoin:
LeftAnti Join: customer.c_custkey = orders.o_custkey
Filter: substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8("13"), Utf8("31"), Utf8("23"), Utf8("29"), Utf8("30"), Utf8("18"), Utf8("17")])
TableScan: customer projection=[c_custkey, c_phone, c_acctbal], partial_filters=[substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8("13"), Utf8("31"), Utf8("23"), Utf8("29"), Utf8("30"), Utf8("18"), Utf8("17")])]
TableScan: orders projection=[o_custkey]
Projection: AVG(customer.c_acctbal) AS __value, alias=__sq_1
Aggregate: groupBy=[[]], aggr=[[AVG(customer.c_acctbal)]]
Filter: customer.c_acctbal > Decimal128(Some(0),15,2) AND substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8("13"), Utf8("31"), Utf8("23"), Utf8("29"), Utf8("30"), Utf8("18"), Utf8("17")])
TableScan: customer projection=[c_phone, c_acctbal], partial_filters=[CAST(customer.c_acctbal AS Decimal128(30, 15)) > Decimal128(Some(0),30,15), substr(customer.c_phone, Int64(1), Int64(2)) IN ([Utf8("13"), Utf8("31"), Utf8("23"), Utf8("29"), Utf8("30"), Utf8("18"), Utf8("17")]), customer.c_acctbal > Decimal128(Some(0),15,2)]"#
.to_string();
assert_eq!(expected, actual);

Expand Down
52 changes: 29 additions & 23 deletions datafusion/core/tests/sql/window.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,17 +336,20 @@ async fn window_expr_eliminate() -> Result<()> {
" Projection: d.b, MAX(d.a) AS max_a [b:Utf8, max_a:Int64;N]",
" Aggregate: groupBy=[[d.b]], aggr=[[MAX(d.a)]] [b:Utf8, MAX(d.a):Int64;N]",
" Projection: _data2.a, _data2.b, alias=d [a:Int64, b:Utf8]",
" Projection: s.a, s.b, alias=_data2 [a:Int64, b:Utf8]",
" Projection: a, b, alias=s [a:Int64, b:Utf8]",
" Union [a:Int64, b:Utf8]",
" Projection: Int64(1) AS a, Utf8(\"aa\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
" Projection: Int64(3) AS a, Utf8(\"aa\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
" Projection: Int64(5) AS a, Utf8(\"bb\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
" Projection: Int64(7) AS a, Utf8(\"bb\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
" Projection: _data2.a, _data2.b [a:Int64, b:Utf8]",
" Projection: s.a, s.b, alias=_data2 [a:Int64, b:Utf8]",
" Projection: s.a, s.b [a:Int64, b:Utf8]",
" Projection: a, b, alias=s [a:Int64, b:Utf8]",
" Projection: a, b [a:Int64, b:Utf8]",
" Union [a:Int64, b:Utf8]",
" Projection: Int64(1) AS a, Utf8(\"aa\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
" Projection: Int64(3) AS a, Utf8(\"aa\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
" Projection: Int64(5) AS a, Utf8(\"bb\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
" Projection: Int64(7) AS a, Utf8(\"bb\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
];
let formatted = plan.display_indent_schema().to_string();
let actual: Vec<&str> = formatted.trim().lines().collect();
Expand Down Expand Up @@ -399,18 +402,21 @@ async fn window_expr_eliminate() -> Result<()> {
" Projection: d.b, MAX(d.a) AS max_a, MAX(d.seq) [b:Utf8, max_a:Int64;N, MAX(d.seq):UInt64;N]",
" Aggregate: groupBy=[[d.b]], aggr=[[MAX(d.a), MAX(d.seq)]] [b:Utf8, MAX(d.a):Int64;N, MAX(d.seq):UInt64;N]",
" Projection: _data2.seq, _data2.a, _data2.b, alias=d [seq:UInt64;N, a:Int64, b:Utf8]",
" Projection: ROW_NUMBER() PARTITION BY [s.b] ORDER BY [s.a ASC NULLS LAST] AS seq, s.a, s.b, alias=_data2 [seq:UInt64;N, a:Int64, b:Utf8]",
" WindowAggr: windowExpr=[[ROW_NUMBER() PARTITION BY [s.b] ORDER BY [s.a ASC NULLS LAST]]] [ROW_NUMBER() PARTITION BY [s.b] ORDER BY [s.a ASC NULLS LAST]:UInt64;N, a:Int64, b:Utf8]",
" Projection: a, b, alias=s [a:Int64, b:Utf8]",
" Union [a:Int64, b:Utf8]",
" Projection: Int64(1) AS a, Utf8(\"aa\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
" Projection: Int64(3) AS a, Utf8(\"aa\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
" Projection: Int64(5) AS a, Utf8(\"bb\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
" Projection: Int64(7) AS a, Utf8(\"bb\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
" Projection: _data2.seq, _data2.a, _data2.b [seq:UInt64;N, a:Int64, b:Utf8]",
" Projection: ROW_NUMBER() PARTITION BY [s.b] ORDER BY [s.a ASC NULLS LAST] AS seq, s.a, s.b, alias=_data2 [seq:UInt64;N, a:Int64, b:Utf8]",
" Projection: s.a, s.b [a:Int64, b:Utf8]",
" WindowAggr: windowExpr=[[ROW_NUMBER() PARTITION BY [s.b] ORDER BY [s.a ASC NULLS LAST]]] [ROW_NUMBER() PARTITION BY [s.b] ORDER BY [s.a ASC NULLS LAST]:UInt64;N, a:Int64, b:Utf8]",
" Projection: a, b, alias=s [a:Int64, b:Utf8]",
" Projection: a, b [a:Int64, b:Utf8]",
" Union [a:Int64, b:Utf8]",
" Projection: Int64(1) AS a, Utf8(\"aa\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
" Projection: Int64(3) AS a, Utf8(\"aa\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
" Projection: Int64(5) AS a, Utf8(\"bb\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
" Projection: Int64(7) AS a, Utf8(\"bb\") AS b [a:Int64, b:Utf8]",
" EmptyRelation []",
];
let formatted = plan.display_indent_schema().to_string();
let actual: Vec<&str> = formatted.trim().lines().collect();
Expand Down
Loading