From 0786966eee0fcf174fd37e3e003e743d2f7660f4 Mon Sep 17 00:00:00 2001 From: irenjj Date: Fri, 7 Mar 2025 23:27:18 +0800 Subject: [PATCH 1/4] Implement tree explain for PartialSortExec --- .../physical-plan/src/sorts/partial_sort.rs | 14 ++++- .../sqllogictest/test_files/explain_tree.slt | 61 ++++++++++++++++++- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/datafusion/physical-plan/src/sorts/partial_sort.rs b/datafusion/physical-plan/src/sorts/partial_sort.rs index bd0e6268de52d..d16963fea6ab2 100644 --- a/datafusion/physical-plan/src/sorts/partial_sort.rs +++ b/datafusion/physical-plan/src/sorts/partial_sort.rs @@ -227,8 +227,18 @@ impl DisplayAs for PartialSortExec { } } DisplayFormatType::TreeRender => { - // TODO: collect info - write!(f, "") + let common_prefix_length = self.common_prefix_length; + match self.fetch { + Some(fetch) => { + writeln!(f, "TopK fetch={fetch}")?; + writeln!(f, "expr=[{}]", self.expr)?; + writeln!(f, "common_prefix_length=[{common_prefix_length}]") + } + None => { + writeln!(f, "expr=[{}]", self.expr)?; + writeln!(f, "common_prefix_length=[{common_prefix_length}]") + } + } } } } diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.slt index 9d50b9bd626e2..965783ad90920 100644 --- a/datafusion/sqllogictest/test_files/explain_tree.slt +++ b/datafusion/sqllogictest/test_files/explain_tree.slt @@ -82,7 +82,18 @@ CREATE EXTERNAL TABLE table5 STORED AS ARROW LOCATION 'test_files/scratch/explain_tree/table5.arrow'; - +statement ok +CREATE UNBOUNDED EXTERNAL TABLE annotated_data_infinite2 ( + a0 INTEGER, + a INTEGER, + b INTEGER, + c INTEGER, + d INTEGER +) +STORED AS CSV +WITH ORDER (a ASC, b ASC, c ASC) +LOCATION '../core/tests/data/window_2.csv' +OPTIONS ('format.has_header' 'true'); ######## Begin Queries ######## @@ -399,6 +410,54 @@ physical_plan 17)│ format: arrow │ 18)└───────────────────────────┘ +# Query with PartialSortExec. +query TT +EXPLAIN SELECT * +FROM annotated_data_infinite2 +ORDER BY a, b, d; +---- +logical_plan +01)Sort: annotated_data_infinite2.a ASC NULLS LAST, annotated_data_infinite2.b ASC NULLS LAST, annotated_data_infinite2.d ASC NULLS LAST +02)--TableScan: annotated_data_infinite2 projection=[a0, a, b, c, d] +physical_plan +01)┌───────────────────────────┐ +02)│ PartialSortExec │ +03)│ -------------------- │ +04)│ common_prefix_length: │ +05)│ [2] │ +06)│ │ +07)│ expr: │ +08)│[a@1 ASC NULLS LAST, b@2...│ +09)└─────────────┬─────────────┘ +10)┌─────────────┴─────────────┐ +11)│ StreamingTableExec │ +12)└───────────────────────────┘ + +query TT +EXPLAIN SELECT * +FROM annotated_data_infinite2 +ORDER BY a, b, d +LIMIT 50; +---- +logical_plan +01)Sort: annotated_data_infinite2.a ASC NULLS LAST, annotated_data_infinite2.b ASC NULLS LAST, annotated_data_infinite2.d ASC NULLS LAST, fetch=50 +02)--TableScan: annotated_data_infinite2 projection=[a0, a, b, c, d] +physical_plan +01)┌───────────────────────────┐ +02)│ PartialSortExec │ +03)│ -------------------- │ +04)│ TopK fetch: 50 │ +05)│ │ +06)│ common_prefix_length: │ +07)│ [2] │ +08)│ │ +09)│ expr: │ +10)│[a@1 ASC NULLS LAST, b@2...│ +11)└─────────────┬─────────────┘ +12)┌─────────────┴─────────────┐ +13)│ StreamingTableExec │ +14)└───────────────────────────┘ + # cleanup statement ok drop table table1; From d5afa3eb16638518b85d622ca3b61c044cc0246d Mon Sep 17 00:00:00 2001 From: irenjj Date: Sat, 8 Mar 2025 08:31:18 +0800 Subject: [PATCH 2/4] simplify --- .../physical-plan/src/sorts/partial_sort.rs | 21 +++++------ .../sqllogictest/test_files/explain_tree.slt | 36 +++++++++---------- 2 files changed, 25 insertions(+), 32 deletions(-) diff --git a/datafusion/physical-plan/src/sorts/partial_sort.rs b/datafusion/physical-plan/src/sorts/partial_sort.rs index d16963fea6ab2..9e97eb264b14c 100644 --- a/datafusion/physical-plan/src/sorts/partial_sort.rs +++ b/datafusion/physical-plan/src/sorts/partial_sort.rs @@ -226,20 +226,15 @@ impl DisplayAs for PartialSortExec { None => write!(f, "PartialSortExec: expr=[{}], common_prefix_length=[{common_prefix_length}]", self.expr), } } - DisplayFormatType::TreeRender => { - let common_prefix_length = self.common_prefix_length; - match self.fetch { - Some(fetch) => { - writeln!(f, "TopK fetch={fetch}")?; - writeln!(f, "expr=[{}]", self.expr)?; - writeln!(f, "common_prefix_length=[{common_prefix_length}]") - } - None => { - writeln!(f, "expr=[{}]", self.expr)?; - writeln!(f, "common_prefix_length=[{common_prefix_length}]") - } + DisplayFormatType::TreeRender => match self.fetch { + Some(fetch) => { + writeln!(f, "fetch={fetch}")?; + writeln!(f, "sort=[{}]", self.expr) } - } + None => { + writeln!(f, "sort=[{}]", self.expr) + } + }, } } } diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.slt index f811bddbe805e..ab5ae26065ea8 100644 --- a/datafusion/sqllogictest/test_files/explain_tree.slt +++ b/datafusion/sqllogictest/test_files/explain_tree.slt @@ -543,15 +543,14 @@ physical_plan 01)┌───────────────────────────┐ 02)│ PartialSortExec │ 03)│ -------------------- │ -04)│ common_prefix_length: │ -05)│ [2] │ -06)│ │ -07)│ expr: │ -08)│[a@1 ASC NULLS LAST, b@2...│ -09)└─────────────┬─────────────┘ -10)┌─────────────┴─────────────┐ -11)│ StreamingTableExec │ -12)└───────────────────────────┘ +04)│ sort: │ +05)│ [a@1 ASC NULLS LAST, b@2 │ +06)│ ASC NULLS LAST, d@4 │ +07)│ ASC NULLS LAST] │ +08)└─────────────┬─────────────┘ +09)┌─────────────┴─────────────┐ +10)│ StreamingTableExec │ +11)└───────────────────────────┘ query TT EXPLAIN SELECT * @@ -566,17 +565,16 @@ physical_plan 01)┌───────────────────────────┐ 02)│ PartialSortExec │ 03)│ -------------------- │ -04)│ TopK fetch: 50 │ +04)│ fetch: 50 │ 05)│ │ -06)│ common_prefix_length: │ -07)│ [2] │ -08)│ │ -09)│ expr: │ -10)│[a@1 ASC NULLS LAST, b@2...│ -11)└─────────────┬─────────────┘ -12)┌─────────────┴─────────────┐ -13)│ StreamingTableExec │ -14)└───────────────────────────┘ +06)│ sort: │ +07)│ [a@1 ASC NULLS LAST, b@2 │ +08)│ ASC NULLS LAST, d@4 │ +09)│ ASC NULLS LAST] │ +10)└─────────────┬─────────────┘ +11)┌─────────────┴─────────────┐ +12)│ StreamingTableExec │ +13)└───────────────────────────┘ # cleanup statement ok From 9670f4170c4998e9312743124468fdfb4b72607f Mon Sep 17 00:00:00 2001 From: irenjj Date: Sat, 8 Mar 2025 13:16:27 +0800 Subject: [PATCH 3/4] fix --- datafusion/physical-plan/src/sorts/partial_sort.rs | 6 +++--- datafusion/sqllogictest/test_files/explain_tree.slt | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datafusion/physical-plan/src/sorts/partial_sort.rs b/datafusion/physical-plan/src/sorts/partial_sort.rs index 9e97eb264b14c..a82e3ee32ac55 100644 --- a/datafusion/physical-plan/src/sorts/partial_sort.rs +++ b/datafusion/physical-plan/src/sorts/partial_sort.rs @@ -228,11 +228,11 @@ impl DisplayAs for PartialSortExec { } DisplayFormatType::TreeRender => match self.fetch { Some(fetch) => { - writeln!(f, "fetch={fetch}")?; - writeln!(f, "sort=[{}]", self.expr) + writeln!(f, "limit={fetch}")?; + writeln!(f, "sort keys=[{}]", self.expr) } None => { - writeln!(f, "sort=[{}]", self.expr) + writeln!(f, "sort keys=[{}]", self.expr) } }, } diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.slt index ab5ae26065ea8..9218c6db931c7 100644 --- a/datafusion/sqllogictest/test_files/explain_tree.slt +++ b/datafusion/sqllogictest/test_files/explain_tree.slt @@ -543,7 +543,7 @@ physical_plan 01)┌───────────────────────────┐ 02)│ PartialSortExec │ 03)│ -------------------- │ -04)│ sort: │ +04)│ sort keys: │ 05)│ [a@1 ASC NULLS LAST, b@2 │ 06)│ ASC NULLS LAST, d@4 │ 07)│ ASC NULLS LAST] │ @@ -565,9 +565,9 @@ physical_plan 01)┌───────────────────────────┐ 02)│ PartialSortExec │ 03)│ -------------------- │ -04)│ fetch: 50 │ +04)│ limit: 50 │ 05)│ │ -06)│ sort: │ +06)│ sort keys: │ 07)│ [a@1 ASC NULLS LAST, b@2 │ 08)│ ASC NULLS LAST, d@4 │ 09)│ ASC NULLS LAST] │ From 2006ebbdad8b9d08ca9bca0a8236fd63a85c7eab Mon Sep 17 00:00:00 2001 From: irenjj Date: Sat, 8 Mar 2025 20:18:43 +0800 Subject: [PATCH 4/4] remove square brackets --- datafusion/physical-plan/src/sorts/partial_sort.rs | 4 ++-- datafusion/sqllogictest/test_files/explain_tree.slt | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/datafusion/physical-plan/src/sorts/partial_sort.rs b/datafusion/physical-plan/src/sorts/partial_sort.rs index a82e3ee32ac55..4cff0fedfaa63 100644 --- a/datafusion/physical-plan/src/sorts/partial_sort.rs +++ b/datafusion/physical-plan/src/sorts/partial_sort.rs @@ -229,10 +229,10 @@ impl DisplayAs for PartialSortExec { DisplayFormatType::TreeRender => match self.fetch { Some(fetch) => { writeln!(f, "limit={fetch}")?; - writeln!(f, "sort keys=[{}]", self.expr) + writeln!(f, "sort keys={}", self.expr) } None => { - writeln!(f, "sort keys=[{}]", self.expr) + writeln!(f, "sort keys={}", self.expr) } }, } diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.slt index 9218c6db931c7..bf935e2ff6586 100644 --- a/datafusion/sqllogictest/test_files/explain_tree.slt +++ b/datafusion/sqllogictest/test_files/explain_tree.slt @@ -544,9 +544,9 @@ physical_plan 02)│ PartialSortExec │ 03)│ -------------------- │ 04)│ sort keys: │ -05)│ [a@1 ASC NULLS LAST, b@2 │ +05)│ a@1 ASC NULLS LAST, b@2 │ 06)│ ASC NULLS LAST, d@4 │ -07)│ ASC NULLS LAST] │ +07)│ ASC NULLS LAST │ 08)└─────────────┬─────────────┘ 09)┌─────────────┴─────────────┐ 10)│ StreamingTableExec │ @@ -568,9 +568,9 @@ physical_plan 04)│ limit: 50 │ 05)│ │ 06)│ sort keys: │ -07)│ [a@1 ASC NULLS LAST, b@2 │ +07)│ a@1 ASC NULLS LAST, b@2 │ 08)│ ASC NULLS LAST, d@4 │ -09)│ ASC NULLS LAST] │ +09)│ ASC NULLS LAST │ 10)└─────────────┬─────────────┘ 11)┌─────────────┴─────────────┐ 12)│ StreamingTableExec │