Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion benchmarks/src/bin/parquet.rs
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,7 @@ async fn exec_sort(
) -> Result<(usize, std::time::Duration)> {
let start = Instant::now();
let scan = test_file.create_scan(None).await?;
let exec = Arc::new(SortExec::try_new(expr.to_owned(), scan, None)?);
let exec = Arc::new(SortExec::new(expr.to_owned(), scan));
let task_ctx = ctx.task_ctx();
let result = collect(exec, task_ctx).await?;
let elapsed = start.elapsed();
Expand Down
6 changes: 3 additions & 3 deletions datafusion/core/benches/sort.rs
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ impl BenchCase {

let exec = MemoryExec::try_new(partitions, schema, None).unwrap();
let exec =
SortExec::new_with_partitioning(sort.clone(), Arc::new(exec), true, None);
SortExec::new(sort.clone(), Arc::new(exec)).with_preserve_partitioning(true);
let plan = Arc::new(SortPreservingMergeExec::new(sort, Arc::new(exec)));

Self {
Expand All @@ -211,7 +211,7 @@ impl BenchCase {

let exec = MemoryExec::try_new(partitions, schema, None).unwrap();
let exec = Arc::new(CoalescePartitionsExec::new(Arc::new(exec)));
let plan = Arc::new(SortExec::try_new(sort, exec, None).unwrap());
let plan = Arc::new(SortExec::new(sort, exec));

Self {
runtime,
Expand All @@ -231,7 +231,7 @@ impl BenchCase {
let sort = make_sort_exprs(schema.as_ref());

let exec = MemoryExec::try_new(partitions, schema, None).unwrap();
let exec = SortExec::new_with_partitioning(sort, Arc::new(exec), true, None);
let exec = SortExec::new(sort, Arc::new(exec)).with_preserve_partitioning(true);
let plan = Arc::new(CoalescePartitionsExec::new(Arc::new(exec)));

Self {
Expand Down
10 changes: 5 additions & 5 deletions datafusion/core/src/physical_optimizer/global_sort_selection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,12 @@ impl PhysicalOptimizerRule for GlobalSortSelection {
&& !sort_exec.preserve_partitioning()
&& (sort_exec.fetch().is_some() || config.optimizer.repartition_sorts)
{
let sort = SortExec::new_with_partitioning(
let sort = SortExec::new(
sort_exec.expr().to_vec(),
sort_exec.input().clone(),
true,
sort_exec.fetch(),
);
sort_exec.input().clone()
)
.with_fetch(sort_exec.fetch())
.with_preserve_partitioning(true);
let global_sort: Arc<dyn ExecutionPlan> =
Arc::new(SortPreservingMergeExec::new(
sort_exec.expr().to_vec(),
Expand Down
9 changes: 3 additions & 6 deletions datafusion/core/src/physical_optimizer/repartition.rs
Original file line number Diff line number Diff line change
Expand Up @@ -462,12 +462,9 @@ mod tests {
expr: col("c1", &schema()).unwrap(),
options: SortOptions::default(),
}];
Arc::new(SortExec::new_with_partitioning(
sort_exprs,
input,
preserve_partitioning,
None,
))
let new_sort = SortExec::new(sort_exprs, input)
.with_preserve_partitioning(preserve_partitioning);
Arc::new(new_sort)
}

fn projection_exec(input: Arc<dyn ExecutionPlan>) -> Arc<dyn ExecutionPlan> {
Expand Down
39 changes: 14 additions & 25 deletions datafusion/core/src/physical_optimizer/sort_enforcement.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1210,12 +1210,10 @@ mod tests {
sort_expr("non_nullable_col", &schema),
];
let repartition_exec = repartition_exec(spm);
let sort2 = Arc::new(SortExec::new_with_partitioning(
sort_exprs.clone(),
repartition_exec,
true,
None,
)) as _;
let sort2 = Arc::new(
SortExec::new(sort_exprs.clone(), repartition_exec)
.with_preserve_partitioning(true),
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this makes it clearer that the sort is being constructed to preserve the input partitioning

) as _;
let spm2 = sort_preserving_merge_exec(sort_exprs, sort2);

let physical_plan = aggregate_exec(spm2);
Expand Down Expand Up @@ -1253,12 +1251,9 @@ mod tests {

let sort_exprs = vec![sort_expr("non_nullable_col", &schema)];
// let sort = sort_exec(sort_exprs.clone(), union);
let sort = Arc::new(SortExec::new_with_partitioning(
sort_exprs.clone(),
union,
true,
None,
)) as _;
let sort = Arc::new(
SortExec::new(sort_exprs.clone(), union).with_preserve_partitioning(true),
) as _;
let spm = sort_preserving_merge_exec(sort_exprs, sort);

let filter = filter_exec(
Expand Down Expand Up @@ -2264,12 +2259,8 @@ mod tests {
let window = bounded_window_exec("nullable_col", sort_exprs.clone(), memory_exec);
let repartition = repartition_exec(window);

let orig_plan = Arc::new(SortExec::new_with_partitioning(
sort_exprs,
repartition,
false,
None,
)) as Arc<dyn ExecutionPlan>;
let orig_plan =
Arc::new(SortExec::new(sort_exprs, repartition)) as Arc<dyn ExecutionPlan>;

let mut plan = orig_plan.clone();
let rules = vec![
Expand Down Expand Up @@ -2305,12 +2296,10 @@ mod tests {
let repartition = repartition_exec(coalesce_partitions);
let sort_exprs = vec![sort_expr("nullable_col", &schema)];
// Add local sort
let sort = Arc::new(SortExec::new_with_partitioning(
sort_exprs.clone(),
repartition,
true,
None,
)) as _;
let sort = Arc::new(
SortExec::new(sort_exprs.clone(), repartition)
.with_preserve_partitioning(true),
) as _;
let spm = sort_preserving_merge_exec(sort_exprs.clone(), sort);
let sort = sort_exec(sort_exprs, spm);

Expand Down Expand Up @@ -2362,7 +2351,7 @@ mod tests {
input: Arc<dyn ExecutionPlan>,
) -> Arc<dyn ExecutionPlan> {
let sort_exprs = sort_exprs.into_iter().collect();
Arc::new(SortExec::try_new(sort_exprs, input, None).unwrap())
Arc::new(SortExec::new(sort_exprs, input))
}

fn sort_preserving_merge_exec(
Expand Down
6 changes: 4 additions & 2 deletions datafusion/core/src/physical_optimizer/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,12 @@ pub fn add_sort_above(
if !ordering_satisfy(node.output_ordering(), Some(&sort_expr), || {
node.equivalence_properties()
}) {
let new_sort = SortExec::new(sort_expr, node.clone());

*node = Arc::new(if node.output_partitioning().partition_count() > 1 {
SortExec::new_with_partitioning(sort_expr, node.clone(), true, None)
new_sort.with_preserve_partitioning(true)
} else {
SortExec::try_new(sort_expr, node.clone(), None)?
new_sort
}) as _
}
Ok(())
Expand Down
2 changes: 1 addition & 1 deletion datafusion/core/src/physical_plan/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -485,7 +485,7 @@ mod tests {
options: SortOptions::default(),
}];
let memory_exec = Arc::new(MemoryExec::try_new(&[], schema.clone(), None)?) as _;
let sort_exec = Arc::new(SortExec::try_new(sort_expr.clone(), memory_exec, None)?)
let sort_exec = Arc::new(SortExec::new(sort_expr.clone(), memory_exec))
as Arc<dyn ExecutionPlan>;
let memory_exec2 = Arc::new(MemoryExec::try_new(&[], schema, None)?) as _;
// memory_exec2 doesn't have output ordering
Expand Down
4 changes: 3 additions & 1 deletion datafusion/core/src/physical_plan/planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -821,7 +821,9 @@ impl DefaultPhysicalPlanner {
)),
})
.collect::<Result<Vec<_>>>()?;
Ok(Arc::new(SortExec::try_new(sort_expr, physical_input, *fetch)?))
let new_sort = SortExec::new(sort_expr, physical_input)
.with_fetch(*fetch);
Ok(Arc::new(new_sort))
}
LogicalPlan::Join(Join {
left,
Expand Down
Loading