diff --git a/benchmarks/src/bin/parquet_filter_pushdown.rs b/benchmarks/src/bin/parquet_filter_pushdown.rs index 86cd79d2ad139..46a65587f194b 100644 --- a/benchmarks/src/bin/parquet_filter_pushdown.rs +++ b/benchmarks/src/bin/parquet_filter_pushdown.rs @@ -22,12 +22,11 @@ use arrow::array::{ use arrow::datatypes::{DataType, Field, Int32Type, Schema, SchemaRef, TimeUnit}; use arrow::record_batch::RecordBatch; use arrow::util::pretty; -use datafusion::common::Result; +use datafusion::common::{Result, ToDFSchema}; use datafusion::datasource::listing::{ListingTableUrl, PartitionedFile}; use datafusion::datasource::object_store::ObjectStoreUrl; use datafusion::execution::context::ExecutionProps; use datafusion::logical_expr::{lit, or, Expr}; -use datafusion::logical_plan::ToDFSchema; use datafusion::optimizer::utils::combine_filters_disjunctive; use datafusion::physical_expr::create_physical_expr; use datafusion::physical_plan::collect; diff --git a/datafusion-examples/examples/custom_datasource.rs b/datafusion-examples/examples/custom_datasource.rs index fd8f945f27701..7256c94ff0cfe 100644 --- a/datafusion-examples/examples/custom_datasource.rs +++ b/datafusion-examples/examples/custom_datasource.rs @@ -20,16 +20,17 @@ use datafusion::arrow::array::{UInt64Builder, UInt8Builder}; use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion::arrow::record_batch::RecordBatch; use datafusion::dataframe::DataFrame; +use datafusion::datasource::provider_as_source; use datafusion::datasource::{TableProvider, TableType}; use datafusion::error::Result; use datafusion::execution::context::{SessionState, TaskContext}; -use datafusion::logical_plan::{provider_as_source, Expr, LogicalPlanBuilder}; use datafusion::physical_plan::expressions::PhysicalSortExpr; use datafusion::physical_plan::memory::MemoryStream; use datafusion::physical_plan::{ project_schema, ExecutionPlan, SendableRecordBatchStream, Statistics, }; use datafusion::prelude::*; +use datafusion_expr::{Expr, LogicalPlanBuilder}; use std::any::Any; use std::collections::{BTreeMap, HashMap}; use std::fmt::{Debug, Formatter}; diff --git a/datafusion-examples/examples/expr_api.rs b/datafusion-examples/examples/expr_api.rs index fe7a7936d5de7..5392e20f264bc 100644 --- a/datafusion-examples/examples/expr_api.rs +++ b/datafusion-examples/examples/expr_api.rs @@ -16,13 +16,12 @@ // under the License. use datafusion::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; - use datafusion::error::Result; -use datafusion::logical_plan::ToDFSchema; use datafusion::optimizer::expr_simplifier::{ExprSimplifier, SimplifyContext}; use datafusion::physical_expr::execution_props::ExecutionProps; use datafusion::prelude::*; -use datafusion::{logical_plan::Operator, scalar::ScalarValue}; +use datafusion_common::{ScalarValue, ToDFSchema}; +use datafusion_expr::Operator; /// This example demonstrates the DataFusion [`Expr`] API. /// diff --git a/datafusion-examples/examples/simple_udaf.rs b/datafusion-examples/examples/simple_udaf.rs index 0afb5727195c4..c35c214059a4b 100644 --- a/datafusion-examples/examples/simple_udaf.rs +++ b/datafusion-examples/examples/simple_udaf.rs @@ -21,11 +21,11 @@ use datafusion::arrow::{ array::ArrayRef, array::Float32Array, array::Float64Array, datatypes::DataType, record_batch::RecordBatch, }; - use datafusion::from_slice::FromSlice; use datafusion::logical_expr::AggregateState; -use datafusion::{error::Result, logical_plan::create_udaf, physical_plan::Accumulator}; +use datafusion::{error::Result, physical_plan::Accumulator}; use datafusion::{logical_expr::Volatility, prelude::*, scalar::ScalarValue}; +use datafusion_expr::create_udaf; use std::sync::Arc; // create local session context with an in-memory table diff --git a/datafusion/core/src/dataframe.rs b/datafusion/core/src/dataframe.rs index a5caad1765581..c52f8259f2d04 100644 --- a/datafusion/core/src/dataframe.rs +++ b/datafusion/core/src/dataframe.rs @@ -27,9 +27,9 @@ use crate::execution::{ context::{SessionState, TaskContext}, FunctionRegistry, }; -use crate::logical_expr::{utils::find_window_exprs, TableType}; -use crate::logical_plan::{ - col, DFSchema, Expr, JoinType, LogicalPlan, LogicalPlanBuilder, Partitioning, +use crate::logical_expr::{ + col, utils::find_window_exprs, Expr, JoinType, LogicalPlan, LogicalPlanBuilder, + Partitioning, TableType, }; use crate::physical_plan::file_format::{plan_to_csv, plan_to_json, plan_to_parquet}; use crate::physical_plan::SendableRecordBatchStream; @@ -38,7 +38,7 @@ use crate::physical_plan::{execute_stream, execute_stream_partitioned, Execution use crate::prelude::SessionContext; use crate::scalar::ScalarValue; use async_trait::async_trait; -use datafusion_common::Column; +use datafusion_common::{Column, DFSchema}; use parking_lot::RwLock; use parquet::file::properties::WriterProperties; use std::any::Any; diff --git a/datafusion/core/src/datasource/datasource.rs b/datafusion/core/src/datasource/datasource.rs index 64e964e0e5aec..300cfecb96313 100644 --- a/datafusion/core/src/datasource/datasource.rs +++ b/datafusion/core/src/datasource/datasource.rs @@ -26,7 +26,7 @@ pub use datafusion_expr::{TableProviderFilterPushDown, TableType}; use crate::arrow::datatypes::SchemaRef; use crate::error::Result; use crate::execution::context::SessionState; -use crate::logical_plan::Expr; +use crate::logical_expr::Expr; use crate::physical_plan::ExecutionPlan; /// Source table diff --git a/datafusion/core/src/datasource/empty.rs b/datafusion/core/src/datasource/empty.rs index b8bdb5fd87886..df459955539e8 100644 --- a/datafusion/core/src/datasource/empty.rs +++ b/datafusion/core/src/datasource/empty.rs @@ -26,7 +26,7 @@ use async_trait::async_trait; use crate::datasource::{TableProvider, TableType}; use crate::error::Result; use crate::execution::context::SessionState; -use crate::logical_plan::Expr; +use crate::logical_expr::Expr; use crate::physical_plan::project_schema; use crate::physical_plan::{empty::EmptyExec, ExecutionPlan}; diff --git a/datafusion/core/src/datasource/file_format/avro.rs b/datafusion/core/src/datasource/file_format/avro.rs index dec368808c442..c2007dc384d88 100644 --- a/datafusion/core/src/datasource/file_format/avro.rs +++ b/datafusion/core/src/datasource/file_format/avro.rs @@ -28,7 +28,7 @@ use object_store::{GetResult, ObjectMeta, ObjectStore}; use super::FileFormat; use crate::avro_to_arrow::read_avro_schema_from_reader; use crate::error::Result; -use crate::logical_plan::Expr; +use crate::logical_expr::Expr; use crate::physical_plan::file_format::{AvroExec, FileScanConfig}; use crate::physical_plan::ExecutionPlan; use crate::physical_plan::Statistics; diff --git a/datafusion/core/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rs index 6a99e35b812cc..f3d170d9ad6e1 100644 --- a/datafusion/core/src/datasource/file_format/csv.rs +++ b/datafusion/core/src/datasource/file_format/csv.rs @@ -35,7 +35,7 @@ use super::FileFormat; use crate::datasource::file_format::file_type::FileCompressionType; use crate::datasource::file_format::DEFAULT_SCHEMA_INFER_MAX_RECORD; use crate::error::Result; -use crate::logical_plan::Expr; +use crate::logical_expr::Expr; use crate::physical_plan::file_format::{CsvExec, FileScanConfig}; use crate::physical_plan::ExecutionPlan; use crate::physical_plan::Statistics; diff --git a/datafusion/core/src/datasource/file_format/json.rs b/datafusion/core/src/datasource/file_format/json.rs index 02a684e852445..05c6f0d662a21 100644 --- a/datafusion/core/src/datasource/file_format/json.rs +++ b/datafusion/core/src/datasource/file_format/json.rs @@ -36,7 +36,7 @@ use super::FileScanConfig; use crate::datasource::file_format::file_type::FileCompressionType; use crate::datasource::file_format::DEFAULT_SCHEMA_INFER_MAX_RECORD; use crate::error::Result; -use crate::logical_plan::Expr; +use crate::logical_expr::Expr; use crate::physical_plan::file_format::NdJsonExec; use crate::physical_plan::ExecutionPlan; use crate::physical_plan::Statistics; diff --git a/datafusion/core/src/datasource/file_format/mod.rs b/datafusion/core/src/datasource/file_format/mod.rs index 7b9421bc7d715..82f5b1df8839e 100644 --- a/datafusion/core/src/datasource/file_format/mod.rs +++ b/datafusion/core/src/datasource/file_format/mod.rs @@ -32,7 +32,7 @@ use std::sync::Arc; use crate::arrow::datatypes::SchemaRef; use crate::error::Result; -use crate::logical_plan::Expr; +use crate::logical_expr::Expr; use crate::physical_plan::file_format::FileScanConfig; use crate::physical_plan::{ExecutionPlan, Statistics}; diff --git a/datafusion/core/src/datasource/file_format/parquet.rs b/datafusion/core/src/datasource/file_format/parquet.rs index bf0488932fa17..23bb3557ac096 100644 --- a/datafusion/core/src/datasource/file_format/parquet.rs +++ b/datafusion/core/src/datasource/file_format/parquet.rs @@ -41,7 +41,7 @@ use crate::arrow::array::{ use crate::arrow::datatypes::{DataType, Field}; use crate::datasource::{create_max_min_accs, get_col_stats}; use crate::error::Result; -use crate::logical_plan::Expr; +use crate::logical_expr::Expr; use crate::physical_plan::expressions::{MaxAccumulator, MinAccumulator}; use crate::physical_plan::file_format::{ParquetExec, SchemaAdapter}; use crate::physical_plan::{Accumulator, ExecutionPlan, Statistics}; diff --git a/datafusion/core/src/datasource/listing/helpers.rs b/datafusion/core/src/datasource/listing/helpers.rs index 9efe490505bfb..23571c7f3daaf 100644 --- a/datafusion/core/src/datasource/listing/helpers.rs +++ b/datafusion/core/src/datasource/listing/helpers.rs @@ -357,7 +357,7 @@ fn parse_partitions_for_path<'a>( #[cfg(test)] mod tests { - use crate::logical_plan::{case, col, lit}; + use crate::logical_expr::{case, col, lit}; use crate::test::object_store::make_test_store; use futures::StreamExt; diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index 72f5b98270305..3a0c4dceea244 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -42,7 +42,7 @@ use crate::logical_expr::TableProviderFilterPushDown; use crate::{ error::{DataFusionError, Result}, execution::context::SessionState, - logical_plan::Expr, + logical_expr::Expr, physical_plan::{ empty::EmptyExec, file_format::{FileScanConfig, DEFAULT_PARTITION_COLUMN_DATATYPE}, @@ -499,7 +499,7 @@ mod tests { use crate::prelude::SessionContext; use crate::{ datasource::file_format::{avro::AvroFormat, parquet::ParquetFormat}, - logical_plan::{col, lit}, + logical_expr::{col, lit}, test::{columns, object_store::register_test_store}, }; use arrow::datatypes::DataType; diff --git a/datafusion/core/src/datasource/memory.rs b/datafusion/core/src/datasource/memory.rs index 62dca1ea04ab9..d965324d818f2 100644 --- a/datafusion/core/src/datasource/memory.rs +++ b/datafusion/core/src/datasource/memory.rs @@ -30,7 +30,7 @@ use async_trait::async_trait; use crate::datasource::{TableProvider, TableType}; use crate::error::{DataFusionError, Result}; use crate::execution::context::{SessionState, TaskContext}; -use crate::logical_plan::Expr; +use crate::logical_expr::Expr; use crate::physical_plan::common; use crate::physical_plan::memory::MemoryExec; use crate::physical_plan::ExecutionPlan; diff --git a/datafusion/core/src/datasource/view.rs b/datafusion/core/src/datasource/view.rs index 534359b837132..42e847b53c3ff 100644 --- a/datafusion/core/src/datasource/view.rs +++ b/datafusion/core/src/datasource/view.rs @@ -25,7 +25,7 @@ use datafusion_expr::LogicalPlanBuilder; use crate::{ error::Result, - logical_plan::{Expr, LogicalPlan}, + logical_expr::{Expr, LogicalPlan}, physical_plan::ExecutionPlan, }; diff --git a/datafusion/core/src/execution/context.rs b/datafusion/core/src/execution/context.rs index 35670f21fcee9..734becca30a6b 100644 --- a/datafusion/core/src/execution/context.rs +++ b/datafusion/core/src/execution/context.rs @@ -29,7 +29,7 @@ use crate::{ }, MemTable, ViewTable, }, - logical_plan::{PlanType, ToStringifiedPlan}, + logical_expr::{PlanType, ToStringifiedPlan}, optimizer::optimizer::Optimizer, physical_optimizer::{ aggregate_statistics::AggregateStatistics, @@ -59,13 +59,15 @@ use crate::catalog::{ schema::{MemorySchemaProvider, SchemaProvider}, }; use crate::dataframe::DataFrame; -use crate::datasource::listing::{ListingTableConfig, ListingTableUrl}; -use crate::datasource::TableProvider; +use crate::datasource::{ + listing::{ListingTableConfig, ListingTableUrl}, + provider_as_source, TableProvider, +}; use crate::error::{DataFusionError, Result}; -use crate::logical_plan::{ - provider_as_source, CreateCatalog, CreateCatalogSchema, CreateExternalTable, - CreateMemoryTable, CreateView, DropTable, FunctionRegistry, LogicalPlan, - LogicalPlanBuilder, UNNAMED_TABLE, +use crate::logical_expr::{ + CreateCatalog, CreateCatalogSchema, CreateExternalTable, CreateMemoryTable, + CreateView, DropTable, DropView, Explain, LogicalPlan, LogicalPlanBuilder, + TableSource, TableType, UNNAMED_TABLE, }; use crate::optimizer::optimizer::{OptimizerConfig, OptimizerRule}; use datafusion_sql::{ResolvedTableReference, TableReference}; @@ -80,8 +82,7 @@ use crate::config::{ }; use crate::datasource::datasource::TableProviderFactory; use crate::datasource::file_format::file_type::{FileCompressionType, FileType}; -use crate::execution::runtime_env::RuntimeEnv; -use crate::logical_expr::Explain; +use crate::execution::{runtime_env::RuntimeEnv, FunctionRegistry}; use crate::physical_plan::file_format::{plan_to_csv, plan_to_json, plan_to_parquet}; use crate::physical_plan::planner::DefaultPhysicalPlanner; use crate::physical_plan::udaf::AggregateUDF; @@ -92,8 +93,6 @@ use crate::variable::{VarProvider, VarType}; use async_trait::async_trait; use chrono::{DateTime, Utc}; use datafusion_common::ScalarValue; -use datafusion_expr::logical_plan::DropView; -use datafusion_expr::{TableSource, TableType}; use datafusion_sql::{ parser::DFParser, planner::{ContextProvider, SqlToRel}, @@ -1868,20 +1867,17 @@ impl FunctionRegistry for TaskContext { #[cfg(test)] mod tests { use super::*; + use crate::assert_batches_eq; use crate::execution::context::QueryPlanner; + use crate::physical_plan::expressions::AvgAccumulator; use crate::test; use crate::test_util::parquet_test_data; use crate::variable::VarType; - use crate::{ - assert_batches_eq, - logical_plan::{create_udf, Expr}, - }; - use crate::{logical_plan::create_udaf, physical_plan::expressions::AvgAccumulator}; use arrow::array::ArrayRef; use arrow::datatypes::*; use arrow::record_batch::RecordBatch; use async_trait::async_trait; - use datafusion_expr::Volatility; + use datafusion_expr::{create_udaf, create_udf, Expr, Volatility}; use datafusion_physical_expr::functions::make_scalar_function; use std::fs::File; use std::sync::Weak; @@ -2398,7 +2394,7 @@ mod tests { fn create_physical_expr( &self, _expr: &Expr, - _input_dfschema: &crate::logical_plan::DFSchema, + _input_dfschema: &crate::common::DFSchema, _input_schema: &Schema, _session_state: &SessionState, ) -> Result> { diff --git a/datafusion/core/src/lib.rs b/datafusion/core/src/lib.rs index 74782359ec8e4..30bf6dc7eb7db 100644 --- a/datafusion/core/src/lib.rs +++ b/datafusion/core/src/lib.rs @@ -221,10 +221,6 @@ pub mod dataframe; pub mod datasource; pub mod error; pub mod execution; -#[deprecated] -// logical_plan module just contains re-exports and will be removed in a future release -// https://github.com/apache/arrow-datafusion/issues/2683 -pub mod logical_plan; pub mod physical_optimizer; pub mod physical_plan; pub mod prelude; diff --git a/datafusion/core/src/logical_plan/expr.rs b/datafusion/core/src/logical_plan/expr.rs deleted file mode 100644 index 771888cbc5237..0000000000000 --- a/datafusion/core/src/logical_plan/expr.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! This is a legacy module that only contains re-exports of other modules - -pub use datafusion_common::{Column, ExprSchema}; -pub use datafusion_expr::{expr_fn::*, lit, lit_timestamp_nano, Expr, Literal, Operator}; diff --git a/datafusion/core/src/logical_plan/mod.rs b/datafusion/core/src/logical_plan/mod.rs deleted file mode 100644 index 57da79a742037..0000000000000 --- a/datafusion/core/src/logical_plan/mod.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! This is a legacy module that only contains re-exports of other modules - -mod expr; -pub mod window_frames; - -pub use crate::datasource::{provider_as_source, source_as_provider}; -pub use crate::execution::FunctionRegistry; -pub use datafusion_common::{ - Column, DFField, DFSchema, DFSchemaRef, ExprSchema, ToDFSchema, -}; -pub use datafusion_expr::{ - abs, acos, and, approx_distinct, approx_percentile_cont, array, ascii, asin, atan, - atan2, avg, bit_length, btrim, call_fn, case, cast, ceil, character_length, chr, - coalesce, col, concat, concat_expr, concat_ws, concat_ws_expr, cos, count, - count_distinct, create_udaf, create_udf, date_part, date_trunc, digest, exists, exp, - expr_rewriter, - expr_rewriter::{ - normalize_col, normalize_col_with_schemas, normalize_cols, replace_col, - rewrite_sort_cols_by_aggs, unnormalize_col, unnormalize_cols, ExprRewritable, - ExprRewriter, RewriteRecursion, - }, - expr_visitor::{ExprVisitable, ExpressionVisitor, Recursion}, - floor, from_unixtime, in_list, in_subquery, initcap, left, length, lit, - lit_timestamp_nano, ln, log10, log2, - logical_plan::{ - builder::{ - build_join_schema, union_with_alias, LogicalPlanBuilder, UNNAMED_TABLE, - }, - CreateCatalog, CreateCatalogSchema, CreateExternalTable, CreateMemoryTable, - CreateView, CrossJoin, DropTable, EmptyRelation, JoinConstraint, JoinType, Limit, - LogicalPlan, Partitioning, PlanType, PlanVisitor, Repartition, StringifiedPlan, - Subquery, TableScan, ToStringifiedPlan, Union, UserDefinedLogicalNode, Values, - }, - lower, lpad, ltrim, max, md5, min, not_exists, not_in_subquery, now, nullif, - octet_length, or, power, random, regexp_match, regexp_replace, repeat, replace, - reverse, right, round, rpad, rtrim, scalar_subquery, sha224, sha256, sha384, sha512, - signum, sin, split_part, sqrt, starts_with, strpos, substr, sum, tan, to_hex, - to_timestamp_micros, to_timestamp_millis, to_timestamp_seconds, translate, trim, - trunc, upper, when, Expr, ExprSchemable, Literal, Operator, -}; diff --git a/datafusion/core/src/logical_plan/window_frames.rs b/datafusion/core/src/logical_plan/window_frames.rs deleted file mode 100644 index 1ff7331b9ff4d..0000000000000 --- a/datafusion/core/src/logical_plan/window_frames.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! This is a legacy module that only contains re-exports of other modules - -pub use datafusion_expr::{WindowFrame, WindowFrameBound, WindowFrameUnits}; diff --git a/datafusion/core/src/physical_optimizer/aggregate_statistics.rs b/datafusion/core/src/physical_optimizer/aggregate_statistics.rs index bb1e49cf3a9d6..e0c81d8677272 100644 --- a/datafusion/core/src/physical_optimizer/aggregate_statistics.rs +++ b/datafusion/core/src/physical_optimizer/aggregate_statistics.rs @@ -265,7 +265,7 @@ mod tests { use datafusion_physical_expr::PhysicalExpr; use crate::error::Result; - use crate::logical_plan::Operator; + use crate::logical_expr::Operator; use crate::physical_plan::aggregates::{AggregateExec, PhysicalGroupBy}; use crate::physical_plan::coalesce_partitions::CoalescePartitionsExec; use crate::physical_plan::common; diff --git a/datafusion/core/src/physical_optimizer/hash_build_probe_order.rs b/datafusion/core/src/physical_optimizer/hash_build_probe_order.rs index 1cd68aaaef959..b4b0deb029c21 100644 --- a/datafusion/core/src/physical_optimizer/hash_build_probe_order.rs +++ b/datafusion/core/src/physical_optimizer/hash_build_probe_order.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use arrow::datatypes::Schema; use crate::execution::context::SessionConfig; -use crate::logical_plan::JoinType; +use crate::logical_expr::JoinType; use crate::physical_plan::cross_join::CrossJoinExec; use crate::physical_plan::expressions::Column; use crate::physical_plan::hash_join::HashJoinExec; diff --git a/datafusion/core/src/physical_optimizer/pruning.rs b/datafusion/core/src/physical_optimizer/pruning.rs index 107bbed461433..8abd5f7c428d3 100644 --- a/datafusion/core/src/physical_optimizer/pruning.rs +++ b/datafusion/core/src/physical_optimizer/pruning.rs @@ -34,8 +34,9 @@ use std::{collections::HashSet, sync::Arc}; use crate::execution::context::ExecutionProps; use crate::prelude::lit; use crate::{ + common::{Column, DFSchema}, error::{DataFusionError, Result}, - logical_plan::{Column, DFSchema, Expr, Operator}, + logical_expr::{Expr, Operator}, physical_plan::{ColumnarValue, PhysicalExpr}, }; use arrow::record_batch::RecordBatchOptions; @@ -46,7 +47,6 @@ use arrow::{ }; use datafusion_common::{downcast_value, ScalarValue}; use datafusion_expr::expr_rewriter::{ExprRewritable, ExprRewriter}; - use datafusion_expr::utils::expr_to_columns; use datafusion_expr::{binary_expr, cast, try_cast, ExprSchemable}; use datafusion_physical_expr::create_physical_expr; @@ -846,7 +846,7 @@ enum StatisticsType { mod tests { use super::*; use crate::from_slice::FromSlice; - use crate::logical_plan::{col, lit}; + use crate::logical_expr::{col, lit}; use crate::{assert_batches_eq, physical_optimizer::pruning::StatisticsType}; use arrow::array::Decimal128Array; use arrow::{ diff --git a/datafusion/core/src/physical_plan/display.rs b/datafusion/core/src/physical_plan/display.rs index aa02af12da2c6..f89512ab447b8 100644 --- a/datafusion/core/src/physical_plan/display.rs +++ b/datafusion/core/src/physical_plan/display.rs @@ -21,7 +21,7 @@ use std::fmt; -use crate::logical_plan::{StringifiedPlan, ToStringifiedPlan}; +use crate::logical_expr::{StringifiedPlan, ToStringifiedPlan}; use super::{accept, ExecutionPlan, ExecutionPlanVisitor}; @@ -198,7 +198,7 @@ impl<'a, 'b> ExecutionPlanVisitor for IndentVisitor<'a, 'b> { impl<'a> ToStringifiedPlan for DisplayableExecutionPlan<'a> { fn to_stringified( &self, - plan_type: crate::logical_plan::PlanType, + plan_type: crate::logical_expr::PlanType, ) -> StringifiedPlan { StringifiedPlan::new(plan_type, self.indent().to_string()) } diff --git a/datafusion/core/src/physical_plan/explain.rs b/datafusion/core/src/physical_plan/explain.rs index 29a17ee870db4..15f459fb045b1 100644 --- a/datafusion/core/src/physical_plan/explain.rs +++ b/datafusion/core/src/physical_plan/explain.rs @@ -22,7 +22,7 @@ use std::sync::Arc; use crate::{ error::{DataFusionError, Result}, - logical_plan::StringifiedPlan, + logical_expr::StringifiedPlan, physical_plan::{ common::SizedRecordBatchStream, DisplayFormatType, ExecutionPlan, Partitioning, Statistics, diff --git a/datafusion/core/src/physical_plan/hash_join.rs b/datafusion/core/src/physical_plan/hash_join.rs index a22bcbc13c7e1..6d7fe582899b2 100644 --- a/datafusion/core/src/physical_plan/hash_join.rs +++ b/datafusion/core/src/physical_plan/hash_join.rs @@ -68,7 +68,7 @@ use super::{ }; use super::{hash_utils::create_hashes, Statistics}; use crate::error::{DataFusionError, Result}; -use crate::logical_plan::JoinType; +use crate::logical_expr::JoinType; use super::{ DisplayFormatType, ExecutionPlan, Partitioning, RecordBatchStream, diff --git a/datafusion/core/src/physical_plan/join_utils.rs b/datafusion/core/src/physical_plan/join_utils.rs index cbabf548c12b3..dc48d2aa8a7d0 100644 --- a/datafusion/core/src/physical_plan/join_utils.rs +++ b/datafusion/core/src/physical_plan/join_utils.rs @@ -18,7 +18,7 @@ //! Join related functionality used both on logical and physical plans use crate::error::{DataFusionError, Result}; -use crate::logical_plan::JoinType; +use crate::logical_expr::JoinType; use crate::physical_plan::expressions::Column; use arrow::datatypes::{Field, Schema}; use arrow::error::ArrowError; diff --git a/datafusion/core/src/physical_plan/planner.rs b/datafusion/core/src/physical_plan/planner.rs index 2a34dd9fbb3a1..4a33b299b96fa 100644 --- a/datafusion/core/src/physical_plan/planner.rs +++ b/datafusion/core/src/physical_plan/planner.rs @@ -31,12 +31,11 @@ use crate::logical_expr::{ Aggregate, Distinct, EmptyRelation, Join, Projection, Sort, SubqueryAlias, TableScan, Window, }; -use crate::logical_plan::{ - unnormalize_cols, CrossJoin, DFSchema, Expr, LogicalPlan, - Partitioning as LogicalPartitioning, PlanType, Repartition, ToStringifiedPlan, Union, - UserDefinedLogicalNode, +use crate::logical_expr::{ + CrossJoin, Expr, LogicalPlan, Partitioning as LogicalPartitioning, PlanType, + Repartition, ToStringifiedPlan, Union, UserDefinedLogicalNode, }; -use crate::logical_plan::{Limit, Values}; +use crate::logical_expr::{Limit, Values}; use crate::physical_expr::create_physical_expr; use crate::physical_optimizer::optimizer::PhysicalOptimizerRule; use crate::physical_plan::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy}; @@ -59,8 +58,9 @@ use crate::{ use arrow::compute::SortOptions; use arrow::datatypes::{Schema, SchemaRef}; use async_trait::async_trait; -use datafusion_common::ScalarValue; +use datafusion_common::{DFSchema, ScalarValue}; use datafusion_expr::expr::GroupingSet; +use datafusion_expr::expr_rewriter::unnormalize_cols; use datafusion_expr::utils::{expand_wildcard, expr_to_columns}; use datafusion_expr::WindowFrameUnits; use datafusion_optimizer::utils::unalias; @@ -1680,22 +1680,18 @@ mod tests { use crate::execution::context::TaskContext; use crate::execution::options::CsvReadOptions; use crate::execution::runtime_env::RuntimeEnv; - use crate::logical_expr::Extension; + use crate::physical_plan::SendableRecordBatchStream; use crate::physical_plan::{ expressions, DisplayFormatType, Partitioning, PhysicalPlanner, Statistics, }; use crate::prelude::{SessionConfig, SessionContext}; use crate::scalar::ScalarValue; use crate::test_util::{scan_empty, scan_empty_with_partitions}; - use crate::{ - logical_plan::LogicalPlanBuilder, physical_plan::SendableRecordBatchStream, - }; use arrow::array::{ArrayRef, DictionaryArray, Int32Array}; use arrow::datatypes::{DataType, Field, Int32Type, SchemaRef}; use arrow::record_batch::RecordBatch; use datafusion_common::{DFField, DFSchema, DFSchemaRef}; - use datafusion_expr::expr::GroupingSet; - use datafusion_expr::{col, lit, sum}; + use datafusion_expr::{col, lit, sum, Extension, GroupingSet, LogicalPlanBuilder}; use fmt::Debug; use std::collections::HashMap; use std::convert::TryFrom; diff --git a/datafusion/core/src/physical_plan/sort_merge_join.rs b/datafusion/core/src/physical_plan/sort_merge_join.rs index d84ea9a52a454..29da01a1474e1 100644 --- a/datafusion/core/src/physical_plan/sort_merge_join.rs +++ b/datafusion/core/src/physical_plan/sort_merge_join.rs @@ -38,7 +38,7 @@ use futures::{Stream, StreamExt}; use crate::error::DataFusionError; use crate::error::Result; use crate::execution::context::TaskContext; -use crate::logical_plan::JoinType; +use crate::logical_expr::JoinType; use crate::physical_plan::common::combine_batches; use crate::physical_plan::expressions::Column; use crate::physical_plan::expressions::PhysicalSortExpr; @@ -1196,7 +1196,7 @@ mod tests { use arrow::record_batch::RecordBatch; use crate::error::Result; - use crate::logical_plan::JoinType; + use crate::logical_expr::JoinType; use crate::physical_plan::expressions::Column; use crate::physical_plan::join_utils::JoinOn; use crate::physical_plan::memory::MemoryExec; diff --git a/datafusion/core/src/test/mod.rs b/datafusion/core/src/test/mod.rs index dfc6d8edca7d6..bb884e142b05b 100644 --- a/datafusion/core/src/test/mod.rs +++ b/datafusion/core/src/test/mod.rs @@ -24,7 +24,7 @@ use crate::datasource::object_store::ObjectStoreUrl; use crate::datasource::{MemTable, TableProvider}; use crate::error::Result; use crate::from_slice::FromSlice; -use crate::logical_plan::LogicalPlan; +use crate::logical_expr::LogicalPlan; use crate::physical_plan::file_format::{CsvExec, FileScanConfig}; use crate::test::object_store::local_unpartitioned_file; use crate::test_util::{aggr_test_schema, arrow_test_data}; diff --git a/datafusion/core/src/test_util.rs b/datafusion/core/src/test_util.rs index ad27ea3c1aafa..d92b9db6082c2 100644 --- a/datafusion/core/src/test_util.rs +++ b/datafusion/core/src/test_util.rs @@ -20,8 +20,8 @@ use std::collections::BTreeMap; use std::{env, error::Error, path::PathBuf, sync::Arc}; -use crate::datasource::empty::EmptyTable; -use crate::logical_plan::{provider_as_source, LogicalPlanBuilder, UNNAMED_TABLE}; +use crate::datasource::{empty::EmptyTable, provider_as_source}; +use crate::logical_expr::{LogicalPlanBuilder, UNNAMED_TABLE}; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion_common::DataFusionError; diff --git a/datafusion/core/tests/dataframe.rs b/datafusion/core/tests/dataframe.rs index 77a4b25bc0692..2925320e064de 100644 --- a/datafusion/core/tests/dataframe.rs +++ b/datafusion/core/tests/dataframe.rs @@ -27,11 +27,11 @@ use datafusion::assert_batches_eq; use datafusion::dataframe::DataFrame; use datafusion::error::Result; use datafusion::execution::context::SessionContext; -use datafusion::logical_plan::{col, Expr}; use datafusion::prelude::CsvReadOptions; use datafusion::prelude::JoinType; use datafusion_expr::expr::GroupingSet; use datafusion_expr::{avg, count, lit, sum}; +use datafusion_expr::{col, Expr}; #[tokio::test] async fn join() -> Result<()> { diff --git a/datafusion/core/tests/join_fuzz.rs b/datafusion/core/tests/join_fuzz.rs index af42c7785c628..9e402896cd09e 100644 --- a/datafusion/core/tests/join_fuzz.rs +++ b/datafusion/core/tests/join_fuzz.rs @@ -24,12 +24,12 @@ use arrow::util::pretty::pretty_format_batches; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; -use datafusion::logical_plan::JoinType; use datafusion::physical_plan::collect; use datafusion::physical_plan::expressions::Column; use datafusion::physical_plan::hash_join::{HashJoinExec, PartitionMode}; use datafusion::physical_plan::memory::MemoryExec; use datafusion::physical_plan::sort_merge_join::SortMergeJoinExec; +use datafusion_expr::JoinType; use datafusion::prelude::{SessionConfig, SessionContext}; use fuzz_utils::add_empty_batches; diff --git a/datafusion/core/tests/parquet_pruning.rs b/datafusion/core/tests/parquet_pruning.rs index a7b9181800161..b70e05c66a334 100644 --- a/datafusion/core/tests/parquet_pruning.rs +++ b/datafusion/core/tests/parquet_pruning.rs @@ -32,10 +32,8 @@ use arrow::{ util::pretty::pretty_format_batches, }; use chrono::{Datelike, Duration}; -use datafusion::logical_plan::provider_as_source; use datafusion::{ - datasource::TableProvider, - logical_plan::{col, lit, Expr, LogicalPlan, LogicalPlanBuilder}, + datasource::{provider_as_source, TableProvider}, physical_plan::{ accept, file_format::ParquetExec, metrics::MetricsSet, ExecutionPlan, ExecutionPlanVisitor, @@ -43,6 +41,7 @@ use datafusion::{ prelude::{ParquetReadOptions, SessionConfig, SessionContext}, scalar::ScalarValue, }; +use datafusion_expr::{col, lit, Expr, LogicalPlan, LogicalPlanBuilder}; use parquet::{arrow::ArrowWriter, file::properties::WriterProperties}; use tempfile::NamedTempFile; diff --git a/datafusion/core/tests/simplification.rs b/datafusion/core/tests/simplification.rs index 8c98b8045dbd6..5a4a2e92fbf98 100644 --- a/datafusion/core/tests/simplification.rs +++ b/datafusion/core/tests/simplification.rs @@ -18,13 +18,9 @@ //! This program demonstrates the DataFusion expression simplification API. use arrow::datatypes::{DataType, Field, Schema}; -use datafusion::logical_plan::ExprSchemable; -use datafusion::{ - error::Result, - execution::context::ExecutionProps, - logical_plan::{DFSchema, Expr}, - prelude::*, -}; +use datafusion::common::DFSchema; +use datafusion::{error::Result, execution::context::ExecutionProps, prelude::*}; +use datafusion_expr::{Expr, ExprSchemable}; use datafusion_optimizer::expr_simplifier::{ExprSimplifier, SimplifyInfo}; /// In order to simplify expressions, DataFusion must have information diff --git a/datafusion/core/tests/sql/explain.rs b/datafusion/core/tests/sql/explain.rs index 97e7e6e761237..2dd4d4f64f3a8 100644 --- a/datafusion/core/tests/sql/explain.rs +++ b/datafusion/core/tests/sql/explain.rs @@ -16,11 +16,9 @@ // under the License. use arrow::datatypes::{DataType, Field, Schema}; +use datafusion::prelude::SessionContext; use datafusion::test_util::scan_empty; -use datafusion::{ - logical_plan::{LogicalPlan, PlanType}, - prelude::SessionContext, -}; +use datafusion_expr::{LogicalPlan, PlanType}; #[test] fn optimize_explain() { diff --git a/datafusion/core/tests/sql/information_schema.rs b/datafusion/core/tests/sql/information_schema.rs index 854166155161d..24c0ce0277f72 100644 --- a/datafusion/core/tests/sql/information_schema.rs +++ b/datafusion/core/tests/sql/information_schema.rs @@ -23,8 +23,8 @@ use datafusion::{ schema::{MemorySchemaProvider, SchemaProvider}, }, datasource::{TableProvider, TableType}, - logical_plan::Expr, }; +use datafusion_expr::Expr; use super::*; diff --git a/datafusion/core/tests/sql/projection.rs b/datafusion/core/tests/sql/projection.rs index 4a339edd213b3..fdc0949997a2e 100644 --- a/datafusion/core/tests/sql/projection.rs +++ b/datafusion/core/tests/sql/projection.rs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -use datafusion::logical_plan::{provider_as_source, LogicalPlanBuilder, UNNAMED_TABLE}; +use datafusion::datasource::provider_as_source; use datafusion::test_util::scan_empty; -use datafusion_expr::when; +use datafusion_expr::{when, LogicalPlanBuilder, UNNAMED_TABLE}; use tempfile::TempDir; use super::*; diff --git a/datafusion/core/tests/sql/udf.rs b/datafusion/core/tests/sql/udf.rs index b2440dc222f52..6e3780642057d 100644 --- a/datafusion/core/tests/sql/udf.rs +++ b/datafusion/core/tests/sql/udf.rs @@ -18,9 +18,10 @@ use super::*; use arrow::compute::add; use datafusion::{ - logical_plan::{create_udaf, FunctionRegistry, LogicalPlanBuilder}, + execution::registry::FunctionRegistry, physical_plan::{expressions::AvgAccumulator, functions::make_scalar_function}, }; +use datafusion_expr::{create_udaf, LogicalPlanBuilder}; /// test that casting happens on udfs. /// c11 is f32, but `custom_sqrt` requires f64. Casting happens but the logical plan and diff --git a/datafusion/core/tests/statistics.rs b/datafusion/core/tests/statistics.rs index 95879ebaf679f..bcf82481cf1f5 100644 --- a/datafusion/core/tests/statistics.rs +++ b/datafusion/core/tests/statistics.rs @@ -23,7 +23,7 @@ use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion::{ datasource::{TableProvider, TableType}, error::Result, - logical_plan::Expr, + logical_expr::Expr, physical_plan::{ expressions::PhysicalSortExpr, project_schema, ColumnStatistics, DisplayFormatType, ExecutionPlan, Partitioning, SendableRecordBatchStream, diff --git a/datafusion/expr/src/lib.rs b/datafusion/expr/src/lib.rs index 56e60f5a28c06..e1bae33143fab 100644 --- a/datafusion/expr/src/lib.rs +++ b/datafusion/expr/src/lib.rs @@ -56,7 +56,7 @@ pub use accumulator::{Accumulator, AggregateState}; pub use aggregate_function::AggregateFunction; pub use built_in_function::BuiltinScalarFunction; pub use columnar_value::{ColumnarValue, NullColumnarValue}; -pub use expr::Expr; +pub use expr::{Case, Expr, GroupingSet}; pub use expr_fn::*; pub use expr_schema::ExprSchemable; pub use function::{ diff --git a/datafusion/proto/src/bytes/mod.rs b/datafusion/proto/src/bytes/mod.rs index 1dd8a5c1a4035..8eab5baebe722 100644 --- a/datafusion/proto/src/bytes/mod.rs +++ b/datafusion/proto/src/bytes/mod.rs @@ -16,22 +16,18 @@ // under the License. //! Serialization / Deserialization to Bytes -use crate::{ - from_proto::parse_expr, - logical_plan::{AsLogicalPlan, LogicalExtensionCodec}, - protobuf, -}; +use crate::logical_plan::{AsLogicalPlan, LogicalExtensionCodec}; +use crate::{from_proto::parse_expr, protobuf}; use datafusion_common::{DataFusionError, Result}; -use datafusion_expr::{Expr, LogicalPlan}; +use datafusion_expr::{Expr, Extension, LogicalPlan}; use prost::{ bytes::{Bytes, BytesMut}, Message, }; // Reexport Bytes which appears in the API -use datafusion::logical_plan::FunctionRegistry; +use datafusion::execution::registry::FunctionRegistry; use datafusion::prelude::SessionContext; -use datafusion_expr::logical_plan::Extension; mod registry; @@ -190,11 +186,9 @@ impl LogicalExtensionCodec for DefaultExtensionCodec { mod test { use super::*; use arrow::{array::ArrayRef, datatypes::DataType}; + use datafusion::physical_plan::functions::make_scalar_function; use datafusion::prelude::SessionContext; - use datafusion::{ - logical_plan::create_udf, physical_plan::functions::make_scalar_function, - }; - use datafusion_expr::{lit, Volatility}; + use datafusion_expr::{create_udf, lit, Volatility}; use std::sync::Arc; #[test] diff --git a/datafusion/proto/src/bytes/registry.rs b/datafusion/proto/src/bytes/registry.rs index 2f701a0c20c16..675e40240a769 100644 --- a/datafusion/proto/src/bytes/registry.rs +++ b/datafusion/proto/src/bytes/registry.rs @@ -17,7 +17,7 @@ use std::{collections::HashSet, sync::Arc}; -use datafusion::logical_plan::FunctionRegistry; +use datafusion::execution::registry::FunctionRegistry; use datafusion_common::{DataFusionError, Result}; use datafusion_expr::{AggregateUDF, ScalarUDF}; diff --git a/datafusion/proto/src/from_proto.rs b/datafusion/proto/src/from_proto.rs index fe1fdfaa0c6ad..ffbf62658f158 100644 --- a/datafusion/proto/src/from_proto.rs +++ b/datafusion/proto/src/from_proto.rs @@ -27,12 +27,10 @@ use crate::protobuf::{ use arrow::datatypes::{ DataType, Field, IntervalMonthDayNanoType, IntervalUnit, Schema, TimeUnit, UnionMode, }; -use datafusion::logical_plan::FunctionRegistry; +use datafusion::execution::registry::FunctionRegistry; use datafusion_common::{ Column, DFField, DFSchema, DFSchemaRef, DataFusionError, ScalarValue, }; -use datafusion_expr::expr::GroupingSet::GroupingSets; -use datafusion_expr::expr::{Case, GroupingSet}; use datafusion_expr::{ abs, acos, array, ascii, asin, atan, atan2, bit_length, btrim, ceil, character_length, chr, coalesce, concat_expr, concat_ws_expr, cos, date_bin, @@ -43,7 +41,9 @@ use datafusion_expr::{ sha384, sha512, signum, sin, split_part, sqrt, starts_with, strpos, substr, tan, to_hex, to_timestamp_micros, to_timestamp_millis, to_timestamp_seconds, translate, trim, trunc, upper, AggregateFunction, BuiltInWindowFunction, BuiltinScalarFunction, - Expr, Operator, WindowFrame, WindowFrameBound, WindowFrameUnits, + Case, Expr, GroupingSet, + GroupingSet::GroupingSets, + Operator, WindowFrame, WindowFrameBound, WindowFrameUnits, }; use std::sync::Arc; diff --git a/datafusion/proto/src/lib.rs b/datafusion/proto/src/lib.rs index 8475d85c82e40..af5205cea071d 100644 --- a/datafusion/proto/src/lib.rs +++ b/datafusion/proto/src/lib.rs @@ -59,10 +59,10 @@ mod roundtrip_tests { TimeUnit, UnionMode, }, }; - use datafusion::logical_plan::create_udaf; use datafusion::physical_plan::functions::make_scalar_function; use datafusion::prelude::{create_udf, CsvReadOptions, SessionContext}; use datafusion_common::{DFSchemaRef, DataFusionError, ScalarValue}; + use datafusion_expr::create_udaf; use datafusion_expr::expr::{Case, GroupingSet}; use datafusion_expr::logical_plan::{Extension, UserDefinedLogicalNode}; use datafusion_expr::{ diff --git a/datafusion/proto/src/logical_plan.rs b/datafusion/proto/src/logical_plan.rs index 7a9d635f8152f..befa3d12c389e 100644 --- a/datafusion/proto/src/logical_plan.rs +++ b/datafusion/proto/src/logical_plan.rs @@ -24,7 +24,6 @@ use crate::{ to_proto, }; use arrow::datatypes::Schema; -use datafusion::prelude::SessionContext; use datafusion::{ datasource::{ file_format::{ @@ -32,7 +31,8 @@ use datafusion::{ }, listing::{ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl}, }, - logical_plan::{provider_as_source, source_as_provider}, + datasource::{provider_as_source, source_as_provider}, + prelude::SessionContext, }; use datafusion_common::{Column, DataFusionError}; use datafusion_expr::{ @@ -420,7 +420,7 @@ impl AsLogicalPlan for LogicalPlanNode { LogicalPlanBuilder::from(input).sort(sort_expr)?.build() } LogicalPlanType::Repartition(repartition) => { - use datafusion::logical_plan::Partitioning; + use datafusion::logical_expr::Partitioning; let input: LogicalPlan = into_logical_plan!(repartition.input, ctx, extension_codec)?; use protobuf::repartition_node::PartitionMethod; @@ -983,7 +983,7 @@ impl AsLogicalPlan for LogicalPlanNode { input, partitioning_scheme, }) => { - use datafusion::logical_plan::Partitioning; + use datafusion::logical_expr::Partitioning; let input: protobuf::LogicalPlanNode = protobuf::LogicalPlanNode::try_from_logical_plan( input.as_ref(),