From c7248fa3a2006a8cc3791ecd877c9e6c2921102b Mon Sep 17 00:00:00 2001 From: Ian Lai Date: Tue, 11 Feb 2025 05:14:49 +0000 Subject: [PATCH 1/4] refactor: remove uses of arrow_schema and use reexport in arrow instead --- Cargo.lock | 31 +++++++------------ datafusion/catalog-listing/Cargo.toml | 1 - datafusion/catalog-listing/src/helpers.rs | 3 +- datafusion/common/Cargo.toml | 1 - datafusion/common/src/column.rs | 5 ++- datafusion/common/src/dfschema.rs | 12 +++---- .../common/src/file_options/parquet_writer.rs | 2 +- datafusion/common/src/param_value.rs | 2 +- datafusion/common/src/scalar/mod.rs | 5 ++- .../common/src/scalar/struct_builder.rs | 3 +- datafusion/common/src/stats.rs | 2 +- datafusion/common/src/test_util.rs | 4 +-- datafusion/common/src/types/field.rs | 2 +- datafusion/common/src/types/logical.rs | 2 +- datafusion/common/src/types/native.rs | 4 +-- datafusion/common/src/utils/mod.rs | 3 +- datafusion/core/Cargo.toml | 3 +- datafusion/core/benches/sql_query_with_io.rs | 2 +- datafusion/core/src/dataframe/mod.rs | 3 +- datafusion/core/src/datasource/data_source.rs | 2 +- .../core/src/datasource/file_format/arrow.rs | 3 +- .../core/src/datasource/file_format/csv.rs | 2 +- .../core/src/datasource/file_format/json.rs | 7 ++--- .../core/src/datasource/file_format/mod.rs | 4 +-- .../src/datasource/file_format/parquet.rs | 2 +- .../src/datasource/file_format/write/demux.rs | 2 +- .../src/datasource/file_format/write/mod.rs | 2 +- .../core/src/datasource/listing/table.rs | 5 ++- datafusion/core/src/datasource/mod.rs | 3 +- .../datasource/physical_plan/arrow_file.rs | 4 +-- .../physical_plan/file_scan_config.rs | 7 ++--- .../datasource/physical_plan/file_stream.rs | 2 +- .../core/src/datasource/physical_plan/mod.rs | 2 +- .../datasource/physical_plan/parquet/mod.rs | 3 +- .../physical_plan/parquet/opener.rs | 3 +- .../physical_plan/parquet/page_filter.rs | 6 ++-- .../physical_plan/parquet/row_filter.rs | 3 +- .../physical_plan/parquet/source.rs | 4 +-- .../datasource/physical_plan/statistics.rs | 2 +- .../core/src/datasource/schema_adapter.rs | 5 ++- datafusion/core/src/datasource/stream.rs | 2 +- datafusion/core/src/execution/context/mod.rs | 5 ++- .../core/src/execution/session_state.rs | 4 +-- datafusion/core/src/physical_planner.rs | 2 +- datafusion/core/src/schema_equivalence.rs | 2 +- datafusion/core/tests/catalog/memory.rs | 2 +- .../tests/dataframe/dataframe_functions.rs | 2 +- datafusion/core/tests/dataframe/mod.rs | 7 +++-- .../core/tests/execution/logical_plan.rs | 2 +- datafusion/core/tests/expr_api/mod.rs | 2 +- .../core/tests/expr_api/parse_sql_expr.rs | 2 +- datafusion/core/tests/fifo/mod.rs | 3 +- .../core/tests/fuzz_cases/aggregate_fuzz.rs | 7 ++--- .../aggregation_fuzzer/context_generator.rs | 2 +- .../aggregation_fuzzer/data_generator.rs | 16 +++++----- .../tests/fuzz_cases/equivalence/ordering.rs | 2 +- .../fuzz_cases/equivalence/projection.rs | 2 +- .../tests/fuzz_cases/equivalence/utils.rs | 4 +-- datafusion/core/tests/fuzz_cases/join_fuzz.rs | 2 +- .../core/tests/fuzz_cases/limit_fuzz.rs | 2 +- datafusion/core/tests/fuzz_cases/pruning.rs | 2 +- .../sort_preserving_repartition_fuzz.rs | 4 +-- datafusion/core/tests/memory_limit/mod.rs | 2 +- datafusion/core/tests/optimizer/mod.rs | 5 +-- .../tests/parquet/external_access_plan.rs | 2 +- .../core/tests/parquet/schema_coercion.rs | 3 +- .../physical_optimizer/enforce_sorting.rs | 2 +- .../physical_optimizer/join_selection.rs | 3 +- .../limited_distinct_aggregation.rs | 2 +- .../physical_optimizer/projection_pushdown.rs | 3 +- .../physical_optimizer/sanity_checker.rs | 3 +- .../tests/physical_optimizer/test_utils.rs | 3 +- .../tests/user_defined/insert_operation.rs | 2 +- .../user_defined/user_defined_aggregates.rs | 3 +- .../user_defined_scalar_functions.rs | 2 +- .../user_defined_window_functions.rs | 2 +- datafusion/ffi/Cargo.toml | 1 - datafusion/ffi/src/tests/async_provider.rs | 2 +- datafusion/functions-aggregate/Cargo.toml | 1 - .../functions-aggregate/benches/array_agg.rs | 3 +- .../functions-aggregate/benches/count.rs | 3 +- datafusion/functions-aggregate/benches/sum.rs | 3 +- .../functions-aggregate/src/approx_median.rs | 4 +-- .../src/approx_percentile_cont.rs | 5 ++- .../functions-aggregate/src/array_agg.rs | 5 ++- .../functions-aggregate/src/bit_and_or_xor.rs | 3 +- datafusion/functions-aggregate/src/min_max.rs | 5 ++- .../src/min_max/min_max_bytes.rs | 2 +- .../functions-aggregate/src/nth_value.rs | 2 +- .../functions-aggregate/src/string_agg.rs | 2 +- datafusion/functions-nested/Cargo.toml | 1 - datafusion/functions-nested/benches/map.rs | 2 +- .../functions-nested/src/cardinality.rs | 6 ++-- datafusion/functions-nested/src/concat.rs | 2 +- datafusion/functions-nested/src/dimension.rs | 8 +++-- datafusion/functions-nested/src/distance.rs | 6 ++-- datafusion/functions-nested/src/empty.rs | 6 ++-- datafusion/functions-nested/src/except.rs | 2 +- datafusion/functions-nested/src/extract.rs | 8 +++-- datafusion/functions-nested/src/flatten.rs | 6 ++-- datafusion/functions-nested/src/length.rs | 6 ++-- datafusion/functions-nested/src/make_array.rs | 7 +++-- datafusion/functions-nested/src/map.rs | 3 +- .../functions-nested/src/map_extract.rs | 4 +-- datafusion/functions-nested/src/map_keys.rs | 2 +- datafusion/functions-nested/src/map_values.rs | 2 +- datafusion/functions-nested/src/position.rs | 7 +++-- datafusion/functions-nested/src/range.rs | 7 ++--- datafusion/functions-nested/src/remove.rs | 2 +- datafusion/functions-nested/src/repeat.rs | 7 +++-- datafusion/functions-nested/src/replace.rs | 3 +- datafusion/functions-nested/src/resize.rs | 7 +++-- datafusion/functions-nested/src/reverse.rs | 4 +-- datafusion/functions-nested/src/set_ops.rs | 2 +- datafusion/functions-nested/src/sort.rs | 6 ++-- datafusion/functions-nested/src/string.rs | 2 +- datafusion/functions-nested/src/utils.rs | 3 +- datafusion/functions/src/datetime/mod.rs | 2 +- datafusion/physical-expr/Cargo.toml | 1 - datafusion/physical-expr/benches/case_when.rs | 3 +- datafusion/physical-expr/benches/is_null.rs | 3 +- datafusion/physical-expr/src/aggregate.rs | 2 +- datafusion/physical-expr/src/analysis.rs | 2 +- .../physical-expr/src/equivalence/mod.rs | 4 +-- .../physical-expr/src/equivalence/ordering.rs | 4 +-- .../src/equivalence/projection.rs | 4 +-- .../src/equivalence/properties.rs | 8 ++--- .../physical-expr/src/expressions/binary.rs | 2 +- .../src/expressions/binary/kernels.rs | 2 +- .../physical-expr/src/expressions/column.rs | 3 +- .../physical-expr/src/expressions/like.rs | 4 +-- .../physical-expr/src/expressions/negative.rs | 2 +- .../physical-expr/src/intervals/cp_solver.rs | 5 ++- .../physical-expr/src/intervals/test_utils.rs | 2 +- .../physical-expr/src/intervals/utils.rs | 2 +- datafusion/physical-expr/src/planner.rs | 2 +- .../physical-expr/src/utils/guarantee.rs | 2 +- datafusion/physical-expr/src/utils/mod.rs | 2 +- .../window/standard_window_function_expr.rs | 3 +- .../physical-expr/src/window/window_expr.rs | 2 +- datafusion/physical-optimizer/Cargo.toml | 1 - .../src/enforce_sorting/sort_pushdown.rs | 2 +- datafusion/physical-plan/Cargo.toml | 1 - .../src/aggregates/group_values/mod.rs | 3 +- .../group_values/multi_group_by/mod.rs | 14 ++++----- .../group_values/multi_group_by/primitive.rs | 5 ++- .../src/aggregates/group_values/row.rs | 4 +-- .../group_values/single_group_by/primitive.rs | 3 +- .../physical-plan/src/aggregates/order/mod.rs | 2 +- .../src/aggregates/order/partial.rs | 2 +- .../physical-plan/src/aggregates/row_hash.rs | 2 +- .../src/aggregates/topk/hash_table.rs | 3 +- .../physical-plan/src/aggregates/topk/heap.rs | 3 +- .../src/aggregates/topk/priority_map.rs | 6 ++-- .../src/aggregates/topk_stream.rs | 2 +- datafusion/physical-plan/src/coalesce/mod.rs | 2 +- datafusion/physical-plan/src/common.rs | 4 +-- datafusion/physical-plan/src/display.rs | 4 +-- .../physical-plan/src/execution_plan.rs | 8 ++--- datafusion/physical-plan/src/filter.rs | 3 +- datafusion/physical-plan/src/insert.rs | 3 +- .../physical-plan/src/joins/hash_join.rs | 2 +- .../physical-plan/src/joins/join_filter.rs | 2 +- .../src/joins/nested_loop_join.rs | 2 +- .../src/joins/stream_join_utils.rs | 3 +- .../physical-plan/src/joins/test_utils.rs | 2 +- datafusion/physical-plan/src/joins/utils.rs | 4 +-- datafusion/physical-plan/src/limit.rs | 2 +- datafusion/physical-plan/src/memory.rs | 8 ++--- datafusion/physical-plan/src/projection.rs | 2 +- .../physical-plan/src/repartition/mod.rs | 3 +- datafusion/physical-plan/src/sorts/sort.rs | 3 +- .../src/sorts/sort_preserving_merge.rs | 3 +- datafusion/physical-plan/src/stream.rs | 2 +- datafusion/physical-plan/src/streaming.rs | 3 +- datafusion/physical-plan/src/test.rs | 2 +- datafusion/physical-plan/src/topk/mod.rs | 2 +- datafusion/physical-plan/src/union.rs | 3 +- datafusion/physical-plan/src/values.rs | 2 +- .../src/windows/bounded_window_agg_exec.rs | 3 +- datafusion/physical-plan/src/windows/mod.rs | 3 +- datafusion/physical-plan/src/windows/utils.rs | 2 +- datafusion/proto/src/generated/pbjson.rs | 2 +- datafusion/sql/Cargo.toml | 1 - datafusion/sql/examples/sql.rs | 2 +- datafusion/sql/src/expr/function.rs | 2 +- datafusion/sql/src/expr/identifier.rs | 2 +- datafusion/sql/src/expr/mod.rs | 3 +- datafusion/sql/src/expr/value.rs | 5 +-- datafusion/sql/src/planner.rs | 2 +- datafusion/sql/src/statement.rs | 2 +- datafusion/sql/src/unparser/dialect.rs | 2 +- datafusion/sql/src/unparser/expr.rs | 6 ++-- datafusion/sql/src/unparser/rewrite.rs | 2 +- datafusion/sql/src/utils.rs | 5 ++- datafusion/sql/tests/cases/plan_to_sql.rs | 2 +- datafusion/sql/tests/common/mod.rs | 2 +- datafusion/sql/tests/sql_integration.rs | 3 +- 198 files changed, 327 insertions(+), 357 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cd6586180d734..27403b0f14047 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -453,7 +453,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "105f01ec0090259e9a33a9263ec18ff223ab91a0ea9fbc18042f7e38005142f6" dependencies = [ "bitflags 2.8.0", - "serde", ] [[package]] @@ -1396,9 +1395,9 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.53" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24a03c8b52922d68a1589ad61032f2c1aa5a8158d2aa0d93c6e9534944bbad6" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" dependencies = [ "cc", ] @@ -1767,6 +1766,7 @@ dependencies = [ "rand_distr", "regex", "rstest", + "serde", "serde_json", "sqlparser", "sysinfo", @@ -1825,7 +1825,6 @@ name = "datafusion-catalog-listing" version = "45.0.0" dependencies = [ "arrow", - "arrow-schema", "async-compression", "async-trait", "bytes", @@ -1887,7 +1886,6 @@ dependencies = [ "apache-avro", "arrow", "arrow-ipc", - "arrow-schema", "base64 0.22.1", "chrono", "half", @@ -2000,7 +1998,6 @@ version = "45.0.0" dependencies = [ "abi_stable", "arrow", - "arrow-schema", "async-ffi", "async-trait", "datafusion", @@ -2049,7 +2046,6 @@ version = "45.0.0" dependencies = [ "ahash 0.8.11", "arrow", - "arrow-schema", "criterion", "datafusion-common", "datafusion-doc", @@ -2084,7 +2080,6 @@ version = "45.0.0" dependencies = [ "arrow", "arrow-ord", - "arrow-schema", "criterion", "datafusion-common", "datafusion-doc", @@ -2177,7 +2172,6 @@ version = "45.0.0" dependencies = [ "ahash 0.8.11", "arrow", - "arrow-schema", "criterion", "datafusion-common", "datafusion-expr", @@ -2213,7 +2207,6 @@ name = "datafusion-physical-optimizer" version = "45.0.0" dependencies = [ "arrow", - "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -2238,7 +2231,6 @@ dependencies = [ "ahash 0.8.11", "arrow", "arrow-ord", - "arrow-schema", "async-trait", "chrono", "criterion", @@ -2306,7 +2298,6 @@ name = "datafusion-sql" version = "45.0.0" dependencies = [ "arrow", - "arrow-schema", "bigdecimal", "ctor", "datafusion-common", @@ -5633,9 +5624,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stacker" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +checksum = "1d08feb8f695b465baed819b03c128dc23f57a694510ab1f06c77f763975685e" dependencies = [ "cc", "cfg-if", @@ -5767,9 +5758,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef201e3234acdb66865840012c0b9c3d04269b74416fb6285cd480b01718c2f9" +checksum = "9a71f43d785f205cd4a685f6cb245a0f0a5be20fbaee956a172e80574f5e8eda" dependencies = [ "heck 0.5.0", "pbjson", @@ -6191,9 +6182,9 @@ checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" [[package]] name = "toml_edit" -version = "0.22.23" +version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02a8b472d1a3d7c18e2d61a489aee3453fd9031c33e4f55bd533f4a7adca1bee" +checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ "indexmap 2.7.1", "toml_datetime", @@ -7002,9 +6993,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e376c75f4f43f44db463cf729e0d3acbf954d13e22c51e26e4c264b4ab545f" +checksum = "59690dea168f2198d1a3b0cac23b8063efcd11012f10ae4698f284808c8ef603" dependencies = [ "memchr", ] diff --git a/datafusion/catalog-listing/Cargo.toml b/datafusion/catalog-listing/Cargo.toml index 99e20aa8f07a0..ad97fda19d711 100644 --- a/datafusion/catalog-listing/Cargo.toml +++ b/datafusion/catalog-listing/Cargo.toml @@ -33,7 +33,6 @@ default = ["compression"] [dependencies] arrow = { workspace = true } -arrow-schema = { workspace = true } async-compression = { version = "0.4.0", features = [ "bzip2", "gzip", diff --git a/datafusion/catalog-listing/src/helpers.rs b/datafusion/catalog-listing/src/helpers.rs index 6cb3f661e652f..ceacde2494e21 100644 --- a/datafusion/catalog-listing/src/helpers.rs +++ b/datafusion/catalog-listing/src/helpers.rs @@ -30,10 +30,9 @@ use datafusion_expr::{BinaryExpr, Operator}; use arrow::{ array::{Array, ArrayRef, AsArray, StringBuilder}, compute::{and, cast, prep_null_mask_filter}, - datatypes::{DataType, Field, Schema}, + datatypes::{DataType, Field, Fields, Schema}, record_batch::RecordBatch, }; -use arrow_schema::Fields; use datafusion_expr::execution_props::ExecutionProps; use futures::stream::FuturesUnordered; use futures::{stream::BoxStream, StreamExt, TryStreamExt}; diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 215a06e81c3dc..3be666ce7974f 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -52,7 +52,6 @@ apache-avro = { version = "0.17", default-features = false, features = [ ], optional = true } arrow = { workspace = true } arrow-ipc = { workspace = true } -arrow-schema = { workspace = true } base64 = "0.22.1" half = { workspace = true } hashbrown = { workspace = true } diff --git a/datafusion/common/src/column.rs b/datafusion/common/src/column.rs index 2c3a84229d306..50a4e257d1c99 100644 --- a/datafusion/common/src/column.rs +++ b/datafusion/common/src/column.rs @@ -20,7 +20,7 @@ use crate::error::{_schema_err, add_possible_columns_to_diag}; use crate::utils::{parse_identifiers_normalized, quote_identifier}; use crate::{DFSchema, Diagnostic, Result, SchemaError, Spans, TableReference}; -use arrow_schema::{Field, FieldRef}; +use arrow::datatypes::{Field, FieldRef}; use std::collections::HashSet; use std::convert::Infallible; use std::fmt; @@ -373,8 +373,7 @@ impl fmt::Display for Column { #[cfg(test)] mod tests { use super::*; - use arrow::datatypes::DataType; - use arrow_schema::SchemaBuilder; + use arrow::datatypes::{DataType, SchemaBuilder}; use std::sync::Arc; fn create_qualified_schema(qualifier: &str, names: Vec<&str>) -> Result { diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs index f6ab5acd975f1..7e9025dee1f4c 100644 --- a/datafusion/common/src/dfschema.rs +++ b/datafusion/common/src/dfschema.rs @@ -30,8 +30,9 @@ use crate::{ }; use arrow::compute::can_cast_types; -use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema, SchemaRef}; -use arrow_schema::SchemaBuilder; +use arrow::datatypes::{ + DataType, Field, FieldRef, Fields, Schema, SchemaBuilder, SchemaRef, +}; /// A reference-counted reference to a [DFSchema]. pub type DFSchemaRef = Arc; @@ -56,7 +57,7 @@ pub type DFSchemaRef = Arc; /// /// ```rust /// use datafusion_common::{DFSchema, Column}; -/// use arrow_schema::{DataType, Field, Schema}; +/// use arrow::datatypes::{DataType, Field, Schema}; /// /// let arrow_schema = Schema::new(vec![ /// Field::new("c1", DataType::Int32, false), @@ -77,7 +78,7 @@ pub type DFSchemaRef = Arc; /// /// ```rust /// use datafusion_common::{DFSchema, Column}; -/// use arrow_schema::{DataType, Field, Schema}; +/// use arrow::datatypes::{DataType, Field, Schema}; /// /// let arrow_schema = Schema::new(vec![ /// Field::new("c1", DataType::Int32, false), @@ -94,8 +95,7 @@ pub type DFSchemaRef = Arc; /// /// ```rust /// use datafusion_common::DFSchema; -/// use arrow_schema::Schema; -/// use arrow::datatypes::Field; +/// use arrow::datatypes::{Schema, Field}; /// use std::collections::HashMap; /// /// let df_schema = DFSchema::from_unqualified_fields(vec![ diff --git a/datafusion/common/src/file_options/parquet_writer.rs b/datafusion/common/src/file_options/parquet_writer.rs index 6a717d3c0c60f..8c785b84313c8 100644 --- a/datafusion/common/src/file_options/parquet_writer.rs +++ b/datafusion/common/src/file_options/parquet_writer.rs @@ -25,7 +25,7 @@ use crate::{ DataFusionError, Result, _internal_datafusion_err, }; -use arrow_schema::Schema; +use arrow::datatypes::Schema; // TODO: handle once deprecated #[allow(deprecated)] use parquet::{ diff --git a/datafusion/common/src/param_value.rs b/datafusion/common/src/param_value.rs index 8d61bad97b9f9..d2802c096da1b 100644 --- a/datafusion/common/src/param_value.rs +++ b/datafusion/common/src/param_value.rs @@ -17,7 +17,7 @@ use crate::error::{_plan_datafusion_err, _plan_err}; use crate::{Result, ScalarValue}; -use arrow_schema::DataType; +use arrow::datatypes::DataType; use std::collections::HashMap; /// The parameter value corresponding to the placeholder diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rs index 5db0f5ed5cc0a..e609e24265803 100644 --- a/datafusion/common/src/scalar/mod.rs +++ b/datafusion/common/src/scalar/mod.rs @@ -55,10 +55,9 @@ use arrow::datatypes::{ Int8Type, IntervalDayTimeType, IntervalMonthDayNanoType, IntervalUnit, IntervalYearMonthType, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, - UInt8Type, DECIMAL128_MAX_PRECISION, + UInt8Type, UnionFields, UnionMode, DECIMAL128_MAX_PRECISION, }; use arrow::util::display::{array_value_to_string, ArrayFormatter, FormatOptions}; -use arrow_schema::{UnionFields, UnionMode}; use crate::format::DEFAULT_CAST_OPTIONS; use half::f16; @@ -3964,9 +3963,9 @@ mod tests { use arrow::array::{types::Float64Type, NullBufferBuilder}; use arrow::buffer::{Buffer, OffsetBuffer}; use arrow::compute::{is_null, kernels}; + use arrow::datatypes::Fields; use arrow::error::ArrowError; use arrow::util::pretty::pretty_format_columns; - use arrow_schema::Fields; use chrono::NaiveDate; use rand::Rng; diff --git a/datafusion/common/src/scalar/struct_builder.rs b/datafusion/common/src/scalar/struct_builder.rs index 4a6a8f0289a7d..5ed464018401d 100644 --- a/datafusion/common/src/scalar/struct_builder.rs +++ b/datafusion/common/src/scalar/struct_builder.rs @@ -20,8 +20,7 @@ use crate::error::_internal_err; use crate::{Result, ScalarValue}; use arrow::array::{ArrayRef, StructArray}; -use arrow::datatypes::{DataType, FieldRef, Fields}; -use arrow_schema::Field; +use arrow::datatypes::{DataType, Field, FieldRef, Fields}; use std::sync::Arc; /// Builder for [`ScalarValue::Struct`]. diff --git a/datafusion/common/src/stats.rs b/datafusion/common/src/stats.rs index dd8848d249233..5b841db53c5ee 100644 --- a/datafusion/common/src/stats.rs +++ b/datafusion/common/src/stats.rs @@ -21,7 +21,7 @@ use std::fmt::{self, Debug, Display}; use crate::{Result, ScalarValue}; -use arrow_schema::{DataType, Schema, SchemaRef}; +use arrow::datatypes::{DataType, Schema, SchemaRef}; /// Represents a value with a degree of certainty. `Precision` is used to /// propagate information the precision of statistical values. diff --git a/datafusion/common/src/test_util.rs b/datafusion/common/src/test_util.rs index 22a7d87a89492..a1f883f20525d 100644 --- a/datafusion/common/src/test_util.rs +++ b/datafusion/common/src/test_util.rs @@ -338,9 +338,9 @@ macro_rules! create_array { macro_rules! record_batch { ($(($name: expr, $type: ident, $values: expr)),*) => { { - let schema = std::sync::Arc::new(arrow_schema::Schema::new(vec![ + let schema = std::sync::Arc::new(arrow::datatypes::Schema::new(vec![ $( - arrow_schema::Field::new($name, arrow_schema::DataType::$type, true), + arrow::datatypes::Field::new($name, arrow::datatypes::DataType::$type, true), )* ])); diff --git a/datafusion/common/src/types/field.rs b/datafusion/common/src/types/field.rs index 85c7c157272ae..5a880ba10a413 100644 --- a/datafusion/common/src/types/field.rs +++ b/datafusion/common/src/types/field.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use arrow_schema::{Field, Fields, UnionFields}; +use arrow::datatypes::{Field, Fields, UnionFields}; use std::hash::{Hash, Hasher}; use std::{ops::Deref, sync::Arc}; diff --git a/datafusion/common/src/types/logical.rs b/datafusion/common/src/types/logical.rs index a65392cae3444..884ce20fd9e29 100644 --- a/datafusion/common/src/types/logical.rs +++ b/datafusion/common/src/types/logical.rs @@ -17,7 +17,7 @@ use super::NativeType; use crate::error::Result; -use arrow_schema::DataType; +use arrow::datatypes::DataType; use core::fmt; use std::{cmp::Ordering, hash::Hash, sync::Arc}; diff --git a/datafusion/common/src/types/native.rs b/datafusion/common/src/types/native.rs index c5f180a150352..a4c4dfc7b1061 100644 --- a/datafusion/common/src/types/native.rs +++ b/datafusion/common/src/types/native.rs @@ -21,7 +21,7 @@ use super::{ }; use crate::error::{Result, _internal_err}; use arrow::compute::can_cast_types; -use arrow_schema::{ +use arrow::datatypes::{ DataType, Field, FieldRef, Fields, IntervalUnit, TimeUnit, UnionFields, }; use std::{fmt::Display, sync::Arc}; @@ -126,7 +126,7 @@ pub enum NativeType { /// nevertheless correct). /// /// ``` - /// # use arrow_schema::{DataType, TimeUnit}; + /// # use arrow::datatypes::{DataType, TimeUnit}; /// DataType::Timestamp(TimeUnit::Second, None); /// DataType::Timestamp(TimeUnit::Second, Some("literal".into())); /// DataType::Timestamp(TimeUnit::Second, Some("string".to_string().into())); diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rs index d66cf75622782..0bf7c03a0a195 100644 --- a/datafusion/common/src/utils/mod.rs +++ b/datafusion/common/src/utils/mod.rs @@ -30,8 +30,7 @@ use arrow::array::{ }; use arrow::buffer::OffsetBuffer; use arrow::compute::{partition, SortColumn, SortOptions}; -use arrow::datatypes::{Field, SchemaRef}; -use arrow_schema::DataType; +use arrow::datatypes::{DataType, Field, SchemaRef}; use sqlparser::ast::Ident; use sqlparser::dialect::GenericDialect; use sqlparser::parser::Parser; diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 3c7287b887a27..2c303fda2eeb1 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -74,7 +74,7 @@ recursive_protection = [ "datafusion-physical-optimizer/recursive_protection", "datafusion-sql/recursive_protection", ] -serde = ["arrow-schema/serde"] +serde = ["dep:serde"] string_expressions = ["datafusion-functions/string_expressions"] unicode_expressions = [ "datafusion-sql/unicode_expressions", @@ -120,6 +120,7 @@ parking_lot = { workspace = true } parquet = { workspace = true, optional = true, default-features = true } rand = { workspace = true } regex = { workspace = true } +serde = { version = "1.0", default-features = false, features = ["derive"], optional = true } sqlparser = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true } diff --git a/datafusion/core/benches/sql_query_with_io.rs b/datafusion/core/benches/sql_query_with_io.rs index 58f8409313aad..58d71ee5b2eb8 100644 --- a/datafusion/core/benches/sql_query_with_io.rs +++ b/datafusion/core/benches/sql_query_with_io.rs @@ -18,7 +18,7 @@ use std::{fmt::Write, sync::Arc, time::Duration}; use arrow::array::{Int64Builder, RecordBatch, UInt64Builder}; -use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use bytes::Bytes; use criterion::{criterion_group, criterion_main, Criterion, SamplingMode}; use datafusion::{ diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rs index 9731b8784076f..b1eb2a19e31d6 100644 --- a/datafusion/core/src/dataframe/mod.rs +++ b/datafusion/core/src/dataframe/mod.rs @@ -45,8 +45,7 @@ use std::sync::Arc; use arrow::array::{Array, ArrayRef, Int64Array, StringArray}; use arrow::compute::{cast, concat}; -use arrow::datatypes::{DataType, Field}; -use arrow_schema::{Schema, SchemaRef}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion_common::config::{CsvOptions, JsonOptions}; use datafusion_common::{ exec_err, not_impl_err, plan_err, Column, DFSchema, DataFusionError, ParamValues, diff --git a/datafusion/core/src/datasource/data_source.rs b/datafusion/core/src/datasource/data_source.rs index 03bfb4175022d..d31b68019e308 100644 --- a/datafusion/core/src/datasource/data_source.rs +++ b/datafusion/core/src/datasource/data_source.rs @@ -24,7 +24,7 @@ use std::sync::Arc; use crate::datasource::physical_plan::{FileOpener, FileScanConfig}; -use arrow_schema::SchemaRef; +use arrow::datatypes::SchemaRef; use datafusion_common::Statistics; use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion_physical_plan::DisplayFormatType; diff --git a/datafusion/core/src/datasource/file_format/arrow.rs b/datafusion/core/src/datasource/file_format/arrow.rs index 5a4bf103e7ce4..dd56b4c137eda 100644 --- a/datafusion/core/src/datasource/file_format/arrow.rs +++ b/datafusion/core/src/datasource/file_format/arrow.rs @@ -37,11 +37,12 @@ use crate::datasource::physical_plan::{ use crate::error::Result; use crate::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan}; +use arrow::datatypes::{Schema, SchemaRef}; +use arrow::error::ArrowError; use arrow::ipc::convert::fb_to_schema; use arrow::ipc::reader::FileReader; use arrow::ipc::writer::IpcWriteOptions; use arrow::ipc::{root_as_message, CompressionType}; -use arrow_schema::{ArrowError, Schema, SchemaRef}; use datafusion_catalog::Session; use datafusion_common::parsers::CompressionTypeVariant; use datafusion_common::{ diff --git a/datafusion/core/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rs index cd8a4dedc1695..7d06648d7ba8f 100644 --- a/datafusion/core/src/datasource/file_format/csv.rs +++ b/datafusion/core/src/datasource/file_format/csv.rs @@ -43,7 +43,7 @@ use crate::physical_plan::{ use arrow::array::RecordBatch; use arrow::csv::WriterBuilder; use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef}; -use arrow_schema::ArrowError; +use arrow::error::ArrowError; use datafusion_catalog::Session; use datafusion_common::config::{ConfigField, ConfigFileType, CsvOptions}; use datafusion_common::file_options::csv_writer::CsvWriterOptions; diff --git a/datafusion/core/src/datasource/file_format/json.rs b/datafusion/core/src/datasource/file_format/json.rs index 3aae692151f6b..32a527bc58766 100644 --- a/datafusion/core/src/datasource/file_format/json.rs +++ b/datafusion/core/src/datasource/file_format/json.rs @@ -43,11 +43,10 @@ use crate::physical_plan::{ }; use arrow::array::RecordBatch; -use arrow::datatypes::Schema; -use arrow::datatypes::SchemaRef; +use arrow::datatypes::{Schema, SchemaRef}; +use arrow::error::ArrowError; use arrow::json; use arrow::json::reader::{infer_json_schema_from_iterator, ValueIter}; -use arrow_schema::ArrowError; use datafusion_catalog::Session; use datafusion_common::config::{ConfigField, ConfigFileType, JsonOptions}; use datafusion_common::file_options::json_writer::JsonWriterOptions; @@ -438,9 +437,9 @@ mod tests { use crate::test::object_store::local_unpartitioned_file; use arrow::compute::concat_batches; + use arrow::datatypes::{DataType, Field}; use arrow::json::ReaderBuilder; use arrow::util::pretty; - use arrow_schema::{DataType, Field}; use datafusion_common::cast::as_int64_array; use datafusion_common::stats::Precision; use datafusion_common::{assert_batches_eq, internal_err}; diff --git a/datafusion/core/src/datasource/file_format/mod.rs b/datafusion/core/src/datasource/file_format/mod.rs index 90a09252e7895..e50043d31c15c 100644 --- a/datafusion/core/src/datasource/file_format/mod.rs +++ b/datafusion/core/src/datasource/file_format/mod.rs @@ -38,12 +38,12 @@ use std::sync::Arc; use std::task::Poll; use crate::arrow::array::RecordBatch; -use crate::arrow::datatypes::SchemaRef; +use crate::arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef}; +use crate::arrow::error::ArrowError; use crate::datasource::physical_plan::{FileScanConfig, FileSinkConfig}; use crate::error::Result; use crate::physical_plan::{ExecutionPlan, Statistics}; -use arrow_schema::{ArrowError, DataType, Field, FieldRef, Schema}; use datafusion_catalog::Session; use datafusion_common::file_options::file_type::FileType; use datafusion_common::{internal_err, not_impl_err, GetExt}; diff --git a/datafusion/core/src/datasource/file_format/parquet.rs b/datafusion/core/src/datasource/file_format/parquet.rs index 0b23fd8da6124..9774792133cd3 100644 --- a/datafusion/core/src/datasource/file_format/parquet.rs +++ b/datafusion/core/src/datasource/file_format/parquet.rs @@ -1313,7 +1313,7 @@ mod tests { types::Int32Type, Array, ArrayRef, DictionaryArray, Int32Array, Int64Array, StringArray, }; - use arrow_schema::{DataType, Field}; + use arrow::datatypes::{DataType, Field}; use async_trait::async_trait; use datafusion_common::cast::{ as_binary_array, as_binary_view_array, as_boolean_array, as_float32_array, diff --git a/datafusion/core/src/datasource/file_format/write/demux.rs b/datafusion/core/src/datasource/file_format/write/demux.rs index 4546660032543..f3798847fe297 100644 --- a/datafusion/core/src/datasource/file_format/write/demux.rs +++ b/datafusion/core/src/datasource/file_format/write/demux.rs @@ -31,7 +31,7 @@ use arrow::array::{ builder::UInt64Builder, cast::AsArray, downcast_dictionary_array, RecordBatch, StringArray, StructArray, }; -use arrow_schema::{DataType, Schema}; +use arrow::datatypes::{DataType, Schema}; use datafusion_common::cast::{ as_boolean_array, as_date32_array, as_date64_array, as_int32_array, as_int64_array, as_string_array, as_string_view_array, diff --git a/datafusion/core/src/datasource/file_format/write/mod.rs b/datafusion/core/src/datasource/file_format/write/mod.rs index 81ecf3f0f88ce..8070887b92dfe 100644 --- a/datafusion/core/src/datasource/file_format/write/mod.rs +++ b/datafusion/core/src/datasource/file_format/write/mod.rs @@ -26,7 +26,7 @@ use crate::datasource::physical_plan::FileSinkConfig; use crate::error::Result; use arrow::array::RecordBatch; -use arrow_schema::Schema; +use arrow::datatypes::Schema; use bytes::Bytes; use object_store::buffered::BufWriter; use object_store::path::Path; diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index 5d3a0f886fe49..642ec93f36712 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -39,8 +39,7 @@ use datafusion_expr::{utils::conjunction, Expr, TableProviderFilterPushDown}; use datafusion_expr::{SortExpr, TableType}; use datafusion_physical_plan::{empty::EmptyExec, ExecutionPlan, Statistics}; -use arrow::datatypes::{DataType, Field, SchemaBuilder, SchemaRef}; -use arrow_schema::Schema; +use arrow::datatypes::{DataType, Field, Schema, SchemaBuilder, SchemaRef}; use datafusion_common::{ config_datafusion_err, internal_err, plan_err, project_schema, Constraints, SchemaExt, ToDFSchema, @@ -1205,8 +1204,8 @@ mod tests { }; use datafusion_physical_plan::collect; + use arrow::compute::SortOptions; use arrow::record_batch::RecordBatch; - use arrow_schema::SortOptions; use datafusion_common::stats::Precision; use datafusion_common::{assert_contains, ScalarValue}; use datafusion_expr::{BinaryExpr, LogicalPlanBuilder, Operator}; diff --git a/datafusion/core/src/datasource/mod.rs b/datafusion/core/src/datasource/mod.rs index 55df55ae3543e..12dd9d7cab389 100644 --- a/datafusion/core/src/datasource/mod.rs +++ b/datafusion/core/src/datasource/mod.rs @@ -47,7 +47,8 @@ pub use crate::logical_expr::TableType; pub use datafusion_execution::object_store; pub use statistics::get_statistics_with_limit; -use arrow_schema::{Schema, SortOptions}; +use arrow::compute::SortOptions; +use arrow::datatypes::Schema; use datafusion_common::{plan_err, Result}; use datafusion_expr::{Expr, SortExpr}; use datafusion_physical_expr::{expressions, LexOrdering, PhysicalSortExpr}; diff --git a/datafusion/core/src/datasource/physical_plan/arrow_file.rs b/datafusion/core/src/datasource/physical_plan/arrow_file.rs index 1a486a54ca396..4a7cdc192cd34 100644 --- a/datafusion/core/src/datasource/physical_plan/arrow_file.rs +++ b/datafusion/core/src/datasource/physical_plan/arrow_file.rs @@ -28,8 +28,8 @@ use crate::datasource::physical_plan::{ use crate::error::Result; use arrow::buffer::Buffer; +use arrow::datatypes::SchemaRef; use arrow_ipc::reader::FileDecoder; -use arrow_schema::SchemaRef; use datafusion_common::config::ConfigOptions; use datafusion_common::{Constraints, Statistics}; use datafusion_execution::{SendableRecordBatchStream, TaskContext}; @@ -321,7 +321,7 @@ impl FileOpener for ArrowOpener { footer_buf[..footer_len].try_into().unwrap(), ) .map_err(|err| { - arrow_schema::ArrowError::ParseError(format!( + arrow::error::ArrowError::ParseError(format!( "Unable to get root as footer: {err:?}" )) })?; diff --git a/datafusion/core/src/datasource/physical_plan/file_scan_config.rs b/datafusion/core/src/datasource/physical_plan/file_scan_config.rs index dc9207da51cb4..c714fad6e9c19 100644 --- a/datafusion/core/src/datasource/physical_plan/file_scan_config.rs +++ b/datafusion/core/src/datasource/physical_plan/file_scan_config.rs @@ -36,8 +36,7 @@ use arrow::array::{ ArrayData, ArrayRef, BufferBuilder, DictionaryArray, RecordBatch, RecordBatchOptions, }; use arrow::buffer::Buffer; -use arrow::datatypes::{ArrowNativeType, UInt16Type}; -use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use arrow::datatypes::{ArrowNativeType, DataType, Field, Schema, SchemaRef, UInt16Type}; use datafusion_common::stats::Precision; use datafusion_common::{ exec_err, ColumnStatistics, Constraints, DataFusionError, Statistics, @@ -84,8 +83,8 @@ pub fn wrap_partition_value_in_dict(val: ScalarValue) -> ScalarValue { /// # Example /// ``` /// # use std::sync::Arc; -/// # use arrow_schema::Schema; -/// use datafusion::datasource::listing::PartitionedFile; +/// # use arrow::datatypes::Schema; +/// # use datafusion::datasource::listing::PartitionedFile; /// # use datafusion::datasource::physical_plan::FileScanConfig; /// # use datafusion_execution::object_store::ObjectStoreUrl; /// # use datafusion::datasource::physical_plan::ArrowSource; diff --git a/datafusion/core/src/datasource/physical_plan/file_stream.rs b/datafusion/core/src/datasource/physical_plan/file_stream.rs index 85b1d714548d7..497af101bee75 100644 --- a/datafusion/core/src/datasource/physical_plan/file_stream.rs +++ b/datafusion/core/src/datasource/physical_plan/file_stream.rs @@ -526,7 +526,7 @@ mod tests { use crate::test::{make_partition, object_store::register_test_store}; use crate::datasource::physical_plan::CsvSource; - use arrow_schema::Schema; + use arrow::datatypes::Schema; use datafusion_common::internal_err; /// Test `FileOpener` which will simulate errors during file opening or scanning diff --git a/datafusion/core/src/datasource/physical_plan/mod.rs b/datafusion/core/src/datasource/physical_plan/mod.rs index 873df859702aa..a40d8680b1a5a 100644 --- a/datafusion/core/src/datasource/physical_plan/mod.rs +++ b/datafusion/core/src/datasource/physical_plan/mod.rs @@ -586,7 +586,7 @@ mod tests { BinaryArray, BooleanArray, Float32Array, Int32Array, Int64Array, RecordBatch, StringArray, UInt64Array, }; - use arrow_schema::{Field, Schema}; + use arrow::datatypes::{Field, Schema}; use crate::datasource::schema_adapter::{ DefaultSchemaAdapterFactory, SchemaAdapterFactory, diff --git a/datafusion/core/src/datasource/physical_plan/parquet/mod.rs b/datafusion/core/src/datasource/physical_plan/parquet/mod.rs index 4ba449e2ee82d..a1c2bb4207efa 100644 --- a/datafusion/core/src/datasource/physical_plan/parquet/mod.rs +++ b/datafusion/core/src/datasource/physical_plan/parquet/mod.rs @@ -575,9 +575,8 @@ mod tests { ArrayRef, Date64Array, Int32Array, Int64Array, Int8Array, StringArray, StructArray, }; - use arrow::datatypes::{Field, Schema, SchemaBuilder}; + use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaBuilder}; use arrow::record_batch::RecordBatch; - use arrow_schema::{DataType, Fields}; use bytes::{BufMut, BytesMut}; use datafusion_common::{assert_contains, ScalarValue}; use datafusion_expr::{col, lit, when, Expr}; diff --git a/datafusion/core/src/datasource/physical_plan/parquet/opener.rs b/datafusion/core/src/datasource/physical_plan/parquet/opener.rs index 02ad9dd551007..138b448979313 100644 --- a/datafusion/core/src/datasource/physical_plan/parquet/opener.rs +++ b/datafusion/core/src/datasource/physical_plan/parquet/opener.rs @@ -32,7 +32,8 @@ use crate::datasource::physical_plan::{ }; use crate::datasource::schema_adapter::SchemaAdapterFactory; -use arrow_schema::{ArrowError, SchemaRef}; +use arrow::datatypes::SchemaRef; +use arrow::error::ArrowError; use datafusion_common::{exec_err, Result}; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; use datafusion_physical_optimizer::pruning::PruningPredicate; diff --git a/datafusion/core/src/datasource/physical_plan/parquet/page_filter.rs b/datafusion/core/src/datasource/physical_plan/parquet/page_filter.rs index dcc4b0bc8150a..02329effb09a0 100644 --- a/datafusion/core/src/datasource/physical_plan/parquet/page_filter.rs +++ b/datafusion/core/src/datasource/physical_plan/parquet/page_filter.rs @@ -24,8 +24,10 @@ use super::metrics::ParquetFileMetrics; use crate::datasource::physical_plan::parquet::ParquetAccessPlan; use arrow::array::BooleanArray; -use arrow::{array::ArrayRef, datatypes::SchemaRef}; -use arrow_schema::Schema; +use arrow::{ + array::ArrayRef, + datatypes::{Schema, SchemaRef}, +}; use datafusion_common::ScalarValue; use datafusion_physical_expr::{split_conjunction, PhysicalExpr}; use datafusion_physical_optimizer::pruning::{PruningPredicate, PruningStatistics}; diff --git a/datafusion/core/src/datasource/physical_plan/parquet/row_filter.rs b/datafusion/core/src/datasource/physical_plan/parquet/row_filter.rs index bcd2c0af6f6f6..ac6eaf2c8f636 100644 --- a/datafusion/core/src/datasource/physical_plan/parquet/row_filter.rs +++ b/datafusion/core/src/datasource/physical_plan/parquet/row_filter.rs @@ -588,8 +588,7 @@ mod test { DefaultSchemaAdapterFactory, SchemaAdapterFactory, }; - use arrow::datatypes::Field; - use arrow_schema::{Fields, TimeUnit::Nanosecond}; + use arrow::datatypes::{Field, Fields, TimeUnit::Nanosecond}; use datafusion_expr::{cast, col, lit, Expr}; use datafusion_physical_expr::planner::logical2physical; use datafusion_physical_plan::metrics::{Count, Time}; diff --git a/datafusion/core/src/datasource/physical_plan/parquet/source.rs b/datafusion/core/src/datasource/physical_plan/parquet/source.rs index 0705a398f4fba..a98524b0bead4 100644 --- a/datafusion/core/src/datasource/physical_plan/parquet/source.rs +++ b/datafusion/core/src/datasource/physical_plan/parquet/source.rs @@ -31,7 +31,7 @@ use crate::datasource::schema_adapter::{ DefaultSchemaAdapterFactory, SchemaAdapterFactory, }; -use arrow_schema::{Schema, SchemaRef}; +use arrow::datatypes::{Schema, SchemaRef}; use datafusion_common::config::TableParquetOptions; use datafusion_common::Statistics; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; @@ -196,7 +196,7 @@ use object_store::ObjectStore; /// /// ``` /// # use std::sync::Arc; -/// # use arrow_schema::{Schema, SchemaRef}; +/// # use arrow::datatypes::{Schema, SchemaRef}; /// # use datafusion::datasource::listing::PartitionedFile; /// # use datafusion::datasource::physical_plan::parquet::ParquetAccessPlan; /// # use datafusion::datasource::physical_plan::FileScanConfig; diff --git a/datafusion/core/src/datasource/physical_plan/statistics.rs b/datafusion/core/src/datasource/physical_plan/statistics.rs index 64eb2b00de948..5811c19be408d 100644 --- a/datafusion/core/src/datasource/physical_plan/statistics.rs +++ b/datafusion/core/src/datasource/physical_plan/statistics.rs @@ -29,11 +29,11 @@ use std::sync::Arc; use crate::datasource::listing::PartitionedFile; use arrow::array::RecordBatch; +use arrow::datatypes::SchemaRef; use arrow::{ compute::SortColumn, row::{Row, Rows}, }; -use arrow_schema::SchemaRef; use datafusion_common::{plan_err, DataFusionError, Result}; use datafusion_physical_expr::{expressions::Column, PhysicalSortExpr}; use datafusion_physical_expr_common::sort_expr::LexOrdering; diff --git a/datafusion/core/src/datasource/schema_adapter.rs b/datafusion/core/src/datasource/schema_adapter.rs index efaae403b4158..e59d7b669ce0a 100644 --- a/datafusion/core/src/datasource/schema_adapter.rs +++ b/datafusion/core/src/datasource/schema_adapter.rs @@ -23,7 +23,7 @@ use arrow::array::{new_null_array, RecordBatch, RecordBatchOptions}; use arrow::compute::{can_cast_types, cast}; -use arrow_schema::{Schema, SchemaRef}; +use arrow::datatypes::{Schema, SchemaRef}; use datafusion_common::plan_err; use std::fmt::Debug; use std::sync::Arc; @@ -435,9 +435,8 @@ mod tests { use crate::assert_batches_sorted_eq; use arrow::array::{Int32Array, StringArray}; - use arrow::datatypes::{Field, Schema}; + use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; - use arrow_schema::{DataType, SchemaRef}; use object_store::path::Path; use object_store::ObjectMeta; diff --git a/datafusion/core/src/datasource/stream.rs b/datafusion/core/src/datasource/stream.rs index ff2e4436e94d6..d5fe070be82c4 100644 --- a/datafusion/core/src/datasource/stream.rs +++ b/datafusion/core/src/datasource/stream.rs @@ -29,7 +29,7 @@ use crate::catalog::{TableProvider, TableProviderFactory}; use crate::datasource::create_ordering; use arrow::array::{RecordBatch, RecordBatchReader, RecordBatchWriter}; -use arrow_schema::SchemaRef; +use arrow::datatypes::SchemaRef; use datafusion_common::{config_err, plan_err, Constraints, DataFusionError, Result}; use datafusion_common_runtime::SpawnedTask; use datafusion_execution::{SendableRecordBatchStream, TaskContext}; diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs index 61ddba10b09cc..c27d1e4fd46b4 100644 --- a/datafusion/core/src/execution/context/mod.rs +++ b/datafusion/core/src/execution/context/mod.rs @@ -49,9 +49,8 @@ use crate::{ variable::{VarProvider, VarType}, }; -use arrow::datatypes::SchemaRef; +use arrow::datatypes::{Schema, SchemaRef}; use arrow::record_batch::RecordBatch; -use arrow_schema::Schema; use datafusion_common::{ config::{ConfigExtension, TableOptions}, exec_datafusion_err, exec_err, not_impl_err, plan_datafusion_err, plan_err, @@ -1817,7 +1816,7 @@ mod tests { use crate::execution::memory_pool::MemoryConsumer; use crate::test; use crate::test_util::{plan_and_collect, populate_csv_partitions}; - use arrow_schema::{DataType, TimeUnit}; + use arrow::datatypes::{DataType, TimeUnit}; use std::env; use std::error::Error; use std::path::PathBuf; diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 85c2b2a0fd786..f1abf30c0c548 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -35,7 +35,7 @@ use datafusion_catalog::information_schema::{ }; use datafusion_catalog::MemoryCatalogProviderList; -use arrow_schema::{DataType, SchemaRef}; +use arrow::datatypes::{DataType, SchemaRef}; use datafusion_catalog::{Session, TableFunction, TableFunctionImpl}; use datafusion_common::alias::AliasGenerator; use datafusion_common::config::{ConfigExtension, ConfigOptions, TableOptions}; @@ -1991,7 +1991,7 @@ mod tests { use crate::datasource::MemTable; use crate::execution::context::SessionState; use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray}; - use arrow_schema::{DataType, Field, Schema}; + use arrow::datatypes::{DataType, Field, Schema}; use datafusion_catalog::MemoryCatalogProviderList; use datafusion_common::DFSchema; use datafusion_common::Result; diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 9fcb9562a485b..47dee391c7511 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -508,7 +508,7 @@ impl DefaultPhysicalPlanner { // the column name rather than column name + explicit data type. let table_partition_cols = partition_by .iter() - .map(|s| (s.to_string(), arrow_schema::DataType::Null)) + .map(|s| (s.to_string(), arrow::datatypes::DataType::Null)) .collect::>(); let keep_partition_by_columns = match source_option_tuples diff --git a/datafusion/core/src/schema_equivalence.rs b/datafusion/core/src/schema_equivalence.rs index f0d2acad6be98..70bee206655bf 100644 --- a/datafusion/core/src/schema_equivalence.rs +++ b/datafusion/core/src/schema_equivalence.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use arrow_schema::{DataType, Field, Fields, Schema}; +use arrow::datatypes::{DataType, Field, Fields, Schema}; /// Verifies whether the original planned schema can be satisfied with data /// adhering to the candidate schema. In practice, this is equality check on the diff --git a/datafusion/core/tests/catalog/memory.rs b/datafusion/core/tests/catalog/memory.rs index bef23fff3e962..3e45fb7532264 100644 --- a/datafusion/core/tests/catalog/memory.rs +++ b/datafusion/core/tests/catalog/memory.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use arrow_schema::Schema; +use arrow::datatypes::Schema; use datafusion::catalog::CatalogProvider; use datafusion::datasource::empty::EmptyTable; use datafusion::datasource::listing::{ diff --git a/datafusion/core/tests/dataframe/dataframe_functions.rs b/datafusion/core/tests/dataframe/dataframe_functions.rs index 89ec5a5908de7..29c24948fbf09 100644 --- a/datafusion/core/tests/dataframe/dataframe_functions.rs +++ b/datafusion/core/tests/dataframe/dataframe_functions.rs @@ -16,12 +16,12 @@ // under the License. use arrow::array::{types::Int32Type, ListArray}; +use arrow::datatypes::SchemaRef; use arrow::datatypes::{DataType, Field, Schema}; use arrow::{ array::{Int32Array, StringArray}, record_batch::RecordBatch, }; -use arrow_schema::SchemaRef; use std::sync::Arc; use datafusion::error::Result; diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs index 772d9dbc8f46c..8155fd6a2ff96 100644 --- a/datafusion/core/tests/dataframe/mod.rs +++ b/datafusion/core/tests/dataframe/mod.rs @@ -26,9 +26,12 @@ use arrow::array::{ StringBuilder, StructBuilder, UInt32Array, UInt32Builder, UnionArray, }; use arrow::buffer::ScalarBuffer; -use arrow::datatypes::{DataType, Field, Float32Type, Int32Type, Schema, UInt64Type}; +use arrow::datatypes::{ + DataType, Field, Float32Type, Int32Type, Schema, SchemaRef, UInt64Type, UnionFields, + UnionMode, +}; +use arrow::error::ArrowError; use arrow::util::pretty::pretty_format_batches; -use arrow_schema::{ArrowError, SchemaRef, UnionFields, UnionMode}; use datafusion_functions_aggregate::count::count_udaf; use datafusion_functions_aggregate::expr_fn::{ array_agg, avg, count, count_distinct, max, median, min, sum, diff --git a/datafusion/core/tests/execution/logical_plan.rs b/datafusion/core/tests/execution/logical_plan.rs index 86acbe16474c2..a521902389dcb 100644 --- a/datafusion/core/tests/execution/logical_plan.rs +++ b/datafusion/core/tests/execution/logical_plan.rs @@ -16,7 +16,7 @@ // under the License. use arrow::array::Int64Array; -use arrow_schema::{DataType, Field}; +use arrow::datatypes::{DataType, Field}; use datafusion::execution::session_state::SessionStateBuilder; use datafusion_common::{Column, DFSchema, Result, ScalarValue, Spans}; use datafusion_execution::TaskContext; diff --git a/datafusion/core/tests/expr_api/mod.rs b/datafusion/core/tests/expr_api/mod.rs index 8f8ca21c206dd..7c0119e8ae83d 100644 --- a/datafusion/core/tests/expr_api/mod.rs +++ b/datafusion/core/tests/expr_api/mod.rs @@ -19,8 +19,8 @@ use arrow::array::{ builder::{ListBuilder, StringBuilder}, ArrayRef, Int64Array, RecordBatch, StringArray, StructArray, }; +use arrow::datatypes::{DataType, Field}; use arrow::util::pretty::{pretty_format_batches, pretty_format_columns}; -use arrow_schema::{DataType, Field}; use datafusion::prelude::*; use datafusion_common::{DFSchema, ScalarValue}; use datafusion_expr::ExprFunctionExt; diff --git a/datafusion/core/tests/expr_api/parse_sql_expr.rs b/datafusion/core/tests/expr_api/parse_sql_expr.rs index cc049f0004d97..92c18204324f7 100644 --- a/datafusion/core/tests/expr_api/parse_sql_expr.rs +++ b/datafusion/core/tests/expr_api/parse_sql_expr.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use arrow_schema::{DataType, Field, Schema}; +use arrow::datatypes::{DataType, Field, Schema}; use datafusion::prelude::{CsvReadOptions, SessionContext}; use datafusion_common::DFSchema; use datafusion_common::{DFSchemaRef, Result, ToDFSchema}; diff --git a/datafusion/core/tests/fifo/mod.rs b/datafusion/core/tests/fifo/mod.rs index cb587e3510c2d..141a3f3b75586 100644 --- a/datafusion/core/tests/fifo/mod.rs +++ b/datafusion/core/tests/fifo/mod.rs @@ -28,8 +28,7 @@ mod unix_test { use arrow::array::Array; use arrow::csv::ReaderBuilder; - use arrow::datatypes::{DataType, Field, Schema}; - use arrow_schema::SchemaRef; + use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion::datasource::stream::{FileStreamProvider, StreamConfig, StreamTable}; use datafusion::datasource::TableProvider; use datafusion::{ diff --git a/datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs b/datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs index 0257850ffc504..5e1f263b4c76b 100644 --- a/datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs +++ b/datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs @@ -24,12 +24,11 @@ use crate::fuzz_cases::aggregation_fuzzer::{ use arrow::array::{types::Int64Type, Array, ArrayRef, AsArray, Int64Array, RecordBatch}; use arrow::compute::{concat_batches, SortOptions}; -use arrow::datatypes::DataType; -use arrow::util::pretty::pretty_format_batches; -use arrow_schema::{ - IntervalUnit, TimeUnit, DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE, +use arrow::datatypes::{ + DataType, IntervalUnit, TimeUnit, DECIMAL128_MAX_PRECISION, DECIMAL128_MAX_SCALE, DECIMAL256_MAX_PRECISION, DECIMAL256_MAX_SCALE, }; +use arrow::util::pretty::pretty_format_batches; use datafusion::common::Result; use datafusion::datasource::MemTable; use datafusion::physical_expr::aggregate::AggregateExprBuilder; diff --git a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/context_generator.rs b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/context_generator.rs index 9c8f83f75ccb4..8a8aa180b3c44 100644 --- a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/context_generator.rs +++ b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/context_generator.rs @@ -254,7 +254,7 @@ impl SkipPartialParams { #[cfg(test)] mod test { use arrow::array::{RecordBatch, StringArray, UInt32Array}; - use arrow_schema::{DataType, Field, Schema}; + use arrow::datatypes::{DataType, Field, Schema}; use crate::fuzz_cases::aggregation_fuzzer::check_equality_of_batches; diff --git a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/data_generator.rs b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/data_generator.rs index 3ebd899f4e157..4d4c6aa79357b 100644 --- a/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/data_generator.rs +++ b/datafusion/core/tests/fuzz_cases/aggregation_fuzzer/data_generator.rs @@ -19,15 +19,15 @@ use std::sync::Arc; use arrow::array::{ArrayRef, RecordBatch}; use arrow::datatypes::{ - BinaryType, BinaryViewType, BooleanType, ByteArrayType, ByteViewType, Date32Type, - Date64Type, Decimal128Type, Decimal256Type, Float32Type, Float64Type, Int16Type, - Int32Type, Int64Type, Int8Type, IntervalDayTimeType, IntervalMonthDayNanoType, - IntervalYearMonthType, LargeBinaryType, LargeUtf8Type, StringViewType, - Time32MillisecondType, Time32SecondType, Time64MicrosecondType, Time64NanosecondType, - TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, - TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, UInt8Type, Utf8Type, + BinaryType, BinaryViewType, BooleanType, ByteArrayType, ByteViewType, DataType, + Date32Type, Date64Type, Decimal128Type, Decimal256Type, Field, Float32Type, + Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, IntervalDayTimeType, + IntervalMonthDayNanoType, IntervalUnit, IntervalYearMonthType, LargeBinaryType, + LargeUtf8Type, Schema, StringViewType, Time32MillisecondType, Time32SecondType, + Time64MicrosecondType, Time64NanosecondType, TimeUnit, TimestampMicrosecondType, + TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, UInt16Type, + UInt32Type, UInt64Type, UInt8Type, Utf8Type, }; -use arrow_schema::{DataType, Field, IntervalUnit, Schema, TimeUnit}; use datafusion_common::{arrow_datafusion_err, DataFusionError, Result}; use datafusion_physical_expr::{expressions::col, PhysicalSortExpr}; use datafusion_physical_expr_common::sort_expr::LexOrdering; diff --git a/datafusion/core/tests/fuzz_cases/equivalence/ordering.rs b/datafusion/core/tests/fuzz_cases/equivalence/ordering.rs index cd9897d43baad..769deef1187d6 100644 --- a/datafusion/core/tests/fuzz_cases/equivalence/ordering.rs +++ b/datafusion/core/tests/fuzz_cases/equivalence/ordering.rs @@ -20,7 +20,7 @@ use crate::fuzz_cases::equivalence::utils::{ generate_table_for_eq_properties, generate_table_for_orderings, is_table_same_after_sort, TestScalarUDF, }; -use arrow_schema::SortOptions; +use arrow::compute::SortOptions; use datafusion_common::Result; use datafusion_expr::{Operator, ScalarUDF}; use datafusion_physical_expr::expressions::{col, BinaryExpr}; diff --git a/datafusion/core/tests/fuzz_cases/equivalence/projection.rs b/datafusion/core/tests/fuzz_cases/equivalence/projection.rs index 78fbda16c0a05..a3fa1157b38f4 100644 --- a/datafusion/core/tests/fuzz_cases/equivalence/projection.rs +++ b/datafusion/core/tests/fuzz_cases/equivalence/projection.rs @@ -19,7 +19,7 @@ use crate::fuzz_cases::equivalence::utils::{ apply_projection, create_random_schema, generate_table_for_eq_properties, is_table_same_after_sort, TestScalarUDF, }; -use arrow_schema::SortOptions; +use arrow::compute::SortOptions; use datafusion_common::Result; use datafusion_expr::{Operator, ScalarUDF}; use datafusion_physical_expr::equivalence::ProjectionMapping; diff --git a/datafusion/core/tests/fuzz_cases/equivalence/utils.rs b/datafusion/core/tests/fuzz_cases/equivalence/utils.rs index b66b7b2aca430..aa6bba8083a1f 100644 --- a/datafusion/core/tests/fuzz_cases/equivalence/utils.rs +++ b/datafusion/core/tests/fuzz_cases/equivalence/utils.rs @@ -23,9 +23,9 @@ use std::cmp::Ordering; use std::sync::Arc; use arrow::array::{ArrayRef, Float32Array, Float64Array, RecordBatch, UInt32Array}; +use arrow::compute::SortOptions; use arrow::compute::{lexsort_to_indices, take_record_batch, SortColumn}; -use arrow::datatypes::{DataType, Field, Schema}; -use arrow_schema::{SchemaRef, SortOptions}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion_common::utils::{compare_rows, get_row_at_idx}; use datafusion_common::{exec_err, plan_datafusion_err, DataFusionError, Result}; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; diff --git a/datafusion/core/tests/fuzz_cases/join_fuzz.rs b/datafusion/core/tests/fuzz_cases/join_fuzz.rs index 8e8178e55d876..5dd29f90ef838 100644 --- a/datafusion/core/tests/fuzz_cases/join_fuzz.rs +++ b/datafusion/core/tests/fuzz_cases/join_fuzz.rs @@ -22,9 +22,9 @@ use crate::fuzz_cases::join_fuzz::JoinTestType::{HjSmj, NljHj}; use arrow::array::{ArrayRef, Int32Array}; use arrow::compute::SortOptions; +use arrow::datatypes::Schema; use arrow::record_batch::RecordBatch; use arrow::util::pretty::pretty_format_batches; -use arrow_schema::Schema; use datafusion::common::JoinSide; use datafusion::logical_expr::{JoinType, Operator}; use datafusion::physical_expr::expressions::BinaryExpr; diff --git a/datafusion/core/tests/fuzz_cases/limit_fuzz.rs b/datafusion/core/tests/fuzz_cases/limit_fuzz.rs index a73845c56a0f0..987a732eb294b 100644 --- a/datafusion/core/tests/fuzz_cases/limit_fuzz.rs +++ b/datafusion/core/tests/fuzz_cases/limit_fuzz.rs @@ -19,8 +19,8 @@ use arrow::array::{Float64Array, Int32Array, Int64Array, RecordBatch, StringArray}; use arrow::compute::concat_batches; +use arrow::datatypes::SchemaRef; use arrow::util::pretty::pretty_format_batches; -use arrow_schema::SchemaRef; use datafusion::datasource::MemTable; use datafusion::prelude::SessionContext; use datafusion_common::assert_contains; diff --git a/datafusion/core/tests/fuzz_cases/pruning.rs b/datafusion/core/tests/fuzz_cases/pruning.rs index fef009fa911c6..e11d472b9b8a8 100644 --- a/datafusion/core/tests/fuzz_cases/pruning.rs +++ b/datafusion/core/tests/fuzz_cases/pruning.rs @@ -18,7 +18,7 @@ use std::sync::{Arc, OnceLock}; use arrow::array::{Array, RecordBatch, StringArray}; -use arrow_schema::{DataType, Field, Schema}; +use arrow::datatypes::{DataType, Field, Schema}; use bytes::{BufMut, Bytes, BytesMut}; use datafusion::{ datasource::{ diff --git a/datafusion/core/tests/fuzz_cases/sort_preserving_repartition_fuzz.rs b/datafusion/core/tests/fuzz_cases/sort_preserving_repartition_fuzz.rs index 8ffc78a9f59db..d23408743f9fa 100644 --- a/datafusion/core/tests/fuzz_cases/sort_preserving_repartition_fuzz.rs +++ b/datafusion/core/tests/fuzz_cases/sort_preserving_repartition_fuzz.rs @@ -20,8 +20,8 @@ mod sp_repartition_fuzz_tests { use std::sync::Arc; use arrow::array::{ArrayRef, Int64Array, RecordBatch, UInt64Array}; - use arrow::compute::{concat_batches, lexsort, SortColumn}; - use arrow_schema::{DataType, Field, Schema, SchemaRef, SortOptions}; + use arrow::compute::{concat_batches, lexsort, SortColumn, SortOptions}; + use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion::physical_plan::{ collect, diff --git a/datafusion/core/tests/memory_limit/mod.rs b/datafusion/core/tests/memory_limit/mod.rs index b7c6566271870..719faed4e4542 100644 --- a/datafusion/core/tests/memory_limit/mod.rs +++ b/datafusion/core/tests/memory_limit/mod.rs @@ -24,8 +24,8 @@ use std::sync::{Arc, LazyLock}; #[cfg(feature = "extended_tests")] mod memory_limit_validation; use arrow::array::{ArrayRef, DictionaryArray, RecordBatch}; +use arrow::compute::SortOptions; use arrow::datatypes::{Int32Type, SchemaRef}; -use arrow_schema::SortOptions; use datafusion::assert_batches_eq; use datafusion::datasource::{MemTable, TableProvider}; use datafusion::execution::disk_manager::DiskManagerConfig; diff --git a/datafusion/core/tests/optimizer/mod.rs b/datafusion/core/tests/optimizer/mod.rs index 37a6ca7f5934e..585540bd58754 100644 --- a/datafusion/core/tests/optimizer/mod.rs +++ b/datafusion/core/tests/optimizer/mod.rs @@ -22,8 +22,9 @@ use std::any::Any; use std::collections::HashMap; use std::sync::Arc; -use arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit}; -use arrow_schema::{Fields, SchemaBuilder}; +use arrow::datatypes::{ + DataType, Field, Fields, Schema, SchemaBuilder, SchemaRef, TimeUnit, +}; use datafusion_common::config::ConfigOptions; use datafusion_common::tree_node::{TransformedResult, TreeNode}; use datafusion_common::{plan_err, DFSchema, Result, ScalarValue, TableReference}; diff --git a/datafusion/core/tests/parquet/external_access_plan.rs b/datafusion/core/tests/parquet/external_access_plan.rs index 216f03aac746e..cbf6580b7e4ba 100644 --- a/datafusion/core/tests/parquet/external_access_plan.rs +++ b/datafusion/core/tests/parquet/external_access_plan.rs @@ -23,8 +23,8 @@ use std::sync::Arc; use crate::parquet::utils::MetricsFinder; use crate::parquet::{create_data_batch, Scenario}; +use arrow::datatypes::SchemaRef; use arrow::util::pretty::pretty_format_batches; -use arrow_schema::SchemaRef; use datafusion::common::Result; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::physical_plan::parquet::{ParquetAccessPlan, RowGroupAccess}; diff --git a/datafusion/core/tests/parquet/schema_coercion.rs b/datafusion/core/tests/parquet/schema_coercion.rs index 9175a6e91e91a..90793028f2095 100644 --- a/datafusion/core/tests/parquet/schema_coercion.rs +++ b/datafusion/core/tests/parquet/schema_coercion.rs @@ -21,8 +21,7 @@ use arrow::array::{ types::Int32Type, ArrayRef, DictionaryArray, Float32Array, Int64Array, RecordBatch, StringArray, }; -use arrow::datatypes::{Field, Schema}; -use arrow_schema::DataType; +use arrow::datatypes::{DataType, Field, Schema}; use datafusion::assert_batches_sorted_eq; use datafusion::datasource::physical_plan::{FileScanConfig, ParquetSource}; use datafusion::physical_plan::collect; diff --git a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs index c30b906d199e3..4b358e47361bb 100644 --- a/datafusion/core/tests/physical_optimizer/enforce_sorting.rs +++ b/datafusion/core/tests/physical_optimizer/enforce_sorting.rs @@ -30,7 +30,7 @@ use crate::physical_optimizer::test_utils::{ use datafusion_physical_plan::displayable; use arrow::compute::SortOptions; -use arrow_schema::SchemaRef; +use arrow::datatypes::SchemaRef; use datafusion_common::Result; use datafusion_expr::JoinType; use datafusion_physical_expr::expressions::{col, Column, NotExpr}; diff --git a/datafusion/core/tests/physical_optimizer/join_selection.rs b/datafusion/core/tests/physical_optimizer/join_selection.rs index ae7adacadb197..375af94acaf49 100644 --- a/datafusion/core/tests/physical_optimizer/join_selection.rs +++ b/datafusion/core/tests/physical_optimizer/join_selection.rs @@ -22,9 +22,8 @@ use std::{ task::{Context, Poll}, }; -use arrow::datatypes::{DataType, Field, Schema}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; -use arrow_schema::SchemaRef; use datafusion_common::config::ConfigOptions; use datafusion_common::JoinSide; use datafusion_common::{stats::Precision, ColumnStatistics, JoinType, ScalarValue}; diff --git a/datafusion/core/tests/physical_optimizer/limited_distinct_aggregation.rs b/datafusion/core/tests/physical_optimizer/limited_distinct_aggregation.rs index 6e5c677541c58..f9810eab8f594 100644 --- a/datafusion/core/tests/physical_optimizer/limited_distinct_aggregation.rs +++ b/datafusion/core/tests/physical_optimizer/limited_distinct_aggregation.rs @@ -24,8 +24,8 @@ use crate::physical_optimizer::test_utils::{ schema, TestAggregate, }; +use arrow::datatypes::DataType; use arrow::{compute::SortOptions, util::pretty::pretty_format_batches}; -use arrow_schema::DataType; use datafusion::prelude::SessionContext; use datafusion_common::Result; use datafusion_execution::config::SessionConfig; diff --git a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs index 12b41196ef374..c9eadf0091300 100644 --- a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs +++ b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs @@ -18,7 +18,8 @@ use std::any::Any; use std::sync::Arc; -use arrow_schema::{DataType, Field, Schema, SchemaRef, SortOptions}; +use arrow::compute::SortOptions; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::physical_plan::{CsvSource, FileScanConfig}; use datafusion_common::config::ConfigOptions; diff --git a/datafusion/core/tests/physical_optimizer/sanity_checker.rs b/datafusion/core/tests/physical_optimizer/sanity_checker.rs index ccfec1fcb10e9..a73d084a081f3 100644 --- a/datafusion/core/tests/physical_optimizer/sanity_checker.rs +++ b/datafusion/core/tests/physical_optimizer/sanity_checker.rs @@ -22,7 +22,8 @@ use crate::physical_optimizer::test_utils::{ repartition_exec, sort_exec, sort_expr_options, sort_merge_join_exec, }; -use arrow_schema::{DataType, Field, Schema, SchemaRef, SortOptions}; +use arrow::compute::SortOptions; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion::datasource::stream::{FileStreamProvider, StreamConfig, StreamTable}; use datafusion::prelude::{CsvReadOptions, SessionContext}; use datafusion_common::config::ConfigOptions; diff --git a/datafusion/core/tests/physical_optimizer/test_utils.rs b/datafusion/core/tests/physical_optimizer/test_utils.rs index d3f1426c641f9..5e486a715b412 100644 --- a/datafusion/core/tests/physical_optimizer/test_utils.rs +++ b/datafusion/core/tests/physical_optimizer/test_utils.rs @@ -22,8 +22,9 @@ use std::fmt::Formatter; use std::sync::Arc; use arrow::array::Int32Array; +use arrow::compute::SortOptions; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::record_batch::RecordBatch; -use arrow_schema::{DataType, Field, Schema, SchemaRef, SortOptions}; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::physical_plan::{FileScanConfig, ParquetSource}; use datafusion_common::config::ConfigOptions; diff --git a/datafusion/core/tests/user_defined/insert_operation.rs b/datafusion/core/tests/user_defined/insert_operation.rs index aa531632c60b7..12f700ce572ba 100644 --- a/datafusion/core/tests/user_defined/insert_operation.rs +++ b/datafusion/core/tests/user_defined/insert_operation.rs @@ -17,7 +17,7 @@ use std::{any::Any, sync::Arc}; -use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use async_trait::async_trait; use datafusion::{ error::Result, diff --git a/datafusion/core/tests/user_defined/user_defined_aggregates.rs b/datafusion/core/tests/user_defined/user_defined_aggregates.rs index aa0f6c8fed8d2..7cda6d410f4e4 100644 --- a/datafusion/core/tests/user_defined/user_defined_aggregates.rs +++ b/datafusion/core/tests/user_defined/user_defined_aggregates.rs @@ -28,8 +28,7 @@ use std::sync::{ use arrow::array::{ types::UInt64Type, AsArray, Int32Array, PrimitiveArray, StringArray, StructArray, }; -use arrow::datatypes::Fields; -use arrow_schema::Schema; +use arrow::datatypes::{Fields, Schema}; use datafusion::dataframe::DataFrame; use datafusion::datasource::MemTable; diff --git a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs index ea83bd16b4684..2c3577a137adc 100644 --- a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs +++ b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs @@ -25,7 +25,7 @@ use arrow::array::{ Int32Array, RecordBatch, StringArray, }; use arrow::compute::kernels::numeric::add; -use arrow_schema::{DataType, Field, Schema}; +use arrow::datatypes::{DataType, Field, Schema}; use datafusion::execution::context::{FunctionFactory, RegisterFunction, SessionState}; use datafusion::prelude::*; use datafusion::{execution::registry::FunctionRegistry, test_util}; diff --git a/datafusion/core/tests/user_defined/user_defined_window_functions.rs b/datafusion/core/tests/user_defined/user_defined_window_functions.rs index 204d786994f8f..9acd17493da43 100644 --- a/datafusion/core/tests/user_defined/user_defined_window_functions.rs +++ b/datafusion/core/tests/user_defined/user_defined_window_functions.rs @@ -19,7 +19,7 @@ //! user defined window functions use arrow::array::{ArrayRef, AsArray, Int64Array, RecordBatch, StringArray}; -use arrow_schema::{DataType, Field, Schema}; +use arrow::datatypes::{DataType, Field, Schema}; use datafusion::{assert_batches_eq, prelude::SessionContext}; use datafusion_common::{Result, ScalarValue}; use datafusion_expr::{ diff --git a/datafusion/ffi/Cargo.toml b/datafusion/ffi/Cargo.toml index c33c87786de8b..4c396144347c4 100644 --- a/datafusion/ffi/Cargo.toml +++ b/datafusion/ffi/Cargo.toml @@ -38,7 +38,6 @@ crate-type = ["cdylib", "rlib"] [dependencies] abi_stable = "0.11.3" arrow = { workspace = true, features = ["ffi"] } -arrow-schema = { workspace = true } async-ffi = { version = "0.5.0", features = ["abi_stable"] } async-trait = { workspace = true } datafusion = { workspace = true, default-features = false } diff --git a/datafusion/ffi/src/tests/async_provider.rs b/datafusion/ffi/src/tests/async_provider.rs index eff3ed61d7391..de13363db04bc 100644 --- a/datafusion/ffi/src/tests/async_provider.rs +++ b/datafusion/ffi/src/tests/async_provider.rs @@ -29,7 +29,7 @@ use std::{any::Any, fmt::Debug, sync::Arc}; use crate::table_provider::FFI_TableProvider; use arrow::array::RecordBatch; -use arrow_schema::Schema; +use arrow::datatypes::Schema; use async_trait::async_trait; use datafusion::{ catalog::{Session, TableProvider}, diff --git a/datafusion/functions-aggregate/Cargo.toml b/datafusion/functions-aggregate/Cargo.toml index 78e22011b61ab..007e1e76a3be2 100644 --- a/datafusion/functions-aggregate/Cargo.toml +++ b/datafusion/functions-aggregate/Cargo.toml @@ -40,7 +40,6 @@ path = "src/lib.rs" [dependencies] ahash = { workspace = true } arrow = { workspace = true } -arrow-schema = { workspace = true } datafusion-common = { workspace = true } datafusion-doc = { workspace = true } datafusion-execution = { workspace = true } diff --git a/datafusion/functions-aggregate/benches/array_agg.rs b/datafusion/functions-aggregate/benches/array_agg.rs index c9792d541a4f3..fb605e87ed0cc 100644 --- a/datafusion/functions-aggregate/benches/array_agg.rs +++ b/datafusion/functions-aggregate/benches/array_agg.rs @@ -20,9 +20,8 @@ use std::sync::Arc; use arrow::array::{ Array, ArrayRef, ArrowPrimitiveType, AsArray, ListArray, NullBufferBuilder, }; -use arrow::datatypes::Int64Type; +use arrow::datatypes::{Field, Int64Type}; use arrow::util::bench_util::create_primitive_array; -use arrow_schema::Field; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use datafusion_expr::Accumulator; use datafusion_functions_aggregate::array_agg::ArrayAggAccumulator; diff --git a/datafusion/functions-aggregate/benches/count.rs b/datafusion/functions-aggregate/benches/count.rs index e6b62e6e1856a..8bde7d04c44d9 100644 --- a/datafusion/functions-aggregate/benches/count.rs +++ b/datafusion/functions-aggregate/benches/count.rs @@ -16,9 +16,8 @@ // under the License. use arrow::array::{ArrayRef, BooleanArray}; -use arrow::datatypes::Int32Type; +use arrow::datatypes::{DataType, Field, Int32Type, Schema}; use arrow::util::bench_util::{create_boolean_array, create_primitive_array}; -use arrow_schema::{DataType, Field, Schema}; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use datafusion_expr::{function::AccumulatorArgs, AggregateUDFImpl, GroupsAccumulator}; use datafusion_functions_aggregate::count::Count; diff --git a/datafusion/functions-aggregate/benches/sum.rs b/datafusion/functions-aggregate/benches/sum.rs index 1c180126a3136..fab53ae94b25d 100644 --- a/datafusion/functions-aggregate/benches/sum.rs +++ b/datafusion/functions-aggregate/benches/sum.rs @@ -16,9 +16,8 @@ // under the License. use arrow::array::{ArrayRef, BooleanArray}; -use arrow::datatypes::Int64Type; +use arrow::datatypes::{DataType, Field, Int64Type, Schema}; use arrow::util::bench_util::{create_boolean_array, create_primitive_array}; -use arrow_schema::{DataType, Field, Schema}; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use datafusion_expr::{function::AccumulatorArgs, AggregateUDFImpl, GroupsAccumulator}; use datafusion_functions_aggregate::sum::Sum; diff --git a/datafusion/functions-aggregate/src/approx_median.rs b/datafusion/functions-aggregate/src/approx_median.rs index 5d174a7522966..787e08bae2867 100644 --- a/datafusion/functions-aggregate/src/approx_median.rs +++ b/datafusion/functions-aggregate/src/approx_median.rs @@ -20,8 +20,8 @@ use std::any::Any; use std::fmt::Debug; -use arrow::{datatypes::DataType, datatypes::Field}; -use arrow_schema::DataType::{Float64, UInt64}; +use arrow::datatypes::DataType::{Float64, UInt64}; +use arrow::datatypes::{DataType, Field}; use datafusion_common::{not_impl_err, plan_err, Result}; use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs}; diff --git a/datafusion/functions-aggregate/src/approx_percentile_cont.rs b/datafusion/functions-aggregate/src/approx_percentile_cont.rs index 000c69d9f3312..237a4f8de6a74 100644 --- a/datafusion/functions-aggregate/src/approx_percentile_cont.rs +++ b/datafusion/functions-aggregate/src/approx_percentile_cont.rs @@ -27,9 +27,8 @@ use arrow::{ ArrayRef, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, UInt16Array, UInt32Array, UInt64Array, UInt8Array, }, - datatypes::DataType, + datatypes::{DataType, Field, Schema}, }; -use arrow_schema::{Field, Schema}; use datafusion_common::{ downcast_value, internal_err, not_impl_datafusion_err, not_impl_err, plan_err, @@ -491,7 +490,7 @@ impl Accumulator for ApproxPercentileAccumulator { #[cfg(test)] mod tests { - use arrow_schema::DataType; + use arrow::datatypes::DataType; use datafusion_functions_aggregate_common::tdigest::TDigest; diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rs index 9fff05999122e..0f12ac34bfd2c 100644 --- a/datafusion/functions-aggregate/src/array_agg.rs +++ b/datafusion/functions-aggregate/src/array_agg.rs @@ -18,9 +18,8 @@ //! `ARRAY_AGG` aggregate implementation: [`ArrayAgg`] use arrow::array::{new_empty_array, Array, ArrayRef, AsArray, ListArray, StructArray}; -use arrow::datatypes::DataType; +use arrow::datatypes::{DataType, Field, Fields}; -use arrow_schema::{Field, Fields}; use datafusion_common::cast::as_list_array; use datafusion_common::utils::{get_row_at_idx, SingleRowListArrayBuilder}; use datafusion_common::{exec_err, ScalarValue}; @@ -604,7 +603,7 @@ mod tests { use std::sync::Arc; use arrow::array::Int64Array; - use arrow_schema::SortOptions; + use arrow::compute::SortOptions; use datafusion_common::utils::get_row_at_idx; use datafusion_common::{Result, ScalarValue}; diff --git a/datafusion/functions-aggregate/src/bit_and_or_xor.rs b/datafusion/functions-aggregate/src/bit_and_or_xor.rs index 6298071a223b2..6319a9c07dd2f 100644 --- a/datafusion/functions-aggregate/src/bit_and_or_xor.rs +++ b/datafusion/functions-aggregate/src/bit_and_or_xor.rs @@ -25,10 +25,9 @@ use std::mem::{size_of, size_of_val}; use ahash::RandomState; use arrow::array::{downcast_integer, Array, ArrayRef, AsArray}; use arrow::datatypes::{ - ArrowNativeType, ArrowNumericType, DataType, Int16Type, Int32Type, Int64Type, + ArrowNativeType, ArrowNumericType, DataType, Field, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type, UInt32Type, UInt64Type, UInt8Type, }; -use arrow_schema::Field; use datafusion_common::cast::as_list_array; use datafusion_common::{exec_err, not_impl_err, Result, ScalarValue}; diff --git a/datafusion/functions-aggregate/src/min_max.rs b/datafusion/functions-aggregate/src/min_max.rs index da5ec739ad8d1..90fb46883de64 100644 --- a/datafusion/functions-aggregate/src/min_max.rs +++ b/datafusion/functions-aggregate/src/min_max.rs @@ -33,10 +33,9 @@ use arrow::array::{ use arrow::compute; use arrow::datatypes::{ DataType, Decimal128Type, Decimal256Type, Float16Type, Float32Type, Float64Type, - Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type, UInt32Type, UInt64Type, - UInt8Type, + Int16Type, Int32Type, Int64Type, Int8Type, IntervalUnit, UInt16Type, UInt32Type, + UInt64Type, UInt8Type, }; -use arrow_schema::IntervalUnit; use datafusion_common::stats::Precision; use datafusion_common::{ downcast_value, exec_err, internal_err, ColumnStatistics, DataFusionError, Result, diff --git a/datafusion/functions-aggregate/src/min_max/min_max_bytes.rs b/datafusion/functions-aggregate/src/min_max/min_max_bytes.rs index 725b7a29bd479..05321c2ff52d2 100644 --- a/datafusion/functions-aggregate/src/min_max/min_max_bytes.rs +++ b/datafusion/functions-aggregate/src/min_max/min_max_bytes.rs @@ -19,7 +19,7 @@ use arrow::array::{ Array, ArrayRef, AsArray, BinaryBuilder, BinaryViewBuilder, BooleanArray, LargeBinaryBuilder, LargeStringBuilder, StringBuilder, StringViewBuilder, }; -use arrow_schema::DataType; +use arrow::datatypes::DataType; use datafusion_common::{internal_err, Result}; use datafusion_expr::{EmitTo, GroupsAccumulator}; use datafusion_functions_aggregate_common::aggregate::groups_accumulator::nulls::apply_filter_as_nulls; diff --git a/datafusion/functions-aggregate/src/nth_value.rs b/datafusion/functions-aggregate/src/nth_value.rs index 8252fd6baaa3e..d84bd02a6bafe 100644 --- a/datafusion/functions-aggregate/src/nth_value.rs +++ b/datafusion/functions-aggregate/src/nth_value.rs @@ -24,7 +24,7 @@ use std::mem::{size_of, size_of_val}; use std::sync::Arc; use arrow::array::{new_empty_array, ArrayRef, AsArray, StructArray}; -use arrow_schema::{DataType, Field, Fields}; +use arrow::datatypes::{DataType, Field, Fields}; use datafusion_common::utils::{get_row_at_idx, SingleRowListArrayBuilder}; use datafusion_common::{exec_err, internal_err, not_impl_err, Result, ScalarValue}; diff --git a/datafusion/functions-aggregate/src/string_agg.rs b/datafusion/functions-aggregate/src/string_agg.rs index 0cd403cff428b..64314ef6df687 100644 --- a/datafusion/functions-aggregate/src/string_agg.rs +++ b/datafusion/functions-aggregate/src/string_agg.rs @@ -18,7 +18,7 @@ //! [`StringAgg`] accumulator for the `string_agg` function use arrow::array::ArrayRef; -use arrow_schema::DataType; +use arrow::datatypes::DataType; use datafusion_common::cast::as_generic_string_array; use datafusion_common::Result; use datafusion_common::{not_impl_err, ScalarValue}; diff --git a/datafusion/functions-nested/Cargo.toml b/datafusion/functions-nested/Cargo.toml index 7835985b297fc..a63175b36e215 100644 --- a/datafusion/functions-nested/Cargo.toml +++ b/datafusion/functions-nested/Cargo.toml @@ -42,7 +42,6 @@ path = "src/lib.rs" [dependencies] arrow = { workspace = true } arrow-ord = { workspace = true } -arrow-schema = { workspace = true } datafusion-common = { workspace = true } datafusion-doc = { workspace = true } datafusion-execution = { workspace = true } diff --git a/datafusion/functions-nested/benches/map.rs b/datafusion/functions-nested/benches/map.rs index e60f7f388ac1c..3726cac0752eb 100644 --- a/datafusion/functions-nested/benches/map.rs +++ b/datafusion/functions-nested/benches/map.rs @@ -19,7 +19,7 @@ extern crate criterion; use arrow::array::{Int32Array, ListArray, StringArray}; use arrow::buffer::{OffsetBuffer, ScalarBuffer}; -use arrow_schema::{DataType, Field}; +use arrow::datatypes::{DataType, Field}; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use rand::prelude::ThreadRng; use rand::Rng; diff --git a/datafusion/functions-nested/src/cardinality.rs b/datafusion/functions-nested/src/cardinality.rs index 21ab9fb359829..f38a2ab5b90a0 100644 --- a/datafusion/functions-nested/src/cardinality.rs +++ b/datafusion/functions-nested/src/cardinality.rs @@ -21,8 +21,10 @@ use crate::utils::make_scalar_function; use arrow::array::{ Array, ArrayRef, GenericListArray, MapArray, OffsetSizeTrait, UInt64Array, }; -use arrow_schema::DataType; -use arrow_schema::DataType::{FixedSizeList, LargeList, List, Map, UInt64}; +use arrow::datatypes::{ + DataType, + DataType::{FixedSizeList, LargeList, List, Map, UInt64}, +}; use datafusion_common::cast::{as_large_list_array, as_list_array, as_map_array}; use datafusion_common::Result; use datafusion_common::{exec_err, plan_err}; diff --git a/datafusion/functions-nested/src/concat.rs b/datafusion/functions-nested/src/concat.rs index 723dab9a76b70..2f1d9e9938a86 100644 --- a/datafusion/functions-nested/src/concat.rs +++ b/datafusion/functions-nested/src/concat.rs @@ -25,7 +25,7 @@ use arrow::array::{ OffsetSizeTrait, }; use arrow::buffer::OffsetBuffer; -use arrow_schema::{DataType, Field}; +use arrow::datatypes::{DataType, Field}; use datafusion_common::Result; use datafusion_common::{ cast::as_generic_list_array, exec_err, not_impl_err, plan_err, utils::list_ndims, diff --git a/datafusion/functions-nested/src/dimension.rs b/datafusion/functions-nested/src/dimension.rs index 702d0fc3a77dd..30b2650bff38a 100644 --- a/datafusion/functions-nested/src/dimension.rs +++ b/datafusion/functions-nested/src/dimension.rs @@ -20,15 +20,17 @@ use arrow::array::{ Array, ArrayRef, GenericListArray, ListArray, OffsetSizeTrait, UInt64Array, }; -use arrow::datatypes::{DataType, UInt64Type}; +use arrow::datatypes::{ + DataType, + DataType::{FixedSizeList, LargeList, List, UInt64}, + Field, UInt64Type, +}; use std::any::Any; use datafusion_common::cast::{as_large_list_array, as_list_array}; use datafusion_common::{exec_err, plan_err, Result}; use crate::utils::{compute_array_dims, make_scalar_function}; -use arrow_schema::DataType::{FixedSizeList, LargeList, List, UInt64}; -use arrow_schema::Field; use datafusion_expr::{ ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility, }; diff --git a/datafusion/functions-nested/src/distance.rs b/datafusion/functions-nested/src/distance.rs index 6a5d6b4fa0ff5..805cfd69c01f3 100644 --- a/datafusion/functions-nested/src/distance.rs +++ b/datafusion/functions-nested/src/distance.rs @@ -21,8 +21,10 @@ use crate::utils::make_scalar_function; use arrow::array::{ Array, ArrayRef, Float64Array, LargeListArray, ListArray, OffsetSizeTrait, }; -use arrow_schema::DataType; -use arrow_schema::DataType::{FixedSizeList, Float64, LargeList, List}; +use arrow::datatypes::{ + DataType, + DataType::{FixedSizeList, Float64, LargeList, List}, +}; use datafusion_common::cast::{ as_float32_array, as_float64_array, as_generic_list_array, as_int32_array, as_int64_array, diff --git a/datafusion/functions-nested/src/empty.rs b/datafusion/functions-nested/src/empty.rs index b5e2df6f89527..eab773819bf73 100644 --- a/datafusion/functions-nested/src/empty.rs +++ b/datafusion/functions-nested/src/empty.rs @@ -19,8 +19,10 @@ use crate::utils::make_scalar_function; use arrow::array::{ArrayRef, BooleanArray, OffsetSizeTrait}; -use arrow_schema::DataType; -use arrow_schema::DataType::{Boolean, FixedSizeList, LargeList, List}; +use arrow::datatypes::{ + DataType, + DataType::{Boolean, FixedSizeList, LargeList, List}, +}; use datafusion_common::cast::as_generic_list_array; use datafusion_common::{exec_err, plan_err, Result}; use datafusion_expr::{ diff --git a/datafusion/functions-nested/src/except.rs b/datafusion/functions-nested/src/except.rs index 79e2c0f23ce30..fda76894507b3 100644 --- a/datafusion/functions-nested/src/except.rs +++ b/datafusion/functions-nested/src/except.rs @@ -20,8 +20,8 @@ use crate::utils::{check_datatypes, make_scalar_function}; use arrow::array::{cast::AsArray, Array, ArrayRef, GenericListArray, OffsetSizeTrait}; use arrow::buffer::OffsetBuffer; +use arrow::datatypes::{DataType, FieldRef}; use arrow::row::{RowConverter, SortField}; -use arrow_schema::{DataType, FieldRef}; use datafusion_common::{exec_err, internal_err, HashSet, Result}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility, diff --git a/datafusion/functions-nested/src/extract.rs b/datafusion/functions-nested/src/extract.rs index 2f59dccad94a7..93c2b45dde9bc 100644 --- a/datafusion/functions-nested/src/extract.rs +++ b/datafusion/functions-nested/src/extract.rs @@ -23,8 +23,10 @@ use arrow::array::{ }; use arrow::buffer::OffsetBuffer; use arrow::datatypes::DataType; -use arrow_schema::DataType::{FixedSizeList, LargeList, List}; -use arrow_schema::Field; +use arrow::datatypes::{ + DataType::{FixedSizeList, LargeList, List}, + Field, +}; use datafusion_common::cast::as_int64_array; use datafusion_common::cast::as_large_list_array; use datafusion_common::cast::as_list_array; @@ -982,7 +984,7 @@ where #[cfg(test)] mod tests { use super::array_element_udf; - use arrow_schema::{DataType, Field}; + use arrow::datatypes::{DataType, Field}; use datafusion_common::{Column, DFSchema, ScalarValue}; use datafusion_expr::expr::ScalarFunction; use datafusion_expr::{cast, Expr, ExprSchemable}; diff --git a/datafusion/functions-nested/src/flatten.rs b/datafusion/functions-nested/src/flatten.rs index 4fd14c79644b0..be5d80ecf4ad0 100644 --- a/datafusion/functions-nested/src/flatten.rs +++ b/datafusion/functions-nested/src/flatten.rs @@ -20,8 +20,10 @@ use crate::utils::make_scalar_function; use arrow::array::{ArrayRef, GenericListArray, OffsetSizeTrait}; use arrow::buffer::OffsetBuffer; -use arrow_schema::DataType; -use arrow_schema::DataType::{FixedSizeList, LargeList, List, Null}; +use arrow::datatypes::{ + DataType, + DataType::{FixedSizeList, LargeList, List, Null}, +}; use datafusion_common::cast::{ as_generic_list_array, as_large_list_array, as_list_array, }; diff --git a/datafusion/functions-nested/src/length.rs b/datafusion/functions-nested/src/length.rs index 1081a682897fc..a8190a6108440 100644 --- a/datafusion/functions-nested/src/length.rs +++ b/datafusion/functions-nested/src/length.rs @@ -21,8 +21,10 @@ use crate::utils::make_scalar_function; use arrow::array::{ Array, ArrayRef, Int64Array, LargeListArray, ListArray, OffsetSizeTrait, UInt64Array, }; -use arrow_schema::DataType; -use arrow_schema::DataType::{FixedSizeList, LargeList, List, UInt64}; +use arrow::datatypes::{ + DataType, + DataType::{FixedSizeList, LargeList, List, UInt64}, +}; use datafusion_common::cast::{as_generic_list_array, as_int64_array}; use datafusion_common::{exec_err, internal_datafusion_err, plan_err, Result}; use datafusion_expr::{ diff --git a/datafusion/functions-nested/src/make_array.rs b/datafusion/functions-nested/src/make_array.rs index 8bf5f37b8adda..7c41238af5c4c 100644 --- a/datafusion/functions-nested/src/make_array.rs +++ b/datafusion/functions-nested/src/make_array.rs @@ -27,8 +27,11 @@ use arrow::array::{ MutableArrayData, NullArray, OffsetSizeTrait, }; use arrow::buffer::OffsetBuffer; -use arrow_schema::DataType::{List, Null}; -use arrow_schema::{DataType, Field}; +use arrow::datatypes::DataType; +use arrow::datatypes::{ + DataType::{List, Null}, + Field, +}; use datafusion_common::utils::SingleRowListArrayBuilder; use datafusion_common::{plan_err, Result}; use datafusion_expr::binary::{ diff --git a/datafusion/functions-nested/src/map.rs b/datafusion/functions-nested/src/map.rs index 77e06b28a8d6c..26e7733d581b8 100644 --- a/datafusion/functions-nested/src/map.rs +++ b/datafusion/functions-nested/src/map.rs @@ -21,8 +21,7 @@ use std::sync::Arc; use arrow::array::{Array, ArrayData, ArrayRef, MapArray, OffsetSizeTrait, StructArray}; use arrow::buffer::Buffer; -use arrow::datatypes::ToByteSlice; -use arrow_schema::{DataType, Field, SchemaBuilder}; +use arrow::datatypes::{DataType, Field, SchemaBuilder, ToByteSlice}; use datafusion_common::utils::{fixed_size_list_to_arrays, list_to_arrays}; use datafusion_common::{exec_err, HashSet, Result, ScalarValue}; diff --git a/datafusion/functions-nested/src/map_extract.rs b/datafusion/functions-nested/src/map_extract.rs index 47d977a8c01cf..98fb8440427bc 100644 --- a/datafusion/functions-nested/src/map_extract.rs +++ b/datafusion/functions-nested/src/map_extract.rs @@ -21,9 +21,7 @@ use arrow::array::{ make_array, Array, ArrayRef, Capacities, ListArray, MapArray, MutableArrayData, }; use arrow::buffer::OffsetBuffer; -use arrow::datatypes::DataType; -use arrow_schema::Field; - +use arrow::datatypes::{DataType, Field}; use datafusion_common::{cast::as_map_array, exec_err, Result}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility, diff --git a/datafusion/functions-nested/src/map_keys.rs b/datafusion/functions-nested/src/map_keys.rs index 60039865daaea..40a936208770c 100644 --- a/datafusion/functions-nested/src/map_keys.rs +++ b/datafusion/functions-nested/src/map_keys.rs @@ -19,7 +19,7 @@ use crate::utils::{get_map_entry_field, make_scalar_function}; use arrow::array::{Array, ArrayRef, ListArray}; -use arrow_schema::{DataType, Field}; +use arrow::datatypes::{DataType, Field}; use datafusion_common::{cast::as_map_array, exec_err, Result}; use datafusion_expr::{ ArrayFunctionSignature, ColumnarValue, Documentation, ScalarUDFImpl, Signature, diff --git a/datafusion/functions-nested/src/map_values.rs b/datafusion/functions-nested/src/map_values.rs index c6d31f3d9067e..48da2cfb68a46 100644 --- a/datafusion/functions-nested/src/map_values.rs +++ b/datafusion/functions-nested/src/map_values.rs @@ -19,7 +19,7 @@ use crate::utils::{get_map_entry_field, make_scalar_function}; use arrow::array::{Array, ArrayRef, ListArray}; -use arrow_schema::{DataType, Field}; +use arrow::datatypes::{DataType, Field}; use datafusion_common::{cast::as_map_array, exec_err, Result}; use datafusion_expr::{ ArrayFunctionSignature, ColumnarValue, Documentation, ScalarUDFImpl, Signature, diff --git a/datafusion/functions-nested/src/position.rs b/datafusion/functions-nested/src/position.rs index d5c9944709b36..95c5fdf9a59d2 100644 --- a/datafusion/functions-nested/src/position.rs +++ b/datafusion/functions-nested/src/position.rs @@ -17,8 +17,11 @@ //! [`ScalarUDFImpl`] definitions for array_position and array_positions functions. -use arrow_schema::DataType::{LargeList, List, UInt64}; -use arrow_schema::{DataType, Field}; +use arrow::datatypes::DataType; +use arrow::datatypes::{ + DataType::{LargeList, List, UInt64}, + Field, +}; use datafusion_expr::{ ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility, }; diff --git a/datafusion/functions-nested/src/range.rs b/datafusion/functions-nested/src/range.rs index c3f52cef33668..3636babe7d5c3 100644 --- a/datafusion/functions-nested/src/range.rs +++ b/datafusion/functions-nested/src/range.rs @@ -27,10 +27,9 @@ use arrow::array::{ TimestampNanosecondArray, }; use arrow::buffer::OffsetBuffer; -use arrow::datatypes::{DataType, Field}; -use arrow_schema::DataType::*; -use arrow_schema::IntervalUnit::MonthDayNano; -use arrow_schema::TimeUnit::Nanosecond; +use arrow::datatypes::{ + DataType, DataType::*, Field, IntervalUnit::MonthDayNano, TimeUnit::Nanosecond, +}; use datafusion_common::cast::{ as_date32_array, as_int64_array, as_interval_mdn_array, as_timestamp_nanosecond_array, }; diff --git a/datafusion/functions-nested/src/remove.rs b/datafusion/functions-nested/src/remove.rs index 099cc7e1131d7..e196e244646ff 100644 --- a/datafusion/functions-nested/src/remove.rs +++ b/datafusion/functions-nested/src/remove.rs @@ -24,7 +24,7 @@ use arrow::array::{ OffsetSizeTrait, }; use arrow::buffer::OffsetBuffer; -use arrow_schema::{DataType, Field}; +use arrow::datatypes::{DataType, Field}; use datafusion_common::cast::as_int64_array; use datafusion_common::{exec_err, Result}; use datafusion_expr::{ diff --git a/datafusion/functions-nested/src/repeat.rs b/datafusion/functions-nested/src/repeat.rs index 4772da9a4bf4c..e26c79ab45c6c 100644 --- a/datafusion/functions-nested/src/repeat.rs +++ b/datafusion/functions-nested/src/repeat.rs @@ -25,8 +25,11 @@ use arrow::array::{ use arrow::buffer::OffsetBuffer; use arrow::compute; use arrow::compute::cast; -use arrow_schema::DataType::{LargeList, List}; -use arrow_schema::{DataType, Field}; +use arrow::datatypes::DataType; +use arrow::datatypes::{ + DataType::{LargeList, List}, + Field, +}; use datafusion_common::cast::{as_large_list_array, as_list_array, as_uint64_array}; use datafusion_common::{exec_err, Result}; use datafusion_expr::{ diff --git a/datafusion/functions-nested/src/replace.rs b/datafusion/functions-nested/src/replace.rs index 939fce6fdf3fe..8213685f71b63 100644 --- a/datafusion/functions-nested/src/replace.rs +++ b/datafusion/functions-nested/src/replace.rs @@ -21,10 +21,9 @@ use arrow::array::{ Array, ArrayRef, AsArray, Capacities, GenericListArray, MutableArrayData, NullBufferBuilder, OffsetSizeTrait, }; -use arrow::datatypes::DataType; +use arrow::datatypes::{DataType, Field}; use arrow::buffer::OffsetBuffer; -use arrow_schema::Field; use datafusion_common::cast::as_int64_array; use datafusion_common::{exec_err, Result}; use datafusion_expr::{ diff --git a/datafusion/functions-nested/src/resize.rs b/datafusion/functions-nested/src/resize.rs index 3cd7bb5dac81e..83d04ec1b7ed2 100644 --- a/datafusion/functions-nested/src/resize.rs +++ b/datafusion/functions-nested/src/resize.rs @@ -24,8 +24,11 @@ use arrow::array::{ }; use arrow::buffer::OffsetBuffer; use arrow::datatypes::ArrowNativeType; -use arrow_schema::DataType::{FixedSizeList, LargeList, List}; -use arrow_schema::{DataType, FieldRef}; +use arrow::datatypes::DataType; +use arrow::datatypes::{ + DataType::{FixedSizeList, LargeList, List}, + FieldRef, +}; use datafusion_common::cast::{as_int64_array, as_large_list_array, as_list_array}; use datafusion_common::{exec_err, internal_datafusion_err, Result, ScalarValue}; use datafusion_expr::{ diff --git a/datafusion/functions-nested/src/reverse.rs b/datafusion/functions-nested/src/reverse.rs index a60f84cb0320f..d2969cff74ce9 100644 --- a/datafusion/functions-nested/src/reverse.rs +++ b/datafusion/functions-nested/src/reverse.rs @@ -22,8 +22,8 @@ use arrow::array::{ Array, ArrayRef, Capacities, GenericListArray, MutableArrayData, OffsetSizeTrait, }; use arrow::buffer::OffsetBuffer; -use arrow_schema::DataType::{LargeList, List, Null}; -use arrow_schema::{DataType, FieldRef}; +use arrow::datatypes::DataType::{LargeList, List, Null}; +use arrow::datatypes::{DataType, FieldRef}; use datafusion_common::cast::{as_large_list_array, as_list_array}; use datafusion_common::{exec_err, Result}; use datafusion_expr::{ diff --git a/datafusion/functions-nested/src/set_ops.rs b/datafusion/functions-nested/src/set_ops.rs index 079e0e3ed214a..afb877d4cf9a9 100644 --- a/datafusion/functions-nested/src/set_ops.rs +++ b/datafusion/functions-nested/src/set_ops.rs @@ -22,9 +22,9 @@ use crate::utils::make_scalar_function; use arrow::array::{new_empty_array, Array, ArrayRef, GenericListArray, OffsetSizeTrait}; use arrow::buffer::OffsetBuffer; use arrow::compute; +use arrow::datatypes::DataType::{FixedSizeList, LargeList, List, Null}; use arrow::datatypes::{DataType, Field, FieldRef}; use arrow::row::{RowConverter, SortField}; -use arrow_schema::DataType::{FixedSizeList, LargeList, List, Null}; use datafusion_common::cast::{as_large_list_array, as_list_array}; use datafusion_common::{exec_err, internal_err, Result}; use datafusion_expr::{ diff --git a/datafusion/functions-nested/src/sort.rs b/datafusion/functions-nested/src/sort.rs index e4dcc02286f3f..163ca0c05bada 100644 --- a/datafusion/functions-nested/src/sort.rs +++ b/datafusion/functions-nested/src/sort.rs @@ -20,9 +20,9 @@ use crate::utils::make_scalar_function; use arrow::array::{Array, ArrayRef, ListArray, NullBufferBuilder}; use arrow::buffer::OffsetBuffer; -use arrow::compute; -use arrow_schema::DataType::{FixedSizeList, LargeList, List}; -use arrow_schema::{DataType, Field, SortOptions}; +use arrow::datatypes::DataType::{FixedSizeList, LargeList, List}; +use arrow::datatypes::{DataType, Field}; +use arrow::{compute, compute::SortOptions}; use datafusion_common::cast::{as_list_array, as_string_array}; use datafusion_common::{exec_err, Result}; use datafusion_expr::{ diff --git a/datafusion/functions-nested/src/string.rs b/datafusion/functions-nested/src/string.rs index 1a0676aa39d59..9b5e0c952a3be 100644 --- a/datafusion/functions-nested/src/string.rs +++ b/datafusion/functions-nested/src/string.rs @@ -39,7 +39,7 @@ use arrow::array::{ GenericStringArray, StringArrayType, StringViewArray, }; use arrow::compute::cast; -use arrow_schema::DataType::{ +use arrow::datatypes::DataType::{ Dictionary, FixedSizeList, LargeList, LargeUtf8, List, Null, Utf8, Utf8View, }; use datafusion_common::cast::{as_large_list_array, as_list_array}; diff --git a/datafusion/functions-nested/src/utils.rs b/datafusion/functions-nested/src/utils.rs index 5dd812a23b9a4..74b21a3ceb479 100644 --- a/datafusion/functions-nested/src/utils.rs +++ b/datafusion/functions-nested/src/utils.rs @@ -19,14 +19,13 @@ use std::sync::Arc; -use arrow::datatypes::DataType; +use arrow::datatypes::{DataType, Field, Fields}; use arrow::array::{ Array, ArrayRef, BooleanArray, GenericListArray, ListArray, OffsetSizeTrait, Scalar, UInt32Array, }; use arrow::buffer::OffsetBuffer; -use arrow_schema::{Field, Fields}; use datafusion_common::cast::{as_large_list_array, as_list_array}; use datafusion_common::{ exec_err, internal_datafusion_err, internal_err, plan_err, Result, ScalarValue, diff --git a/datafusion/functions/src/datetime/mod.rs b/datafusion/functions/src/datetime/mod.rs index eec5e3cef6244..dee40215c9ea5 100644 --- a/datafusion/functions/src/datetime/mod.rs +++ b/datafusion/functions/src/datetime/mod.rs @@ -137,7 +137,7 @@ pub mod expr_fn { /// # use datafusion_common::ScalarValue::TimestampNanosecond; /// # use std::sync::Arc; /// # use arrow::array::{Date32Array, RecordBatch, StringArray}; - /// # use arrow_schema::{DataType, Field, Schema}; + /// # use arrow::datatypes::{DataType, Field, Schema}; /// # #[tokio::main] /// # async fn main() -> Result<()> { /// let schema = Arc::new(Schema::new(vec![ diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml index 33983676472b1..38e8b44791ab8 100644 --- a/datafusion/physical-expr/Cargo.toml +++ b/datafusion/physical-expr/Cargo.toml @@ -38,7 +38,6 @@ path = "src/lib.rs" [dependencies] ahash = { workspace = true } arrow = { workspace = true } -arrow-schema = { workspace = true } datafusion-common = { workspace = true, default-features = true } datafusion-expr = { workspace = true } datafusion-expr-common = { workspace = true } diff --git a/datafusion/physical-expr/benches/case_when.rs b/datafusion/physical-expr/benches/case_when.rs index 480b1043fbf50..5a88604716d21 100644 --- a/datafusion/physical-expr/benches/case_when.rs +++ b/datafusion/physical-expr/benches/case_when.rs @@ -16,9 +16,8 @@ // under the License. use arrow::array::builder::{Int32Builder, StringBuilder}; -use arrow::datatypes::{Field, Schema}; +use arrow::datatypes::{DataType, Field, Schema}; use arrow::record_batch::RecordBatch; -use arrow_schema::DataType; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use datafusion_common::ScalarValue; use datafusion_expr::Operator; diff --git a/datafusion/physical-expr/benches/is_null.rs b/datafusion/physical-expr/benches/is_null.rs index ed393b8900f1f..ce6ad6eac2c71 100644 --- a/datafusion/physical-expr/benches/is_null.rs +++ b/datafusion/physical-expr/benches/is_null.rs @@ -16,8 +16,7 @@ // under the License. use arrow::array::{builder::Int32Builder, RecordBatch}; -use arrow::datatypes::{Field, Schema}; -use arrow_schema::DataType; +use arrow::datatypes::{DataType, Field, Schema}; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use datafusion_physical_expr::expressions::{Column, IsNotNullExpr, IsNullExpr}; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; diff --git a/datafusion/physical-expr/src/aggregate.rs b/datafusion/physical-expr/src/aggregate.rs index 84406f50051f8..07a98340dbe71 100644 --- a/datafusion/physical-expr/src/aggregate.rs +++ b/datafusion/physical-expr/src/aggregate.rs @@ -40,8 +40,8 @@ use std::sync::Arc; use crate::expressions::Column; +use arrow::compute::SortOptions; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; -use arrow_schema::SortOptions; use datafusion_common::{internal_err, not_impl_err, Result, ScalarValue}; use datafusion_expr::{AggregateUDF, ReversedUDAF, SetMonotonicity}; use datafusion_expr_common::accumulator::Accumulator; diff --git a/datafusion/physical-expr/src/analysis.rs b/datafusion/physical-expr/src/analysis.rs index ceec21c711719..5abd50f6d1b4f 100644 --- a/datafusion/physical-expr/src/analysis.rs +++ b/datafusion/physical-expr/src/analysis.rs @@ -301,7 +301,7 @@ fn calculate_selectivity( mod tests { use std::sync::Arc; - use arrow_schema::{DataType, Field, Schema}; + use arrow::datatypes::{DataType, Field, Schema}; use datafusion_common::{assert_contains, DFSchema}; use datafusion_expr::{ col, execution_props::ExecutionProps, interval_arithmetic::Interval, lit, Expr, diff --git a/datafusion/physical-expr/src/equivalence/mod.rs b/datafusion/physical-expr/src/equivalence/mod.rs index a5b85064e6252..fcc1c564d8c8f 100644 --- a/datafusion/physical-expr/src/equivalence/mod.rs +++ b/datafusion/physical-expr/src/equivalence/mod.rs @@ -72,8 +72,8 @@ mod tests { use crate::expressions::col; use crate::PhysicalSortExpr; - use arrow::datatypes::{DataType, Field, Schema}; - use arrow_schema::{SchemaRef, SortOptions}; + use arrow::compute::SortOptions; + use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion_common::{plan_datafusion_err, Result}; use datafusion_physical_expr_common::sort_expr::{ LexOrdering, PhysicalSortRequirement, diff --git a/datafusion/physical-expr/src/equivalence/ordering.rs b/datafusion/physical-expr/src/equivalence/ordering.rs index a72759b5d49ac..0f9743aecce3a 100644 --- a/datafusion/physical-expr/src/equivalence/ordering.rs +++ b/datafusion/physical-expr/src/equivalence/ordering.rs @@ -22,7 +22,7 @@ use std::vec::IntoIter; use crate::equivalence::add_offset_to_expr; use crate::{LexOrdering, PhysicalExpr}; -use arrow_schema::SortOptions; +use arrow::compute::SortOptions; /// An `OrderingEquivalenceClass` object keeps track of different alternative /// orderings than can describe a schema. For example, consider the following table: @@ -279,8 +279,8 @@ mod tests { ScalarFunctionExpr, }; + use arrow::compute::SortOptions; use arrow::datatypes::{DataType, Field, Schema}; - use arrow_schema::SortOptions; use datafusion_common::Result; use datafusion_expr::{Operator, ScalarUDF}; use datafusion_physical_expr_common::sort_expr::LexOrdering; diff --git a/datafusion/physical-expr/src/equivalence/projection.rs b/datafusion/physical-expr/src/equivalence/projection.rs index d1e7625525ae6..035678fbf1f39 100644 --- a/datafusion/physical-expr/src/equivalence/projection.rs +++ b/datafusion/physical-expr/src/equivalence/projection.rs @@ -146,8 +146,8 @@ mod tests { use crate::utils::tests::TestScalarUDF; use crate::{PhysicalExprRef, ScalarFunctionExpr}; - use arrow::datatypes::{DataType, Field, Schema}; - use arrow_schema::{SortOptions, TimeUnit}; + use arrow::compute::SortOptions; + use arrow::datatypes::{DataType, Field, Schema, TimeUnit}; use datafusion_expr::{Operator, ScalarUDF}; #[test] diff --git a/datafusion/physical-expr/src/equivalence/properties.rs b/datafusion/physical-expr/src/equivalence/properties.rs index a6417044a0616..96208cc5e32c9 100755 --- a/datafusion/physical-expr/src/equivalence/properties.rs +++ b/datafusion/physical-expr/src/equivalence/properties.rs @@ -32,7 +32,8 @@ use crate::{ PhysicalExprRef, PhysicalSortExpr, PhysicalSortRequirement, }; -use arrow_schema::{SchemaRef, SortOptions}; +use arrow::compute::SortOptions; +use arrow::datatypes::SchemaRef; use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode}; use datafusion_common::{ internal_err, plan_err, Constraint, Constraints, HashMap, JoinSide, JoinType, Result, @@ -99,7 +100,7 @@ use itertools::Itertools; /// # Code Example /// ``` /// # use std::sync::Arc; -/// # use arrow_schema::{Schema, Field, DataType, SchemaRef}; +/// # use arrow::datatypes::{Schema, Field, DataType, SchemaRef}; /// # use datafusion_physical_expr::{ConstExpr, EquivalenceProperties}; /// # use datafusion_physical_expr::expressions::col; /// use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr}; @@ -2403,8 +2404,7 @@ mod tests { use crate::expressions::{col, BinaryExpr, Column}; use crate::ScalarFunctionExpr; - use arrow::datatypes::{DataType, Field, Schema}; - use arrow_schema::{Fields, TimeUnit}; + use arrow::datatypes::{DataType, Field, Fields, Schema, TimeUnit}; use datafusion_common::{Constraint, ScalarValue}; use datafusion_expr::Operator; diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index 1713842f410ef..052054bad6c1c 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -30,7 +30,7 @@ use arrow::compute::kernels::comparison::{regexp_is_match, regexp_is_match_scala use arrow::compute::kernels::concat_elements::concat_elements_utf8; use arrow::compute::{cast, ilike, like, nilike, nlike}; use arrow::datatypes::*; -use arrow_schema::ArrowError; +use arrow::error::ArrowError; use datafusion_common::cast::as_boolean_array; use datafusion_common::{internal_err, Result, ScalarValue}; use datafusion_expr::binary::BinaryTypeCoercer; diff --git a/datafusion/physical-expr/src/expressions/binary/kernels.rs b/datafusion/physical-expr/src/expressions/binary/kernels.rs index c0685c6decde7..ae26f3e842418 100644 --- a/datafusion/physical-expr/src/expressions/binary/kernels.rs +++ b/datafusion/physical-expr/src/expressions/binary/kernels.rs @@ -27,7 +27,7 @@ use arrow::datatypes::DataType; use datafusion_common::plan_err; use datafusion_common::{Result, ScalarValue}; -use arrow_schema::ArrowError; +use arrow::error::ArrowError; use std::sync::Arc; /// Downcasts $LEFT and $RIGHT to $ARRAY_TYPE and then calls $KERNEL($LEFT, $RIGHT) diff --git a/datafusion/physical-expr/src/expressions/column.rs b/datafusion/physical-expr/src/expressions/column.rs index 0649cbd65d34d..0ec985887c3f8 100644 --- a/datafusion/physical-expr/src/expressions/column.rs +++ b/datafusion/physical-expr/src/expressions/column.rs @@ -23,10 +23,9 @@ use std::sync::Arc; use crate::physical_expr::PhysicalExpr; use arrow::{ - datatypes::{DataType, Schema}, + datatypes::{DataType, Schema, SchemaRef}, record_batch::RecordBatch, }; -use arrow_schema::SchemaRef; use datafusion_common::tree_node::{Transformed, TreeNode}; use datafusion_common::{internal_err, plan_err, Result}; use datafusion_expr::ColumnarValue; diff --git a/datafusion/physical-expr/src/expressions/like.rs b/datafusion/physical-expr/src/expressions/like.rs index d61cd63c35b1e..b26927b77f1ff 100644 --- a/datafusion/physical-expr/src/expressions/like.rs +++ b/datafusion/physical-expr/src/expressions/like.rs @@ -19,8 +19,8 @@ use std::hash::Hash; use std::{any::Any, sync::Arc}; use crate::PhysicalExpr; +use arrow::datatypes::{DataType, Schema}; use arrow::record_batch::RecordBatch; -use arrow_schema::{DataType, Schema}; use datafusion_common::{internal_err, Result}; use datafusion_expr::ColumnarValue; use datafusion_physical_expr_common::datum::apply_cmp; @@ -183,7 +183,7 @@ mod test { use super::*; use crate::expressions::col; use arrow::array::*; - use arrow_schema::Field; + use arrow::datatypes::Field; use datafusion_common::cast::as_boolean_array; macro_rules! test_like { diff --git a/datafusion/physical-expr/src/expressions/negative.rs b/datafusion/physical-expr/src/expressions/negative.rs index 03f2111aca330..dc863ccff511b 100644 --- a/datafusion/physical-expr/src/expressions/negative.rs +++ b/datafusion/physical-expr/src/expressions/negative.rs @@ -178,8 +178,8 @@ mod tests { use crate::expressions::{col, Column}; use arrow::array::*; + use arrow::datatypes::DataType::{Float32, Float64, Int16, Int32, Int64, Int8}; use arrow::datatypes::*; - use arrow_schema::DataType::{Float32, Float64, Int16, Int32, Int64, Int8}; use datafusion_common::cast::as_primitive_array; use datafusion_common::DataFusionError; diff --git a/datafusion/physical-expr/src/intervals/cp_solver.rs b/datafusion/physical-expr/src/intervals/cp_solver.rs index 166d2564fdf35..cb29109684fef 100644 --- a/datafusion/physical-expr/src/intervals/cp_solver.rs +++ b/datafusion/physical-expr/src/intervals/cp_solver.rs @@ -29,7 +29,7 @@ use crate::expressions::Literal; use crate::utils::{build_dag, ExprTreeNode}; use crate::PhysicalExpr; -use arrow_schema::{DataType, Schema}; +use arrow::datatypes::{DataType, Schema}; use datafusion_common::{internal_err, Result}; use datafusion_expr::interval_arithmetic::{apply_operator, satisfy_greater, Interval}; use datafusion_expr::Operator; @@ -723,8 +723,7 @@ mod tests { use crate::intervals::test_utils::gen_conjunctive_numerical_expr; use arrow::array::types::{IntervalDayTime, IntervalMonthDayNano}; - use arrow::datatypes::TimeUnit; - use arrow_schema::Field; + use arrow::datatypes::{Field, TimeUnit}; use datafusion_common::ScalarValue; use itertools::Itertools; diff --git a/datafusion/physical-expr/src/intervals/test_utils.rs b/datafusion/physical-expr/src/intervals/test_utils.rs index fbd018fb9e803..c3d38a974ab02 100644 --- a/datafusion/physical-expr/src/intervals/test_utils.rs +++ b/datafusion/physical-expr/src/intervals/test_utils.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use crate::expressions::{binary, BinaryExpr, Literal}; use crate::PhysicalExpr; -use arrow_schema::Schema; +use arrow::datatypes::Schema; use datafusion_common::{DataFusionError, ScalarValue}; use datafusion_expr::Operator; diff --git a/datafusion/physical-expr/src/intervals/utils.rs b/datafusion/physical-expr/src/intervals/utils.rs index 56af8238c04e6..910631ef4a43f 100644 --- a/datafusion/physical-expr/src/intervals/utils.rs +++ b/datafusion/physical-expr/src/intervals/utils.rs @@ -25,7 +25,7 @@ use crate::{ }; use arrow::array::types::{IntervalDayTime, IntervalMonthDayNano}; -use arrow_schema::{DataType, SchemaRef}; +use arrow::datatypes::{DataType, SchemaRef}; use datafusion_common::{internal_err, Result, ScalarValue}; use datafusion_expr::interval_arithmetic::Interval; use datafusion_expr::Operator; diff --git a/datafusion/physical-expr/src/planner.rs b/datafusion/physical-expr/src/planner.rs index 8504705f2a09f..fac83dfc45247 100644 --- a/datafusion/physical-expr/src/planner.rs +++ b/datafusion/physical-expr/src/planner.rs @@ -393,7 +393,7 @@ pub fn logical2physical(expr: &Expr, schema: &Schema) -> Arc { #[cfg(test)] mod tests { use arrow::array::{ArrayRef, BooleanArray, RecordBatch, StringArray}; - use arrow_schema::{DataType, Field}; + use arrow::datatypes::{DataType, Field}; use datafusion_expr::{col, lit}; diff --git a/datafusion/physical-expr/src/utils/guarantee.rs b/datafusion/physical-expr/src/utils/guarantee.rs index 7afb78b8bf2ed..8092dc3c1a614 100644 --- a/datafusion/physical-expr/src/utils/guarantee.rs +++ b/datafusion/physical-expr/src/utils/guarantee.rs @@ -417,7 +417,7 @@ mod test { use super::*; use crate::planner::logical2physical; - use arrow_schema::{DataType, Field, Schema, SchemaRef}; + use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion_expr::expr_fn::*; use datafusion_expr::{lit, Expr}; diff --git a/datafusion/physical-expr/src/utils/mod.rs b/datafusion/physical-expr/src/utils/mod.rs index 25769bef7200c..71c9253782184 100644 --- a/datafusion/physical-expr/src/utils/mod.rs +++ b/datafusion/physical-expr/src/utils/mod.rs @@ -259,7 +259,7 @@ pub(crate) mod tests { use crate::expressions::{binary, cast, col, in_list, lit, Literal}; use arrow::array::{ArrayRef, Float32Array, Float64Array}; - use arrow_schema::{DataType, Field, Schema}; + use arrow::datatypes::{DataType, Field, Schema}; use datafusion_common::{exec_err, DataFusionError, ScalarValue}; use datafusion_expr::sort_properties::{ExprProperties, SortProperties}; use datafusion_expr::{ColumnarValue, ScalarUDFImpl, Signature, Volatility}; diff --git a/datafusion/physical-expr/src/window/standard_window_function_expr.rs b/datafusion/physical-expr/src/window/standard_window_function_expr.rs index d308812a0e351..624b747d93f9a 100644 --- a/datafusion/physical-expr/src/window/standard_window_function_expr.rs +++ b/datafusion/physical-expr/src/window/standard_window_function_expr.rs @@ -18,9 +18,8 @@ use crate::{PhysicalExpr, PhysicalSortExpr}; use arrow::array::ArrayRef; -use arrow::datatypes::Field; +use arrow::datatypes::{Field, SchemaRef}; use arrow::record_batch::RecordBatch; -use arrow_schema::SchemaRef; use datafusion_common::Result; use datafusion_expr::PartitionEvaluator; diff --git a/datafusion/physical-expr/src/window/window_expr.rs b/datafusion/physical-expr/src/window/window_expr.rs index be7d080b683c0..793f2e5ee5867 100644 --- a/datafusion/physical-expr/src/window/window_expr.rs +++ b/datafusion/physical-expr/src/window/window_expr.rs @@ -553,7 +553,7 @@ mod tests { use crate::window::window_expr::is_row_ahead; use arrow::array::{ArrayRef, Float64Array}; - use arrow_schema::SortOptions; + use arrow::compute::SortOptions; use datafusion_common::Result; #[test] diff --git a/datafusion/physical-optimizer/Cargo.toml b/datafusion/physical-optimizer/Cargo.toml index 4dc9ac22f173a..d189dc3920af9 100644 --- a/datafusion/physical-optimizer/Cargo.toml +++ b/datafusion/physical-optimizer/Cargo.toml @@ -36,7 +36,6 @@ recursive_protection = ["dep:recursive"] [dependencies] arrow = { workspace = true } -arrow-schema = { workspace = true } datafusion-common = { workspace = true, default-features = true } datafusion-execution = { workspace = true } datafusion-expr = { workspace = true } diff --git a/datafusion/physical-optimizer/src/enforce_sorting/sort_pushdown.rs b/datafusion/physical-optimizer/src/enforce_sorting/sort_pushdown.rs index a0414af13daf0..13d46940c87c4 100644 --- a/datafusion/physical-optimizer/src/enforce_sorting/sort_pushdown.rs +++ b/datafusion/physical-optimizer/src/enforce_sorting/sort_pushdown.rs @@ -22,7 +22,7 @@ use crate::utils::{ add_sort_above, is_sort, is_sort_preserving_merge, is_union, is_window, }; -use arrow_schema::SchemaRef; +use arrow::datatypes::SchemaRef; use datafusion_common::tree_node::{ ConcreteTreeNode, Transformed, TreeNode, TreeNodeRecursion, }; diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml index b84243b1b56b0..410863272b310 100644 --- a/datafusion/physical-plan/Cargo.toml +++ b/datafusion/physical-plan/Cargo.toml @@ -42,7 +42,6 @@ path = "src/lib.rs" ahash = { workspace = true } arrow = { workspace = true } arrow-ord = { workspace = true } -arrow-schema = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } datafusion-common = { workspace = true, default-features = true } diff --git a/datafusion/physical-plan/src/aggregates/group_values/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/mod.rs index 4cbeed9951f9e..ce56ca4f7dfd7 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/mod.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/mod.rs @@ -23,8 +23,7 @@ use arrow::array::types::{ TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, }; use arrow::array::{downcast_primitive, ArrayRef, RecordBatch}; -use arrow_schema::TimeUnit; -use arrow_schema::{DataType, SchemaRef}; +use arrow::datatypes::{DataType, SchemaRef, TimeUnit}; use datafusion_common::Result; use datafusion_expr::EmitTo; diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs index 96885f03146c6..ac96a98edfe11 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs @@ -32,13 +32,13 @@ use ahash::RandomState; use arrow::array::{Array, ArrayRef, RecordBatch}; use arrow::compute::cast; use arrow::datatypes::{ - BinaryViewType, Date32Type, Date64Type, Decimal128Type, Float32Type, Float64Type, - Int16Type, Int32Type, Int64Type, Int8Type, StringViewType, Time32MillisecondType, - Time32SecondType, Time64MicrosecondType, Time64NanosecondType, - TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType, - TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, UInt8Type, + BinaryViewType, DataType, Date32Type, Date64Type, Decimal128Type, Float32Type, + Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, Schema, SchemaRef, + StringViewType, Time32MillisecondType, Time32SecondType, Time64MicrosecondType, + Time64NanosecondType, TimeUnit, TimestampMicrosecondType, TimestampMillisecondType, + TimestampNanosecondType, TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, + UInt8Type, }; -use arrow_schema::{DataType, Schema, SchemaRef, TimeUnit}; use datafusion_common::hash_utils::create_hashes; use datafusion_common::{not_impl_err, DataFusionError, Result}; use datafusion_execution::memory_pool::proxy::{HashTableAllocExt, VecAllocExt}; @@ -1236,8 +1236,8 @@ mod tests { use std::{collections::HashMap, sync::Arc}; use arrow::array::{ArrayRef, Int64Array, RecordBatch, StringArray, StringViewArray}; + use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::{compute::concat_batches, util::pretty::pretty_format_batches}; - use arrow_schema::{DataType, Field, Schema, SchemaRef}; use datafusion_common::utils::proxy::HashTableAllocExt; use datafusion_expr::EmitTo; diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/primitive.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/primitive.rs index c85245d05592e..005dcc8da3863 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/primitive.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/primitive.rs @@ -19,7 +19,7 @@ use crate::aggregates::group_values::multi_group_by::{nulls_equal_to, GroupColum use crate::aggregates::group_values::null_builder::MaybeNullBufferBuilder; use arrow::array::{cast::AsArray, Array, ArrayRef, ArrowPrimitiveType, PrimitiveArray}; use arrow::buffer::ScalarBuffer; -use arrow_schema::DataType; +use arrow::datatypes::DataType; use datafusion_execution::memory_pool::proxy::VecAllocExt; use itertools::izip; use std::iter; @@ -212,8 +212,7 @@ mod tests { use crate::aggregates::group_values::multi_group_by::primitive::PrimitiveGroupValueBuilder; use arrow::array::{ArrayRef, Int64Array, NullBufferBuilder}; - use arrow::datatypes::Int64Type; - use arrow_schema::DataType; + use arrow::datatypes::{DataType, Int64Type}; use super::GroupColumn; diff --git a/datafusion/physical-plan/src/aggregates/group_values/row.rs b/datafusion/physical-plan/src/aggregates/group_values/row.rs index a0331bf3fa3d2..63751d4703135 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/row.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/row.rs @@ -19,8 +19,8 @@ use crate::aggregates::group_values::GroupValues; use ahash::RandomState; use arrow::array::{Array, ArrayRef, ListArray, RecordBatch, StructArray}; use arrow::compute::cast; +use arrow::datatypes::{DataType, SchemaRef}; use arrow::row::{RowConverter, Rows, SortField}; -use arrow_schema::{DataType, SchemaRef}; use datafusion_common::hash_utils::create_hashes; use datafusion_common::Result; use datafusion_execution::memory_pool::proxy::{HashTableAllocExt, VecAllocExt}; @@ -285,7 +285,7 @@ fn dictionary_encode_if_necessary( let list = array.as_any().downcast_ref::().unwrap(); Ok(Arc::new(ListArray::try_new( - Arc::::clone(expected_field), + Arc::::clone(expected_field), list.offsets().clone(), dictionary_encode_if_necessary( Arc::::clone(list.values()), diff --git a/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rs b/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rs index 5a6235edb25a4..d945d3ddcbf5c 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/single_group_by/primitive.rs @@ -22,9 +22,8 @@ use arrow::array::{ cast::AsArray, ArrayRef, ArrowNativeTypeOp, ArrowPrimitiveType, NullBufferBuilder, PrimitiveArray, }; -use arrow::datatypes::i256; +use arrow::datatypes::{i256, DataType}; use arrow::record_batch::RecordBatch; -use arrow_schema::DataType; use datafusion_common::Result; use datafusion_execution::memory_pool::proxy::VecAllocExt; use datafusion_expr::EmitTo; diff --git a/datafusion/physical-plan/src/aggregates/order/mod.rs b/datafusion/physical-plan/src/aggregates/order/mod.rs index 61a0ab8b247d3..0b742b3d20fdc 100644 --- a/datafusion/physical-plan/src/aggregates/order/mod.rs +++ b/datafusion/physical-plan/src/aggregates/order/mod.rs @@ -16,7 +16,7 @@ // under the License. use arrow::array::ArrayRef; -use arrow_schema::Schema; +use arrow::datatypes::Schema; use datafusion_common::Result; use datafusion_expr::EmitTo; use datafusion_physical_expr_common::sort_expr::LexOrdering; diff --git a/datafusion/physical-plan/src/aggregates/order/partial.rs b/datafusion/physical-plan/src/aggregates/order/partial.rs index 30655cd0ad59c..f19aefbcf47e4 100644 --- a/datafusion/physical-plan/src/aggregates/order/partial.rs +++ b/datafusion/physical-plan/src/aggregates/order/partial.rs @@ -16,8 +16,8 @@ // under the License. use arrow::array::ArrayRef; +use arrow::datatypes::Schema; use arrow::row::{OwnedRow, RowConverter, Rows, SortField}; -use arrow_schema::Schema; use datafusion_common::Result; use datafusion_execution::memory_pool::proxy::VecAllocExt; use datafusion_expr::EmitTo; diff --git a/datafusion/physical-plan/src/aggregates/row_hash.rs b/datafusion/physical-plan/src/aggregates/row_hash.rs index cc95ce51c15b3..a0251857c272f 100644 --- a/datafusion/physical-plan/src/aggregates/row_hash.rs +++ b/datafusion/physical-plan/src/aggregates/row_hash.rs @@ -36,8 +36,8 @@ use crate::{aggregates, metrics, ExecutionPlan, PhysicalExpr}; use crate::{RecordBatchStream, SendableRecordBatchStream}; use arrow::array::*; +use arrow::compute::SortOptions; use arrow::datatypes::SchemaRef; -use arrow_schema::SortOptions; use datafusion_common::{internal_err, DataFusionError, Result}; use datafusion_execution::disk_manager::RefCountedTempFile; use datafusion_execution::memory_pool::proxy::VecAllocExt; diff --git a/datafusion/physical-plan/src/aggregates/topk/hash_table.rs b/datafusion/physical-plan/src/aggregates/topk/hash_table.rs index 8c7ba21b37c07..c818b4608de71 100644 --- a/datafusion/physical-plan/src/aggregates/topk/hash_table.rs +++ b/datafusion/physical-plan/src/aggregates/topk/hash_table.rs @@ -25,8 +25,7 @@ use arrow::array::{ builder::PrimitiveBuilder, cast::AsArray, downcast_primitive, Array, ArrayRef, ArrowPrimitiveType, PrimitiveArray, StringArray, }; -use arrow::datatypes::i256; -use arrow_schema::DataType; +use arrow::datatypes::{i256, DataType}; use datafusion_common::DataFusionError; use datafusion_common::Result; use half::f16; diff --git a/datafusion/physical-plan/src/aggregates/topk/heap.rs b/datafusion/physical-plan/src/aggregates/topk/heap.rs index 09dae3df0a96a..b202f812c6e10 100644 --- a/datafusion/physical-plan/src/aggregates/topk/heap.rs +++ b/datafusion/physical-plan/src/aggregates/topk/heap.rs @@ -23,8 +23,7 @@ use arrow::array::{ }; use arrow::array::{downcast_primitive, ArrayRef, ArrowPrimitiveType, PrimitiveArray}; use arrow::buffer::ScalarBuffer; -use arrow::datatypes::i256; -use arrow_schema::DataType; +use arrow::datatypes::{i256, DataType}; use datafusion_common::DataFusionError; use datafusion_common::Result; diff --git a/datafusion/physical-plan/src/aggregates/topk/priority_map.rs b/datafusion/physical-plan/src/aggregates/topk/priority_map.rs index 3cb12f0af0896..3b954c4c72d34 100644 --- a/datafusion/physical-plan/src/aggregates/topk/priority_map.rs +++ b/datafusion/physical-plan/src/aggregates/topk/priority_map.rs @@ -20,7 +20,7 @@ use crate::aggregates::topk::hash_table::{new_hash_table, ArrowHashTable}; use crate::aggregates::topk::heap::{new_heap, ArrowHeap}; use arrow::array::ArrayRef; -use arrow_schema::DataType; +use arrow::datatypes::DataType; use datafusion_common::Result; /// A `Map` / `PriorityQueue` combo that evicts the worst values after reaching `capacity` @@ -109,10 +109,8 @@ impl PriorityMap { mod tests { use super::*; use arrow::array::{Int64Array, RecordBatch, StringArray}; + use arrow::datatypes::{Field, Schema, SchemaRef}; use arrow::util::pretty::pretty_format_batches; - use arrow_schema::Field; - use arrow_schema::Schema; - use arrow_schema::SchemaRef; use std::sync::Arc; #[test] diff --git a/datafusion/physical-plan/src/aggregates/topk_stream.rs b/datafusion/physical-plan/src/aggregates/topk_stream.rs index 8a984fc0d27b2..bf02692486cc6 100644 --- a/datafusion/physical-plan/src/aggregates/topk_stream.rs +++ b/datafusion/physical-plan/src/aggregates/topk_stream.rs @@ -24,8 +24,8 @@ use crate::aggregates::{ }; use crate::{RecordBatchStream, SendableRecordBatchStream}; use arrow::array::{Array, ArrayRef, RecordBatch}; +use arrow::datatypes::SchemaRef; use arrow::util::pretty::print_batches; -use arrow_schema::SchemaRef; use datafusion_common::DataFusionError; use datafusion_common::Result; use datafusion_execution::TaskContext; diff --git a/datafusion/physical-plan/src/coalesce/mod.rs b/datafusion/physical-plan/src/coalesce/mod.rs index ed60a350e3003..eb4a7d875c95a 100644 --- a/datafusion/physical-plan/src/coalesce/mod.rs +++ b/datafusion/physical-plan/src/coalesce/mod.rs @@ -20,7 +20,7 @@ use arrow::array::{ RecordBatchOptions, }; use arrow::compute::concat_batches; -use arrow_schema::SchemaRef; +use arrow::datatypes::SchemaRef; use std::sync::Arc; /// Concatenate multiple [`RecordBatch`]es diff --git a/datafusion/physical-plan/src/common.rs b/datafusion/physical-plan/src/common.rs index 541f8bcae1223..b83641acf2ce4 100644 --- a/datafusion/physical-plan/src/common.rs +++ b/datafusion/physical-plan/src/common.rs @@ -253,7 +253,7 @@ impl IPCWriter { /// Checks if the given projection is valid for the given schema. pub fn can_project( - schema: &arrow_schema::SchemaRef, + schema: &arrow::datatypes::SchemaRef, projection: Option<&Vec>, ) -> Result<()> { match projection { @@ -263,7 +263,7 @@ pub fn can_project( .max() .is_some_and(|&i| i >= schema.fields().len()) { - Err(arrow_schema::ArrowError::SchemaError(format!( + Err(arrow::error::ArrowError::SchemaError(format!( "project index {} out of bounds, max field {}", columns.iter().max().unwrap(), schema.fields().len() diff --git a/datafusion/physical-plan/src/display.rs b/datafusion/physical-plan/src/display.rs index dbf82eee05eb0..0cc1cb02438a5 100644 --- a/datafusion/physical-plan/src/display.rs +++ b/datafusion/physical-plan/src/display.rs @@ -21,7 +21,7 @@ use std::fmt; use std::fmt::Formatter; -use arrow_schema::SchemaRef; +use arrow::datatypes::SchemaRef; use datafusion_common::display::{GraphvizBuilder, PlanType, StringifiedPlan}; use datafusion_expr::display_schema; @@ -44,7 +44,7 @@ pub enum DisplayFormatType { /// # Example /// ``` /// # use std::sync::Arc; -/// # use arrow_schema::{Field, Schema, DataType}; +/// # use arrow::datatypes::{Field, Schema, DataType}; /// # use datafusion_expr::Operator; /// # use datafusion_physical_expr::expressions::{binary, col, lit}; /// # use datafusion_physical_plan::{displayable, ExecutionPlan}; diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rs index 0104c91eb5d5d..851e504b69afc 100644 --- a/datafusion/physical-plan/src/execution_plan.rs +++ b/datafusion/physical-plan/src/execution_plan.rs @@ -283,7 +283,7 @@ pub trait ExecutionPlan: Debug + DisplayAs + Send + Sync { /// ``` /// # use std::sync::Arc; /// # use arrow::array::RecordBatch; - /// # use arrow_schema::SchemaRef; + /// # use arrow::datatypes::SchemaRef; /// # use datafusion_common::Result; /// # use datafusion_execution::{SendableRecordBatchStream, TaskContext}; /// # use datafusion_physical_plan::memory::MemoryStream; @@ -313,7 +313,7 @@ pub trait ExecutionPlan: Debug + DisplayAs + Send + Sync { /// ``` /// # use std::sync::Arc; /// # use arrow::array::RecordBatch; - /// # use arrow_schema::SchemaRef; + /// # use arrow::datatypes::SchemaRef; /// # use datafusion_common::Result; /// # use datafusion_execution::{SendableRecordBatchStream, TaskContext}; /// # use datafusion_physical_plan::memory::MemoryStream; @@ -348,7 +348,7 @@ pub trait ExecutionPlan: Debug + DisplayAs + Send + Sync { /// ``` /// # use std::sync::Arc; /// # use arrow::array::RecordBatch; - /// # use arrow_schema::SchemaRef; + /// # use arrow::datatypes::SchemaRef; /// # use futures::TryStreamExt; /// # use datafusion_common::Result; /// # use datafusion_execution::{SendableRecordBatchStream, TaskContext}; @@ -1055,7 +1055,7 @@ pub enum CardinalityEffect { mod tests { use super::*; use arrow::array::{DictionaryArray, Int32Array, NullArray, RunArray}; - use arrow_schema::{DataType, Field, Schema, SchemaRef}; + use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use std::any::Any; use std::sync::Arc; diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 91c44a4139d2a..5866f0938e41f 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -625,8 +625,7 @@ mod tests { use crate::test; use crate::test::exec::StatisticsExec; - use arrow::datatypes::{Field, Schema}; - use arrow_schema::{UnionFields, UnionMode}; + use arrow::datatypes::{Field, Schema, UnionFields, UnionMode}; use datafusion_common::ScalarValue; #[tokio::test] diff --git a/datafusion/physical-plan/src/insert.rs b/datafusion/physical-plan/src/insert.rs index 0b8c1eede36c6..63c9c99212483 100644 --- a/datafusion/physical-plan/src/insert.rs +++ b/datafusion/physical-plan/src/insert.rs @@ -31,8 +31,7 @@ use crate::stream::RecordBatchStreamAdapter; use crate::ExecutionPlanProperties; use arrow::array::{ArrayRef, RecordBatch, UInt64Array}; -use arrow::datatypes::SchemaRef; -use arrow_schema::{DataType, Field, Schema}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion_common::{internal_err, Result}; use datafusion_execution::TaskContext; use datafusion_physical_expr::{Distribution, EquivalenceProperties}; diff --git a/datafusion/physical-plan/src/joins/hash_join.rs b/datafusion/physical-plan/src/joins/hash_join.rs index 6cdd91bb1721b..2983478ada743 100644 --- a/datafusion/physical-plan/src/joins/hash_join.rs +++ b/datafusion/physical-plan/src/joins/hash_join.rs @@ -64,9 +64,9 @@ use arrow::array::{ use arrow::compute::kernels::cmp::{eq, not_distinct}; use arrow::compute::{and, concat_batches, take, FilterBuilder}; use arrow::datatypes::{Schema, SchemaRef}; +use arrow::error::ArrowError; use arrow::record_batch::RecordBatch; use arrow::util::bit_util; -use arrow_schema::ArrowError; use datafusion_common::utils::memory::estimate_memory_size; use datafusion_common::{ internal_datafusion_err, internal_err, plan_err, project_schema, DataFusionError, diff --git a/datafusion/physical-plan/src/joins/join_filter.rs b/datafusion/physical-plan/src/joins/join_filter.rs index cfc7ad2c10e0a..0e46a971d90bb 100644 --- a/datafusion/physical-plan/src/joins/join_filter.rs +++ b/datafusion/physical-plan/src/joins/join_filter.rs @@ -16,7 +16,7 @@ // under the License. use crate::joins::utils::ColumnIndex; -use arrow_schema::SchemaRef; +use arrow::datatypes::SchemaRef; use datafusion_common::JoinSide; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; use std::sync::Arc; diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs index 07289d861bcf9..6de6b3b4dff44 100644 --- a/datafusion/physical-plan/src/joins/nested_loop_join.rs +++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs @@ -1037,8 +1037,8 @@ pub(crate) mod tests { }; use arrow::array::Int32Array; + use arrow::compute::SortOptions; use arrow::datatypes::{DataType, Field}; - use arrow_schema::SortOptions; use datafusion_common::{assert_batches_sorted_eq, assert_contains, ScalarValue}; use datafusion_execution::runtime_env::RuntimeEnvBuilder; use datafusion_expr::Operator; diff --git a/datafusion/physical-plan/src/joins/stream_join_utils.rs b/datafusion/physical-plan/src/joins/stream_join_utils.rs index a3b3a37aa7efd..61a71315846ca 100644 --- a/datafusion/physical-plan/src/joins/stream_join_utils.rs +++ b/datafusion/physical-plan/src/joins/stream_join_utils.rs @@ -30,8 +30,7 @@ use arrow::array::{ ArrowPrimitiveType, BooleanBufferBuilder, NativeAdapter, PrimitiveArray, RecordBatch, }; use arrow::compute::concat_batches; -use arrow::datatypes::ArrowNativeType; -use arrow_schema::{Schema, SchemaRef}; +use arrow::datatypes::{ArrowNativeType, Schema, SchemaRef}; use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode}; use datafusion_common::{ arrow_datafusion_err, DataFusionError, HashSet, JoinSide, Result, ScalarValue, diff --git a/datafusion/physical-plan/src/joins/test_utils.rs b/datafusion/physical-plan/src/joins/test_utils.rs index 9e34c27ee7f43..9932c647be0ab 100644 --- a/datafusion/physical-plan/src/joins/test_utils.rs +++ b/datafusion/physical-plan/src/joins/test_utils.rs @@ -32,8 +32,8 @@ use arrow::array::{ types::IntervalDayTime, ArrayRef, Float64Array, Int32Array, IntervalDayTimeArray, RecordBatch, TimestampMillisecondArray, }; +use arrow::datatypes::{DataType, Schema}; use arrow::util::pretty::pretty_format_batches; -use arrow_schema::{DataType, Schema}; use datafusion_common::{Result, ScalarValue}; use datafusion_execution::TaskContext; use datafusion_expr::{JoinType, Operator}; diff --git a/datafusion/physical-plan/src/joins/utils.rs b/datafusion/physical-plan/src/joins/utils.rs index bccfd2a693834..00edda8fa3a85 100644 --- a/datafusion/physical-plan/src/joins/utils.rs +++ b/datafusion/physical-plan/src/joins/utils.rs @@ -624,7 +624,7 @@ pub fn build_join_schema( JoinType::LeftSemi | JoinType::LeftAnti => left_fields().unzip(), JoinType::LeftMark => { let right_field = once(( - Field::new("mark", arrow_schema::DataType::Boolean, false), + Field::new("mark", arrow::datatypes::DataType::Boolean, false), ColumnIndex { index: 0, side: JoinSide::None, @@ -1822,9 +1822,9 @@ mod tests { use std::pin::Pin; use arrow::array::Int32Array; + use arrow::compute::SortOptions; use arrow::datatypes::{DataType, Fields}; use arrow::error::{ArrowError, Result as ArrowResult}; - use arrow_schema::SortOptions; use datafusion_common::stats::Precision::{Absent, Exact, Inexact}; use datafusion_common::{arrow_datafusion_err, arrow_err, ScalarValue}; diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rs index 15f19f6456a5b..f720294c7ad90 100644 --- a/datafusion/physical-plan/src/limit.rs +++ b/datafusion/physical-plan/src/limit.rs @@ -481,7 +481,7 @@ mod tests { use crate::aggregates::{AggregateExec, AggregateMode, PhysicalGroupBy}; use arrow::array::RecordBatchOptions; - use arrow_schema::Schema; + use arrow::datatypes::Schema; use datafusion_common::stats::Precision; use datafusion_physical_expr::expressions::col; use datafusion_physical_expr::PhysicalExpr; diff --git a/datafusion/physical-plan/src/memory.rs b/datafusion/physical-plan/src/memory.rs index 3d161a0478535..0077804bdfc9a 100644 --- a/datafusion/physical-plan/src/memory.rs +++ b/datafusion/physical-plan/src/memory.rs @@ -34,8 +34,7 @@ use crate::projection::{ use crate::source::{DataSource, DataSourceExec}; use arrow::array::{RecordBatch, RecordBatchOptions}; -use arrow::datatypes::SchemaRef; -use arrow_schema::Schema; +use arrow::datatypes::{Schema, SchemaRef}; use datafusion_common::{ internal_err, plan_err, project_schema, Constraints, Result, ScalarValue, }; @@ -971,7 +970,8 @@ mod memory_exec_tests { use crate::source::DataSourceExec; use crate::ExecutionPlan; - use arrow_schema::{DataType, Field, Schema, SortOptions}; + use arrow::compute::SortOptions; + use arrow::datatypes::{DataType, Field, Schema}; use datafusion_physical_expr::expressions::col; use datafusion_physical_expr::PhysicalSortExpr; use datafusion_physical_expr_common::sort_expr::LexOrdering; @@ -1144,7 +1144,7 @@ mod tests { use crate::expressions::lit; use crate::test::{self, make_partition}; - use arrow_schema::{DataType, Field}; + use arrow::datatypes::{DataType, Field}; use datafusion_common::assert_batches_eq; use datafusion_common::stats::{ColumnStatistics, Precision}; use futures::StreamExt; diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs index 3ebfd8f8ca803..08c4d24f4c7fe 100644 --- a/datafusion/physical-plan/src/projection.rs +++ b/datafusion/physical-plan/src/projection.rs @@ -1003,7 +1003,7 @@ mod tests { use crate::common::collect; use crate::test; - use arrow_schema::DataType; + use arrow::datatypes::DataType; use datafusion_common::ScalarValue; use datafusion_expr::Operator; diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs index ffd1a5b520faa..25668fa67d5b0 100644 --- a/datafusion/physical-plan/src/repartition/mod.rs +++ b/datafusion/physical-plan/src/repartition/mod.rs @@ -1600,7 +1600,8 @@ mod tests { #[cfg(test)] mod test { - use arrow_schema::{DataType, Field, Schema, SortOptions}; + use arrow::compute::SortOptions; + use arrow::datatypes::{DataType, Field, Schema}; use super::*; use crate::memory::MemorySourceConfig; diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index 6c538801d71a6..9f7e82f026bd7 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -46,9 +46,8 @@ use crate::{ use arrow::array::{Array, RecordBatch, RecordBatchOptions, UInt32Array}; use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays, SortColumn}; -use arrow::datatypes::SchemaRef; +use arrow::datatypes::{DataType, SchemaRef}; use arrow::row::{RowConverter, SortField}; -use arrow_schema::DataType; use datafusion_common::{internal_err, Result}; use datafusion_execution::disk_manager::RefCountedTempFile; use datafusion_execution::memory_pool::{MemoryConsumer, MemoryReservation}; diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs index c2cd9d7db3c65..2cc55d60292a3 100644 --- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs +++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs @@ -398,8 +398,7 @@ mod tests { TimestampNanosecondArray, }; use arrow::compute::SortOptions; - use arrow::datatypes::{DataType, Field, Schema}; - use arrow_schema::SchemaRef; + use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion_common::{assert_batches_eq, assert_contains, DataFusionError}; use datafusion_common_runtime::SpawnedTask; use datafusion_execution::config::SessionConfig; diff --git a/datafusion/physical-plan/src/stream.rs b/datafusion/physical-plan/src/stream.rs index 5c941c76ae47d..23cbb1ce49c18 100644 --- a/datafusion/physical-plan/src/stream.rs +++ b/datafusion/physical-plan/src/stream.rs @@ -512,7 +512,7 @@ mod test { assert_strong_count_converges_to_zero, BlockingExec, MockExec, PanicExec, }; - use arrow_schema::{DataType, Field, Schema}; + use arrow::datatypes::{DataType, Field, Schema}; use datafusion_common::exec_err; fn schema() -> SchemaRef { diff --git a/datafusion/physical-plan/src/streaming.rs b/datafusion/physical-plan/src/streaming.rs index 751af99214482..8bdfca2a89076 100644 --- a/datafusion/physical-plan/src/streaming.rs +++ b/datafusion/physical-plan/src/streaming.rs @@ -32,8 +32,7 @@ use crate::projection::{ use crate::stream::RecordBatchStreamAdapter; use crate::{ExecutionPlan, Partitioning, SendableRecordBatchStream}; -use arrow::datatypes::SchemaRef; -use arrow_schema::Schema; +use arrow::datatypes::{Schema, SchemaRef}; use datafusion_common::{internal_err, plan_err, Result}; use datafusion_execution::TaskContext; use datafusion_physical_expr::{EquivalenceProperties, LexOrdering, PhysicalSortExpr}; diff --git a/datafusion/physical-plan/src/test.rs b/datafusion/physical-plan/src/test.rs index 757e2df658313..e73d6d97e9866 100644 --- a/datafusion/physical-plan/src/test.rs +++ b/datafusion/physical-plan/src/test.rs @@ -22,7 +22,7 @@ use std::pin::Pin; use std::sync::Arc; use arrow::array::{ArrayRef, Int32Array, RecordBatch}; -use arrow_schema::{DataType, Field, Schema, SchemaRef}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion_execution::{SendableRecordBatchStream, TaskContext}; use futures::{Future, FutureExt}; diff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rs index 4cc8fc8711dea..85de1eefce2e4 100644 --- a/datafusion/physical-plan/src/topk/mod.rs +++ b/datafusion/physical-plan/src/topk/mod.rs @@ -28,7 +28,7 @@ use super::metrics::{BaselineMetrics, Count, ExecutionPlanMetricsSet, MetricBuil use crate::spill::get_record_batch_memory_size; use crate::{stream::RecordBatchStreamAdapter, SendableRecordBatchStream}; use arrow::array::{Array, ArrayRef, RecordBatch}; -use arrow_schema::SchemaRef; +use arrow::datatypes::SchemaRef; use datafusion_common::HashMap; use datafusion_common::Result; use datafusion_execution::{ diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs index 7e0f88784644b..e1972d267b972 100644 --- a/datafusion/physical-plan/src/union.rs +++ b/datafusion/physical-plan/src/union.rs @@ -644,7 +644,8 @@ mod tests { use crate::test; use crate::source::DataSourceExec; - use arrow_schema::{DataType, SortOptions}; + use arrow::compute::SortOptions; + use arrow::datatypes::DataType; use datafusion_common::ScalarValue; use datafusion_physical_expr::expressions::col; use datafusion_physical_expr::{PhysicalExpr, PhysicalSortExpr}; diff --git a/datafusion/physical-plan/src/values.rs b/datafusion/physical-plan/src/values.rs index ea1086c0a3d63..6ab5cc84a21f5 100644 --- a/datafusion/physical-plan/src/values.rs +++ b/datafusion/physical-plan/src/values.rs @@ -233,7 +233,7 @@ mod tests { use crate::expressions::lit; use crate::test::{self, make_partition}; - use arrow_schema::{DataType, Field}; + use arrow::datatypes::{DataType, Field}; use datafusion_common::stats::{ColumnStatistics, Precision}; #[tokio::test] diff --git a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs index a734feae5533c..1e21d0757c414 100644 --- a/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs +++ b/datafusion/physical-plan/src/windows/bounded_window_agg_exec.rs @@ -1202,7 +1202,8 @@ mod tests { builder::{Int64Builder, UInt64Builder}, RecordBatch, }; - use arrow_schema::{DataType, Field, Schema, SchemaRef, SortOptions}; + use arrow::compute::SortOptions; + use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion_common::{ assert_batches_eq, exec_datafusion_err, Result, ScalarValue, }; diff --git a/datafusion/physical-plan/src/windows/mod.rs b/datafusion/physical-plan/src/windows/mod.rs index cdab1fa5929d5..8225007366364 100644 --- a/datafusion/physical-plan/src/windows/mod.rs +++ b/datafusion/physical-plan/src/windows/mod.rs @@ -25,8 +25,7 @@ use crate::{ InputOrderMode, PhysicalExpr, }; -use arrow::datatypes::Schema; -use arrow_schema::{DataType, Field, SchemaRef}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use datafusion_common::{exec_err, Result}; use datafusion_expr::{ PartitionEvaluator, ReversedUDWF, WindowFrame, WindowFunctionDefinition, WindowUDF, diff --git a/datafusion/physical-plan/src/windows/utils.rs b/datafusion/physical-plan/src/windows/utils.rs index 13332ea82fa14..be38976b35513 100644 --- a/datafusion/physical-plan/src/windows/utils.rs +++ b/datafusion/physical-plan/src/windows/utils.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use arrow_schema::{Schema, SchemaBuilder}; +use arrow::datatypes::{Schema, SchemaBuilder}; use datafusion_common::Result; use datafusion_physical_expr::window::WindowExpr; use std::sync::Arc; diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs index add72e4f777e5..837a766ff3909 100644 --- a/datafusion/proto/src/generated/pbjson.rs +++ b/datafusion/proto/src/generated/pbjson.rs @@ -97,7 +97,7 @@ impl serde::Serialize for AggregateExecNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where - S: serde::Serializer, + S: serde::Serializer, { use serde::ser::SerializeStruct; let mut len = 0; diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml index b53f3674d13ae..c4a404975d291 100644 --- a/datafusion/sql/Cargo.toml +++ b/datafusion/sql/Cargo.toml @@ -43,7 +43,6 @@ recursive_protection = ["dep:recursive"] [dependencies] arrow = { workspace = true } -arrow-schema = { workspace = true } bigdecimal = { workspace = true } datafusion-common = { workspace = true, default-features = true } datafusion-expr = { workspace = true } diff --git a/datafusion/sql/examples/sql.rs b/datafusion/sql/examples/sql.rs index 7f1e6bf8f28cc..2c0bb86cd8087 100644 --- a/datafusion/sql/examples/sql.rs +++ b/datafusion/sql/examples/sql.rs @@ -17,7 +17,7 @@ use std::{collections::HashMap, sync::Arc}; -use arrow_schema::{DataType, Field, Schema}; +use arrow::datatypes::{DataType, Field, Schema}; use datafusion_common::config::ConfigOptions; use datafusion_common::{plan_err, Result, TableReference}; diff --git a/datafusion/sql/src/expr/function.rs b/datafusion/sql/src/expr/function.rs index da1a4ba81f5a9..1cf3dcb289a62 100644 --- a/datafusion/sql/src/expr/function.rs +++ b/datafusion/sql/src/expr/function.rs @@ -17,7 +17,7 @@ use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; -use arrow_schema::DataType; +use arrow::datatypes::DataType; use datafusion_common::{ internal_datafusion_err, internal_err, not_impl_err, plan_datafusion_err, plan_err, DFSchema, Dependency, Result, diff --git a/datafusion/sql/src/expr/identifier.rs b/datafusion/sql/src/expr/identifier.rs index ab5c550691bd2..7d358d0b66241 100644 --- a/datafusion/sql/src/expr/identifier.rs +++ b/datafusion/sql/src/expr/identifier.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use arrow_schema::Field; +use arrow::datatypes::Field; use datafusion_common::{ internal_err, not_impl_err, plan_datafusion_err, plan_err, Column, DFSchema, DataFusionError, Result, Span, TableReference, diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index de753da895d30..fa2619111e7e9 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -15,8 +15,7 @@ // specific language governing permissions and limitations // under the License. -use arrow_schema::DataType; -use arrow_schema::TimeUnit; +use arrow::datatypes::{DataType, TimeUnit}; use datafusion_expr::planner::{ PlannerResult, RawBinaryExpr, RawDictionaryExpr, RawFieldAccessExpr, }; diff --git a/datafusion/sql/src/expr/value.rs b/datafusion/sql/src/expr/value.rs index 847163c6d3b3e..e81bfa0dc55f3 100644 --- a/datafusion/sql/src/expr/value.rs +++ b/datafusion/sql/src/expr/value.rs @@ -19,8 +19,9 @@ use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; use arrow::compute::kernels::cast_utils::{ parse_interval_month_day_nano_config, IntervalParseConfig, IntervalUnit, }; -use arrow::datatypes::{i256, DECIMAL128_MAX_PRECISION}; -use arrow_schema::{DataType, DECIMAL256_MAX_PRECISION}; +use arrow::datatypes::{ + i256, DataType, DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION, +}; use bigdecimal::num_bigint::BigInt; use bigdecimal::{BigDecimal, Signed, ToPrimitive}; use datafusion_common::{ diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rs index b376d0c5d82e3..a3ac831a1f783 100644 --- a/datafusion/sql/src/planner.rs +++ b/datafusion/sql/src/planner.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::vec; -use arrow_schema::*; +use arrow::datatypes::*; use datafusion_common::error::add_possible_columns_to_diag; use datafusion_common::{ field_not_found, internal_err, plan_datafusion_err, DFSchemaRef, Diagnostic, diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index d48cc93ee39e3..eed39be575a01 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -29,7 +29,7 @@ use crate::planner::{ }; use crate::utils::normalize_ident; -use arrow_schema::{DataType, Fields}; +use arrow::datatypes::{DataType, Fields}; use datafusion_common::error::_plan_err; use datafusion_common::parsers::CompressionTypeVariant; use datafusion_common::{ diff --git a/datafusion/sql/src/unparser/dialect.rs b/datafusion/sql/src/unparser/dialect.rs index adfb7a0d0cd2c..399f0df0a699b 100644 --- a/datafusion/sql/src/unparser/dialect.rs +++ b/datafusion/sql/src/unparser/dialect.rs @@ -18,7 +18,7 @@ use std::{collections::HashMap, sync::Arc}; use super::{utils::character_length_to_sql, utils::date_part_to_sql, Unparser}; -use arrow_schema::TimeUnit; +use arrow::datatypes::TimeUnit; use datafusion_common::Result; use datafusion_expr::Expr; use regex::Regex; diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rs index 9095337734356..72618c2b6ab4f 100644 --- a/datafusion/sql/src/unparser/expr.rs +++ b/datafusion/sql/src/unparser/expr.rs @@ -34,9 +34,8 @@ use arrow::array::{ }, ArrayRef, Date32Array, Date64Array, PrimitiveArray, }; -use arrow::datatypes::{Decimal128Type, Decimal256Type, DecimalType}; +use arrow::datatypes::{DataType, Decimal128Type, Decimal256Type, DecimalType}; use arrow::util::display::array_value_to_string; -use arrow_schema::DataType; use datafusion_common::{ internal_datafusion_err, internal_err, not_impl_err, plan_err, Column, Result, ScalarValue, @@ -1649,8 +1648,7 @@ mod tests { use std::{any::Any, sync::Arc, vec}; use arrow::array::{LargeListArray, ListArray}; - use arrow::datatypes::{Field, Int32Type, Schema, TimeUnit}; - use arrow_schema::DataType::Int8; + use arrow::datatypes::{DataType::Int8, Field, Int32Type, Schema, TimeUnit}; use ast::ObjectName; use datafusion_common::{Spans, TableReference}; use datafusion_expr::expr::WildcardOptions; diff --git a/datafusion/sql/src/unparser/rewrite.rs b/datafusion/sql/src/unparser/rewrite.rs index db98374831683..2e3c8e9e9484e 100644 --- a/datafusion/sql/src/unparser/rewrite.rs +++ b/datafusion/sql/src/unparser/rewrite.rs @@ -17,7 +17,7 @@ use std::{collections::HashSet, sync::Arc}; -use arrow_schema::Schema; +use arrow::datatypes::Schema; use datafusion_common::tree_node::TreeNodeContainer; use datafusion_common::{ tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRewriter}, diff --git a/datafusion/sql/src/utils.rs b/datafusion/sql/src/utils.rs index ab3e75a960a09..a9838ee68c447 100644 --- a/datafusion/sql/src/utils.rs +++ b/datafusion/sql/src/utils.rs @@ -19,7 +19,7 @@ use std::vec; -use arrow_schema::{ +use arrow::datatypes::{ DataType, DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION, DECIMAL_DEFAULT_SCALE, }; use datafusion_common::tree_node::{ @@ -650,8 +650,7 @@ pub(crate) fn rewrite_recursive_unnest_bottom_up( mod tests { use std::{ops::Add, sync::Arc}; - use arrow::datatypes::{DataType as ArrowDataType, Field, Schema}; - use arrow_schema::Fields; + use arrow::datatypes::{DataType as ArrowDataType, Field, Fields, Schema}; use datafusion_common::{Column, DFSchema, Result}; use datafusion_expr::{ col, lit, unnest, ColumnUnnestList, EmptyRelation, LogicalPlan, diff --git a/datafusion/sql/tests/cases/plan_to_sql.rs b/datafusion/sql/tests/cases/plan_to_sql.rs index 80af4d367b1ae..5af93a01e6c99 100644 --- a/datafusion/sql/tests/cases/plan_to_sql.rs +++ b/datafusion/sql/tests/cases/plan_to_sql.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use arrow_schema::{DataType, Field, Schema}; +use arrow::datatypes::{DataType, Field, Schema}; use datafusion_common::{assert_contains, DFSchema, DFSchemaRef, Result, TableReference}; use datafusion_expr::test::function_stub::{ count_udaf, max_udaf, min_udaf, sum, sum_udaf, diff --git a/datafusion/sql/tests/common/mod.rs b/datafusion/sql/tests/common/mod.rs index 337fc2ce4f678..ee1b761970def 100644 --- a/datafusion/sql/tests/common/mod.rs +++ b/datafusion/sql/tests/common/mod.rs @@ -21,7 +21,7 @@ use std::collections::HashMap; use std::fmt::{Debug, Display}; use std::{sync::Arc, vec}; -use arrow_schema::*; +use arrow::datatypes::*; use datafusion_common::config::ConfigOptions; use datafusion_common::file_options::file_type::FileType; use datafusion_common::{plan_err, DFSchema, GetExt, Result, TableReference}; diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs index c514458d4a277..5f460eb94a3ff 100644 --- a/datafusion/sql/tests/sql_integration.rs +++ b/datafusion/sql/tests/sql_integration.rs @@ -21,8 +21,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::vec; -use arrow_schema::TimeUnit::Nanosecond; -use arrow_schema::*; +use arrow::datatypes::{TimeUnit::Nanosecond, *}; use common::MockContextProvider; use datafusion_common::{ assert_contains, DataFusionError, ParamValues, Result, ScalarValue, From 373ef1df7193cdf32638a270c3f9102e6baada3b Mon Sep 17 00:00:00 2001 From: Ian Lai Date: Tue, 11 Feb 2025 05:28:09 +0000 Subject: [PATCH 2/4] fix: CI error --- datafusion/proto/src/generated/pbjson.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs index 837a766ff3909..add72e4f777e5 100644 --- a/datafusion/proto/src/generated/pbjson.rs +++ b/datafusion/proto/src/generated/pbjson.rs @@ -97,7 +97,7 @@ impl serde::Serialize for AggregateExecNode { #[allow(deprecated)] fn serialize(&self, serializer: S) -> std::result::Result where - S: serde::Serializer, + S: serde::Serializer, { use serde::ser::SerializeStruct; let mut len = 0; From f1d56175b9814ec9385812ef2e6cc2c5af93c70e Mon Sep 17 00:00:00 2001 From: Ian Lai Date: Tue, 11 Feb 2025 05:37:08 +0000 Subject: [PATCH 3/4] fix: remove arrow_schema in async_provider --- datafusion/ffi/src/tests/async_provider.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/ffi/src/tests/async_provider.rs b/datafusion/ffi/src/tests/async_provider.rs index de13363db04bc..cf05d596308f3 100644 --- a/datafusion/ffi/src/tests/async_provider.rs +++ b/datafusion/ffi/src/tests/async_provider.rs @@ -238,7 +238,7 @@ struct AsyncTestRecordBatchStream { } impl RecordBatchStream for AsyncTestRecordBatchStream { - fn schema(&self) -> arrow_schema::SchemaRef { + fn schema(&self) -> arrow::datatypes::SchemaRef { super::create_test_schema() } } From 50b4dc773c30119bb6f10019f0ef277afb50b21b Mon Sep 17 00:00:00 2001 From: Ian Lai Date: Tue, 11 Feb 2025 08:46:40 +0000 Subject: [PATCH 4/4] fix: add arrow_schema physical-plan due to doc test error in CI --- Cargo.lock | 1 + datafusion/physical-plan/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 27403b0f14047..96ad2527f7574 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2231,6 +2231,7 @@ dependencies = [ "ahash 0.8.11", "arrow", "arrow-ord", + "arrow-schema", "async-trait", "chrono", "criterion", diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml index 410863272b310..b84243b1b56b0 100644 --- a/datafusion/physical-plan/Cargo.toml +++ b/datafusion/physical-plan/Cargo.toml @@ -42,6 +42,7 @@ path = "src/lib.rs" ahash = { workspace = true } arrow = { workspace = true } arrow-ord = { workspace = true } +arrow-schema = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } datafusion-common = { workspace = true, default-features = true }