Upgrade to arrow 21

datafusion-cli/Cargo.toml

@@ -29,7 +29,7 @@ rust-version = "1.62"
 readme = "README.md"
 
 [dependencies]
-arrow = { version = "20.0.0" }
+arrow = "21.0.0"
 clap = { version = "3", features = ["derive", "cargo"] }
 datafusion = { path = "../datafusion/core", version = "11.0.0" }
 dirs = "4.0.0"

datafusion-examples/Cargo.toml

@@ -34,7 +34,7 @@ path = "examples/avro_sql.rs"
 required-features = ["datafusion/avro"]
 
 [dev-dependencies]
-arrow-flight = { version = "20.0.0" }
+arrow-flight = "21.0.0"
 async-trait = "0.1.41"
 datafusion = { path = "../datafusion/core" }
 futures = "0.3"

datafusion/common/Cargo.toml

@@ -39,12 +39,12 @@ pyarrow = ["pyo3"]
 
 [dependencies]
 apache-avro = { version = "0.14", features = ["snappy"], optional = true }
-arrow = { version = "20.0.0", features = ["prettyprint"] }
+arrow = { version = "21.0.0", features = ["prettyprint"] }
 avro-rs = { version = "0.13", features = ["snappy"], optional = true }
 cranelift-module = { version = "0.87.0", optional = true }
-object_store = { version = "0.4", optional = true }
+object_store = { version = "0.4.0", optional = true }
 ordered-float = "3.0"
-parquet = { version = "20.0.0", features = ["arrow"], optional = true }
+parquet = { version = "21.0.0", optional = true, features = ["arrow"] }
 pyo3 = { version = "0.16", optional = true }
 serde_json = "1.0"
 sqlparser = "0.22"

datafusion/common/src/scalar.rs

@@ -29,7 +29,7 @@ use arrow::{
         TimestampSecondType, UInt16Type, UInt32Type, UInt64Type, UInt8Type,
         DECIMAL128_MAX_PRECISION,
     },
-    util::decimal::{BasicDecimal, Decimal128},
+    util::decimal::Decimal128,
 };
 use ordered_float::OrderedFloat;
 use std::cmp::Ordering;
@@ -1236,8 +1236,9 @@ impl ScalarValue {
         scale: &usize,
         size: usize,
     ) -> Decimal128Array {
-        std::iter::repeat(value)
+        std::iter::repeat(*value)
             .take(size)
+            .into_iter()
             .collect::<Decimal128Array>()
             .with_precision_and_scale(*precision, *scale)
             .unwrap()

datafusion/core/Cargo.toml

@@ -56,7 +56,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion
 [dependencies]
 ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
 apache-avro = { version = "0.14", optional = true }
-arrow = { version = "20.0.0", features = ["prettyprint"] }
+arrow = { version = "21.0.0", features = ["prettyprint"] }
 async-trait = "0.1.41"
 bytes = "1.1"
 chrono = { version = "0.4", default-features = false }
@@ -78,7 +78,7 @@ num_cpus = "1.13.0"
 object_store = "0.4.0"
 ordered-float = "3.0"
 parking_lot = "0.12"
-parquet = { version = "20.0.0", features = ["arrow", "async"] }
+parquet = { version = "21.0.0", features = ["arrow", "async"] }
 paste = "^1.0"
 pin-project-lite = "^0.2.7"
 pyo3 = { version = "0.16", optional = true }

datafusion/core/fuzz-utils/Cargo.toml

@@ -23,6 +23,6 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-arrow = { version = "20.0.0", features = ["prettyprint"] }
+arrow = { features = ["prettyprint"], version = "21.0.0" }
 env_logger = "0.9.0"
 rand = "0.8"

datafusion/core/src/physical_optimizer/pruning.rs

@@ -800,7 +800,7 @@ mod tests {
     use crate::from_slice::FromSlice;
     use crate::logical_plan::{col, lit};
     use crate::{assert_batches_eq, physical_optimizer::pruning::StatisticsType};
-    use arrow::array::{BasicDecimalArray, Decimal128Array};
+    use arrow::array::Decimal128Array;
     use arrow::{
         array::{BinaryArray, Int32Array, Int64Array, StringArray},
         datatypes::{DataType, TimeUnit},

datafusion/core/src/physical_plan/hash_join.rs

@@ -22,11 +22,11 @@ use ahash::RandomState;
 
 use arrow::{
     array::{
-        as_dictionary_array, as_string_array, ArrayData, ArrayRef, BasicDecimalArray,
-        BooleanArray, Date32Array, Date64Array, Decimal128Array, DictionaryArray,
-        LargeStringArray, PrimitiveArray, TimestampMicrosecondArray,
-        TimestampMillisecondArray, TimestampSecondArray, UInt32BufferBuilder,
-        UInt32Builder, UInt64BufferBuilder, UInt64Builder,
+        as_dictionary_array, as_string_array, ArrayData, ArrayRef, BooleanArray,
+        Date32Array, Date64Array, Decimal128Array, DictionaryArray, LargeStringArray,
+        PrimitiveArray, TimestampMicrosecondArray, TimestampMillisecondArray,
+        TimestampSecondArray, UInt32BufferBuilder, UInt32Builder, UInt64BufferBuilder,
+        UInt64Builder,
     },
     compute,
     datatypes::{

datafusion/core/src/physical_plan/hash_utils.rs

@@ -20,9 +20,9 @@
 use crate::error::{DataFusionError, Result};
 use ahash::RandomState;
 use arrow::array::{
-    Array, ArrayRef, BasicDecimalArray, BooleanArray, Date32Array, Date64Array,
-    Decimal128Array, DictionaryArray, Float32Array, Float64Array, Int16Array, Int32Array,
-    Int64Array, Int8Array, LargeStringArray, StringArray, TimestampMicrosecondArray,
+    Array, ArrayRef, BooleanArray, Date32Array, Date64Array, Decimal128Array,
+    DictionaryArray, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array,
+    Int8Array, LargeStringArray, StringArray, TimestampMicrosecondArray,
     TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray,
     UInt16Array, UInt32Array, UInt64Array, UInt8Array,
 };

datafusion/core/src/test/mod.rs

@@ -26,8 +26,8 @@ use crate::logical_plan::LogicalPlan;
 use crate::physical_plan::file_format::{CsvExec, FileScanConfig};
 use crate::test::object_store::local_unpartitioned_file;
 use crate::test_util::aggr_test_schema;
-use array::{Array, ArrayRef};
-use arrow::array::{self, Decimal128Builder, Int32Array};
+use array::ArrayRef;
+use arrow::array::{self, Array, Decimal128Builder, Int32Array};
 use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
 use arrow::record_batch::RecordBatch;
 use futures::{Future, FutureExt};

datafusion/core/tests/parquet_pruning.rs

@@ -20,7 +20,7 @@
 //! expected.
 use std::sync::Arc;
 
-use arrow::array::{BasicDecimalArray, Decimal128Array};
+use arrow::array::Decimal128Array;
 use arrow::{
     array::{
         Array, ArrayRef, Date32Array, Date64Array, Float64Array, Int32Array, StringArray,

datafusion/core/tests/sql/select.rs

@@ -631,8 +631,10 @@ async fn query_nested_get_indexed_field_on_struct() -> Result<()> {
             lb.values().append_value(int);
         }
         lb.append(true);
+        sb.append(true);
     }
-    let data = RecordBatch::try_new(schema.clone(), vec![Arc::new(sb.finish())])?;
+    let s = sb.finish();
+    let data = RecordBatch::try_new(schema.clone(), vec![Arc::new(s)])?;
     let table = MemTable::try_new(schema, vec![vec![data]])?;
     let table_a = Arc::new(table);
 

datafusion/expr/Cargo.toml

@@ -36,6 +36,6 @@ path = "src/lib.rs"
 
 [dependencies]
 ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
-arrow = { version = "20.0.0", features = ["prettyprint"] }
+arrow = { version = "21.0.0", features = ["prettyprint"] }
 datafusion-common = { path = "../common", version = "11.0.0" }
 sqlparser = "0.22"

datafusion/jit/Cargo.toml

@@ -36,7 +36,7 @@ path = "src/lib.rs"
 jit = []
 
 [dependencies]
-arrow = { version = "20.0.0" }
+arrow = "21.0.0"
 cranelift = "0.87.0"
 cranelift-jit = "0.87.0"
 cranelift-module = "0.87.0"

datafusion/optimizer/Cargo.toml

@@ -37,7 +37,7 @@ default = ["unicode_expressions"]
 unicode_expressions = []
 
 [dependencies]
-arrow = { version = "20.0.0", features = ["prettyprint"] }
+arrow = { features = ["prettyprint"], version = "21.0.0" }
 async-trait = "0.1.41"
 chrono = { version = "0.4", default-features = false }
 datafusion-common = { path = "../common", version = "11.0.0" }

datafusion/physical-expr/Cargo.toml

@@ -40,7 +40,7 @@ unicode_expressions = ["unicode-segmentation"]
 
 [dependencies]
 ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
-arrow = { version = "20.0.0", features = ["prettyprint"] }
+arrow = { version = "21.0.0", features = ["prettyprint"] }
 blake2 = { version = "^0.10.2", optional = true }
 blake3 = { version = "1.0", optional = true }
 chrono = { version = "0.4", default-features = false }

datafusion/physical-expr/src/aggregate/average.rs

@@ -329,8 +329,9 @@ mod tests {
     fn avg_decimal_all_nulls() -> Result<()> {
         // test agg
         let array: ArrayRef = Arc::new(
-            std::iter::repeat(None)
+            std::iter::repeat::<Option<i128>>(None)
                 .take(6)
+                .into_iter()
                 .collect::<Decimal128Array>()
                 .with_precision_and_scale(10, 0)?,
         );

datafusion/physical-expr/src/aggregate/min_max.rs

@@ -26,11 +26,11 @@ use arrow::compute;
 use arrow::datatypes::{DataType, TimeUnit};
 use arrow::{
     array::{
-        ArrayRef, BasicDecimalArray, Date32Array, Date64Array, Float32Array,
-        Float64Array, Int16Array, Int32Array, Int64Array, Int8Array, LargeStringArray,
-        StringArray, Time64NanosecondArray, TimestampMicrosecondArray,
-        TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray,
-        UInt16Array, UInt32Array, UInt64Array, UInt8Array,
+        ArrayRef, Date32Array, Date64Array, Float32Array, Float64Array, Int16Array,
+        Int32Array, Int64Array, Int8Array, LargeStringArray, StringArray,
+        Time64NanosecondArray, TimestampMicrosecondArray, TimestampMillisecondArray,
+        TimestampNanosecondArray, TimestampSecondArray, UInt16Array, UInt32Array,
+        UInt64Array, UInt8Array,
     },
     datatypes::Field,
 };
@@ -795,8 +795,9 @@ mod tests {
 
         // min batch without values
         let array: ArrayRef = Arc::new(
-            std::iter::repeat(None)
+            std::iter::repeat::<Option<i128>>(None)
                 .take(0)
+                .into_iter()
                 .collect::<Decimal128Array>()
                 .with_precision_and_scale(10, 0)?,
         );
@@ -823,8 +824,9 @@ mod tests {
     fn min_decimal_all_nulls() -> Result<()> {
         // min batch all nulls
         let array: ArrayRef = Arc::new(
-            std::iter::repeat(None)
+            std::iter::repeat::<Option<i128>>(None)
                 .take(6)
+                .into_iter()
                 .collect::<Decimal128Array>()
                 .with_precision_and_scale(10, 0)?,
         );
@@ -884,8 +886,9 @@ mod tests {
 
         // max batch without values
         let array: ArrayRef = Arc::new(
-            std::iter::repeat(None)
+            std::iter::repeat::<Option<i128>>(None)
                 .take(0)
+                .into_iter()
                 .collect::<Decimal128Array>()
                 .with_precision_and_scale(10, 0)?,
         );
@@ -928,8 +931,9 @@ mod tests {
     #[test]
     fn max_decimal_all_nulls() -> Result<()> {
         let array: ArrayRef = Arc::new(
-            std::iter::repeat(None)
+            std::iter::repeat::<Option<i128>>(None)
                 .take(6)
+                .into_iter()
                 .collect::<Decimal128Array>()
                 .with_precision_and_scale(10, 0)?,
         );

datafusion/physical-expr/src/aggregate/sum.rs

@@ -26,8 +26,8 @@ use arrow::compute;
 use arrow::datatypes::DataType;
 use arrow::{
     array::{
-        ArrayRef, BasicDecimalArray, Float32Array, Float64Array, Int16Array, Int32Array,
-        Int64Array, Int8Array, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
+        ArrayRef, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array,
+        Int8Array, UInt16Array, UInt32Array, UInt64Array, UInt8Array,
     },
     datatypes::Field,
 };
@@ -608,8 +608,9 @@ mod tests {
 
         // test with batch
         let array: ArrayRef = Arc::new(
-            std::iter::repeat(None)
+            std::iter::repeat::<Option<i128>>(None)
                 .take(6)
+                .into_iter()
                 .collect::<Decimal128Array>()
                 .with_precision_and_scale(10, 0)?,
         );

datafusion/physical-expr/src/expressions/cast.rs

@@ -161,12 +161,12 @@ mod tests {
     use crate::expressions::col;
     use arrow::{
         array::{
-            Array, BasicDecimalArray, Decimal128Array, Float32Array, Float64Array,
-            Int16Array, Int32Array, Int64Array, Int8Array, StringArray,
-            Time64NanosecondArray, TimestampNanosecondArray, UInt32Array,
+            Array, Decimal128Array, Float32Array, Float64Array, Int16Array, Int32Array,
+            Int64Array, Int8Array, StringArray, Time64NanosecondArray,
+            TimestampNanosecondArray, UInt32Array,
         },
         datatypes::*,
-        util::decimal::{BasicDecimal, Decimal128},
+        util::decimal::Decimal128,
     };
     use datafusion_common::Result;
 
@@ -281,7 +281,8 @@ mod tests {
         ];
 
         let decimal_array = array
-            .iter()
+            .clone()
+            .into_iter()
             .collect::<Decimal128Array>()
             .with_precision_and_scale(10, 3)?;
 
@@ -305,7 +306,7 @@ mod tests {
         );
 
         let decimal_array = array
-            .iter()
+            .into_iter()
             .collect::<Decimal128Array>()
             .with_precision_and_scale(10, 3)?;
 
@@ -334,7 +335,8 @@ mod tests {
         let array = vec![Some(1), Some(2), Some(3), Some(4), Some(5), None];
         // decimal to i8
         let decimal_array = array
-            .iter()
+            .clone()
+            .into_iter()
             .collect::<Decimal128Array>()
             .with_precision_and_scale(10, 0)?;
         generic_decimal_to_other_test_cast!(
@@ -355,7 +357,8 @@ mod tests {
 
         // decimal to i16
         let decimal_array = array
-            .iter()
+            .clone()
+            .into_iter()
             .collect::<Decimal128Array>()
             .with_precision_and_scale(10, 0)?;
         generic_decimal_to_other_test_cast!(
@@ -376,7 +379,8 @@ mod tests {
 
         // decimal to i32
         let decimal_array = array
-            .iter()
+            .clone()
+            .into_iter()
             .collect::<Decimal128Array>()
             .with_precision_and_scale(10, 0)?;
         generic_decimal_to_other_test_cast!(
@@ -397,7 +401,7 @@ mod tests {
 
         // decimal to i64
         let decimal_array = array
-            .iter()
+            .into_iter()
             .collect::<Decimal128Array>()
             .with_precision_and_scale(10, 0)?;
         generic_decimal_to_other_test_cast!(
@@ -426,7 +430,8 @@ mod tests {
             None,
         ];
         let decimal_array = array
-            .iter()
+            .clone()
+            .into_iter()
             .collect::<Decimal128Array>()
             .with_precision_and_scale(10, 3)?;
         generic_decimal_to_other_test_cast!(

datafusion/physical-expr/src/expressions/try_cast.rs

@@ -119,10 +119,9 @@ mod tests {
     use super::*;
     use crate::expressions::col;
     use arrow::array::{
-        BasicDecimalArray, Decimal128Array, Decimal128Builder, StringArray,
-        Time64NanosecondArray,
+        Decimal128Array, Decimal128Builder, StringArray, Time64NanosecondArray,
     };
-    use arrow::util::decimal::{BasicDecimal, Decimal128};
+    use arrow::util::decimal::Decimal128;
     use arrow::{
         array::{
             Array, Float32Array, Float64Array, Int16Array, Int32Array, Int64Array,

datafusion/proto/Cargo.toml

@@ -37,7 +37,7 @@ default = []
 json = ["pbjson", "pbjson-build", "serde", "serde_json"]
 
 [dependencies]
-arrow = { version = "20.0.0" }
+arrow = "21.0.0"
 datafusion = { path = "../core", version = "11.0.0" }
 datafusion-common = { path = "../common", version = "11.0.0" }
 datafusion-expr = { path = "../expr", version = "11.0.0" }