From a4b918ca039bae5c2084276d625f4e0fe53e31a5 Mon Sep 17 00:00:00 2001 From: Marko Grujic Date: Tue, 3 Dec 2024 12:16:58 +0100 Subject: [PATCH] Consolidate construction of the list field This also makes the naming of the nested field implicit, and removes a bunch of item literals. --- datafusion/common/src/dfschema.rs | 12 ++--- datafusion/common/src/scalar/mod.rs | 44 +++++++++---------- datafusion/core/tests/sql/aggregates.rs | 2 +- .../expr-common/src/type_coercion/binary.rs | 5 +-- datafusion/expr/src/logical_plan/builder.rs | 2 +- .../expr/src/type_coercion/functions.rs | 17 +++---- datafusion/functions-aggregate/COMMENTS.md | 2 +- .../functions-aggregate/src/approx_median.rs | 2 +- .../src/approx_percentile_cont.rs | 2 +- .../functions-aggregate/src/array_agg.rs | 9 ++-- .../functions-aggregate/src/bit_and_or_xor.rs | 2 +- datafusion/functions-aggregate/src/count.rs | 2 +- datafusion/functions-aggregate/src/median.rs | 2 +- .../functions-aggregate/src/nth_value.rs | 4 +- datafusion/functions-aggregate/src/sum.rs | 2 +- datafusion/functions-nested/benches/map.rs | 4 +- datafusion/functions-nested/src/concat.rs | 4 +- datafusion/functions-nested/src/dimension.rs | 2 +- datafusion/functions-nested/src/extract.rs | 2 +- datafusion/functions-nested/src/make_array.rs | 7 ++- .../functions-nested/src/map_extract.rs | 5 +-- datafusion/functions-nested/src/map_keys.rs | 5 +-- datafusion/functions-nested/src/map_values.rs | 5 +-- datafusion/functions-nested/src/position.rs | 2 +- datafusion/functions-nested/src/range.rs | 8 ++-- datafusion/functions-nested/src/remove.rs | 2 +- datafusion/functions-nested/src/repeat.rs | 9 ++-- datafusion/functions-nested/src/replace.rs | 2 +- datafusion/functions-nested/src/set_ops.rs | 15 +++---- datafusion/functions-nested/src/sort.rs | 13 +++--- datafusion/functions-nested/src/string.rs | 2 +- datafusion/functions-nested/src/utils.rs | 2 +- datafusion/functions/src/regex/regexpmatch.rs | 2 +- .../optimizer/src/analyzer/type_coercion.rs | 28 ++++++------ .../group_values/multi_group_by/mod.rs | 2 +- datafusion/physical-plan/src/sorts/sort.rs | 4 +- datafusion/physical-plan/src/spill.rs | 4 +- datafusion/physical-plan/src/unnest.rs | 6 +-- .../tests/cases/roundtrip_logical_plan.rs | 4 +- datafusion/sql/src/utils.rs | 3 +- datafusion/sql/tests/common/mod.rs | 15 +++++-- .../substrait/src/logical_plan/producer.rs | 2 +- .../tests/cases/roundtrip_logical_plan.rs | 8 +++- 43 files changed, 133 insertions(+), 143 deletions(-) diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs index 45620c3cacc8a..b5f7b5681eefc 100644 --- a/datafusion/common/src/dfschema.rs +++ b/datafusion/common/src/dfschema.rs @@ -1382,14 +1382,14 @@ mod tests { // Succeeds if both have the same element type, disregards names and nullability assert!(DFSchema::datatype_is_logically_equal( - &DataType::List(Field::new("item", DataType::Int8, true).into()), + &DataType::List(Field::new_list_field(DataType::Int8, true).into()), &DataType::List(Field::new("element", DataType::Int8, false).into()) )); // Fails if element type is different assert!(!DFSchema::datatype_is_logically_equal( - &DataType::List(Field::new("item", DataType::Int8, true).into()), - &DataType::List(Field::new("item", DataType::Int16, true).into()) + &DataType::List(Field::new_list_field(DataType::Int8, true).into()), + &DataType::List(Field::new_list_field(DataType::Int16, true).into()) )); // Test maps @@ -1522,14 +1522,14 @@ mod tests { // Succeeds if both have the same element type, disregards names and nullability assert!(DFSchema::datatype_is_semantically_equal( - &DataType::List(Field::new("item", DataType::Int8, true).into()), + &DataType::List(Field::new_list_field(DataType::Int8, true).into()), &DataType::List(Field::new("element", DataType::Int8, false).into()) )); // Fails if element type is different assert!(!DFSchema::datatype_is_semantically_equal( - &DataType::List(Field::new("item", DataType::Int8, true).into()), - &DataType::List(Field::new("item", DataType::Int16, true).into()) + &DataType::List(Field::new_list_field(DataType::Int8, true).into()), + &DataType::List(Field::new_list_field(DataType::Int16, true).into()) )); // Test maps diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rs index edba0b84431fb..cdc278fde4bf2 100644 --- a/datafusion/common/src/scalar/mod.rs +++ b/datafusion/common/src/scalar/mod.rs @@ -4037,7 +4037,7 @@ mod tests { #[test] fn test_to_array_of_size_for_fsl() { let values = Int32Array::from_iter([Some(1), None, Some(2)]); - let field = Arc::new(Field::new("item", DataType::Int32, true)); + let field = Arc::new(Field::new_list_field(DataType::Int32, true)); let arr = FixedSizeListArray::new(Arc::clone(&field), 3, Arc::new(values), None); let sv = ScalarValue::FixedSizeList(Arc::new(arr)); let actual_arr = sv @@ -4093,8 +4093,7 @@ mod tests { ) } else if O::IS_LARGE { new_null_array( - &DataType::LargeList(Arc::new(Field::new( - "item", + &DataType::LargeList(Arc::new(Field::new_list_field( DataType::Int64, true, ))), @@ -4102,8 +4101,7 @@ mod tests { ) } else { new_null_array( - &DataType::List(Arc::new(Field::new( - "item", + &DataType::List(Arc::new(Field::new_list_field( DataType::Int64, true, ))), @@ -4122,7 +4120,7 @@ mod tests { #[test] fn test_iter_to_array_fixed_size_list() { - let field = Arc::new(Field::new("item", DataType::Int32, true)); + let field = Arc::new(Field::new_list_field(DataType::Int32, true)); let f1 = Arc::new(FixedSizeListArray::new( Arc::clone(&field), 3, @@ -4961,7 +4959,7 @@ mod tests { let null_list_scalar = ScalarValue::try_from_array(&list, 1).unwrap(); let data_type = - DataType::List(Arc::new(Field::new("item", DataType::Int32, true))); + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))); assert_eq!(non_null_list_scalar.data_type(), data_type); assert_eq!(null_list_scalar.data_type(), data_type); @@ -4969,7 +4967,7 @@ mod tests { #[test] fn scalar_try_from_list_datatypes() { - let inner_field = Arc::new(Field::new("item", DataType::Int32, true)); + let inner_field = Arc::new(Field::new_list_field(DataType::Int32, true)); // Test for List let data_type = &DataType::List(Arc::clone(&inner_field)); @@ -5010,9 +5008,8 @@ mod tests { #[test] fn scalar_try_from_list_of_list() { - let data_type = DataType::List(Arc::new(Field::new( - "item", - DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + let data_type = DataType::List(Arc::new(Field::new_list_field( + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))), true, ))); let data_type = &data_type; @@ -5020,9 +5017,11 @@ mod tests { let expected = ScalarValue::List( new_null_array( - &DataType::List(Arc::new(Field::new( - "item", - DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + &DataType::List(Arc::new(Field::new_list_field( + DataType::List(Arc::new(Field::new_list_field( + DataType::Int32, + true, + ))), true, ))), 1, @@ -5038,13 +5037,12 @@ mod tests { #[test] fn scalar_try_from_not_equal_list_nested_list() { let list_data_type = - DataType::List(Arc::new(Field::new("item", DataType::Int32, true))); + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))); let data_type = &list_data_type; let list_scalar: ScalarValue = data_type.try_into().unwrap(); - let nested_list_data_type = DataType::List(Arc::new(Field::new( - "item", - DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + let nested_list_data_type = DataType::List(Arc::new(Field::new_list_field( + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))), true, ))); let data_type = &nested_list_data_type; @@ -5678,7 +5676,7 @@ mod tests { let field_a = Arc::new(Field::new("A", DataType::Utf8, false)); let field_primitive_list = Arc::new(Field::new( "primitive_list", - DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))), false, )); @@ -5879,9 +5877,8 @@ mod tests { fn build_2d_list(data: Vec>) -> ListArray { let a1 = ListArray::from_iter_primitive::(vec![Some(data)]); ListArray::new( - Arc::new(Field::new( - "item", - DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + Arc::new(Field::new_list_field( + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))), true, )), OffsetBuffer::::from_lengths([1]), @@ -6876,8 +6873,7 @@ mod tests { assert_eq!(1, arr.len()); assert_eq!( arr.data_type(), - &DataType::List(Arc::new(Field::new( - "item", + &DataType::List(Arc::new(Field::new_list_field( DataType::Timestamp(TimeUnit::Millisecond, Some(s.into())), true, ))) diff --git a/datafusion/core/tests/sql/aggregates.rs b/datafusion/core/tests/sql/aggregates.rs index 1f10cb244e83c..7b1f349e15b5a 100644 --- a/datafusion/core/tests/sql/aggregates.rs +++ b/datafusion/core/tests/sql/aggregates.rs @@ -36,7 +36,7 @@ async fn csv_query_array_agg_distinct() -> Result<()> { *actual[0].schema(), Schema::new(vec![Field::new_list( "array_agg(DISTINCT aggregate_test_100.c2)", - Field::new("item", DataType::UInt32, true), + Field::new_list_field(DataType::UInt32, true), true ),]) ); diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rs index 4b2f3b5e46b58..7a6e9841e22c8 100644 --- a/datafusion/expr-common/src/type_coercion/binary.rs +++ b/datafusion/expr-common/src/type_coercion/binary.rs @@ -2098,7 +2098,7 @@ mod tests { ); // list - let inner_field = Arc::new(Field::new("item", DataType::Int64, true)); + let inner_field = Arc::new(Field::new_list_field(DataType::Int64, true)); test_coercion_binary_rule!( DataType::List(Arc::clone(&inner_field)), DataType::List(Arc::clone(&inner_field)), @@ -2155,8 +2155,7 @@ mod tests { ); // Negative test: inner_timestamp_field and inner_field are not compatible because their inner types are not compatible - let inner_timestamp_field = Arc::new(Field::new( - "item", + let inner_timestamp_field = Arc::new(Field::new_list_field( DataType::Timestamp(TimeUnit::Microsecond, None), true, )); diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs index 90235e3f84c48..e69f26b626a11 100644 --- a/datafusion/expr/src/logical_plan/builder.rs +++ b/datafusion/expr/src/logical_plan/builder.rs @@ -2432,7 +2432,7 @@ mod tests { ], false, ); - let string_field = Field::new("item", DataType::Utf8, false); + let string_field = Field::new_list_field(DataType::Utf8, false); let strings_field = Field::new_list("item", string_field.clone(), false); let schema = Schema::new(vec![ Field::new("scalar", DataType::UInt32, false), diff --git a/datafusion/expr/src/type_coercion/functions.rs b/datafusion/expr/src/type_coercion/functions.rs index 5f52c7ccc20ef..3dbffce3c98f4 100644 --- a/datafusion/expr/src/type_coercion/functions.rs +++ b/datafusion/expr/src/type_coercion/functions.rs @@ -958,7 +958,7 @@ mod tests { #[test] fn test_fixed_list_wildcard_coerce() -> Result<()> { - let inner = Arc::new(Field::new("item", DataType::Int32, false)); + let inner = Arc::new(Field::new_list_field(DataType::Int32, false)); let current_types = vec![ DataType::FixedSizeList(Arc::clone(&inner), 2), // able to coerce for any size ]; @@ -996,10 +996,9 @@ mod tests { #[test] fn test_nested_wildcard_fixed_size_lists() -> Result<()> { let type_into = DataType::FixedSizeList( - Arc::new(Field::new( - "item", + Arc::new(Field::new_list_field( DataType::FixedSizeList( - Arc::new(Field::new("item", DataType::Int32, false)), + Arc::new(Field::new_list_field(DataType::Int32, false)), FIXED_SIZE_LIST_WILDCARD, ), false, @@ -1008,10 +1007,9 @@ mod tests { ); let type_from = DataType::FixedSizeList( - Arc::new(Field::new( - "item", + Arc::new(Field::new_list_field( DataType::FixedSizeList( - Arc::new(Field::new("item", DataType::Int8, false)), + Arc::new(Field::new_list_field(DataType::Int8, false)), 4, ), false, @@ -1022,10 +1020,9 @@ mod tests { assert_eq!( coerced_from(&type_into, &type_from), Some(DataType::FixedSizeList( - Arc::new(Field::new( - "item", + Arc::new(Field::new_list_field( DataType::FixedSizeList( - Arc::new(Field::new("item", DataType::Int32, false)), + Arc::new(Field::new_list_field(DataType::Int32, false)), 4, ), false, diff --git a/datafusion/functions-aggregate/COMMENTS.md b/datafusion/functions-aggregate/COMMENTS.md index e669e13557115..1cb4cdd7d5a45 100644 --- a/datafusion/functions-aggregate/COMMENTS.md +++ b/datafusion/functions-aggregate/COMMENTS.md @@ -54,7 +54,7 @@ first argument and the definition looks like this: // `input_type` : data type of the first argument let mut fields = vec![Field::new_list( format_state_name(self.name(), "nth_value"), - Field::new("item", args.input_types[0].clone(), true /* nullable of list item */ ), + Field::new_list_field(args.input_types[0].clone(), true /* nullable of list item */ ), false, // nullable of list itself )]; ``` diff --git a/datafusion/functions-aggregate/src/approx_median.rs b/datafusion/functions-aggregate/src/approx_median.rs index 8920c8e5f0c48..f653e94740b31 100644 --- a/datafusion/functions-aggregate/src/approx_median.rs +++ b/datafusion/functions-aggregate/src/approx_median.rs @@ -87,7 +87,7 @@ impl AggregateUDFImpl for ApproxMedian { Field::new(format_state_name(args.name, "min"), Float64, false), Field::new_list( format_state_name(args.name, "centroids"), - Field::new("item", Float64, true), + Field::new_list_field(Float64, true), false, ), ]) diff --git a/datafusion/functions-aggregate/src/approx_percentile_cont.rs b/datafusion/functions-aggregate/src/approx_percentile_cont.rs index 6edae6344ab15..197c459d26e57 100644 --- a/datafusion/functions-aggregate/src/approx_percentile_cont.rs +++ b/datafusion/functions-aggregate/src/approx_percentile_cont.rs @@ -240,7 +240,7 @@ impl AggregateUDFImpl for ApproxPercentileCont { ), Field::new_list( format_state_name(args.name, "centroids"), - Field::new("item", DataType::Float64, true), + Field::new_list_field(DataType::Float64, true), false, ), ]) diff --git a/datafusion/functions-aggregate/src/array_agg.rs b/datafusion/functions-aggregate/src/array_agg.rs index 3b9a521ec9721..ac781daaff907 100644 --- a/datafusion/functions-aggregate/src/array_agg.rs +++ b/datafusion/functions-aggregate/src/array_agg.rs @@ -77,8 +77,7 @@ impl AggregateUDFImpl for ArrayAgg { } fn return_type(&self, arg_types: &[DataType]) -> Result { - Ok(DataType::List(Arc::new(Field::new( - "item", + Ok(DataType::List(Arc::new(Field::new_list_field( arg_types[0].clone(), true, )))) @@ -89,7 +88,7 @@ impl AggregateUDFImpl for ArrayAgg { return Ok(vec![Field::new_list( format_state_name(args.name, "distinct_array_agg"), // See COMMENTS.md to understand why nullable is set to true - Field::new("item", args.input_types[0].clone(), true), + Field::new_list_field(args.input_types[0].clone(), true), true, )]); } @@ -97,7 +96,7 @@ impl AggregateUDFImpl for ArrayAgg { let mut fields = vec![Field::new_list( format_state_name(args.name, "array_agg"), // See COMMENTS.md to understand why nullable is set to true - Field::new("item", args.input_types[0].clone(), true), + Field::new_list_field(args.input_types[0].clone(), true), true, )]; @@ -108,7 +107,7 @@ impl AggregateUDFImpl for ArrayAgg { let orderings = args.ordering_fields.to_vec(); fields.push(Field::new_list( format_state_name(args.name, "array_agg_orderings"), - Field::new("item", DataType::Struct(Fields::from(orderings)), true), + Field::new_list_field(DataType::Struct(Fields::from(orderings)), true), false, )); diff --git a/datafusion/functions-aggregate/src/bit_and_or_xor.rs b/datafusion/functions-aggregate/src/bit_and_or_xor.rs index aeed78737c1d5..6298071a223b2 100644 --- a/datafusion/functions-aggregate/src/bit_and_or_xor.rs +++ b/datafusion/functions-aggregate/src/bit_and_or_xor.rs @@ -272,7 +272,7 @@ impl AggregateUDFImpl for BitwiseOperation { format!("{} distinct", self.name()).as_str(), ), // See COMMENTS.md to understand why nullable is set to true - Field::new("item", args.return_type.clone(), true), + Field::new_list_field(args.return_type.clone(), true), false, )]) } else { diff --git a/datafusion/functions-aggregate/src/count.rs b/datafusion/functions-aggregate/src/count.rs index c8f8c8153ce11..a191f7bad477b 100644 --- a/datafusion/functions-aggregate/src/count.rs +++ b/datafusion/functions-aggregate/src/count.rs @@ -135,7 +135,7 @@ impl AggregateUDFImpl for Count { Ok(vec![Field::new_list( format_state_name(args.name, "count distinct"), // See COMMENTS.md to understand why nullable is set to true - Field::new("item", args.input_types[0].clone(), true), + Field::new_list_field(args.input_types[0].clone(), true), false, )]) } else { diff --git a/datafusion/functions-aggregate/src/median.rs b/datafusion/functions-aggregate/src/median.rs index bcffb19b75593..0fd9d4ca63e4b 100644 --- a/datafusion/functions-aggregate/src/median.rs +++ b/datafusion/functions-aggregate/src/median.rs @@ -104,7 +104,7 @@ impl AggregateUDFImpl for Median { fn state_fields(&self, args: StateFieldsArgs) -> Result> { //Intermediate state is a list of the elements we have collected so far - let field = Field::new("item", args.input_types[0].clone(), true); + let field = Field::new_list_field(args.input_types[0].clone(), true); let state_name = if args.is_distinct { "distinct_median" } else { diff --git a/datafusion/functions-aggregate/src/nth_value.rs b/datafusion/functions-aggregate/src/nth_value.rs index 0c72939633b16..aa84e0dac7ed0 100644 --- a/datafusion/functions-aggregate/src/nth_value.rs +++ b/datafusion/functions-aggregate/src/nth_value.rs @@ -142,14 +142,14 @@ impl AggregateUDFImpl for NthValueAgg { let mut fields = vec![Field::new_list( format_state_name(self.name(), "nth_value"), // See COMMENTS.md to understand why nullable is set to true - Field::new("item", args.input_types[0].clone(), true), + Field::new_list_field(args.input_types[0].clone(), true), false, )]; let orderings = args.ordering_fields.to_vec(); if !orderings.is_empty() { fields.push(Field::new_list( format_state_name(self.name(), "nth_value_orderings"), - Field::new("item", DataType::Struct(Fields::from(orderings)), true), + Field::new_list_field(DataType::Struct(Fields::from(orderings)), true), false, )); } diff --git a/datafusion/functions-aggregate/src/sum.rs b/datafusion/functions-aggregate/src/sum.rs index 447b5d8a57c44..df87a593cf35a 100644 --- a/datafusion/functions-aggregate/src/sum.rs +++ b/datafusion/functions-aggregate/src/sum.rs @@ -183,7 +183,7 @@ impl AggregateUDFImpl for Sum { Ok(vec![Field::new_list( format_state_name(args.name, "sum distinct"), // See COMMENTS.md to understand why nullable is set to true - Field::new("item", args.return_type.clone(), true), + Field::new_list_field(args.return_type.clone(), true), false, )]) } else { diff --git a/datafusion/functions-nested/benches/map.rs b/datafusion/functions-nested/benches/map.rs index 0f1d9ed506365..f92bb6cecf9c5 100644 --- a/datafusion/functions-nested/benches/map.rs +++ b/datafusion/functions-nested/benches/map.rs @@ -75,7 +75,7 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("map_1000", |b| { let mut rng = rand::thread_rng(); - let field = Arc::new(Field::new("item", DataType::Utf8, true)); + let field = Arc::new(Field::new_list_field(DataType::Utf8, true)); let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, 1000])); let key_list = ListArray::new( field, @@ -83,7 +83,7 @@ fn criterion_benchmark(c: &mut Criterion) { Arc::new(StringArray::from(keys(&mut rng))), None, ); - let field = Arc::new(Field::new("item", DataType::Int32, true)); + let field = Arc::new(Field::new_list_field(DataType::Int32, true)); let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0, 1000])); let value_list = ListArray::new( field, diff --git a/datafusion/functions-nested/src/concat.rs b/datafusion/functions-nested/src/concat.rs index ac9b5b2a16eff..3e8a5877fb336 100644 --- a/datafusion/functions-nested/src/concat.rs +++ b/datafusion/functions-nested/src/concat.rs @@ -429,7 +429,7 @@ fn concat_internal(args: &[ArrayRef]) -> Result { .collect::>(); let list_arr = GenericListArray::::new( - Arc::new(Field::new("item", data_type, true)), + Arc::new(Field::new_list_field(data_type, true)), OffsetBuffer::from_lengths(array_lengths), Arc::new(arrow::compute::concat(elements.as_slice())?), Some(NullBuffer::new(buffer)), @@ -558,7 +558,7 @@ where let data = mutable.freeze(); Ok(Arc::new(GenericListArray::::try_new( - Arc::new(Field::new("item", data_type.to_owned(), true)), + Arc::new(Field::new_list_field(data_type.to_owned(), true)), OffsetBuffer::new(offsets.into()), arrow_array::make_array(data), None, diff --git a/datafusion/functions-nested/src/dimension.rs b/datafusion/functions-nested/src/dimension.rs index 9ca7c87aba559..2d2f90e9c7cb4 100644 --- a/datafusion/functions-nested/src/dimension.rs +++ b/datafusion/functions-nested/src/dimension.rs @@ -73,7 +73,7 @@ impl ScalarUDFImpl for ArrayDims { fn return_type(&self, arg_types: &[DataType]) -> Result { Ok(match arg_types[0] { List(_) | LargeList(_) | FixedSizeList(_, _) => { - List(Arc::new(Field::new("item", UInt64, true))) + List(Arc::new(Field::new_list_field(UInt64, true))) } _ => { return plan_err!("The array_dims function can only accept List/LargeList/FixedSizeList."); diff --git a/datafusion/functions-nested/src/extract.rs b/datafusion/functions-nested/src/extract.rs index 13095bc4ba3fc..fc35f00763305 100644 --- a/datafusion/functions-nested/src/extract.rs +++ b/datafusion/functions-nested/src/extract.rs @@ -622,7 +622,7 @@ where let data = mutable.freeze(); Ok(Arc::new(GenericListArray::::try_new( - Arc::new(Field::new("item", array.value_type(), true)), + Arc::new(Field::new_list_field(array.value_type(), true)), OffsetBuffer::::new(offsets.into()), arrow_array::make_array(data), None, diff --git a/datafusion/functions-nested/src/make_array.rs b/datafusion/functions-nested/src/make_array.rs index 825824a82d200..7473c05648b14 100644 --- a/datafusion/functions-nested/src/make_array.rs +++ b/datafusion/functions-nested/src/make_array.rs @@ -89,8 +89,7 @@ impl ScalarUDFImpl for MakeArray { 0 => Ok(empty_array_type()), _ => { // At this point, all the type in array should be coerced to the same one - Ok(List(Arc::new(Field::new( - "item", + Ok(List(Arc::new(Field::new_list_field( arg_types[0].to_owned(), true, )))) @@ -172,7 +171,7 @@ fn get_make_array_doc() -> &'static Documentation { // Empty array is a special case that is useful for many other array functions pub(super) fn empty_array_type() -> DataType { - List(Arc::new(Field::new("item", DataType::Int64, true))) + List(Arc::new(Field::new_list_field(DataType::Int64, true))) } /// `make_array_inner` is the implementation of the `make_array` function. @@ -285,7 +284,7 @@ fn array_array( let data = mutable.freeze(); Ok(Arc::new(GenericListArray::::try_new( - Arc::new(Field::new("item", data_type, true)), + Arc::new(Field::new_list_field(data_type, true)), OffsetBuffer::new(offsets.into()), arrow_array::make_array(data), None, diff --git a/datafusion/functions-nested/src/map_extract.rs b/datafusion/functions-nested/src/map_extract.rs index d2ab078cb22dd..24f396e741b2d 100644 --- a/datafusion/functions-nested/src/map_extract.rs +++ b/datafusion/functions-nested/src/map_extract.rs @@ -78,8 +78,7 @@ impl ScalarUDFImpl for MapExtract { } let map_type = &arg_types[0]; let map_fields = get_map_entry_field(map_type)?; - Ok(DataType::List(Arc::new(Field::new( - "item", + Ok(DataType::List(Arc::new(Field::new_list_field( map_fields.last().unwrap().data_type().clone(), true, )))) @@ -187,7 +186,7 @@ fn general_map_extract_inner( let data = mutable.freeze(); Ok(Arc::new(ListArray::new( - Arc::new(Field::new("item", map_array.value_type().clone(), true)), + Arc::new(Field::new_list_field(map_array.value_type().clone(), true)), OffsetBuffer::::new(offsets.into()), Arc::new(make_array(data)), None, diff --git a/datafusion/functions-nested/src/map_keys.rs b/datafusion/functions-nested/src/map_keys.rs index 4abdbcad1e825..1d19cb8492f03 100644 --- a/datafusion/functions-nested/src/map_keys.rs +++ b/datafusion/functions-nested/src/map_keys.rs @@ -72,8 +72,7 @@ impl ScalarUDFImpl for MapKeysFunc { } let map_type = &arg_types[0]; let map_fields = get_map_entry_field(map_type)?; - Ok(DataType::List(Arc::new(Field::new( - "item", + Ok(DataType::List(Arc::new(Field::new_list_field( map_fields.first().unwrap().data_type().clone(), false, )))) @@ -130,7 +129,7 @@ fn map_keys_inner(args: &[ArrayRef]) -> Result { }; Ok(Arc::new(ListArray::new( - Arc::new(Field::new("item", map_array.key_type().clone(), false)), + Arc::new(Field::new_list_field(map_array.key_type().clone(), false)), map_array.offsets().clone(), Arc::clone(map_array.keys()), None, diff --git a/datafusion/functions-nested/src/map_values.rs b/datafusion/functions-nested/src/map_values.rs index f1cc36cade636..816ebe74aff0a 100644 --- a/datafusion/functions-nested/src/map_values.rs +++ b/datafusion/functions-nested/src/map_values.rs @@ -72,8 +72,7 @@ impl ScalarUDFImpl for MapValuesFunc { } let map_type = &arg_types[0]; let map_fields = get_map_entry_field(map_type)?; - Ok(DataType::List(Arc::new(Field::new( - "item", + Ok(DataType::List(Arc::new(Field::new_list_field( map_fields.last().unwrap().data_type().clone(), true, )))) @@ -131,7 +130,7 @@ fn map_values_inner(args: &[ArrayRef]) -> Result { }; Ok(Arc::new(ListArray::new( - Arc::new(Field::new("item", map_array.value_type().clone(), true)), + Arc::new(Field::new_list_field(map_array.value_type().clone(), true)), map_array.offsets().clone(), Arc::clone(map_array.values()), None, diff --git a/datafusion/functions-nested/src/position.rs b/datafusion/functions-nested/src/position.rs index 9ed4b4c42d14a..feacc70061926 100644 --- a/datafusion/functions-nested/src/position.rs +++ b/datafusion/functions-nested/src/position.rs @@ -252,7 +252,7 @@ impl ScalarUDFImpl for ArrayPositions { } fn return_type(&self, _arg_types: &[DataType]) -> Result { - Ok(List(Arc::new(Field::new("item", UInt64, true)))) + Ok(List(Arc::new(Field::new_list_field(UInt64, true)))) } fn invoke_batch( diff --git a/datafusion/functions-nested/src/range.rs b/datafusion/functions-nested/src/range.rs index 360f0023dc477..8344c1a261db0 100644 --- a/datafusion/functions-nested/src/range.rs +++ b/datafusion/functions-nested/src/range.rs @@ -109,8 +109,7 @@ impl ScalarUDFImpl for Range { if arg_types.iter().any(|t| t.is_null()) { Ok(Null) } else { - Ok(List(Arc::new(Field::new( - "item", + Ok(List(Arc::new(Field::new_list_field( arg_types[0].clone(), true, )))) @@ -249,8 +248,7 @@ impl ScalarUDFImpl for GenSeries { if arg_types.iter().any(|t| t.is_null()) { Ok(Null) } else { - Ok(List(Arc::new(Field::new( - "item", + Ok(List(Arc::new(Field::new_list_field( arg_types[0].clone(), true, )))) @@ -393,7 +391,7 @@ pub(super) fn gen_range_inner( }; } let arr = Arc::new(ListArray::try_new( - Arc::new(Field::new("item", Int64, true)), + Arc::new(Field::new_list_field(Int64, true)), OffsetBuffer::new(offsets.into()), Arc::new(Int64Array::from(values)), Some(NullBuffer::new(valid.finish())), diff --git a/datafusion/functions-nested/src/remove.rs b/datafusion/functions-nested/src/remove.rs index df0edc99bc62a..e5521706bece2 100644 --- a/datafusion/functions-nested/src/remove.rs +++ b/datafusion/functions-nested/src/remove.rs @@ -433,7 +433,7 @@ fn general_remove( }; Ok(Arc::new(GenericListArray::::try_new( - Arc::new(Field::new("item", data_type, true)), + Arc::new(Field::new_list_field(data_type, true)), OffsetBuffer::new(offsets.into()), values, list_array.nulls().cloned(), diff --git a/datafusion/functions-nested/src/repeat.rs b/datafusion/functions-nested/src/repeat.rs index f67ab83b1d390..2842b91a781b5 100644 --- a/datafusion/functions-nested/src/repeat.rs +++ b/datafusion/functions-nested/src/repeat.rs @@ -72,8 +72,7 @@ impl ScalarUDFImpl for ArrayRepeat { } fn return_type(&self, arg_types: &[DataType]) -> Result { - Ok(List(Arc::new(Field::new( - "item", + Ok(List(Arc::new(Field::new_list_field( arg_types[0].clone(), true, )))) @@ -204,7 +203,7 @@ fn general_repeat( let values = compute::concat(&new_values)?; Ok(Arc::new(GenericListArray::::try_new( - Arc::new(Field::new("item", data_type.to_owned(), true)), + Arc::new(Field::new_list_field(data_type.to_owned(), true)), OffsetBuffer::from_lengths(count_vec), values, None, @@ -255,7 +254,7 @@ fn general_list_repeat( let repeated_array = arrow_array::make_array(data); let list_arr = GenericListArray::::try_new( - Arc::new(Field::new("item", value_type.clone(), true)), + Arc::new(Field::new_list_field(value_type.clone(), true)), OffsetBuffer::::from_lengths(vec![original_data.len(); count]), repeated_array, None, @@ -272,7 +271,7 @@ fn general_list_repeat( let values = compute::concat(&new_values)?; Ok(Arc::new(ListArray::try_new( - Arc::new(Field::new("item", data_type.to_owned(), true)), + Arc::new(Field::new_list_field(data_type.to_owned(), true)), OffsetBuffer::::from_lengths(lengths), values, None, diff --git a/datafusion/functions-nested/src/replace.rs b/datafusion/functions-nested/src/replace.rs index 01811b77734da..e971d97dbf2b9 100644 --- a/datafusion/functions-nested/src/replace.rs +++ b/datafusion/functions-nested/src/replace.rs @@ -414,7 +414,7 @@ fn general_replace( let data = mutable.freeze(); Ok(Arc::new(GenericListArray::::try_new( - Arc::new(Field::new("item", list_array.value_type(), true)), + Arc::new(Field::new_list_field(list_array.value_type(), true)), OffsetBuffer::::new(offsets.into()), arrow_array::make_array(data), Some(NullBuffer::new(valid.finish())), diff --git a/datafusion/functions-nested/src/set_ops.rs b/datafusion/functions-nested/src/set_ops.rs index 889eaed4edcc6..642fd9ad54cd3 100644 --- a/datafusion/functions-nested/src/set_ops.rs +++ b/datafusion/functions-nested/src/set_ops.rs @@ -270,13 +270,10 @@ impl ScalarUDFImpl for ArrayDistinct { fn return_type(&self, arg_types: &[DataType]) -> Result { match &arg_types[0] { - List(field) | FixedSizeList(field, _) => Ok(List(Arc::new(Field::new( - "item", - field.data_type().clone(), - true, - )))), - LargeList(field) => Ok(LargeList(Arc::new(Field::new( - "item", + List(field) | FixedSizeList(field, _) => Ok(List(Arc::new( + Field::new_list_field(field.data_type().clone(), true), + ))), + LargeList(field) => Ok(LargeList(Arc::new(Field::new_list_field( field.data_type().clone(), true, )))), @@ -376,10 +373,10 @@ fn generic_set_lists( set_op: SetOp, ) -> Result { if matches!(l.value_type(), Null) { - let field = Arc::new(Field::new("item", r.value_type(), true)); + let field = Arc::new(Field::new_list_field(r.value_type(), true)); return general_array_distinct::(r, &field); } else if matches!(r.value_type(), Null) { - let field = Arc::new(Field::new("item", l.value_type(), true)); + let field = Arc::new(Field::new_list_field(l.value_type(), true)); return general_array_distinct::(l, &field); } diff --git a/datafusion/functions-nested/src/sort.rs b/datafusion/functions-nested/src/sort.rs index cdd6842acc0ee..043fedd89bf8a 100644 --- a/datafusion/functions-nested/src/sort.rs +++ b/datafusion/functions-nested/src/sort.rs @@ -70,13 +70,10 @@ impl ScalarUDFImpl for ArraySort { fn return_type(&self, arg_types: &[DataType]) -> Result { match &arg_types[0] { - List(field) | FixedSizeList(field, _) => Ok(List(Arc::new(Field::new( - "item", - field.data_type().clone(), - true, - )))), - LargeList(field) => Ok(LargeList(Arc::new(Field::new( - "item", + List(field) | FixedSizeList(field, _) => Ok(List(Arc::new( + Field::new_list_field(field.data_type().clone(), true), + ))), + LargeList(field) => Ok(LargeList(Arc::new(Field::new_list_field( field.data_type().clone(), true, )))), @@ -198,7 +195,7 @@ pub fn array_sort_inner(args: &[ArrayRef]) -> Result { .collect::>(); let list_arr = ListArray::new( - Arc::new(Field::new("item", data_type, true)), + Arc::new(Field::new_list_field(data_type, true)), OffsetBuffer::from_lengths(array_lengths), Arc::new(compute::concat(elements.as_slice())?), Some(NullBuffer::new(buffer)), diff --git a/datafusion/functions-nested/src/string.rs b/datafusion/functions-nested/src/string.rs index 8c6cb73e97c9b..143a3d06a32a2 100644 --- a/datafusion/functions-nested/src/string.rs +++ b/datafusion/functions-nested/src/string.rs @@ -252,7 +252,7 @@ impl ScalarUDFImpl for StringToArray { fn return_type(&self, arg_types: &[DataType]) -> Result { Ok(match arg_types[0] { Utf8 | Utf8View | LargeUtf8 => { - List(Arc::new(Field::new("item", arg_types[0].clone(), true))) + List(Arc::new(Field::new_list_field(arg_types[0].clone(), true))) } _ => { return plan_err!( diff --git a/datafusion/functions-nested/src/utils.rs b/datafusion/functions-nested/src/utils.rs index b9a75724bcdee..5e67cb4228ff6 100644 --- a/datafusion/functions-nested/src/utils.rs +++ b/datafusion/functions-nested/src/utils.rs @@ -114,7 +114,7 @@ pub(crate) fn align_array_dimensions( let offsets = OffsetBuffer::::from_lengths(array_lengths); aligned_array = Arc::new(GenericListArray::::try_new( - Arc::new(Field::new("item", data_type, true)), + Arc::new(Field::new_list_field(data_type, true)), offsets, aligned_array, None, diff --git a/datafusion/functions/src/regex/regexpmatch.rs b/datafusion/functions/src/regex/regexpmatch.rs index df4f294bb9507..93178d23de4f0 100644 --- a/datafusion/functions/src/regex/regexpmatch.rs +++ b/datafusion/functions/src/regex/regexpmatch.rs @@ -80,7 +80,7 @@ impl ScalarUDFImpl for RegexpMatchFunc { fn return_type(&self, arg_types: &[DataType]) -> Result { Ok(match &arg_types[0] { DataType::Null => DataType::Null, - other => DataType::List(Arc::new(Field::new("item", other.clone(), true))), + other => DataType::List(Arc::new(Field::new_list_field(other.clone(), true))), }) } fn invoke_batch( diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs index 58e390ee8bdba..71abfda5fc2ed 100644 --- a/datafusion/optimizer/src/analyzer/type_coercion.rs +++ b/datafusion/optimizer/src/analyzer/type_coercion.rs @@ -1847,7 +1847,7 @@ mod test { #[test] fn tes_case_when_list() -> Result<()> { - let inner_field = Arc::new(Field::new("item", DataType::Int64, true)); + let inner_field = Arc::new(Field::new_list_field(DataType::Int64, true)); let schema = Arc::new(DFSchema::from_unqualified_fields( vec![ Field::new( @@ -1869,7 +1869,7 @@ mod test { test_case_expression!( Some("list"), vec![(Box::new(col("large_list")), Box::new(lit("1")))], - DataType::LargeList(Arc::new(Field::new("item", DataType::Int64, true))), + DataType::LargeList(Arc::new(Field::new_list_field(DataType::Int64, true))), Utf8, schema ); @@ -1877,7 +1877,7 @@ mod test { test_case_expression!( Some("large_list"), vec![(Box::new(col("list")), Box::new(lit("1")))], - DataType::LargeList(Arc::new(Field::new("item", DataType::Int64, true))), + DataType::LargeList(Arc::new(Field::new_list_field(DataType::Int64, true))), Utf8, schema ); @@ -1885,7 +1885,7 @@ mod test { test_case_expression!( Some("list"), vec![(Box::new(col("fixed_list")), Box::new(lit("1")))], - DataType::List(Arc::new(Field::new("item", DataType::Int64, true))), + DataType::List(Arc::new(Field::new_list_field(DataType::Int64, true))), Utf8, schema ); @@ -1893,7 +1893,7 @@ mod test { test_case_expression!( Some("fixed_list"), vec![(Box::new(col("list")), Box::new(lit("1")))], - DataType::List(Arc::new(Field::new("item", DataType::Int64, true))), + DataType::List(Arc::new(Field::new_list_field(DataType::Int64, true))), Utf8, schema ); @@ -1901,7 +1901,7 @@ mod test { test_case_expression!( Some("fixed_list"), vec![(Box::new(col("large_list")), Box::new(lit("1")))], - DataType::LargeList(Arc::new(Field::new("item", DataType::Int64, true))), + DataType::LargeList(Arc::new(Field::new_list_field(DataType::Int64, true))), Utf8, schema ); @@ -1909,7 +1909,7 @@ mod test { test_case_expression!( Some("large_list"), vec![(Box::new(col("fixed_list")), Box::new(lit("1")))], - DataType::LargeList(Arc::new(Field::new("item", DataType::Int64, true))), + DataType::LargeList(Arc::new(Field::new_list_field(DataType::Int64, true))), Utf8, schema ); @@ -1918,7 +1918,7 @@ mod test { #[test] fn test_then_else_list() -> Result<()> { - let inner_field = Arc::new(Field::new("item", DataType::Int64, true)); + let inner_field = Arc::new(Field::new_list_field(DataType::Int64, true)); let schema = Arc::new(DFSchema::from_unqualified_fields( vec![ Field::new("boolean", DataType::Boolean, true), @@ -1946,7 +1946,7 @@ mod test { (Box::new(col("boolean")), Box::new(col("list"))) ], DataType::Boolean, - DataType::LargeList(Arc::new(Field::new("item", DataType::Int64, true))), + DataType::LargeList(Arc::new(Field::new_list_field(DataType::Int64, true))), schema ); @@ -1957,7 +1957,7 @@ mod test { (Box::new(col("boolean")), Box::new(col("large_list"))) ], DataType::Boolean, - DataType::LargeList(Arc::new(Field::new("item", DataType::Int64, true))), + DataType::LargeList(Arc::new(Field::new_list_field(DataType::Int64, true))), schema ); @@ -1969,7 +1969,7 @@ mod test { (Box::new(col("boolean")), Box::new(col("list"))) ], DataType::Boolean, - DataType::List(Arc::new(Field::new("item", DataType::Int64, true))), + DataType::List(Arc::new(Field::new_list_field(DataType::Int64, true))), schema ); @@ -1980,7 +1980,7 @@ mod test { (Box::new(col("boolean")), Box::new(col("fixed_list"))) ], DataType::Boolean, - DataType::List(Arc::new(Field::new("item", DataType::Int64, true))), + DataType::List(Arc::new(Field::new_list_field(DataType::Int64, true))), schema ); @@ -1992,7 +1992,7 @@ mod test { (Box::new(col("boolean")), Box::new(col("large_list"))) ], DataType::Boolean, - DataType::LargeList(Arc::new(Field::new("item", DataType::Int64, true))), + DataType::LargeList(Arc::new(Field::new_list_field(DataType::Int64, true))), schema ); @@ -2003,7 +2003,7 @@ mod test { (Box::new(col("boolean")), Box::new(col("fixed_list"))) ], DataType::Boolean, - DataType::LargeList(Arc::new(Field::new("item", DataType::Int64, true))), + DataType::LargeList(Arc::new(Field::new_list_field(DataType::Int64, true))), schema ); Ok(()) diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs index 89041eb0f04ec..aac186a6f65bb 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs @@ -1309,7 +1309,7 @@ mod tests { // 6. Only decrease group indices in non-inlined group index view // 7. Erase all things - let field = Field::new("item", DataType::Int32, true); + let field = Field::new_list_field(DataType::Int32, true); let schema = Arc::new(Schema::new_with_metadata(vec![field], HashMap::new())); let mut group_values = GroupValuesColumn::::try_new(schema).unwrap(); diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index 2e97334493dd6..c2d8b093a9234 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -1344,7 +1344,7 @@ mod tests { Field::new("a", DataType::Int32, true), Field::new( "b", - DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))), true, ), ])); @@ -1389,7 +1389,7 @@ mod tests { assert_eq!(DataType::Int32, *sort_exec.schema().field(0).data_type()); assert_eq!( - DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))), *sort_exec.schema().field(1).data_type() ); diff --git a/datafusion/physical-plan/src/spill.rs b/datafusion/physical-plan/src/spill.rs index dee2d09862d36..f20adb0d2fab9 100644 --- a/datafusion/physical-plan/src/spill.rs +++ b/datafusion/physical-plan/src/spill.rs @@ -357,12 +357,12 @@ mod tests { let schema = Arc::new(Schema::new(vec![ Field::new( "nested_int", - DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))), false, ), Field::new( "nested_int2", - DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))), false, ), ])); diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rs index 0615e6738a1fc..9f03385f09d1d 100644 --- a/datafusion/physical-plan/src/unnest.rs +++ b/datafusion/physical-plan/src/unnest.rs @@ -991,7 +991,7 @@ mod tests { offsets.push(OffsetSize::from_usize(values.len()).unwrap()); valid.append(true); - let field = Arc::new(Field::new("item", DataType::Utf8, true)); + let field = Arc::new(Field::new_list_field(DataType::Utf8, true)); GenericListArray::::new( field, OffsetBuffer::new(offsets.into()), @@ -1017,7 +1017,7 @@ mod tests { None, None, ])); - let field = Arc::new(Field::new("item", DataType::Utf8, true)); + let field = Arc::new(Field::new_list_field(DataType::Utf8, true)); let valid = NullBuffer::from(vec![true, false, true, false, true, true]); FixedSizeListArray::new(field, 2, values, Some(valid)) } @@ -1100,7 +1100,7 @@ mod tests { let out_schema = Arc::new(Schema::new(vec![ Field::new( "col1_unnest_placeholder_depth_1", - DataType::List(Arc::new(Field::new("item", DataType::Int32, true))), + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))), true, ), Field::new("col1_unnest_placeholder_depth_2", DataType::Int32, true), diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs index cfb6862a0ca3e..8c150b20dd009 100644 --- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs @@ -787,7 +787,7 @@ async fn roundtrip_logical_plan_unnest() -> Result<()> { Field::new("a", DataType::Int64, true), Field::new( "b", - DataType::List(Arc::new(Field::new("item", DataType::Int32, false))), + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, false))), true, ), ]); @@ -1611,7 +1611,7 @@ fn round_trip_scalar_types() { ]; for test_case in should_pass.into_iter() { - let field = Field::new("item", test_case, true); + let field = Field::new_list_field(test_case, true); let proto: protobuf::Field = (&field).try_into().unwrap(); let roundtrip: Field = (&proto).try_into().unwrap(); assert_eq!(format!("{field:?}"), format!("{roundtrip:?}")); diff --git a/datafusion/sql/src/utils.rs b/datafusion/sql/src/utils.rs index 2b3ee8be3ccd4..4e99b362ec795 100644 --- a/datafusion/sql/src/utils.rs +++ b/datafusion/sql/src/utils.rs @@ -763,8 +763,7 @@ mod tests { ), Field::new( "array_col", - ArrowDataType::List(Arc::new(Field::new( - "item", + ArrowDataType::List(Arc::new(Field::new_list_field( ArrowDataType::Int64, true, ))), diff --git a/datafusion/sql/tests/common/mod.rs b/datafusion/sql/tests/common/mod.rs index 63c296dfbc2f9..337fc2ce4f678 100644 --- a/datafusion/sql/tests/common/mod.rs +++ b/datafusion/sql/tests/common/mod.rs @@ -162,12 +162,18 @@ impl ContextProvider for MockContextProvider { "array" => Ok(Schema::new(vec![ Field::new( "left", - DataType::List(Arc::new(Field::new("item", DataType::Int64, true))), + DataType::List(Arc::new(Field::new_list_field( + DataType::Int64, + true, + ))), false, ), Field::new( "right", - DataType::List(Arc::new(Field::new("item", DataType::Int64, true))), + DataType::List(Arc::new(Field::new_list_field( + DataType::Int64, + true, + ))), false, ), ])), @@ -198,7 +204,10 @@ impl ContextProvider for MockContextProvider { "unnest_table" => Ok(Schema::new(vec![ Field::new( "array_col", - DataType::List(Arc::new(Field::new("item", DataType::Int64, true))), + DataType::List(Arc::new(Field::new_list_field( + DataType::Int64, + true, + ))), false, ), Field::new( diff --git a/datafusion/substrait/src/logical_plan/producer.rs b/datafusion/substrait/src/logical_plan/producer.rs index c432a144127c7..26d71c7fd3e24 100644 --- a/datafusion/substrait/src/logical_plan/producer.rs +++ b/datafusion/substrait/src/logical_plan/producer.rs @@ -2430,7 +2430,7 @@ mod test { "struct", DataType::Struct(Fields::from(vec![Field::new( "inner", - DataType::List(Arc::new(Field::new("item", DataType::Utf8, true))), + DataType::List(Arc::new(Field::new_list_field(DataType::Utf8, true))), true, )])), true, diff --git a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs index 0141e6a08647b..f836dea03c61a 100644 --- a/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs +++ b/datafusion/substrait/tests/cases/roundtrip_logical_plan.rs @@ -1453,10 +1453,14 @@ async fn create_all_type_context() -> Result { Field::new("utf8_col", DataType::Utf8, true), Field::new("large_utf8_col", DataType::LargeUtf8, true), Field::new("view_utf8_col", DataType::Utf8View, true), - Field::new_list("list_col", Field::new("item", DataType::Int64, true), true), + Field::new_list( + "list_col", + Field::new_list_field(DataType::Int64, true), + true, + ), Field::new_list( "large_list_col", - Field::new("item", DataType::Int64, true), + Field::new_list_field(DataType::Int64, true), true, ), Field::new("decimal_128_col", DataType::Decimal128(10, 2), true),