From c3a60bc3689fab35370dbe34d7140c695ac67e29 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 10 Oct 2025 12:32:38 -0600 Subject: [PATCH 1/8] fix --- .../spark/src/function/datetime/date_add.rs | 15 ++++++++---- .../spark/src/function/datetime/date_sub.rs | 22 +++++++++++++----- .../test_files/spark/datetime/date_add.slt | 23 +++++++++++++++++++ 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/datafusion/spark/src/function/datetime/date_add.rs b/datafusion/spark/src/function/datetime/date_add.rs index 58633e15f9628..06a250720261b 100644 --- a/datafusion/spark/src/function/datetime/date_add.rs +++ b/datafusion/spark/src/function/datetime/date_add.rs @@ -21,6 +21,7 @@ use std::sync::Arc; use arrow::array::ArrayRef; use arrow::compute; use arrow::datatypes::{DataType, Date32Type}; +use arrow::error::ArrowError; use datafusion_common::cast::{ as_date32_array, as_int16_array, as_int32_array, as_int8_array, }; @@ -96,10 +97,13 @@ fn spark_date_add(args: &[ArrayRef]) -> Result { let result = match days_arg.data_type() { DataType::Int8 => { let days_array = as_int8_array(days_arg)?; - compute::binary::<_, _, _, Date32Type>( + compute::try_binary::<_, _, _, Date32Type>( date_array, days_array, - |date, days| date + days as i32, + |date, days| { + date.checked_add(days as i32) + .ok_or_else(|| ArrowError::ArithmeticOverflow("".to_string())) + }, )? } DataType::Int16 => { @@ -112,10 +116,13 @@ fn spark_date_add(args: &[ArrayRef]) -> Result { } DataType::Int32 => { let days_array = as_int32_array(days_arg)?; - compute::binary::<_, _, _, Date32Type>( + compute::try_binary::<_, _, _, Date32Type>( date_array, days_array, - |date, days| date + days, + |date, days| { + date.checked_add(days) + .ok_or_else(|| ArrowError::ArithmeticOverflow("".to_string())) + }, )? } _ => { diff --git a/datafusion/spark/src/function/datetime/date_sub.rs b/datafusion/spark/src/function/datetime/date_sub.rs index c19d04e617a41..fe77c7c9aba89 100644 --- a/datafusion/spark/src/function/datetime/date_sub.rs +++ b/datafusion/spark/src/function/datetime/date_sub.rs @@ -21,6 +21,7 @@ use std::sync::Arc; use arrow::array::ArrayRef; use arrow::compute; use arrow::datatypes::{DataType, Date32Type}; +use arrow::error::ArrowError; use datafusion_common::cast::{ as_date32_array, as_int16_array, as_int32_array, as_int8_array, }; @@ -90,26 +91,35 @@ fn spark_date_sub(args: &[ArrayRef]) -> Result { let result = match days_arg.data_type() { DataType::Int8 => { let days_array = as_int8_array(days_arg)?; - compute::binary::<_, _, _, Date32Type>( + compute::try_binary::<_, _, _, Date32Type>( date_array, days_array, - |date, days| date - days as i32, + |date, days| { + date.checked_sub(days as i32) + .ok_or_else(|| ArrowError::ArithmeticOverflow("".to_string())) + }, )? } DataType::Int16 => { let days_array = as_int16_array(days_arg)?; - compute::binary::<_, _, _, Date32Type>( + compute::try_binary::<_, _, _, Date32Type>( date_array, days_array, - |date, days| date - days as i32, + |date, days| { + date.checked_sub(days as i32) + .ok_or_else(|| ArrowError::ArithmeticOverflow("".to_string())) + }, )? } DataType::Int32 => { let days_array = as_int32_array(days_arg)?; - compute::binary::<_, _, _, Date32Type>( + compute::try_binary::<_, _, _, Date32Type>( date_array, days_array, - |date, days| date - days, + |date, days| { + date.checked_sub(days) + .ok_or_else(|| ArrowError::ArithmeticOverflow("".to_string())) + }, )? } _ => { diff --git a/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt b/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt index 146f97016638d..0653522280c12 100644 --- a/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt +++ b/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt @@ -45,6 +45,29 @@ SELECT date_sub('2016-07-30'::date, 0::int); ---- 2016-07-30 +query error DataFusion error: Arrow error: Arithmetic overflow: +SELECT date_add('2016-07-30'::date, 2147483647::int); + +query D +SELECT date_sub('2016-07-30'::date, 2147483647::int); +---- +ERROR: Cast error: Failed to convert -2147466635 to temporal for Date32 + +query D +SELECT date_add('2016-07-30'::date, 100000::int); +---- +2290-05-15 + +query D +SELECT date_add('2016-07-30'::date, 100000::int); +---- +2290-05-15 + +query D +SELECT date_sub('2016-07-30'::date, 100000::int); +---- +1742-10-15 + # Test with negative day values (should subtract days) query D SELECT date_add('2016-07-30'::date, -5::int); From 08713864163f6201ae2bc4a038fc9c53d6ccced2 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 10 Oct 2025 12:34:59 -0600 Subject: [PATCH 2/8] fix --- datafusion/spark/src/function/datetime/date_add.rs | 11 +++++++---- datafusion/spark/src/function/datetime/date_sub.rs | 6 +++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/datafusion/spark/src/function/datetime/date_add.rs b/datafusion/spark/src/function/datetime/date_add.rs index 06a250720261b..c43306b0cc70e 100644 --- a/datafusion/spark/src/function/datetime/date_add.rs +++ b/datafusion/spark/src/function/datetime/date_add.rs @@ -102,16 +102,19 @@ fn spark_date_add(args: &[ArrayRef]) -> Result { days_array, |date, days| { date.checked_add(days as i32) - .ok_or_else(|| ArrowError::ArithmeticOverflow("".to_string())) + .ok_or_else(|| ArrowError::ArithmeticOverflow("date_add".to_string())) }, )? } DataType::Int16 => { let days_array = as_int16_array(days_arg)?; - compute::binary::<_, _, _, Date32Type>( + compute::ret_binary::<_, _, _, Date32Type>( date_array, days_array, - |date, days| date + days as i32, + |date, days| { + date.checked_add(days as i32) + .ok_or_else(|| ArrowError::ArithmeticOverflow("date_add".to_string())) + }, )? } DataType::Int32 => { @@ -121,7 +124,7 @@ fn spark_date_add(args: &[ArrayRef]) -> Result { days_array, |date, days| { date.checked_add(days) - .ok_or_else(|| ArrowError::ArithmeticOverflow("".to_string())) + .ok_or_else(|| ArrowError::ArithmeticOverflow("date_add".to_string())) }, )? } diff --git a/datafusion/spark/src/function/datetime/date_sub.rs b/datafusion/spark/src/function/datetime/date_sub.rs index fe77c7c9aba89..fec5b56d836b0 100644 --- a/datafusion/spark/src/function/datetime/date_sub.rs +++ b/datafusion/spark/src/function/datetime/date_sub.rs @@ -96,7 +96,7 @@ fn spark_date_sub(args: &[ArrayRef]) -> Result { days_array, |date, days| { date.checked_sub(days as i32) - .ok_or_else(|| ArrowError::ArithmeticOverflow("".to_string())) + .ok_or_else(|| ArrowError::ArithmeticOverflow("date_sub".to_string())) }, )? } @@ -107,7 +107,7 @@ fn spark_date_sub(args: &[ArrayRef]) -> Result { days_array, |date, days| { date.checked_sub(days as i32) - .ok_or_else(|| ArrowError::ArithmeticOverflow("".to_string())) + .ok_or_else(|| ArrowError::ArithmeticOverflow("date_sub".to_string())) }, )? } @@ -118,7 +118,7 @@ fn spark_date_sub(args: &[ArrayRef]) -> Result { days_array, |date, days| { date.checked_sub(days) - .ok_or_else(|| ArrowError::ArithmeticOverflow("".to_string())) + .ok_or_else(|| ArrowError::ArithmeticOverflow("date_sub".to_string())) }, )? } From fd1807d8f7e1d379aed55803e3f84ff7bb3f5ed8 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 10 Oct 2025 12:35:05 -0600 Subject: [PATCH 3/8] fix --- .../spark/src/function/datetime/date_add.rs | 15 +++++++++------ .../spark/src/function/datetime/date_sub.rs | 15 +++++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/datafusion/spark/src/function/datetime/date_add.rs b/datafusion/spark/src/function/datetime/date_add.rs index c43306b0cc70e..98a1b2e0727bd 100644 --- a/datafusion/spark/src/function/datetime/date_add.rs +++ b/datafusion/spark/src/function/datetime/date_add.rs @@ -101,8 +101,9 @@ fn spark_date_add(args: &[ArrayRef]) -> Result { date_array, days_array, |date, days| { - date.checked_add(days as i32) - .ok_or_else(|| ArrowError::ArithmeticOverflow("date_add".to_string())) + date.checked_add(days as i32).ok_or_else(|| { + ArrowError::ArithmeticOverflow("date_add".to_string()) + }) }, )? } @@ -112,8 +113,9 @@ fn spark_date_add(args: &[ArrayRef]) -> Result { date_array, days_array, |date, days| { - date.checked_add(days as i32) - .ok_or_else(|| ArrowError::ArithmeticOverflow("date_add".to_string())) + date.checked_add(days as i32).ok_or_else(|| { + ArrowError::ArithmeticOverflow("date_add".to_string()) + }) }, )? } @@ -123,8 +125,9 @@ fn spark_date_add(args: &[ArrayRef]) -> Result { date_array, days_array, |date, days| { - date.checked_add(days) - .ok_or_else(|| ArrowError::ArithmeticOverflow("date_add".to_string())) + date.checked_add(days).ok_or_else(|| { + ArrowError::ArithmeticOverflow("date_add".to_string()) + }) }, )? } diff --git a/datafusion/spark/src/function/datetime/date_sub.rs b/datafusion/spark/src/function/datetime/date_sub.rs index fec5b56d836b0..a3b26661d196c 100644 --- a/datafusion/spark/src/function/datetime/date_sub.rs +++ b/datafusion/spark/src/function/datetime/date_sub.rs @@ -95,8 +95,9 @@ fn spark_date_sub(args: &[ArrayRef]) -> Result { date_array, days_array, |date, days| { - date.checked_sub(days as i32) - .ok_or_else(|| ArrowError::ArithmeticOverflow("date_sub".to_string())) + date.checked_sub(days as i32).ok_or_else(|| { + ArrowError::ArithmeticOverflow("date_sub".to_string()) + }) }, )? } @@ -106,8 +107,9 @@ fn spark_date_sub(args: &[ArrayRef]) -> Result { date_array, days_array, |date, days| { - date.checked_sub(days as i32) - .ok_or_else(|| ArrowError::ArithmeticOverflow("date_sub".to_string())) + date.checked_sub(days as i32).ok_or_else(|| { + ArrowError::ArithmeticOverflow("date_sub".to_string()) + }) }, )? } @@ -117,8 +119,9 @@ fn spark_date_sub(args: &[ArrayRef]) -> Result { date_array, days_array, |date, days| { - date.checked_sub(days) - .ok_or_else(|| ArrowError::ArithmeticOverflow("date_sub".to_string())) + date.checked_sub(days).ok_or_else(|| { + ArrowError::ArithmeticOverflow("date_sub".to_string()) + }) }, )? } From 93fe20b27da8c927f152111e30c231e3197776e3 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 10 Oct 2025 12:36:21 -0600 Subject: [PATCH 4/8] fix --- datafusion/spark/src/function/datetime/date_add.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/spark/src/function/datetime/date_add.rs b/datafusion/spark/src/function/datetime/date_add.rs index 98a1b2e0727bd..a00430febcdb0 100644 --- a/datafusion/spark/src/function/datetime/date_add.rs +++ b/datafusion/spark/src/function/datetime/date_add.rs @@ -109,7 +109,7 @@ fn spark_date_add(args: &[ArrayRef]) -> Result { } DataType::Int16 => { let days_array = as_int16_array(days_arg)?; - compute::ret_binary::<_, _, _, Date32Type>( + compute::try_binary::<_, _, _, Date32Type>( date_array, days_array, |date, days| { From 57a6c12522476f398e5595a109338cdcde4fff52 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 10 Oct 2025 12:36:58 -0600 Subject: [PATCH 5/8] fix --- .../sqllogictest/test_files/spark/datetime/date_add.slt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt b/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt index 0653522280c12..5a5582b87d04d 100644 --- a/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt +++ b/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt @@ -58,11 +58,6 @@ SELECT date_add('2016-07-30'::date, 100000::int); ---- 2290-05-15 -query D -SELECT date_add('2016-07-30'::date, 100000::int); ----- -2290-05-15 - query D SELECT date_sub('2016-07-30'::date, 100000::int); ---- From dbba50c1c1ff5918552412a721c526f0dbc2643a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 10 Oct 2025 13:57:01 -0600 Subject: [PATCH 6/8] add negative overflow test --- datafusion/sqllogictest/test_files/spark/datetime/date_add.slt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt b/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt index 5a5582b87d04d..d4d202bfd00ca 100644 --- a/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt +++ b/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt @@ -53,6 +53,9 @@ SELECT date_sub('2016-07-30'::date, 2147483647::int); ---- ERROR: Cast error: Failed to convert -2147466635 to temporal for Date32 +query error DataFusion error: Arrow error: Arithmetic overflow: date_sub +SELECT date_sub('1969-01-01'::date, 2147483647::int); + query D SELECT date_add('2016-07-30'::date, 100000::int); ---- From f3a754a43a7253f95740654cdfd12baa8b02ba2e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 10 Oct 2025 13:58:24 -0600 Subject: [PATCH 7/8] remove unrelated test --- .../sqllogictest/test_files/spark/datetime/date_add.slt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt b/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt index d4d202bfd00ca..2b40fed0c270e 100644 --- a/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt +++ b/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt @@ -48,11 +48,6 @@ SELECT date_sub('2016-07-30'::date, 0::int); query error DataFusion error: Arrow error: Arithmetic overflow: SELECT date_add('2016-07-30'::date, 2147483647::int); -query D -SELECT date_sub('2016-07-30'::date, 2147483647::int); ----- -ERROR: Cast error: Failed to convert -2147466635 to temporal for Date32 - query error DataFusion error: Arrow error: Arithmetic overflow: date_sub SELECT date_sub('1969-01-01'::date, 2147483647::int); From 51f896d672774a8661e5f187a981a825356c484d Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 10 Oct 2025 14:00:48 -0600 Subject: [PATCH 8/8] update test --- datafusion/sqllogictest/test_files/spark/datetime/date_add.slt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt b/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt index 2b40fed0c270e..2e9851ca1e595 100644 --- a/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt +++ b/datafusion/sqllogictest/test_files/spark/datetime/date_add.slt @@ -45,7 +45,7 @@ SELECT date_sub('2016-07-30'::date, 0::int); ---- 2016-07-30 -query error DataFusion error: Arrow error: Arithmetic overflow: +query error DataFusion error: Arrow error: Arithmetic overflow: date_add SELECT date_add('2016-07-30'::date, 2147483647::int); query error DataFusion error: Arrow error: Arithmetic overflow: date_sub