diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index ac09bc7f833a7..51d36c5a90d6b 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -77,9 +77,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" +checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46" [[package]] name = "arrayref" @@ -323,9 +323,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d495b6dc0184693324491a5ac05f559acc97bf937ab31d7a1c33dd0016be6d2b" +checksum = "bb42b2197bf15ccb092b62c74515dbd8b86d0effd934795f6687c93b6e679a2c" dependencies = [ "bzip2", "flate2", @@ -347,7 +347,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -674,9 +674,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.3" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "base64-simd" @@ -766,9 +766,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "byteorder" @@ -831,9 +831,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.30" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defd4e7873dbddba6c7c91e199c7fcb946abc4a6a4ac3195400bcfb01b5de877" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1031,7 +1031,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f34ba9a9bcb8645379e9de8cb3ecfcf4d1c85ba66d90deb3259206fa5aa193b" dependencies = [ "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -1503,7 +1503,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -1938,9 +1938,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.147" +version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" [[package]] name = "libm" @@ -1950,9 +1950,9 @@ checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" [[package]] name = "libmimalloc-sys" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25d058a81af0d1c22d7a1c948576bee6d673f7af3c0f35564abd6c81122f513d" +checksum = "3979b5c37ece694f1f5e51e7ecc871fdb0f517ed04ee45f88d15d6d553cb9664" dependencies = [ "cc", "libc", @@ -1960,9 +1960,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" [[package]] name = "lock_api" @@ -2028,9 +2028,9 @@ checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" [[package]] name = "mimalloc" -version = "0.1.38" +version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "972e5f23f6716f62665760b0f4cbf592576a80c7b879ba9beaafc0e558894127" +checksum = "fa01922b5ea280a911e323e4d2fd24b7fe5cc4042e0d2cda3c40775cdc4bdc9c" dependencies = [ "libmimalloc-sys", ] @@ -2389,7 +2389,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -2479,9 +2479,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" dependencies = [ "unicode-ident", ] @@ -2702,9 +2702,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.11" +version = "0.38.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453" +checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" dependencies = [ "bitflags 2.4.0", "errno", @@ -2760,9 +2760,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.4" +version = "0.101.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" +checksum = "45a27e3b59326c16e23d30aeb7a36a24cc0d29e71d68ff611cdfb4a01d013bed" dependencies = [ "ring", "untrusted", @@ -2889,14 +2889,14 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", @@ -2987,9 +2987,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", "windows-sys", @@ -3078,7 +3078,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -3100,9 +3100,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.31" +version = "2.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398" +checksum = "9caece70c63bfba29ec2fed841a09851b14a235c60010fa4de58089b6c025668" dependencies = [ "proc-macro2", "quote", @@ -3160,7 +3160,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -3238,7 +3238,7 @@ dependencies = [ "num_cpus", "parking_lot", "pin-project-lite", - "socket2 0.5.3", + "socket2 0.5.4", "tokio-macros", "windows-sys", ] @@ -3251,7 +3251,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -3349,7 +3349,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", ] [[package]] @@ -3379,9 +3379,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" @@ -3391,9 +3391,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -3521,7 +3521,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", "wasm-bindgen-shared", ] @@ -3555,7 +3555,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.31", + "syn 2.0.33", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs index fa2175c223388..32343b98fa247 100644 --- a/datafusion/common/src/scalar.rs +++ b/datafusion/common/src/scalar.rs @@ -4965,7 +4965,8 @@ mod tests { .unwrap() .and_hms_nano_opt(hour, minute, second, nanosec) .unwrap() - .timestamp_nanos(), + .timestamp_nanos_opt() + .unwrap(), ), None, )) diff --git a/datafusion/core/src/physical_plan/metrics/value.rs b/datafusion/core/src/physical_plan/metrics/value.rs index 59b012f25a27d..899ceb60b49f7 100644 --- a/datafusion/core/src/physical_plan/metrics/value.rs +++ b/datafusion/core/src/physical_plan/metrics/value.rs @@ -430,11 +430,13 @@ impl MetricValue { Self::Time { time, .. } => time.value(), Self::StartTimestamp(timestamp) => timestamp .value() - .map(|ts| ts.timestamp_nanos() as usize) + .and_then(|ts| ts.timestamp_nanos_opt()) + .map(|nanos| nanos as usize) .unwrap_or(0), Self::EndTimestamp(timestamp) => timestamp .value() - .map(|ts| ts.timestamp_nanos() as usize) + .and_then(|ts| ts.timestamp_nanos_opt()) + .map(|nanos| nanos as usize) .unwrap_or(0), } } diff --git a/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs b/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs index c9f693a8a24e4..fbad5acb016fc 100644 --- a/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs +++ b/datafusion/core/src/physical_plan/sorts/sort_preserving_merge.rs @@ -283,7 +283,7 @@ mod tests { use crate::physical_plan::coalesce_partitions::CoalescePartitionsExec; use crate::physical_plan::expressions::col; use crate::physical_plan::memory::MemoryExec; - use crate::physical_plan::metrics::MetricValue; + use crate::physical_plan::metrics::{MetricValue, Timestamp}; use crate::physical_plan::sorts::sort::SortExec; use crate::physical_plan::stream::RecordBatchReceiverStream; use crate::physical_plan::{collect, common}; @@ -939,11 +939,11 @@ mod tests { metrics.iter().for_each(|m| match m.value() { MetricValue::StartTimestamp(ts) => { saw_start = true; - assert!(ts.value().unwrap().timestamp_nanos() > 0); + assert!(nanos_from_timestamp(ts) > 0); } MetricValue::EndTimestamp(ts) => { saw_end = true; - assert!(ts.value().unwrap().timestamp_nanos() > 0); + assert!(nanos_from_timestamp(ts) > 0); } _ => {} }); @@ -952,6 +952,10 @@ mod tests { assert!(saw_end); } + fn nanos_from_timestamp(ts: &Timestamp) -> i64 { + ts.value().unwrap().timestamp_nanos_opt().unwrap() + } + #[tokio::test] async fn test_drop_cancel() -> Result<()> { let task_ctx = Arc::new(TaskContext::default()); diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rs index 6f289e0c064bd..db7349851ba28 100644 --- a/datafusion/core/tests/parquet/mod.rs +++ b/datafusion/core/tests/parquet/mod.rs @@ -290,7 +290,8 @@ fn make_timestamp_batch(offset: Duration) -> RecordBatch { offset_nanos + t.parse::() .unwrap() - .timestamp_nanos() + .timestamp_nanos_opt() + .unwrap() }) }) .collect::>(); diff --git a/datafusion/core/tests/sql/explain_analyze.rs b/datafusion/core/tests/sql/explain_analyze.rs index f32ffc1642cda..06120c01ce864 100644 --- a/datafusion/core/tests/sql/explain_analyze.rs +++ b/datafusion/core/tests/sql/explain_analyze.rs @@ -18,6 +18,7 @@ use super::*; use datafusion::config::ConfigOptions; use datafusion::physical_plan::display::DisplayableExecutionPlan; +use datafusion::physical_plan::metrics::Timestamp; #[tokio::test] async fn explain_analyze_baseline_metrics() { @@ -142,11 +143,11 @@ async fn explain_analyze_baseline_metrics() { metrics.iter().for_each(|m| match m.value() { MetricValue::StartTimestamp(ts) => { saw_start = true; - assert!(ts.value().unwrap().timestamp_nanos() > 0); + assert!(nanos_from_timestamp(ts) > 0); } MetricValue::EndTimestamp(ts) => { saw_end = true; - assert!(ts.value().unwrap().timestamp_nanos() > 0); + assert!(nanos_from_timestamp(ts) > 0); } _ => {} }); @@ -161,7 +162,9 @@ async fn explain_analyze_baseline_metrics() { datafusion::physical_plan::accept(physical_plan.as_ref(), &mut TimeValidator {}) .unwrap(); } - +fn nanos_from_timestamp(ts: &Timestamp) -> i64 { + ts.value().unwrap().timestamp_nanos_opt().unwrap() +} #[tokio::test] async fn csv_explain_plans() { // This test verify the look of each plan in its full cycle plan creation diff --git a/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs b/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs index e6d66720ee1b4..59c1c5a1266d3 100644 --- a/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs +++ b/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs @@ -473,8 +473,8 @@ mod tests { let expected = format!( "Projection: TimestampNanosecond({}, Some(\"+00:00\")) AS now(), TimestampNanosecond({}, Some(\"+00:00\")) AS t2\ \n TableScan: test", - time.timestamp_nanos(), - time.timestamp_nanos() + time.timestamp_nanos_opt().unwrap(), + time.timestamp_nanos_opt().unwrap() ); assert_eq!(expected, actual); diff --git a/datafusion/physical-expr/src/datetime_expressions.rs b/datafusion/physical-expr/src/datetime_expressions.rs index bf90a7da9c0bd..63f7645fea5db 100644 --- a/datafusion/physical-expr/src/datetime_expressions.rs +++ b/datafusion/physical-expr/src/datetime_expressions.rs @@ -168,7 +168,7 @@ pub fn to_timestamp_seconds(args: &[ColumnarValue]) -> Result { pub fn make_now( now_ts: DateTime, ) -> impl Fn(&[ColumnarValue]) -> Result { - let now_ts = Some(now_ts.timestamp_nanos()); + let now_ts = now_ts.timestamp_nanos_opt(); move |_arg| { Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( now_ts, @@ -204,7 +204,7 @@ pub fn make_current_date( pub fn make_current_time( now_ts: DateTime, ) -> impl Fn(&[ColumnarValue]) -> Result { - let nano = Some(now_ts.timestamp_nanos() % 86400000000000); + let nano = now_ts.timestamp_nanos_opt().map(|ts| ts % 86400000000000); move |_arg| Ok(ColumnarValue::Scalar(ScalarValue::Time64Nanosecond(nano))) } @@ -271,8 +271,9 @@ fn date_trunc_coarse(granularity: &str, value: i64) -> Result { return exec_err!("Unsupported date_trunc granularity: {unsupported}"); } }; + let value = value.and_then(|value| value.timestamp_nanos_opt()); // `with_x(0)` are infallible because `0` are always a valid - Ok(value.unwrap().timestamp_nanos()) + Ok(value.unwrap()) } // truncates a single value with the given timeunit to the specified granularity @@ -459,7 +460,7 @@ fn date_bin_months_interval(stride_months: i64, source: i64, origin: i64) -> i64 }; } - bin_time.timestamp_nanos() + bin_time.timestamp_nanos_opt().unwrap() } fn to_utc_date_time(nanos: i64) -> DateTime { diff --git a/datafusion/proto/src/physical_plan/to_proto.rs b/datafusion/proto/src/physical_plan/to_proto.rs index 62221f9d754c2..a5b1300360fe7 100644 --- a/datafusion/proto/src/physical_plan/to_proto.rs +++ b/datafusion/proto/src/physical_plan/to_proto.rs @@ -422,10 +422,16 @@ impl TryFrom<&PartitionedFile> for protobuf::PartitionedFile { type Error = DataFusionError; fn try_from(pf: &PartitionedFile) -> Result { + let last_modified = pf.object_meta.last_modified; + let last_modified_ns = last_modified.timestamp_nanos_opt().ok_or_else(|| { + DataFusionError::Plan(format!( + "Invalid timestamp on PartitionedFile::ObjectMeta: {last_modified}" + )) + })? as u64; Ok(protobuf::PartitionedFile { path: pf.object_meta.location.as_ref().to_owned(), size: pf.object_meta.size as u64, - last_modified_ns: pf.object_meta.last_modified.timestamp_nanos() as u64, + last_modified_ns, partition_values: pf .partition_values .iter()