From 124c1662b4ad5a65c507786067c68f08543febd9 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 24 Apr 2026 04:22:49 -0600 Subject: [PATCH 1/2] chore: bump DataFusion rev to 85e75e2 --- native/Cargo.lock | 65 +++++++++++++++++++++--------------------- native/Cargo.toml | 8 +++--- native/core/Cargo.toml | 2 +- 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index fecf189630..30ebf48446 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -357,6 +357,7 @@ dependencies = [ "arrow-select", "flatbuffers", "lz4_flex", + "zstd", ] [[package]] @@ -1859,7 +1860,7 @@ dependencies = [ [[package]] name = "datafusion" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "arrow-schema", @@ -1906,7 +1907,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "async-trait", @@ -1930,7 +1931,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "async-trait", @@ -2123,7 +2124,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "arrow-ipc", @@ -2148,7 +2149,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "futures", "log", @@ -2158,7 +2159,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "async-compression", @@ -2193,7 +2194,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "arrow-ipc", @@ -2216,7 +2217,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "async-trait", @@ -2238,7 +2239,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "async-trait", @@ -2260,7 +2261,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "async-trait", @@ -2290,12 +2291,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" [[package]] name = "datafusion-execution" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "arrow-buffer", @@ -2317,7 +2318,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "arrow-schema", @@ -2338,7 +2339,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "datafusion-common", @@ -2349,7 +2350,7 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "arrow-buffer", @@ -2379,7 +2380,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "datafusion-common", @@ -2399,7 +2400,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "datafusion-common", @@ -2410,7 +2411,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "arrow-ord", @@ -2434,7 +2435,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "async-trait", @@ -2448,7 +2449,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "datafusion-common", @@ -2464,7 +2465,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2473,7 +2474,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "datafusion-doc", "quote", @@ -2483,7 +2484,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "chrono", @@ -2501,7 +2502,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "datafusion-common", @@ -2521,7 +2522,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "datafusion-common", @@ -2535,7 +2536,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "chrono", @@ -2550,7 +2551,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "datafusion-common", @@ -2567,7 +2568,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "arrow-data", @@ -2598,7 +2599,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "datafusion-common", @@ -2613,7 +2614,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "async-trait", "datafusion-common", @@ -2626,7 +2627,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "bigdecimal", @@ -2653,7 +2654,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "53.1.0" -source = "git+https://github.com/apache/datafusion.git?rev=067ba4b#067ba4b02e180ccd3f7e43f087698463d53a566e" +source = "git+https://github.com/apache/datafusion.git?rev=85e75e2#85e75e2a19d2039fa4fdb72252795b447aac6300" dependencies = [ "arrow", "bigdecimal", diff --git a/native/Cargo.toml b/native/Cargo.toml index 2e772160d4..3d9f06686e 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -38,10 +38,10 @@ arrow = { version = "58.1.0", features = ["prettyprint", "ffi", "chrono-tz"] } async-trait = { version = "0.1" } bytes = { version = "1.11.1" } parquet = { version = "58.1.0", default-features = false, features = ["experimental"] } -datafusion = { git = "https://github.com/apache/datafusion.git", rev = "067ba4b", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } -datafusion-datasource = { git = "https://github.com/apache/datafusion.git", rev = "067ba4b" } -datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion.git", rev = "067ba4b" } -datafusion-spark = { git = "https://github.com/apache/datafusion.git", rev = "067ba4b", features = ["core"] } +datafusion = { git = "https://github.com/apache/datafusion.git", rev = "85e75e2", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } +datafusion-datasource = { git = "https://github.com/apache/datafusion.git", rev = "85e75e2" } +datafusion-physical-expr-adapter = { git = "https://github.com/apache/datafusion.git", rev = "85e75e2" } +datafusion-spark = { git = "https://github.com/apache/datafusion.git", rev = "85e75e2", features = ["core"] } datafusion-comet-spark-expr = { path = "spark-expr" } datafusion-comet-common = { path = "common" } datafusion-comet-jni-bridge = { path = "jni-bridge" } diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index 4adab838ae..26c665050c 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -91,7 +91,7 @@ jni = { version = "0.22.4", features = ["invocation"] } lazy_static = "1.4" assertables = "9" hex = "0.4.3" -datafusion-functions-nested = { git = "https://github.com/apache/datafusion.git", rev = "067ba4b" } +datafusion-functions-nested = { git = "https://github.com/apache/datafusion.git", rev = "85e75e2" } [features] backtrace = ["datafusion/backtrace"] From d5d440ebfc98a47ced74e2304b6d6c63a9dd3982 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 24 Apr 2026 07:27:34 -0600 Subject: [PATCH 2/2] test: skip shuffle_partitioner_memory under miri (ZSTD FFI) --- native/shuffle/src/shuffle_writer.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/native/shuffle/src/shuffle_writer.rs b/native/shuffle/src/shuffle_writer.rs index 7fed1cf9ff..a89abbb1e3 100644 --- a/native/shuffle/src/shuffle_writer.rs +++ b/native/shuffle/src/shuffle_writer.rs @@ -345,6 +345,7 @@ mod test { } #[tokio::test] + #[cfg_attr(miri, ignore)] // miri can't call foreign function `ZSTD_createCCtx` async fn shuffle_partitioner_memory() { let batch = create_batch(900); assert_eq!(8316, batch.get_array_memory_size()); // Not stable across Arrow versions