From 96c38836286eb6fe47a231cd90e494358eb0aaef Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Fri, 24 Mar 2023 18:05:38 +0000 Subject: [PATCH 1/5] Use consistent arrow version --- datafusion/common/Cargo.toml | 2 +- datafusion/core/tests/sql/set_variable.rs | 24 +++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 7d78ed70eb35e..54c9d395fde1b 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -41,7 +41,7 @@ pyarrow = ["pyo3", "arrow/pyarrow"] [dependencies] apache-avro = { version = "0.14", default-features = false, features = ["snappy"], optional = true } arrow = { workspace = true, default-features = false } -arrow-array = { version = "35.0.0", default-features = false, features = ["chrono-tz"] } +arrow-array = { version = "34.0.0", default-features = false, features = ["chrono-tz"] } chrono = { version = "0.4", default-features = false } cranelift-module = { version = "0.92.0", optional = true } num_cpus = "1.13.0" diff --git a/datafusion/core/tests/sql/set_variable.rs b/datafusion/core/tests/sql/set_variable.rs index b7161eb2b1622..53552d3288bab 100644 --- a/datafusion/core/tests/sql/set_variable.rs +++ b/datafusion/core/tests/sql/set_variable.rs @@ -414,7 +414,7 @@ async fn set_time_zone_bad_time_zone_format() { .await .unwrap(); let err = pretty_format_batches(&result).err().unwrap().to_string(); - assert_eq!(err, "Parser error: Invalid timezone \"+08:00:00\": Expected format [+-]XX:XX, [+-]XX, or [+-]XXXX"); + assert_eq!(err, "Parser error: Invalid timezone \"+08:00:00\": '+08:00:00' is not a valid timezone"); plan_and_collect(&ctx, "SET TIME ZONE = '08:00'") .await @@ -427,7 +427,10 @@ async fn set_time_zone_bad_time_zone_format() { .unwrap(); let err = pretty_format_batches(&result).err().unwrap().to_string(); - assert_eq!(err, "Parser error: Invalid timezone \"08:00\": only offset based timezones supported without chrono-tz feature"); + assert_eq!( + err, + "Parser error: Invalid timezone \"08:00\": '08:00' is not a valid timezone" + ); plan_and_collect(&ctx, "SET TIME ZONE = '08'") .await @@ -440,7 +443,10 @@ async fn set_time_zone_bad_time_zone_format() { .unwrap(); let err = pretty_format_batches(&result).err().unwrap().to_string(); - assert_eq!(err, "Parser error: Invalid timezone \"08\": only offset based timezones supported without chrono-tz feature"); + assert_eq!( + err, + "Parser error: Invalid timezone \"08\": '08' is not a valid timezone" + ); // we dont support named time zone yet plan_and_collect(&ctx, "SET TIME ZONE = 'Asia/Taipei'") @@ -453,8 +459,14 @@ async fn set_time_zone_bad_time_zone_format() { .await .unwrap(); - let err = pretty_format_batches(&result).err().unwrap().to_string(); - assert_eq!(err, "Parser error: Invalid timezone \"Asia/Taipei\": only offset based timezones supported without chrono-tz feature"); + let expected = vec![ + "+-----------------------------+", + "| Utf8(\"2000-01-01T00:00:00\") |", + "+-----------------------------+", + "| 2000-01-01T08:00:00+08:00 |", + "+-----------------------------+", + ]; + assert_batches_eq!(expected, &result); // this is invalid even after we support named time zone plan_and_collect(&ctx, "SET TIME ZONE = 'Asia/Taipei2'") @@ -467,5 +479,5 @@ async fn set_time_zone_bad_time_zone_format() { .await .unwrap(); let err = pretty_format_batches(&result).err().unwrap().to_string(); - assert_eq!(err, "Parser error: Invalid timezone \"Asia/Taipei2\": only offset based timezones supported without chrono-tz feature"); + assert_eq!(err, "Parser error: Invalid timezone \"Asia/Taipei2\": 'Asia/Taipei2' is not a valid timezone"); } From 911bf0987bd097d02b2f7bf21cf11af1ace63fd4 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Fri, 24 Mar 2023 18:17:55 +0000 Subject: [PATCH 2/5] Update test --- datafusion/core/tests/sql/set_variable.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/core/tests/sql/set_variable.rs b/datafusion/core/tests/sql/set_variable.rs index 53552d3288bab..09149f77cf8cf 100644 --- a/datafusion/core/tests/sql/set_variable.rs +++ b/datafusion/core/tests/sql/set_variable.rs @@ -414,7 +414,7 @@ async fn set_time_zone_bad_time_zone_format() { .await .unwrap(); let err = pretty_format_batches(&result).err().unwrap().to_string(); - assert_eq!(err, "Parser error: Invalid timezone \"+08:00:00\": '+08:00:00' is not a valid timezone"); + assert_eq!(err, "Parser error: Invalid timezone \"+08:00:00\": Expected format [+-]XX:XX, [+-]XX, or [+-]XXXX"); plan_and_collect(&ctx, "SET TIME ZONE = '08:00'") .await From cecf334e6e61f4ada9e4765fb5b41a2fccdc5adc Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Fri, 24 Mar 2023 18:21:53 +0000 Subject: [PATCH 3/5] Use workspace dependencies --- Cargo.toml | 6 ++++-- datafusion-examples/Cargo.toml | 4 ++-- datafusion/common/Cargo.toml | 6 +++--- datafusion/expr/Cargo.toml | 4 ++-- datafusion/jit/Cargo.toml | 4 ++-- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 643c88700e247..e53865a368817 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,8 +46,10 @@ rust-version = "1.64" [workspace.dependencies] arrow = { version = "34.0.0", features = ["prettyprint"] } -arrow-buffer = "34.0.0" -arrow-schema = "34.0.0" +arrow-flight = { version = "34.0.0", features = ["flight-sql-experimental"] } +arrow-buffer = { version = "34.0.0", default-features = false } +arrow-schema = { version = "34.0.0", default-features = false } +arrow-array = { version = "34.0.0", default-features = false, features = ["chrono-tz"] } parquet = { version = "34.0.0", features = ["arrow", "async"] } [profile.release] diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml index 6e5c62ce8287d..d459bb52fe433 100644 --- a/datafusion-examples/Cargo.toml +++ b/datafusion-examples/Cargo.toml @@ -18,7 +18,7 @@ [package] name = "datafusion-examples" description = "DataFusion usage examples" -keywords = [ "arrow", "query", "sql" ] +keywords = ["arrow", "query", "sql"] publish = false version = { workspace = true } edition = { workspace = true } @@ -36,7 +36,7 @@ required-features = ["datafusion/avro"] [dev-dependencies] arrow = { workspace = true } -arrow-flight = { version = "34.0.0", features = ["flight-sql-experimental"] } +arrow-flight = { workspace = true } arrow-schema = { workspace = true } async-trait = "0.1.41" dashmap = "5.4" diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 54c9d395fde1b..fabec52fc3b04 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -40,13 +40,13 @@ pyarrow = ["pyo3", "arrow/pyarrow"] [dependencies] apache-avro = { version = "0.14", default-features = false, features = ["snappy"], optional = true } -arrow = { workspace = true, default-features = false } -arrow-array = { version = "34.0.0", default-features = false, features = ["chrono-tz"] } +arrow = { workspace = true } +arrow-array = { workspace = true } chrono = { version = "0.4", default-features = false } cranelift-module = { version = "0.92.0", optional = true } num_cpus = "1.13.0" object_store = { version = "0.5.4", default-features = false, optional = true } -parquet = { workspace = true, default-features = false, optional = true } +parquet = { workspace = true, optional = true } pyo3 = { version = "0.18.0", optional = true } sqlparser = "0.32" diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml index eb0a3e7d5e5eb..586c90f02a8e3 100644 --- a/datafusion/expr/Cargo.toml +++ b/datafusion/expr/Cargo.toml @@ -18,7 +18,7 @@ [package] name = "datafusion-expr" description = "Logical plan and expression representation for DataFusion query engine" -keywords = [ "datafusion", "logical", "plan", "expressions" ] +keywords = ["datafusion", "logical", "plan", "expressions"] version = { workspace = true } edition = { workspace = true } readme = { workspace = true } @@ -36,7 +36,7 @@ path = "src/lib.rs" [dependencies] ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] } -arrow = { workspace = true, default-features = false } +arrow = { workspace = true } datafusion-common = { path = "../common", version = "21.0.0" } sqlparser = "0.32" diff --git a/datafusion/jit/Cargo.toml b/datafusion/jit/Cargo.toml index 2966baf3d4cc2..dd7bc3c6a6c01 100644 --- a/datafusion/jit/Cargo.toml +++ b/datafusion/jit/Cargo.toml @@ -18,7 +18,7 @@ [package] name = "datafusion-jit" description = "Just In Time (JIT) compilation support for DataFusion query engine" -keywords = [ "arrow", "query", "sql" ] +keywords = ["arrow", "query", "sql"] version = { workspace = true } edition = { workspace = true } readme = { workspace = true } @@ -36,7 +36,7 @@ path = "src/lib.rs" jit = [] [dependencies] -arrow = { workspace = true, default-features = false } +arrow = { workspace = true } cranelift = "0.89.0" cranelift-jit = "0.89.0" cranelift-module = "0.89.0" From 18bae196879095709e069e7a48c2acd1be8222e0 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Fri, 24 Mar 2023 18:32:20 +0000 Subject: [PATCH 4/5] Update comment --- datafusion/core/tests/sql/set_variable.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/core/tests/sql/set_variable.rs b/datafusion/core/tests/sql/set_variable.rs index 09149f77cf8cf..8e3511e768d0e 100644 --- a/datafusion/core/tests/sql/set_variable.rs +++ b/datafusion/core/tests/sql/set_variable.rs @@ -448,7 +448,7 @@ async fn set_time_zone_bad_time_zone_format() { "Parser error: Invalid timezone \"08\": '08' is not a valid timezone" ); - // we dont support named time zone yet + // we support named timezones plan_and_collect(&ctx, "SET TIME ZONE = 'Asia/Taipei'") .await .unwrap(); From ba1e3b607a2f76c01cea470851a96226c7251d35 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies Date: Fri, 24 Mar 2023 18:54:37 +0000 Subject: [PATCH 5/5] Update Cargo.lock --- datafusion-cli/Cargo.lock | 150 ++++++++++++++------------------------ 1 file changed, 53 insertions(+), 97 deletions(-) diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 6f07f8355c90e..6de8c98f391f5 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -74,16 +74,16 @@ checksum = "f410d3907b6b3647b9e7bca4551274b2e3d716aa940afb67b7287257401da921" dependencies = [ "ahash", "arrow-arith", - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", + "arrow-array", + "arrow-buffer", "arrow-cast", "arrow-csv", - "arrow-data 34.0.0", + "arrow-data", "arrow-ipc", "arrow-json", "arrow-ord", "arrow-row", - "arrow-schema 34.0.0", + "arrow-schema", "arrow-select", "arrow-string", "comfy-table", @@ -95,10 +95,10 @@ version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f87391cf46473c9bc53dab68cb8872c3a81d4dfd1703f1c8aa397dba9880a043" dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "half", "num", @@ -111,25 +111,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d35d5475e65c57cffba06d0022e3006b677515f99b54af33a7cd54f6cdd4a5b5" dependencies = [ "ahash", - "arrow-buffer 34.0.0", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", - "chrono", - "half", - "hashbrown 0.13.2", - "num", -] - -[[package]] -name = "arrow-array" -version = "35.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43489bbff475545b78b0e20bde1d22abd6c99e54499839f9e815a2fa5134a51b" -dependencies = [ - "ahash", - "arrow-buffer 35.0.0", - "arrow-data 35.0.0", - "arrow-schema 35.0.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "chrono-tz", "half", @@ -147,26 +131,16 @@ dependencies = [ "num", ] -[[package]] -name = "arrow-buffer" -version = "35.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3759e4a52c593281184787af5435671dc8b1e78333e5a30242b2e2d6e3c9d1f" -dependencies = [ - "half", - "num", -] - [[package]] name = "arrow-cast" version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a7285272c9897321dfdba59de29f5b05aeafd3cdedf104a941256d155f6d304" dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "arrow-select", "chrono", "lexical-core", @@ -179,11 +153,11 @@ version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "981ee4e7f6a120da04e00d0b39182e1eeacccb59c8da74511de753c56b7fddf7" dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", + "arrow-array", + "arrow-buffer", "arrow-cast", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-data", + "arrow-schema", "chrono", "csv", "csv-core", @@ -198,20 +172,8 @@ version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27cc673ee6989ea6e4b4e8c7d461f7e06026a096c8f0b1a7288885ff71ae1e56" dependencies = [ - "arrow-buffer 34.0.0", - "arrow-schema 34.0.0", - "half", - "num", -] - -[[package]] -name = "arrow-data" -version = "35.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19c7787c6cdbf9539b1ffb860bfc18c5848926ec3d62cbd52dc3b1ea35c874fd" -dependencies = [ - "arrow-buffer 35.0.0", - "arrow-schema 35.0.0", + "arrow-buffer", + "arrow-schema", "half", "num", ] @@ -222,11 +184,11 @@ version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e37b8b69d9e59116b6b538e8514e0ec63a30f08b617ce800d31cb44e3ef64c1a" dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", + "arrow-array", + "arrow-buffer", "arrow-cast", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-data", + "arrow-schema", "flatbuffers", ] @@ -236,11 +198,11 @@ version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80c3fa0bed7cfebf6d18e46b733f9cb8a1cb43ce8e6539055ca3e1e48a426266" dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", + "arrow-array", + "arrow-buffer", "arrow-cast", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-data", + "arrow-schema", "chrono", "half", "indexmap", @@ -255,10 +217,10 @@ version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d247dce7bed6a8d6a3c6debfa707a3a2f694383f0c692a39d736a593eae5ef94" dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "arrow-select", "num", ] @@ -270,10 +232,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d609c0181f963cea5c70fddf9a388595b5be441f3aa1d1cdbf728ca834bbd3a" dependencies = [ "ahash", - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", "hashbrown 0.13.2", ] @@ -284,22 +246,16 @@ version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64951898473bfb8e22293e83a44f02874d2257514d49cd95f9aa4afcff183fbc" -[[package]] -name = "arrow-schema" -version = "35.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf6b26f6a6f8410e3b9531cbd1886399b99842701da77d4b4cf2013f7708f20f" - [[package]] name = "arrow-select" version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a513d89c2e1ac22b28380900036cf1f3992c6443efc5e079de631dcf83c6888" dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num", ] @@ -309,10 +265,10 @@ version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5288979b2705dae1114c864d73150629add9153b9b8f1d7ee3963db94c372ba5" dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "arrow-select", "regex", "regex-syntax", @@ -804,7 +760,7 @@ name = "datafusion-common" version = "21.0.0" dependencies = [ "arrow", - "arrow-array 35.0.0", + "arrow-array", "chrono", "num_cpus", "object_store", @@ -860,8 +816,8 @@ version = "21.0.0" dependencies = [ "ahash", "arrow", - "arrow-buffer 34.0.0", - "arrow-schema 34.0.0", + "arrow-buffer", + "arrow-schema", "blake2", "blake3", "chrono", @@ -897,7 +853,7 @@ dependencies = [ name = "datafusion-sql" version = "21.0.0" dependencies = [ - "arrow-schema 34.0.0", + "arrow-schema", "datafusion-common", "datafusion-expr", "log", @@ -1859,12 +1815,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ac135ecf63ebb5f53dda0921b0b76d6048b3ef631a5f4760b9e8f863ff00cfa" dependencies = [ "ahash", - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", + "arrow-array", + "arrow-buffer", "arrow-cast", - "arrow-data 34.0.0", + "arrow-data", "arrow-ipc", - "arrow-schema 34.0.0", + "arrow-schema", "arrow-select", "base64", "brotli",