diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 26d6fb9c41463..ad6fb557fe76d 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -43,6 +43,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "anyhow" +version = "1.0.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" + [[package]] name = "arrayref" version = "0.3.6" @@ -57,9 +63,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "14.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0612b6a634de6c3f5e63fdaa6932f7bc598f92de0462ac6e69b0aebd77e093aa" +checksum = "5c6bee230122beb516ead31935a61f683715f987c6f003eff44ad6986624105a" dependencies = [ "bitflags", "chrono", @@ -80,6 +86,45 @@ dependencies = [ "serde_json", ] +[[package]] +name = "arrow-flight" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a3666d2dbc637fa979d1f0bf3031d39a80e709f3b9ec88e3d573c1d666bf553" +dependencies = [ + "arrow", + "base64", + "bytes", + "futures", + "proc-macro2", + "prost", + "prost-derive", + "tokio", + "tonic", + "tonic-build", +] + +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.53" @@ -108,6 +153,93 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00f1e8a972137fad81e2a1a60b86ff17ce0338f8017264e45a9723d0083c39a1" +dependencies = [ + "async-trait", + "axum-core", + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "itoa 1.0.1", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da31c0ed7b4690e2c78fe4b880d21cd7db04a346ebc658b4270251b695437f17" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", +] + +[[package]] +name = "ballista" +version = "0.6.0" +dependencies = [ + "ballista-core", + "datafusion", + "futures", + "log", + "parking_lot", + "sqlparser", + "tempfile", + "tokio", +] + +[[package]] +name = "ballista-core" +version = "0.6.0" +dependencies = [ + "ahash", + "arrow-flight", + "async-trait", + "chrono", + "clap", + "datafusion", + "datafusion-proto", + "futures", + "hashbrown 0.12.1", + "libloading", + "log", + "once_cell", + "parking_lot", + "parse_arg", + "prost", + "prost-types", + "rustc_version", + "serde", + "sqlparser", + "tokio", + "tonic", + "tonic-build", + "uuid", + "walkdir", +] + [[package]] name = "base64" version = "0.13.0" @@ -226,9 +358,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.1.18" +version = "3.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b" +checksum = "47582c09be7c8b32c0ab3a6181825ababb713fde6fff20fc573a3870dd45c6a0" dependencies = [ "atty", "bitflags", @@ -243,9 +375,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.1.18" +version = "3.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25320346e922cffe59c0bbc5410c8d8784509efb321488971081313cb1e1a33c" +checksum = "a3aab4734e083b809aaf5794e14e756d1c798d2c69c7f7de7a09a2f5214993c1" dependencies = [ "heck 0.4.0", "proc-macro-error", @@ -274,6 +406,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "cmake" +version = "0.1.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a" +dependencies = [ + "cc", +] + [[package]] name = "comfy-table" version = "5.0.1" @@ -343,7 +484,7 @@ dependencies = [ [[package]] name = "datafusion" -version = "8.0.0" +version = "7.0.0" dependencies = [ "ahash", "arrow", @@ -375,9 +516,10 @@ dependencies = [ [[package]] name = "datafusion-cli" -version = "8.0.0" +version = "7.0.0" dependencies = [ "arrow", + "ballista", "clap", "datafusion", "dirs", @@ -389,7 +531,7 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "8.0.0" +version = "7.0.0" dependencies = [ "arrow", "ordered-float 3.0.0", @@ -399,7 +541,7 @@ dependencies = [ [[package]] name = "datafusion-data-access" -version = "8.0.0" +version = "7.0.0" dependencies = [ "async-trait", "chrono", @@ -412,7 +554,7 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "8.0.0" +version = "7.0.0" dependencies = [ "ahash", "arrow", @@ -422,7 +564,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "8.0.0" +version = "7.0.0" dependencies = [ "ahash", "arrow", @@ -443,9 +585,18 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "datafusion-proto" +version = "7.0.0" +dependencies = [ + "datafusion", + "prost", + "tonic-build", +] + [[package]] name = "datafusion-row" -version = "8.0.0" +version = "7.0.0" dependencies = [ "arrow", "datafusion-common", @@ -505,6 +656,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + [[package]] name = "endian-type" version = "0.1.2" @@ -575,6 +732,12 @@ dependencies = [ "windows-sys 0.30.0", ] +[[package]] +name = "fixedbitset" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" + [[package]] name = "flatbuffers" version = "2.1.2" @@ -598,6 +761,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "futures" version = "0.3.21" @@ -705,7 +874,7 @@ checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.10.2+wasi-snapshot-preview1", ] [[package]] @@ -714,6 +883,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +[[package]] +name = "h2" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "1.8.2" @@ -765,12 +953,88 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "http" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" +dependencies = [ + "bytes", + "fnv", + "itoa 1.0.1", +] + +[[package]] +name = "http-body" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + +[[package]] +name = "httparse" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + [[package]] name = "humantime" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa 1.0.1", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "indexmap" version = "1.8.1" @@ -802,6 +1066,15 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9448015e586b611e5d322f6703812bbca2f1e709d5773ecd38ddb4e3bb649504" +[[package]] +name = "itertools" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "0.4.8" @@ -810,9 +1083,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" [[package]] name = "jobserver" @@ -899,6 +1172,16 @@ version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" +[[package]] +name = "libloading" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" +dependencies = [ + "cfg-if", + "winapi", +] + [[package]] name = "libmimalloc-sys" version = "0.1.25" @@ -953,6 +1236,12 @@ dependencies = [ "libc", ] +[[package]] +name = "matchit" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" + [[package]] name = "md-5" version = "0.10.1" @@ -986,6 +1275,12 @@ dependencies = [ "libmimalloc-sys", ] +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "miniz_oxide" version = "0.5.1" @@ -995,6 +1290,24 @@ dependencies = [ "adler", ] +[[package]] +name = "mio" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.36.1", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + [[package]] name = "multiversion" version = "0.6.1" @@ -1149,9 +1462,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.0.1" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435" +checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" [[package]] name = "parking_lot" @@ -1178,15 +1491,14 @@ dependencies = [ [[package]] name = "parquet" -version = "14.0.0" +version = "13.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1185ee1da5091e40b86519265a44d2704e3916ff867059c915141cab14d413" +checksum = "6c6d737baed48775e87a69aa262f1fa2f1d6bd074dedbe9cac244b9aabf2a0b4" dependencies = [ "arrow", "base64", "brotli", "byteorder", - "bytes", "chrono", "flate2", "lz4", @@ -1208,12 +1520,54 @@ dependencies = [ "thrift", ] +[[package]] +name = "parse_arg" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14248cc8eced350e20122a291613de29e4fa129ba2731818c4cdbb44fccd3e55" + [[package]] name = "paste" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "petgraph" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -1232,6 +1586,16 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +[[package]] +name = "prettyplease" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9e07e3a46d0771a8a06b5f4441527802830b43e679ba12f44960f48dd4c6803" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1265,6 +1629,61 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "prost" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc03e116981ff7d8da8e5c220e374587b98d294af7ba7dd7fda761158f00086f" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65a1118354442de7feb8a2a76f3d80ef01426bd45542c8c1fdffca41a758f846" +dependencies = [ + "bytes", + "cfg-if", + "cmake", + "heck 0.4.0", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "regex", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68" +dependencies = [ + "bytes", + "prost", +] + [[package]] name = "quote" version = "1.0.18" @@ -1366,6 +1785,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.34.6" @@ -1412,9 +1840,18 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.10" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" + +[[package]] +name = "same-file" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] [[package]] name = "scopeguard" @@ -1422,11 +1859,20 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "semver" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" + [[package]] name = "serde" version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" +dependencies = [ + "serde_derive", +] [[package]] name = "serde_derive" @@ -1446,7 +1892,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ "indexmap", - "itoa 1.0.2", + "itoa 1.0.1", "ryu", "serde", ] @@ -1480,6 +1926,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" +[[package]] +name = "socket2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "sqlparser" version = "0.17.0" @@ -1534,15 +1990,21 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.94" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a07e33e919ebcd69113d5be0e4d70c5707004ff45188910106854f38b960df4a" +checksum = "04066589568b72ec65f42d65a1a52436e954b168773148893c020269563decf2" dependencies = [ "proc-macro2", "quote", "unicode-xid", ] +[[package]] +name = "sync_wrapper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" + [[package]] name = "tempfile" version = "3.3.0" @@ -1620,11 +2082,27 @@ version = "1.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4903bf0427cf68dddd5aa6a93220756f8be0c34fcfa9f5e6191e103e15a31395" dependencies = [ + "bytes", + "libc", + "memchr", + "mio", "num_cpus", "once_cell", "parking_lot", "pin-project-lite", + "socket2", "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite", + "tokio", ] [[package]] @@ -1649,6 +2127,165 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "tonic" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be9d60db39854b30b835107500cf0aca0b0d14d6e1c3de124217c23a29c2ddb" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "prost-derive", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic-build" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9263bf4c9bfaae7317c1c2faf7f18491d2fe476f70c414b73bf5d445b00ffa1" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn", +] + +[[package]] +name = "tower" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d342c6d58709c0a6d48d48dabbb62d4ef955cf5f0f3bbfd845838e7ae88dbae" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62" + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" +dependencies = [ + "cfg-if", + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.15.0" @@ -1694,12 +2331,50 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "which" +version = "4.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" +dependencies = [ + "either", + "lazy_static", + "libc", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1819,18 +2494,18 @@ checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" [[package]] name = "zstd" -version = "0.11.2+zstd.1.5.2" +version = "0.11.1+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +checksum = "77a16b8414fde0414e90c612eba70985577451c4c504b99885ebed24762cb81a" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" +version = "5.0.1+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +checksum = "7c12659121420dd6365c5c3de4901f97145b79651fb1d25814020ed2ed0585ae" dependencies = [ "libc", "zstd-sys", diff --git a/datafusion/core/src/physical_plan/file_format/parquet.rs b/datafusion/core/src/physical_plan/file_format/parquet.rs index 8ed1bcee527b8..ab4ab3fbe908f 100644 --- a/datafusion/core/src/physical_plan/file_format/parquet.rs +++ b/datafusion/core/src/physical_plan/file_format/parquet.rs @@ -1465,9 +1465,10 @@ mod tests { .enumerate() .map(|(i, g)| row_group_predicate(g, i)) .collect::>(); - // no row group is filtered out because the predicate expression can't be evaluated - // when a null array is generated for a statistics column, - assert_eq!(row_group_filter, vec![true, true]); + + // bool = NULL always evaluates to NULL (and thus will not + // pass predicates. Ideally these should both be false + assert_eq!(row_group_filter, vec![false, true]); Ok(()) } diff --git a/datafusion/core/tests/sql/expr.rs b/datafusion/core/tests/sql/expr.rs index b8a00265766b0..ca1f824ac36d0 100644 --- a/datafusion/core/tests/sql/expr.rs +++ b/datafusion/core/tests/sql/expr.rs @@ -1203,121 +1203,54 @@ async fn nested_subquery() -> Result<()> { } #[tokio::test] -async fn comparisons_with_null() -> Result<()> { +async fn comparisons_with_null_lt() { let ctx = SessionContext::new(); - // 1. Numeric comparison with NULL - let sql = "select column1 < NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t"; - let actual = execute_to_batches(&ctx, sql).await; - let expected = vec![ - "+-------------------+", - "| t.column1 Lt NULL |", - "+-------------------+", - "| |", - "| |", - "+-------------------+", - ]; - assert_batches_eq!(expected, &actual); - let sql = - "select column1 <= NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t"; - let actual = execute_to_batches(&ctx, sql).await; - let expected = vec![ - "+---------------------+", - "| t.column1 LtEq NULL |", - "+---------------------+", - "| |", - "| |", - "+---------------------+", + // we expect all the following queries to yield a two null values + let cases = vec![ + // 1. Numeric comparison with NULL + "select column1 < NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + "select column1 <= NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + "select column1 > NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + "select column1 >= NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + "select column1 = NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + "select column1 != NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + // 1.1 Float value comparison with NULL + "select column3 < NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + // String comparison with NULL + "select column2 < NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + // Boolean comparison with NULL + "select column1 < NULL from (VALUES (true), (false)) as t", + // ---- + // ---- same queries, reversed argument order (as they go through + // ---- a different evaluation path) + // ---- + + // 1. Numeric comparison with NULL + "select NULL < column1 from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + "select NULL <= column1 from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + "select NULL > column1 from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + "select NULL >= column1 from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + "select NULL = column1 from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + "select NULL != column1 from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + // 1.1 Float value comparison with NULL + "select NULL < column3 from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + // String comparison with NULL + "select NULL < column2 from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t", + // Boolean comparison with NULL + "select NULL < column1 from (VALUES (true), (false)) as t", ]; - assert_batches_eq!(expected, &actual); - let sql = "select column1 > NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t"; - let actual = execute_to_batches(&ctx, sql).await; - let expected = vec![ - "+-------------------+", - "| t.column1 Gt NULL |", - "+-------------------+", - "| |", - "| |", - "+-------------------+", - ]; - assert_batches_eq!(expected, &actual); + for sql in cases { + println!("Computing: {}", sql); - let sql = - "select column1 >= NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t"; - let actual = execute_to_batches(&ctx, sql).await; - let expected = vec![ - "+---------------------+", - "| t.column1 GtEq NULL |", - "+---------------------+", - "| |", - "| |", - "+---------------------+", - ]; - assert_batches_eq!(expected, &actual); + let mut actual = execute_to_batches(&ctx, sql).await; + assert_eq!(actual.len(), 1); - let sql = "select column1 = NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t"; - let actual = execute_to_batches(&ctx, sql).await; - let expected = vec![ - "+-------------------+", - "| t.column1 Eq NULL |", - "+-------------------+", - "| |", - "| |", - "+-------------------+", - ]; - assert_batches_eq!(expected, &actual); - - let sql = - "select column1 != NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t"; - let actual = execute_to_batches(&ctx, sql).await; - let expected = vec![ - "+----------------------+", - "| t.column1 NotEq NULL |", - "+----------------------+", - "| |", - "| |", - "+----------------------+", - ]; - assert_batches_eq!(expected, &actual); - - // 1.1 Float value comparison with NULL - let sql = "select column3 < NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t"; - let actual = execute_to_batches(&ctx, sql).await; - let expected = vec![ - "+-------------------+", - "| t.column3 Lt NULL |", - "+-------------------+", - "| |", - "| |", - "+-------------------+", - ]; - assert_batches_eq!(expected, &actual); - - // String comparison with NULL - let sql = "select column2 < NULL from (VALUES (1, 'foo' ,2.3), (2, 'bar', 5.4)) as t"; - let actual = execute_to_batches(&ctx, sql).await; - let expected = vec![ - "+-------------------+", - "| t.column2 Lt NULL |", - "+-------------------+", - "| |", - "| |", - "+-------------------+", - ]; - assert_batches_eq!(expected, &actual); - - // Boolean comparison with NULL - let sql = "select column1 < NULL from (VALUES (true), (false)) as t"; - let actual = execute_to_batches(&ctx, sql).await; - let expected = vec![ - "+-------------------+", - "| t.column1 Lt NULL |", - "+-------------------+", - "| |", - "| |", - "+-------------------+", - ]; - assert_batches_eq!(expected, &actual); - Ok(()) + let batch = actual.pop().unwrap(); + assert_eq!(batch.num_rows(), 2); + assert_eq!(batch.num_columns(), 1); + assert!(batch.columns()[0].is_null(0)); + assert!(batch.columns()[0].is_null(1)); + } } diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index e507f8fe8c3a2..b3de2461cdd0c 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -25,10 +25,6 @@ use arrow::compute::kernels::arithmetic::{ multiply_scalar, subtract, subtract_scalar, }; use arrow::compute::kernels::boolean::{and_kleene, not, or_kleene}; -use arrow::compute::kernels::comparison::{ - eq_bool_scalar, gt_bool_scalar, gt_eq_bool_scalar, lt_bool_scalar, lt_eq_bool_scalar, - neq_bool_scalar, -}; use arrow::compute::kernels::comparison::{ eq_dyn_bool_scalar, gt_dyn_bool_scalar, gt_eq_dyn_bool_scalar, lt_dyn_bool_scalar, lt_eq_dyn_bool_scalar, neq_dyn_bool_scalar, @@ -44,12 +40,10 @@ use arrow::compute::kernels::comparison::{ use arrow::compute::kernels::comparison::{ eq_scalar, gt_eq_scalar, gt_scalar, lt_eq_scalar, lt_scalar, neq_scalar, }; +use arrow::compute::kernels::comparison::{like_utf8, nlike_utf8, regexp_is_match_utf8}; use arrow::compute::kernels::comparison::{ - eq_utf8_scalar, gt_eq_utf8_scalar, gt_utf8_scalar, like_utf8_scalar, - lt_eq_utf8_scalar, lt_utf8_scalar, neq_utf8_scalar, nlike_utf8_scalar, - regexp_is_match_utf8_scalar, + like_utf8_scalar, nlike_utf8_scalar, regexp_is_match_utf8_scalar, }; -use arrow::compute::kernels::comparison::{like_utf8, nlike_utf8, regexp_is_match_utf8}; use arrow::datatypes::{ArrowNumericType, DataType, Schema, TimeUnit}; use arrow::error::ArrowError::DivideByZero; use arrow::record_batch::RecordBatch; @@ -738,7 +732,7 @@ macro_rules! compute_op_scalar { } /// Invoke a dyn compute kernel on a data array and a scalar value -/// LEFT is Primitive or Dictionart array of numeric values, RIGHT is scalar value +/// LEFT is Primitive or Dictionary array of numeric values, RIGHT is scalar value /// OP_TYPE is the return type of scalar function macro_rules! compute_op_dyn_scalar { ($LEFT:expr, $RIGHT:expr, $OP:ident, $OP_TYPE:expr) => {{ @@ -1148,39 +1142,25 @@ impl BinaryExpr { array: &dyn Array, scalar: &ScalarValue, ) -> Result>> { + let bool_type = &DataType::Boolean; let scalar_result = match &self.op { Operator::Lt => { - binary_array_op_dyn_scalar!(array, scalar.clone(), lt, &DataType::Boolean) + binary_array_op_dyn_scalar!(array, scalar.clone(), lt, bool_type) } Operator::LtEq => { - binary_array_op_dyn_scalar!( - array, - scalar.clone(), - lt_eq, - &DataType::Boolean - ) + binary_array_op_dyn_scalar!(array, scalar.clone(), lt_eq, bool_type) } Operator::Gt => { - binary_array_op_dyn_scalar!(array, scalar.clone(), gt, &DataType::Boolean) + binary_array_op_dyn_scalar!(array, scalar.clone(), gt, bool_type) } Operator::GtEq => { - binary_array_op_dyn_scalar!( - array, - scalar.clone(), - gt_eq, - &DataType::Boolean - ) + binary_array_op_dyn_scalar!(array, scalar.clone(), gt_eq, bool_type) } Operator::Eq => { - binary_array_op_dyn_scalar!(array, scalar.clone(), eq, &DataType::Boolean) + binary_array_op_dyn_scalar!(array, scalar.clone(), eq, bool_type) } Operator::NotEq => { - binary_array_op_dyn_scalar!( - array, - scalar.clone(), - neq, - &DataType::Boolean - ) + binary_array_op_dyn_scalar!(array, scalar.clone(), neq, bool_type) } Operator::Like => { binary_string_array_op_scalar!(array, scalar.clone(), like) @@ -1247,14 +1227,25 @@ impl BinaryExpr { scalar: &ScalarValue, array: &ArrayRef, ) -> Result>> { + let bool_type = &DataType::Boolean; let scalar_result = match &self.op { - Operator::Lt => binary_array_op_scalar!(array, scalar.clone(), gt), - Operator::LtEq => binary_array_op_scalar!(array, scalar.clone(), gt_eq), - Operator::Gt => binary_array_op_scalar!(array, scalar.clone(), lt), - Operator::GtEq => binary_array_op_scalar!(array, scalar.clone(), lt_eq), - Operator::Eq => binary_array_op_scalar!(array, scalar.clone(), eq), + Operator::Lt => { + binary_array_op_dyn_scalar!(array, scalar.clone(), gt, bool_type) + } + Operator::LtEq => { + binary_array_op_dyn_scalar!(array, scalar.clone(), gt_eq, bool_type) + } + Operator::Gt => { + binary_array_op_dyn_scalar!(array, scalar.clone(), lt, bool_type) + } + Operator::GtEq => { + binary_array_op_dyn_scalar!(array, scalar.clone(), lt_eq, bool_type) + } + Operator::Eq => { + binary_array_op_dyn_scalar!(array, scalar.clone(), eq, bool_type) + } Operator::NotEq => { - binary_array_op_scalar!(array, scalar.clone(), neq) + binary_array_op_dyn_scalar!(array, scalar.clone(), neq, bool_type) } // if scalar operation is not supported - fallback to array implementation _ => None,