diff --git a/Cargo.lock b/Cargo.lock index 654dedd..a20af45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,15 +57,16 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "6.5.0" +version = "9.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216c6846a292bdd93c2b93c1baab58c32ff50e2ab5e8d50db333ab518535dd8b" +checksum = "374ec8e5f39015ab754cfc63398a897423c877b66aecbfd036903b2c9c07f244" dependencies = [ "bitflags", "chrono", "comfy-table", "csv", "flatbuffers", + "half", "hex", "indexmap", "lazy_static", @@ -111,13 +112,11 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "blake2" -version = "0.9.2" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4e37d16930f5459780f5621038b6382b9bb37c19016f39fb6b5808d831f174" +checksum = "b94ba84325db59637ffc528bbe8c7f86c02c57cff5c0e2b9b00f9a851f42f309" dependencies = [ - "crypto-mac", - "digest", - "opaque-debug", + "digest 0.10.1", ] [[package]] @@ -131,14 +130,14 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "digest", + "digest 0.9.0", ] [[package]] name = "block-buffer" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +checksum = "f1d36a02058e76b040de25a4464ba1c80935655595b661505c8b39b664828b95" dependencies = [ "generic-array", ] @@ -206,7 +205,6 @@ dependencies = [ "libc", "num-integer", "num-traits", - "time", "winapi", ] @@ -246,13 +244,12 @@ dependencies = [ ] [[package]] -name = "crypto-mac" -version = "0.8.0" +name = "crypto-common" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +checksum = "683d6b536309245c849479fba3da410962a43ed8e51c26b729208ec0ac2798d0" dependencies = [ "generic-array", - "subtle", ] [[package]] @@ -279,9 +276,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "6.0.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e4a8a1f1ee057b2c27a01f050b9dffe56e8d43605d0201234b353a3cc1eb2f" +checksum = "30cf8e6735817bb021748d72cecc33e468d8775bf749470c52aa7f55ee5cdf9e" dependencies = [ "ahash", "arrow", @@ -289,13 +286,16 @@ dependencies = [ "blake2", "blake3", "chrono", + "datafusion-common", + "datafusion-expr", "futures", - "hashbrown", + "hashbrown 0.12.0", "lazy_static", "log", "md-5", "num_cpus", "ordered-float 2.10.0", + "parking_lot 0.12.0", "parquet", "paste 1.0.6", "pin-project-lite", @@ -305,16 +305,44 @@ dependencies = [ "sha2", "smallvec", "sqlparser", + "tempfile", "tokio", "tokio-stream", "unicode-segmentation", ] +[[package]] +name = "datafusion-common" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40d2e2a1a6508f9e66c6deb84ab655be0e9db177f617ec1904de458439981b03" +dependencies = [ + "arrow", + "ordered-float 2.10.0", + "parquet", + "pyo3", + "sqlparser", +] + +[[package]] +name = "datafusion-expr" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1061dc27f2e4843ddb0f93a8e208d1984260c18c8bbf5e67598be9a111259fe2" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "sqlparser", +] + [[package]] name = "datafusion-python" version = "0.4.0" dependencies = [ "datafusion", + "datafusion-common", + "datafusion-expr", "pyo3", "rand 0.7.3", "tokio", @@ -330,6 +358,27 @@ dependencies = [ "generic-array", ] +[[package]] +name = "digest" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b697d66081d42af4fba142d56918a3cb21dc8eb63372c6b85d14f44fb9c5979b" +dependencies = [ + "block-buffer", + "crypto-common", + "generic-array", + "subtle", +] + +[[package]] +name = "fastrand" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +dependencies = [ + "instant", +] + [[package]] name = "flatbuffers" version = "2.0.0" @@ -474,11 +523,23 @@ dependencies = [ "wasi 0.10.2+wasi-snapshot-preview1", ] +[[package]] +name = "half" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" + [[package]] name = "hashbrown" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "hashbrown" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c21d40587b92fa6a6c6e3c1bdbf87d75511db5672f9c93175574b3a00df1758" dependencies = [ "ahash", ] @@ -514,7 +575,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.11.2", ] [[package]] @@ -654,9 +715,9 @@ checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125" [[package]] name = "lock_api" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109" +checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" dependencies = [ "scopeguard", ] @@ -692,13 +753,11 @@ dependencies = [ [[package]] name = "md-5" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +checksum = "e6a38fc55c8bbc10058782919516f88826e70320db6d206aebc49611d24216ae" dependencies = [ - "block-buffer", - "digest", - "opaque-debug", + "digest 0.10.1", ] [[package]] @@ -829,12 +888,6 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "ordered-float" version = "1.1.1" @@ -861,7 +914,17 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core", + "parking_lot_core 0.8.5", +] + +[[package]] +name = "parking_lot" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.1", ] [[package]] @@ -878,11 +941,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "parking_lot_core" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + [[package]] name = "parquet" -version = "6.5.0" +version = "9.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "788d9953f4cfbe9db1beff7bebd54299d105e34680d78b82b1ddc85d432cac9d" +checksum = "36531313e3e81a646672001c02d1764fabc1320055fe8b176e696b08d3cf44cd" dependencies = [ "arrow", "base64", @@ -901,9 +977,9 @@ dependencies = [ [[package]] name = "parquet-format" -version = "2.6.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5bc6b23543b5dedc8f6cce50758a35e5582e148e0cfa26bd0cacd569cda5b71" +checksum = "1f0c06cdcd5460967c485f9c40a821746f5955ad81990533c7fae95dbd9bc0b5" dependencies = [ "thrift", ] @@ -968,14 +1044,14 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.14.5" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35100f9347670a566a67aa623369293703322bb9db77d99d7df7313b575ae0c8" +checksum = "7cf01dbf1c05af0a14c7779ed6f3aa9deac9c3419606ac9de537a2d649005720" dependencies = [ "cfg-if", "indoc", "libc", - "parking_lot", + "parking_lot 0.11.2", "paste 0.1.18", "pyo3-build-config", "pyo3-macros", @@ -984,18 +1060,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.14.5" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d12961738cacbd7f91b7c43bc25cfeeaa2698ad07a04b3be0aa88b950865738f" +checksum = "dbf9e4d128bfbddc898ad3409900080d8d5095c379632fbbfbb9c8cfb1fb852b" dependencies = [ "once_cell", ] [[package]] name = "pyo3-macros" -version = "0.14.5" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0bc5215d704824dfddddc03f93cb572e1155c68b6761c37005e1c288808ea8" +checksum = "67701eb32b1f9a9722b4bc54b548ff9d7ebfded011c12daece7b9063be1fd755" dependencies = [ "pyo3-macros-backend", "quote", @@ -1004,9 +1080,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.14.5" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71623fc593224afaab918aa3afcaf86ed2f43d34f6afde7f3922608f253240df" +checksum = "f44f09e825ee49a105f2c7b23ebee50886a9aee0746f4dd5a704138a64b0218a" dependencies = [ "proc-macro2", "pyo3-build-config", @@ -1136,6 +1212,15 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "ryu" version = "1.0.9" @@ -1179,15 +1264,13 @@ dependencies = [ [[package]] name = "sha2" -version = "0.9.8" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa" +checksum = "99c3bd8169c58782adad9290a9af5939994036b76187f7b4f0e6de91dbbfc0ec" dependencies = [ - "block-buffer", "cfg-if", "cpufeatures", - "digest", - "opaque-debug", + "digest 0.10.1", ] [[package]] @@ -1210,9 +1293,9 @@ checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" [[package]] name = "sqlparser" -version = "0.12.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760e624412a15d5838ae04fad01037beeff1047781431d74360cddd6b3c1c784" +checksum = "b8f192f29f4aa49e57bebd0aa05858e0a1f32dd270af36efe49edb82cbfffab6" dependencies = [ "log", ] @@ -1258,6 +1341,20 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + [[package]] name = "thiserror" version = "1.0.30" @@ -1300,16 +1397,6 @@ dependencies = [ "threadpool", ] -[[package]] -name = "time" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "tokio" version = "1.15.0" @@ -1317,6 +1404,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838" dependencies = [ "num_cpus", + "parking_lot 0.11.2", "pin-project-lite", "tokio-macros", ] @@ -1422,20 +1510,63 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" + +[[package]] +name = "windows_i686_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" + +[[package]] +name = "windows_i686_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" + [[package]] name = "zstd" -version = "0.9.1+zstd.1.5.1" +version = "0.10.0+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "538b8347df9257b7fbce37677ef7535c00a3c7bf1f81023cc328ed7fe4b41de8" +checksum = "3b1365becbe415f3f0fcd024e2f7b45bacfb5bdd055f0dc113571394114e7bdd" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "4.1.2+zstd.1.5.1" +version = "4.1.4+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb4cfe2f6e6d35c5d27ecd9d256c4b6f7933c4895654917460ec56c29336cc1" +checksum = "2f7cd17c9af1a4d6c24beb1cc54b17e2ef7b593dc92f19e9d9acad8b182bbaee" dependencies = [ "libc", "zstd-sys", @@ -1443,9 +1574,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "1.6.2+zstd.1.5.1" +version = "1.6.3+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2daf2f248d9ea44454bfcb2516534e8b8ad2fc91bf818a1885495fc42bc8ac9f" +checksum = "fc49afa5c8d634e75761feda8c592051e7eeb4683ba827211eb0d731d3402ea8" dependencies = [ "cc", "libc", diff --git a/Cargo.toml b/Cargo.toml index aa16236..bfff3ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,8 +30,10 @@ rust-version = "1.57" [dependencies] tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.7" -pyo3 = { version = "0.14", features = ["extension-module", "abi3", "abi3-py36"] } -datafusion = { version = "6.0.0", features = ["pyarrow"] } +pyo3 = { version = "0.15", features = ["extension-module", "abi3", "abi3-py36"] } +datafusion = { version = "^7.0.0", features = ["pyarrow"] } +datafusion-expr = { version = "^7.0.0" } +datafusion-common = { version = "^7.0.0", features = ["pyarrow"] } uuid = { version = "0.8", features = ["v4"] } [lib] diff --git a/src/udaf.rs b/src/udaf.rs index 89c479f..3ac9d77 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -24,10 +24,10 @@ use datafusion::arrow::datatypes::DataType; use datafusion::arrow::pyarrow::PyArrowConvert; use datafusion::error::{DataFusionError, Result}; use datafusion::logical_plan; -use datafusion::physical_plan::aggregates::AccumulatorFunctionImplementation; -use datafusion::physical_plan::udaf::AggregateUDF; -use datafusion::physical_plan::Accumulator; -use datafusion::scalar::ScalarValue; +use datafusion_common::ScalarValue; +use datafusion_expr::Accumulator; +use datafusion_expr::AccumulatorFunctionImplementation; +use datafusion_expr::AggregateUDF; use crate::expression::PyExpr; use crate::utils::parse_volatility; @@ -49,16 +49,6 @@ impl Accumulator for RustAccumulator { .map_err(|e| DataFusionError::Execution(format!("{}", e))) } - fn update(&mut self, _values: &[ScalarValue]) -> Result<()> { - // no need to implement as datafusion does not use it - todo!() - } - - fn merge(&mut self, _states: &[ScalarValue]) -> Result<()> { - // no need to implement as datafusion does not use it - todo!() - } - fn evaluate(&self) -> Result { Python::with_gil(|py| self.accum.as_ref(py).call_method0("evaluate")?.extract()) .map_err(|e| DataFusionError::Execution(format!("{}", e))) @@ -144,7 +134,6 @@ impl PyAggregateUDF { } /// creates a new PyExpr with the call of the udf - #[call] #[args(args = "*")] fn __call__(&self, args: Vec) -> PyResult { let args = args.iter().map(|e| e.expr.clone()).collect(); diff --git a/src/udf.rs b/src/udf.rs index bfc7e9c..8251739 100644 --- a/src/udf.rs +++ b/src/udf.rs @@ -87,7 +87,6 @@ impl PyScalarUDF { } /// creates a new PyExpr with the call of the udf - #[call] #[args(args = "*")] fn __call__(&self, args: Vec) -> PyResult { let args = args.iter().map(|e| e.expr.clone()).collect();