From 49653db927577b245d4f9c1663b4d22398fbd24d Mon Sep 17 00:00:00 2001 From: Matthew Turner Date: Wed, 23 Feb 2022 08:52:41 -0500 Subject: [PATCH 1/2] Add sn-malloc --- Cargo.lock | 28 ++++++++++++++++++++++++++++ Cargo.toml | 1 + src/lib.rs | 3 +++ 3 files changed, 32 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index a20af45..514e1db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -208,6 +208,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "cmake" +version = "0.1.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a" +dependencies = [ + "cc", +] + [[package]] name = "comfy-table" version = "5.0.0" @@ -345,6 +354,7 @@ dependencies = [ "datafusion-expr", "pyo3", "rand 0.7.3", + "snmalloc-rs", "tokio", "uuid", ] @@ -1291,6 +1301,24 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" +[[package]] +name = "snmalloc-rs" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36acaace2719c972eab3ef6a6b3aee4495f0bf300f59715bb9cff6c5acf4ae20" +dependencies = [ + "snmalloc-sys", +] + +[[package]] +name = "snmalloc-sys" +version = "0.2.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a7e6e7d5fe756bee058ddedefc7e0a9f9c8dbaa9401b48ed3c17d6578e40b5" +dependencies = [ + "cmake", +] + [[package]] name = "sqlparser" version = "0.14.0" diff --git a/Cargo.toml b/Cargo.toml index bfff3ab..aabdbac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ datafusion = { version = "^7.0.0", features = ["pyarrow"] } datafusion-expr = { version = "^7.0.0" } datafusion-common = { version = "^7.0.0", features = ["pyarrow"] } uuid = { version = "0.8", features = ["v4"] } +snmalloc-rs = {version = "0.2", features = ["cache-friendly"]} [lib] name = "_internal" diff --git a/src/lib.rs b/src/lib.rs index d40bae2..c8755c1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,6 +27,9 @@ mod udaf; mod udf; mod utils; +#[global_allocator] +static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc; + /// Low-level DataFusion internal package. /// /// The higher-level public API is defined in pure python files under the From 4b2e7f6cf541674c57f17c6d6665a7b5ed441c49 Mon Sep 17 00:00:00 2001 From: Matthew Turner Date: Thu, 24 Feb 2022 16:08:34 -0500 Subject: [PATCH 2/2] Mimalloc --- Cargo.lock | 47 +++++++++++++++++++---------------------------- Cargo.toml | 2 +- src/lib.rs | 3 ++- 3 files changed, 22 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 514e1db..ce96b94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -208,15 +208,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "cmake" -version = "0.1.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a" -dependencies = [ - "cc", -] - [[package]] name = "comfy-table" version = "5.0.0" @@ -352,9 +343,9 @@ dependencies = [ "datafusion", "datafusion-common", "datafusion-expr", + "mimalloc", "pyo3", "rand 0.7.3", - "snmalloc-rs", "tokio", "uuid", ] @@ -723,6 +714,15 @@ version = "0.2.112" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125" +[[package]] +name = "libmimalloc-sys" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7705fc40f6ed493f73584abbb324e74f96b358ff60dfe5659a0f8fc12c590a69" +dependencies = [ + "cc", +] + [[package]] name = "lock_api" version = "0.4.6" @@ -776,6 +776,15 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "mimalloc" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0dfa131390c2f6bdb3242f65ff271fcdaca5ff7b6c08f28398be7f2280e3926" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "miniz_oxide" version = "0.4.4" @@ -1301,24 +1310,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45456094d1983e2ee2a18fdfebce3189fa451699d0502cb8e3b49dba5ba41451" -[[package]] -name = "snmalloc-rs" -version = "0.2.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36acaace2719c972eab3ef6a6b3aee4495f0bf300f59715bb9cff6c5acf4ae20" -dependencies = [ - "snmalloc-sys", -] - -[[package]] -name = "snmalloc-sys" -version = "0.2.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a7e6e7d5fe756bee058ddedefc7e0a9f9c8dbaa9401b48ed3c17d6578e40b5" -dependencies = [ - "cmake", -] - [[package]] name = "sqlparser" version = "0.14.0" diff --git a/Cargo.toml b/Cargo.toml index aabdbac..82e9141 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ datafusion = { version = "^7.0.0", features = ["pyarrow"] } datafusion-expr = { version = "^7.0.0" } datafusion-common = { version = "^7.0.0", features = ["pyarrow"] } uuid = { version = "0.8", features = ["v4"] } -snmalloc-rs = {version = "0.2", features = ["cache-friendly"]} +mimalloc = { version = "*", default-features = false } [lib] name = "_internal" diff --git a/src/lib.rs b/src/lib.rs index c8755c1..ab528a1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use mimalloc::MiMalloc; use pyo3::prelude::*; mod catalog; @@ -28,7 +29,7 @@ mod udf; mod utils; #[global_allocator] -static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc; +static GLOBAL: MiMalloc = MiMalloc; /// Low-level DataFusion internal package. ///