From 6ba139f1404cfdcf7f70c8ec7ec7d36ceba7310b Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Thu, 14 Jan 2021 22:37:49 +0100 Subject: [PATCH 1/4] Add simd and snmalloc flags to benchmarks --- rust/benchmarks/Cargo.toml | 7 ++++++- rust/benchmarks/src/bin/nyctaxi.rs | 4 ++++ rust/benchmarks/src/bin/tpch.rs | 4 ++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/rust/benchmarks/Cargo.toml b/rust/benchmarks/Cargo.toml index 74b74ece35a..fb68dca6266 100644 --- a/rust/benchmarks/Cargo.toml +++ b/rust/benchmarks/Cargo.toml @@ -26,6 +26,10 @@ repository = "https://github.com/apache/arrow" license = "Apache-2.0" publish = false +[features] +simd = ["datafusion/simd"] +snmalloc = ["snmalloc-rs"] + [dependencies] arrow = { path = "../arrow" } parquet = { path = "../parquet" } @@ -33,4 +37,5 @@ datafusion = { path = "../datafusion" } structopt = { version = "0.3", default-features = false } tokio = { version = "0.2", features = ["macros", "rt-core", "rt-threaded"] } futures = "0.3" -env_logger = "^0.8" \ No newline at end of file +env_logger = "^0.8" +snmalloc-rs = {version = "0.2", optional = true } \ No newline at end of file diff --git a/rust/benchmarks/src/bin/nyctaxi.rs b/rust/benchmarks/src/bin/nyctaxi.rs index a3cac44afc3..3391c38c8d3 100644 --- a/rust/benchmarks/src/bin/nyctaxi.rs +++ b/rust/benchmarks/src/bin/nyctaxi.rs @@ -31,6 +31,10 @@ use datafusion::physical_plan::collect; use datafusion::physical_plan::csv::CsvReadOptions; use structopt::StructOpt; +#[cfg(feature = "snmalloc")] +#[global_allocator] +static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc; + #[derive(Debug, StructOpt)] #[structopt(name = "Benchmarks", about = "Apache Arrow Rust Benchmarks.")] struct Opt { diff --git a/rust/benchmarks/src/bin/tpch.rs b/rust/benchmarks/src/bin/tpch.rs index 539b8d23d08..835c89396c1 100644 --- a/rust/benchmarks/src/bin/tpch.rs +++ b/rust/benchmarks/src/bin/tpch.rs @@ -33,6 +33,10 @@ use parquet::basic::Compression; use parquet::file::properties::WriterProperties; use structopt::StructOpt; +#[cfg(feature = "snmalloc")] +#[global_allocator] +static ALLOC: snmalloc_rs::SnMalloc = snmalloc_rs::SnMalloc; + #[derive(Debug, StructOpt)] struct BenchmarkOpt { /// Query number From defdf92ae76ab5fe13c7c6b630fef30e97771c38 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Thu, 14 Jan 2021 23:03:53 +0100 Subject: [PATCH 2/4] Add instructions to use features --- rust/benchmarks/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rust/benchmarks/README.md b/rust/benchmarks/README.md index 2ae035b9fc4..bb56b75a1bb 100644 --- a/rust/benchmarks/README.md +++ b/rust/benchmarks/README.md @@ -53,6 +53,12 @@ The benchmark can then be run (assuming the data created from `dbgen` is in `/mn cargo run --release --bin tpch -- benchmark --iterations 3 --path /mnt/tpch-dbgen --format tbl --query 1 --batch-size 4096 ``` +You can enable the features `simd` (to use SIMD instructions) and/or `snmalloc` (to use the snmalloc allocator) as features by passing them in as `--features`: + +``` +cargo run --release --features "simd snmalloc" --bin tpch -- benchmark --iterations 3 --path /mnt/tpch-dbgen --format tbl --query 1 --batch-size 4096 +``` + The benchmark program also supports CSV and Parquet input file formats and a utility is provided to convert from `tbl` (generated by the `dbgen` utility) to CSV and Parquet. From 4d5d11a5780327617a88394e0352d300778ce946 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Fri, 15 Jan 2021 06:36:01 +0100 Subject: [PATCH 3/4] Add line ending --- rust/benchmarks/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/benchmarks/Cargo.toml b/rust/benchmarks/Cargo.toml index fb68dca6266..29e714e1ff4 100644 --- a/rust/benchmarks/Cargo.toml +++ b/rust/benchmarks/Cargo.toml @@ -38,4 +38,4 @@ structopt = { version = "0.3", default-features = false } tokio = { version = "0.2", features = ["macros", "rt-core", "rt-threaded"] } futures = "0.3" env_logger = "^0.8" -snmalloc-rs = {version = "0.2", optional = true } \ No newline at end of file +snmalloc-rs = {version = "0.2", optional = true } From 7c9c349a8529edf59784c64b1a5f95adb8179331 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Tue, 19 Jan 2021 08:18:20 +0100 Subject: [PATCH 4/4] Enable `cache-friendly` option --- rust/benchmarks/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/benchmarks/Cargo.toml b/rust/benchmarks/Cargo.toml index 29e714e1ff4..60675efb913 100644 --- a/rust/benchmarks/Cargo.toml +++ b/rust/benchmarks/Cargo.toml @@ -38,4 +38,4 @@ structopt = { version = "0.3", default-features = false } tokio = { version = "0.2", features = ["macros", "rt-core", "rt-threaded"] } futures = "0.3" env_logger = "^0.8" -snmalloc-rs = {version = "0.2", optional = true } +snmalloc-rs = {version = "0.2", optional = true, features= ["cache-friendly"] }