From e9965ce4e1e5da9cdc292336fbe3074e2cd61a82 Mon Sep 17 00:00:00 2001 From: zhuliquan Date: Sat, 30 Nov 2024 22:34:01 +0800 Subject: [PATCH 1/2] test: support run filter_pushdown on windows --- datafusion/core/src/test_util/parquet.rs | 12 +++++++++++- datafusion/core/tests/parquet/filter_pushdown.rs | 6 ++++-- datafusion/core/tests/parquet/mod.rs | 2 +- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/datafusion/core/src/test_util/parquet.rs b/datafusion/core/src/test_util/parquet.rs index fc6b4ad9d0b4e..685ed14777b40 100644 --- a/datafusion/core/src/test_util/parquet.rs +++ b/datafusion/core/src/test_util/parquet.rs @@ -103,7 +103,17 @@ impl TestParquetFile { let size = std::fs::metadata(&path)?.len() as usize; - let canonical_path = path.canonicalize()?; + let mut canonical_path = path.canonicalize()?; + + if cfg!(target_os = "windows") { + canonical_path = canonical_path + .to_str() + .unwrap() + .replace("\\", "/") + .strip_prefix("//?/") + .unwrap() + .into(); + }; let object_store_url = ListingTableUrl::parse(canonical_path.to_str().unwrap_or_default())? diff --git a/datafusion/core/tests/parquet/filter_pushdown.rs b/datafusion/core/tests/parquet/filter_pushdown.rs index 8def192f9331d..02fb59740493f 100644 --- a/datafusion/core/tests/parquet/filter_pushdown.rs +++ b/datafusion/core/tests/parquet/filter_pushdown.rs @@ -26,6 +26,8 @@ //! select * from data limit 10; //! ``` +use std::path::Path; + use arrow::compute::concat_batches; use arrow::record_batch::RecordBatch; use datafusion::physical_plan::collect; @@ -67,7 +69,7 @@ fn generate_file(tempdir: &TempDir, props: WriterProperties) -> TestParquetFile async fn single_file() { // Only create the parquet file once as it is fairly large - let tempdir = TempDir::new().unwrap(); + let tempdir = TempDir::new_in(Path::new(".")).unwrap(); // Set row group size smaller so can test with fewer rows let props = WriterProperties::builder() .set_max_row_group_size(1024) @@ -223,7 +225,7 @@ async fn single_file() { #[tokio::test] async fn single_file_small_data_pages() { - let tempdir = TempDir::new().unwrap(); + let tempdir = TempDir::new_in(Path::new(".")).unwrap(); // Set low row count limit to improve page filtering let props = WriterProperties::builder() diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rs index 3f68222a2ce3c..1ed56965b7471 100644 --- a/datafusion/core/tests/parquet/mod.rs +++ b/datafusion/core/tests/parquet/mod.rs @@ -45,7 +45,7 @@ use tempfile::NamedTempFile; mod custom_reader; mod external_access_plan; mod file_statistics; -#[cfg(not(target_family = "windows"))] +// #[cfg(not(target_family = "windows"))] mod filter_pushdown; mod page_pruning; mod row_group_pruning; From 3e476d57799663a32ca4343ba9cb0e880e446f36 Mon Sep 17 00:00:00 2001 From: zhuliquan Date: Mon, 2 Dec 2024 21:36:18 +0800 Subject: [PATCH 2/2] minor: remove useless comment for filter_pushdown --- datafusion/core/tests/parquet/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rs index 1ed56965b7471..46be2433116a4 100644 --- a/datafusion/core/tests/parquet/mod.rs +++ b/datafusion/core/tests/parquet/mod.rs @@ -45,7 +45,6 @@ use tempfile::NamedTempFile; mod custom_reader; mod external_access_plan; mod file_statistics; -// #[cfg(not(target_family = "windows"))] mod filter_pushdown; mod page_pruning; mod row_group_pruning;