From f5b1f64a58b1d5b59c52ad703e322b87fbec6ca9 Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 23 Feb 2026 20:00:02 +0000 Subject: [PATCH 01/10] runs-on for extended --- .github/workflows/extended.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index e9eb27dd96527..b3c537d7f769f 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -66,7 +66,7 @@ jobs: # Check crate compiles and base cargo check passes linux-build-lib: name: linux build test - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} # note: do not use amd/rust container to preserve disk space steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -90,7 +90,8 @@ jobs: linux-test-extended: name: cargo test 'extended_tests' (amd64) needs: [linux-build-lib] - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion,spot=false', github.run_id) || 'ubuntu-latest' }} + # spot=false because the tests are long, https://runs-on.com/configuration/spot-instances/#disable-spot-pricing # note: do not use amd/rust container to preserve disk space steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -107,6 +108,11 @@ jobs: rustup toolchain install - name: Install Protobuf Compiler run: sudo apt-get install -y protobuf-compiler + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + with: + save-if: true # TODO: set ${{ github.ref_name == 'main' }} + shared-key: "linux-tests-extended" # For debugging, test binaries can be large. - name: Show available disk space run: | @@ -154,7 +160,8 @@ jobs: sqllogictest-sqlite: name: "Run sqllogictests with the sqlite test suite" - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=48,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion,spot=false', github.run_id) || 'ubuntu-latest' }} + # spot=false because the tests are long, https://runs-on.com/configuration/spot-instances/#disable-spot-pricing container: image: amd64/rust steps: @@ -167,6 +174,11 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Rust Dependency Cache + uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 + with: + save-if: true # TODO: set ${{ github.ref_name == 'main' }} + shared-key: "sqllogictest-sqlite" - name: Run sqllogictest run: | cargo test --features backtrace,parquet_encryption --profile release-nonlto --test sqllogictests -- --include-sqlite From 6012e8880b4c76fce7adba5178c694c04fb917b1 Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 23 Feb 2026 20:04:09 +0000 Subject: [PATCH 02/10] update --- .github/workflows/extended.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index b3c537d7f769f..e81e577af673a 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -80,7 +80,9 @@ jobs: source $HOME/.cargo/env rustup toolchain install - name: Install Protobuf Compiler - run: sudo apt-get install -y protobuf-compiler + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler - name: Prepare cargo build run: | cargo check --profile ci --all-targets From e6589d4634de85456899c9bdb2557357f00ccfbd Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 23 Feb 2026 20:09:07 +0000 Subject: [PATCH 03/10] one more update --- .github/workflows/extended.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index e81e577af673a..fd950a1f65da0 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -109,7 +109,9 @@ jobs: source $HOME/.cargo/env rustup toolchain install - name: Install Protobuf Compiler - run: sudo apt-get install -y protobuf-compiler + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler - name: Rust Dependency Cache uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 with: From e12dd2a2264dfe4d66a959903c8139d45b5be87a Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 23 Feb 2026 20:12:36 +0000 Subject: [PATCH 04/10] runs on action! --- .github/workflows/extended.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index fd950a1f65da0..c54be7f0fd2fc 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -69,6 +69,7 @@ jobs: runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} # note: do not use amd/rust container to preserve disk space steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push @@ -96,6 +97,7 @@ jobs: # spot=false because the tests are long, https://runs-on.com/configuration/spot-instances/#disable-spot-pricing # note: do not use amd/rust container to preserve disk space steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push @@ -143,10 +145,11 @@ jobs: # Check answers are correct when hash values collide hash-collisions: name: cargo test hash collisions (amd64) - runs-on: ubuntu-latest + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} container: image: amd64/rust steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push @@ -169,6 +172,7 @@ jobs: container: image: amd64/rust steps: + - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3 - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push From c3216fc998d39a6ef4b8fcc56119c0d5c5687c37 Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 23 Feb 2026 21:38:15 +0000 Subject: [PATCH 05/10] no need for cache --- .github/workflows/extended.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index c54be7f0fd2fc..9768d475c9e8c 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -114,11 +114,6 @@ jobs: run: | sudo apt-get update sudo apt-get install -y protobuf-compiler - - name: Rust Dependency Cache - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - with: - save-if: true # TODO: set ${{ github.ref_name == 'main' }} - shared-key: "linux-tests-extended" # For debugging, test binaries can be large. - name: Show available disk space run: | @@ -182,11 +177,6 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable - - name: Rust Dependency Cache - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - with: - save-if: true # TODO: set ${{ github.ref_name == 'main' }} - shared-key: "sqllogictest-sqlite" - name: Run sqllogictest run: | cargo test --features backtrace,parquet_encryption --profile release-nonlto --test sqllogictests -- --include-sqlite From f15326f2331920bed6433c37aa3a795722945c47 Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 23 Feb 2026 21:38:38 +0000 Subject: [PATCH 06/10] Add CI progress logging for sqllogictests Print test thread count and available parallelism at startup to help debug CI performance issues. Also add periodic progress output for CI environments (no TTY) to show test completion status. Co-Authored-By: Claude Opus 4.5 --- datafusion/sqllogictest/bin/sqllogictests.rs | 30 +++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index 3571377354eb4..09df3ccda80a2 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -44,9 +44,11 @@ use datafusion::common::runtime::SpawnedTask; use futures::FutureExt; use std::ffi::OsStr; use std::fs; -use std::io::{IsTerminal, stdout}; +use std::io::{IsTerminal, stderr, stdout}; use std::path::{Path, PathBuf}; use std::str::FromStr; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; #[cfg(feature = "postgres")] mod postgres_container; @@ -110,6 +112,13 @@ async fn run_tests() -> Result<()> { options.warn_on_ignored(); + // Print parallelism info for debugging CI performance + eprintln!( + "Running with {} test threads (available parallelism: {})", + options.test_threads, + get_available_parallelism() + ); + #[cfg(feature = "postgres")] initialize_postgres_container(&options).await?; @@ -147,6 +156,10 @@ async fn run_tests() -> Result<()> { } let num_tests = test_files.len(); + // For CI environments without TTY, print progress periodically + let is_ci = !stderr().is_terminal(); + let completed_count = Arc::new(AtomicUsize::new(0)); + let errors: Vec<_> = futures::stream::iter(test_files) .map(|test_file| { let validator = if options.include_sqlite @@ -235,6 +248,21 @@ async fn run_tests() -> Result<()> { }) // run up to num_cpus streams in parallel .buffer_unordered(options.test_threads) + .inspect({ + let completed_count = Arc::clone(&completed_count); + move |_| { + let completed = completed_count.fetch_add(1, Ordering::Relaxed) + 1; + // In CI (no TTY), print progress every 10% or every 50 files + if is_ci && (completed % 50 == 0 || completed == num_tests) { + eprintln!( + "Progress: {}/{} files completed ({:.0}%)", + completed, + num_tests, + (completed as f64 / num_tests as f64) * 100.0 + ); + } + } + }) .flat_map(|(result, test_file_path, current_sql)| { // Filter out any Ok() leaving only the DataFusionErrors futures::stream::iter(match result { From 8de43d72327fd1fae82062d017ee59cb55be5929 Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 23 Feb 2026 21:51:06 +0000 Subject: [PATCH 07/10] Log slow test files (>30s) in sqllogictests Add timing information to identify which test files are taking the longest to complete, helping diagnose CI performance bottlenecks. Co-Authored-By: Claude Opus 4.5 --- datafusion/sqllogictest/bin/sqllogictests.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index 09df3ccda80a2..b6ee10e5cbb58 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -175,12 +175,14 @@ async fn run_tests() -> Result<()> { let filters = options.filters.clone(); let relative_path = test_file.relative_path.clone(); + let relative_path_for_timing = test_file.relative_path.clone(); let currently_running_sql_tracker = CurrentlyExecutingSqlTracker::new(); let currently_running_sql_tracker_clone = currently_running_sql_tracker.clone(); + let file_start = Instant::now(); SpawnedTask::spawn(async move { - match ( + let result = match ( options.postgres_runner, options.complete, options.substrait_round_trip, @@ -240,8 +242,17 @@ async fn run_tests() -> Result<()> { ) .await? } + }; + // Log slow files (>30s) for CI debugging + let elapsed = file_start.elapsed(); + if elapsed.as_secs() > 30 { + eprintln!( + "Slow file: {} took {:.1}s", + relative_path_for_timing.display(), + elapsed.as_secs_f64() + ); } - Ok(()) as Result<()> + Ok(result) }) .join() .map(move |result| (result, relative_path, currently_running_sql_tracker)) From d6ce63eb6f25102db62bb3e60c98fa8d03a6c2e6 Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 23 Feb 2026 21:56:25 +0000 Subject: [PATCH 08/10] Use c7i (Intel) for sqlite sqllogictests Switch from m8a.12xlarge (48 AMD cores) to c7i.4xlarge (16 Intel cores) for better single-thread performance. The sqlite tests are bottlenecked by large files running sequentially, so faster cores matter more than core count. Co-Authored-By: Claude Opus 4.5 --- .github/workflows/extended.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index 9768d475c9e8c..b74107007c3ad 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -162,8 +162,8 @@ jobs: sqllogictest-sqlite: name: "Run sqllogictests with the sqlite test suite" - runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=48,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion,spot=false', github.run_id) || 'ubuntu-latest' }} - # spot=false because the tests are long, https://runs-on.com/configuration/spot-instances/#disable-spot-pricing + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=c7i,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion,spot=false', github.run_id) || 'ubuntu-latest' }} + # c7i (Intel) for better single-thread performance; spot=false because tests are long container: image: amd64/rust steps: From caaf5f23bdd14a5b8b3fdb6310416f765e352f5c Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 23 Feb 2026 23:04:19 +0000 Subject: [PATCH 09/10] Revert "Use c7i (Intel) for sqlite sqllogictests" This reverts commit d6ce63eb6f25102db62bb3e60c98fa8d03a6c2e6. --- .github/workflows/extended.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index b74107007c3ad..9768d475c9e8c 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -162,8 +162,8 @@ jobs: sqllogictest-sqlite: name: "Run sqllogictests with the sqlite test suite" - runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=c7i,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion,spot=false', github.run_id) || 'ubuntu-latest' }} - # c7i (Intel) for better single-thread performance; spot=false because tests are long + runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=48,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion,spot=false', github.run_id) || 'ubuntu-latest' }} + # spot=false because the tests are long, https://runs-on.com/configuration/spot-instances/#disable-spot-pricing container: image: amd64/rust steps: From 936e85b91184d1ae15efdfc549bf17e333eaccea Mon Sep 17 00:00:00 2001 From: blaginin Date: Mon, 23 Feb 2026 23:07:51 +0000 Subject: [PATCH 10/10] clippy --- datafusion/sqllogictest/bin/sqllogictests.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index b6ee10e5cbb58..463b7b03a760c 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -182,7 +182,7 @@ async fn run_tests() -> Result<()> { currently_running_sql_tracker.clone(); let file_start = Instant::now(); SpawnedTask::spawn(async move { - let result = match ( + match ( options.postgres_runner, options.complete, options.substrait_round_trip, @@ -252,7 +252,7 @@ async fn run_tests() -> Result<()> { elapsed.as_secs_f64() ); } - Ok(result) + Ok(()) }) .join() .map(move |result| (result, relative_path, currently_running_sql_tracker)) @@ -264,7 +264,7 @@ async fn run_tests() -> Result<()> { move |_| { let completed = completed_count.fetch_add(1, Ordering::Relaxed) + 1; // In CI (no TTY), print progress every 10% or every 50 files - if is_ci && (completed % 50 == 0 || completed == num_tests) { + if is_ci && (completed.is_multiple_of(50) || completed == num_tests) { eprintln!( "Progress: {}/{} files completed ({:.0}%)", completed,