ci: cache rust dependencies, enable parallelism (1.9x to 5.5x faster)

.github/workflows/java.yml

@@ -14,14 +14,7 @@ env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
   # key, so we set it to make sure it is always consistent.
   CARGO_TERM_COLOR: always
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
   RUST_BACKTRACE: "1"
-  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
-  # CI builds are faster with incremental disabled.
-  CARGO_INCREMENTAL: "0"
-  CARGO_BUILD_JOBS: "1"
 
 jobs:
   rust-clippy-fmt:
@@ -88,8 +81,6 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: java/lance-jni -> ../target/rust-maven-plugin/lance-jni
-          cache-targets: false
-          cache-workspace-crates: true
       - name: Set up Java ${{ matrix.java-version }}
         uses: actions/setup-java@v4
         with:

.github/workflows/python.yml

@@ -23,9 +23,6 @@ env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
   # key, so we set it to make sure it is always consistent.
   CARGO_TERM_COLOR: always
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=line-tables-only"
   RUST_BACKTRACE: "1"
   CI: "true"
   # Color output for pytest is off by default.
@@ -60,8 +57,6 @@ jobs:
         with:
           workspaces: python
           prefix-key: ${{ env.CACHE_PREFIX }}
-          cache-targets: false
-          cache-workspace-crates: true
       - name: Install linting tools
         run: |
           pip install ruff==0.11.2 maturin tensorflow tqdm ray[data] pyright datasets polars[pyarrow,pandas]
@@ -79,14 +74,14 @@ jobs:
         run: |
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -`
           cargo fmt --all -- --check
-          cargo clippy --locked --features ${ALL_FEATURES} --tests -- -D warnings
+          cargo clippy  --profile ci --locked --features ${ALL_FEATURES} --tests -- -D warnings
       - name: Build
         run: |
           python -m venv venv
           source venv/bin/activate
           pip install torch tqdm --index-url https://download.pytorch.org/whl/cpu
           pip install maturin
-          maturin develop --locked --extras tests,ray
+          maturin develop --profile ci --locked --extras tests,ray
       - name: Run doctest
         run: |
           source venv/bin/activate
@@ -115,9 +110,9 @@ jobs:
         with:
           workspaces: python
           prefix-key: ${{ env.CACHE_PREFIX }}
-          cache-targets: false
-          cache-workspace-crates: true
       - uses: ./.github/workflows/build_linux_wheel
+        with:
+          args: "--profile ci"
       - uses: ./.github/workflows/run_tests
       - name: Upload wheels as artifacts
         if: ${{ matrix.python-minor-version == '13' }}
@@ -179,12 +174,11 @@ jobs:
         with:
           workspaces: python
           prefix-key: ${{ env.CACHE_PREFIX }}
-          cache-targets: false
-          cache-workspace-crates: true
       - uses: ./.github/workflows/build_linux_wheel
         with:
           arm-build: "true"
           manylinux: "2_28"
+          args: "--profile ci"
       - name: Install dependencies
         run: |
           sudo apt update -y -qq
@@ -211,9 +205,9 @@ jobs:
         with:
           workspaces: python
           prefix-key: ${{ env.CACHE_PREFIX }}
-          cache-targets: false
-          cache-workspace-crates: true
       - uses: ./.github/workflows/build_mac_wheel
+        with:
+          args: "--profile ci"
       - uses: ./.github/workflows/run_tests
         with:
           skip-torch: "true"
@@ -234,9 +228,9 @@ jobs:
         with:
           workspaces: python
           prefix-key: ${{ env.CACHE_PREFIX }}
-          cache-targets: false
-          cache-workspace-crates: true
       - uses: ./.github/workflows/build_windows_wheel
+        with:
+          args: "--profile ci"
       - uses: ./.github/workflows/run_tests
   aws-integtest:
     timeout-minutes: 45
@@ -258,9 +252,9 @@ jobs:
         with:
           workspaces: python
           prefix-key: ${{ env.CACHE_PREFIX }}
-          cache-targets: false
-          cache-workspace-crates: true
       - uses: ./.github/workflows/build_linux_wheel
+        with:
+          args: "--profile ci"
       - name: Install dependencies
         run: |
           pip install ray[data]

.github/workflows/rust.yml

@@ -20,14 +20,7 @@ env:
   # This env var is used by Swatinem/rust-cache@v2 for the cache
   # key, so we set it to make sure it is always consistent.
   CARGO_TERM_COLOR: always
-  # Disable full debug symbol generation to speed up CI build and keep memory down
-  # "1" means line tables only, which is useful for panic tracebacks.
-  RUSTFLAGS: "-C debuginfo=1"
   RUST_BACKTRACE: "1"
-  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
-  # CI builds are faster with incremental disabled.
-  CARGO_INCREMENTAL: "0"
-  CARGO_BUILD_JOBS: "1"
 
 jobs:
   format:
@@ -47,9 +40,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
-        with:
-          cache-targets: false
-          cache-workspace-crates: true
       - name: Install dependencies
         run: |
           sudo apt update
@@ -59,7 +49,7 @@ jobs:
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -`
           echo "ALL_FEATURES=${ALL_FEATURES}" >> $GITHUB_ENV
       - name: Clippy
-        run: cargo clippy --locked --features ${{ env.ALL_FEATURES }} --all-targets -- -D warnings
+        run: cargo clippy --profile ci --locked --features ${{ env.ALL_FEATURES }} --all-targets -- -D warnings
 
   cargo-deny:
     name: Check Rust dependencies (cargo-deny)
@@ -72,8 +62,7 @@ jobs:
           command: check
 
   linux-build:
-    # Make our CI happy until https://github.com/lancedb/lance/issues/5218 addressed.
-    runs-on: warp-ubuntu-2404-x64-8x
+    runs-on: "ubuntu-24.04"
     timeout-minutes: 60
     strategy:
       matrix:
@@ -92,9 +81,6 @@ jobs:
           rustup default ${{ matrix.toolchain }}
       - uses: rui314/setup-mold@v1
       - uses: Swatinem/rust-cache@v2
-        with:
-          cache-targets: false
-          cache-workspace-crates: true
       - name: Install dependencies
         run: |
           sudo apt update
@@ -107,7 +93,7 @@ jobs:
         if: ${{ matrix.toolchain == 'stable' }}
         run: |
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -`
-          cargo llvm-cov  --locked --workspace --codecov --output-path coverage.codecov --features ${ALL_FEATURES}
+          cargo llvm-cov --profile ci --locked --workspace --codecov --output-path coverage.codecov --features ${ALL_FEATURES}
       - name: Upload coverage to Codecov
         if: ${{ matrix.toolchain == 'stable' }}
         uses: codecov/codecov-action@v4
@@ -128,23 +114,20 @@ jobs:
           rustup default stable
       - uses: rui314/setup-mold@v1
       - uses: Swatinem/rust-cache@v2
-        with:
-          cache-targets: false
-          cache-workspace-crates: true
       - name: Install dependencies
         run: |
           sudo apt -y -qq update
           sudo apt install -y protobuf-compiler libssl-dev pkg-config
       - name: Build tests
         run: |
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -`
-          cargo test --locked --features ${ALL_FEATURES} --no-run
+          cargo test --profile ci --locked --features ${ALL_FEATURES} --no-run
       - name: Start DynamodDB and S3
         run: docker compose -f docker-compose.yml up -d --wait
       - name: Run tests
         run: |
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -`
-          cargo test --locked --features ${ALL_FEATURES}
+          cargo test --profile ci --locked --features ${ALL_FEATURES}
   build-no-lock:
     runs-on: warp-ubuntu-2404-x64-8x
     timeout-minutes: 30
@@ -165,7 +148,7 @@ jobs:
       - name: Build all
         run: |
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -`
-          cargo build --benches --features ${ALL_FEATURES} --tests
+          cargo build --profile ci --benches --features ${ALL_FEATURES} --tests
   mac-build:
     runs-on: "warp-macos-14-arm64-6x"
     timeout-minutes: 45
@@ -189,18 +172,15 @@ jobs:
         run: |
           rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
       - uses: Swatinem/rust-cache@v2
-        with:
-          cache-targets: false
-          cache-workspace-crates: true
       - name: Build tests
         run: |
-          cargo test --locked --features fp16kernels,cli,tensorflow,dynamodb,substrait --no-run
+          cargo test --profile ci --locked --features fp16kernels,cli,tensorflow,dynamodb,substrait --no-run
       - name: Run tests
         run: |
-          cargo test --features fp16kernels,cli,tensorflow,dynamodb,substrait
+          cargo test --profile ci --features fp16kernels,cli,tensorflow,dynamodb,substrait
       - name: Check benchmarks
         run: |
-          cargo check --benches --features fp16kernels,cli,tensorflow,dynamodb,substrait
+          cargo check --profile ci --benches --features fp16kernels,cli,tensorflow,dynamodb,substrait
   windows-build:
     runs-on: warp-windows-latest-x64-4x
     defaults:
@@ -209,9 +189,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
-        with:
-          cache-targets: false
-          cache-workspace-crates: true
       - name: Install Protoc v21.12
         working-directory: C:\
         run: |
@@ -222,11 +199,11 @@ jobs:
           Add-Content $env:GITHUB_PATH "C:\protoc\bin"
         shell: powershell
       - name: Build tests
-        run: cargo test --locked --no-run
+        run: cargo test --profile ci --locked --no-run
       - name: Run tests
-        run: cargo test
+        run: cargo test --profile ci
       - name: Check benchmarks
-        run: cargo check --benches
+        run: cargo check --profile ci --benches
 
   msrv:
     # Check the minimum supported Rust version
@@ -244,9 +221,6 @@ jobs:
         with:
           submodules: true
       - uses: Swatinem/rust-cache@v2
-        with:
-          cache-targets: false
-          cache-workspace-crates: true
       - name: Install dependencies
         run: |
           sudo apt update
@@ -258,4 +232,4 @@ jobs:
       - name: cargo +${{ matrix.msrv }} check
         run: |
           ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -`
-          cargo check --workspace --tests --benches --features ${ALL_FEATURES}
+          cargo check --profile ci --workspace --tests --benches --features ${ALL_FEATURES}

Cargo.toml

@@ -193,6 +193,20 @@ opt-level = 3
 debug = true
 strip = false
 
+[profile.ci]
+debug = "line-tables-only"
+inherits = "dev"
+incremental = false
+
+# This rule applies to every package except workspace members (dependencies
+# such as `arrow` and `tokio`). It disables debug info and related features on
+# dependencies so their binaries stay smaller, improving cache reuse.
+[profile.ci.package."*"]
+debug = false
+debug-assertions = false
+strip = "debuginfo"
+incremental = false
+
 [workspace.lints.clippy]
 all = { level = "deny", priority = -1 }
 style = { level = "deny", priority = -1 }

java/lance-jni/Cargo.toml

@@ -42,3 +42,16 @@ prost = "0.13.5"
 roaring = "0.10.1"
 prost-types = "0.13.5"
 chrono = "0.4.41"
+
+[profile.dev]
+debug = "line-tables-only"
+incremental = false
+
+# This rule applies to every package except workspace members (dependencies
+# such as `arrow` and `tokio`). It disables debug info and related features on
+# dependencies so their binaries stay smaller, improving cache reuse.
+[profile.dev.package."*"]
+debug = false
+debug-assertions = false
+strip = "debuginfo"
+incremental = false

python/Cargo.toml

@@ -79,3 +79,17 @@ datagen = ["lance-datagen"]
 fp16kernels = ["lance/fp16kernels"]
 rest = ["lance-namespace-impls/rest"]
 rest-adapter = ["lance-namespace-impls/rest-adapter"]
+
+[profile.ci]
+debug = "line-tables-only"
+inherits = "dev"
+incremental = false
+
+# This rule applies to every package except workspace members (dependencies
+# such as `arrow` and `tokio`). It disables debug info and related features on
+# dependencies so their binaries stay smaller, improving cache reuse.
+[profile.ci.package."*"]
+debug = false
+debug-assertions = false
+strip = "debuginfo"
+incremental = false