diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..4705c14e --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,106 @@ +name: Benchmarks + +permissions: + contents: read + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +env: + CARGO_TERM_COLOR: always + +jobs: + bench_main: + if: github.event_name == 'push' + permissions: + checks: write + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + os: ubuntu-latest + runs-on: ${{ matrix.os }} + + steps: + - name: Harden Runner + uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2 + with: + egress-policy: audit + + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Install Rust + uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 # stable + with: + toolchain: stable + targets: ${{ matrix.target }} + + - name: Rust Cache + uses: swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5 + + - uses: bencherdev/bencher@main + + - name: Track base branch benchmarks with Bencher + run: | + bencher run \ + --project keystone \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch main \ + --testbed ${{ matrix.os }} \ + --threshold-measure latency \ + --threshold-test t_test \ + --threshold-max-sample-size 64 \ + --threshold-upper-boundary 0.99 \ + --thresholds-reset \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --adapter rust_criterion \ + cargo bench + + bench_pr: + # DO NOT REMOVE: For handling Fork PRs see Pull Requests from Forks + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' + permissions: + pull-requests: write + strategy: + matrix: + include: + - target: x86_64-unknown-linux-gnu + os: ubuntu-latest + runs-on: ${{ matrix.os }} + + steps: + - name: Harden Runner + uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2 + with: + egress-policy: audit + + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Install Rust + uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 # stable + with: + toolchain: stable + targets: ${{ matrix.target }} + + - name: Rust Cache + uses: swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5 + + - uses: bencherdev/bencher@main + + - name: Track base branch benchmarks with Bencher + run: | + bencher run \ + --project keystone \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch "$GITHUB_HEAD_REF" \ + --start-point "$GITHUB_BASE_REF" \ + --start-point-hash '${{ github.event.pull_request.base.sha }}' \ + --start-point-clone-thresholds \ + --start-point-reset \ + --testbed ${{ matrix.os }} \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --adapter rust_criterion \ + cargo bench diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77de3fa5..b9eb3980 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,95 +49,3 @@ jobs: - name: Run Doc tests run: cargo test --doc - - bench_main: - if: github.event_name == 'push' - permissions: - checks: write - strategy: - matrix: - include: - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - runs-on: ${{ matrix.os }} - - steps: - - name: Harden Runner - uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2 - with: - egress-policy: audit - - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Install Rust - uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 # stable - with: - toolchain: stable - targets: ${{ matrix.target }} - - - name: Rust Cache - uses: swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5 - - - uses: bencherdev/bencher@main - - - name: Track base branch benchmarks with Bencher - run: | - bencher run \ - --project keystone \ - --token '${{ secrets.BENCHER_API_TOKEN }}' \ - --branch main \ - --testbed ${{ matrix.os }} \ - --threshold-measure latency \ - --threshold-test t_test \ - --threshold-max-sample-size 64 \ - --threshold-upper-boundary 0.99 \ - --thresholds-reset \ - --github-actions '${{ secrets.GITHUB_TOKEN }}' \ - --adapter rust_criterion \ - cargo bench - - bench_pr: - # DO NOT REMOVE: For handling Fork PRs see Pull Requests from Forks - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository - permissions: - pull-requests: write - strategy: - matrix: - include: - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - runs-on: ${{ matrix.os }} - - steps: - - name: Harden Runner - uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2 - with: - egress-policy: audit - - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Install Rust - uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 # stable - with: - toolchain: stable - targets: ${{ matrix.target }} - - - name: Rust Cache - uses: swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5 - - - uses: bencherdev/bencher@main - - - name: Track base branch benchmarks with Bencher - run: | - bencher run \ - --project keystone \ - --token '${{ secrets.BENCHER_API_TOKEN }}' \ - --branch "$GITHUB_HEAD_REF" \ - --start-point "$GITHUB_BASE_REF" \ - --start-point-hash '${{ github.event.pull_request.base.sha }}' \ - --start-point-clone-thresholds \ - --start-point-reset \ - --testbed ${{ matrix.os }} \ - --github-actions '${{ secrets.GITHUB_TOKEN }}' \ - --adapter rust_criterion \ - cargo bench diff --git a/.github/workflows/fork_pr_benchmark_run.yml b/.github/workflows/fork_pr_benchmark_run.yml new file mode 100644 index 00000000..9d4c3b80 --- /dev/null +++ b/.github/workflows/fork_pr_benchmark_run.yml @@ -0,0 +1,39 @@ +name: Run Benchmarks + +on: + pull_request: + types: [opened, reopened, edited, synchronize] + +jobs: + benchmark_fork_pr_branch: + name: Run Fork PR Benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Install Rust + uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 # stable + with: + toolchain: stable + targets: ${{ matrix.target }} + + - name: Rust Cache + uses: swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5 + + - uses: bencherdev/bencher@main + + - name: Track base branch benchmarks with Bencher + run: | + cargo bench > benchmark_results.log + + - name: Upload Benchmark Results + uses: actions/upload-artifact@v4 + with: + name: benchmark_results.log + path: ./benchmark_results.log + + - name: Upload GitHub Pull Request Event + uses: actions/upload-artifact@v4 + with: + name: event.json + path: ${{ github.event_path }} diff --git a/.github/workflows/fork_pr_benchmark_track.yml b/.github/workflows/fork_pr_benchmark_track.yml new file mode 100644 index 00000000..4f23b96c --- /dev/null +++ b/.github/workflows/fork_pr_benchmark_track.yml @@ -0,0 +1,59 @@ +name: Track Benchmarks with Bencher + +on: + workflow_run: + workflows: [Run Benchmarks] + types: [completed] + +jobs: + track_fork_pr_branch: + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + env: + BENCHMARK_RESULTS: benchmark_results.log + PR_EVENT: event.json + steps: + - name: Harden Runner + uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2 + with: + egress-policy: audit + + - name: Download Benchmark Results + uses: dawidd6/action-download-artifact@v6 + with: + name: ${{ env.BENCHMARK_RESULTS }} + run_id: ${{ github.event.workflow_run.id }} + + - name: Download PR Event + uses: dawidd6/action-download-artifact@v6 + with: + name: ${{ env.PR_EVENT }} + run_id: ${{ github.event.workflow_run.id }} + + - name: Export PR Event Data + uses: actions/github-script@v6 + with: + script: | + let fs = require('fs'); + let prEvent = JSON.parse(fs.readFileSync(process.env.PR_EVENT, {encoding: 'utf8'})); + core.exportVariable("PR_HEAD", prEvent.pull_request.head.ref); + core.exportVariable("PR_BASE", prEvent.pull_request.base.ref); + core.exportVariable("PR_BASE_SHA", prEvent.pull_request.base.sha); + core.exportVariable("PR_NUMBER", prEvent.number); + + - uses: bencherdev/bencher@main + - name: Track Benchmarks with Bencher + run: | + bencher run \ + --project keystone \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch "$PR_HEAD" \ + --start-point "$PR_BASE" \ + --start-point-hash "$PR_BASE_SHA" \ + --start-point-clone-thresholds \ + --start-point-reset \ + --testbed ubuntu-latest \ + --adapter rust_criterion \ + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --ci-number "$PR_NUMBER" \ + --file "$BENCHMARK_RESULTS"