Use the gungraun benchmark framework to get reliable instruction count numbers on CI

.cargo/config.toml

@@ -1,2 +1,3 @@
 [alias]
 uniffi-bindgen = ["run", "--package", "uniffi-bindgen", "--"]
+benchmark = ["bench", "--manifest-path", "glean-core/benchmark/Cargo.toml"]

.github/workflows/bench.yml

@@ -0,0 +1,27 @@
+name: bench
+on:
+  pull_request:
+  push:
+    branches:
+      main
+
+jobs:
+  bench:
+    name: Benchmarks
+    environment: ${{ github.ref_name == 'main' && 'production' || 'pull_request' }}
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+    - uses: dtolnay/rust-toolchain@stable
+    - uses: Swatinem/rust-cache@v2
+    - name: Set up dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y valgrind gdb libc6-dbg # Needed for gungraun
+    - name: Install gungraun-runner
+      run: |
+        cargo install gungraun-runner --version 0.17.0
+
+    - name: Benchmark
+      run: |
+        cargo benchmark --bench instruction-count --features gungraun

bin/benchmark-compare.sh

@@ -33,14 +33,14 @@ BASELINE_NAME="bn-${END_COMMIT}"
 stash_commit=$(git stash create)
 
 run git checkout "$START_COMMIT"
-run cargo bench -p glean-core --features benchmark -- --save-baseline "$BASELINE_NAME"
+run cargo benchmark --bench dispatcher -- --save-baseline "$BASELINE_NAME"
 
 run git checkout "$END_COMMIT"
-run cargo bench -p glean-core --features benchmark -- --baseline "$BASELINE_NAME"
+run cargo benchmark --bench dispatcher -- --baseline "$BASELINE_NAME"
 
 if [[ -n "$stash_commit" ]]; then
   git stash apply "$stash_commit"
 fi
 
 run echo "You can rerun the benchmark against the baseline using:"
-run echo "cargo bench -p glean-core -- --baseline $BASELINE_NAME"
+run echo "cargo benchmark --bench dispatcher -- --baseline $BASELINE_NAME"

docs/dev/core/internal/benchmarks.md

@@ -20,16 +20,16 @@ We use [criterion] for micro-benchmarking of the Rust code directly.
 You can run all benchmarks using:
 
 ```
-cargo bench -p glean-core --features benchmark
+cargo benchmark
 ```
 
 To run an individual benchmark pass its name after `--bench`:
 
 ```
-cargo bench -p glean-core --features benchmark --bench dispatcher
+cargo benchmark --bench dispatcher
 ```
 
-To add a new benchmark create a file in `glean-core/benches`, then add a new entry to the `glean-core/Cargo.toml` file:
+To add a new benchmark create a file in `glean-core/benchmark/benches`, then add a new entry to the `glean-core/benchmark/Cargo.toml` file:
 
 ```toml
 [[bench]]
@@ -49,9 +49,10 @@ bin/benchmark-compare.sh main HEAD
 The output will be similar to this:
 
 ```
-   Compiling glean-core v66.1.1 (/home/jer/src/mozilla/glean/glean-core)
+   Compiling glean-core v66.3.0 (/home/user/src/glean/glean-core)
+   Compiling benchmark v0.1.0 (/home/user/jer/src/glean/glean-core/benchmark)
     Finished `bench` profile [optimized] target(s) in 18.62s
-     Running benches/dispatcher.rs (target/release/deps/dispatcher-b3ccfcc39ee552d0)
+     Running benches/dispatcher.rs (glean-core/benchmark/target/release/deps/dispatcher-a44e0c186f62af70)
 Gnuplot not found, using plotters backend
 empty fn                time:   [61.181 ns 61.231 ns 61.285 ns]
                         change: [−23.764% −23.575% −23.390%] (p = 0.00 < 0.05)

glean-core/Cargo.toml

@@ -13,7 +13,6 @@ include = [
   "/LICENSE",
   "/src",
   "/examples",
-  "/benches",
   "/tests",
   "/Cargo.toml",
   "/uniffi.toml",
@@ -68,10 +67,3 @@ enable_env_logger = ["env_logger"]
 gecko = []
 # Expose some internals for easier benchmarking
 benchmark = []
-
-[lib]
-bench = false
-
-[[bench]]
-name = "dispatcher"
-harness = false