Add the devhub command: Gather metrics and upload them to a devhub database

.cargo/config.toml

@@ -1,3 +1,4 @@
 [alias]
 uniffi-bindgen = ["run", "--package", "uniffi-bindgen", "--"]
 benchmark = ["bench", "--manifest-path", "glean-core/benchmark/Cargo.toml"]
+devhub = ["run", "--manifest-path", "tools/devhub/Cargo.toml", "--"]

.dictionary

@@ -1,4 +1,4 @@
-personal_ws-1.1 en 312 utf-8
+personal_ws-1.1 en 317 utf-8
 AAR
 AARs
 ABI
@@ -24,6 +24,7 @@ DOM
 Datetime
 Datetimes
 Dependabot
+Devhub
 Droettboom
 FOG
 FTE
@@ -69,6 +70,7 @@ Mutex
 NDK
 POSIX
 PRs
+PerfHerder
 PingEncryptionPlugin
 PowerShell
 ProGuard
@@ -92,6 +94,7 @@ TSan
 TWiG
 TaskCluster
 Taskcluster
+TigerBeetle's
 UA
 UI
 URIs
@@ -161,6 +164,8 @@ datetime
 destructor
 deterministically
 dev
+devhub
+devhubdb
 devtools
 dexter
 distributable

.github/workflows/bench.yml

@@ -22,6 +22,59 @@ jobs:
       run: |
         cargo install gungraun-runner --version 0.17.0
 
+    - name: Benchmark (machine-readable)
+      if: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+      run: |
+        set -x
+        cargo benchmark --bench instruction-count --features gungraun -- --output-format=json > gungraun-output.json
+        cat gungraun-output.json
+
     - name: Benchmark
+      if: ${{ github.event_name == 'pull_request' }}
       run: |
+        set -x
         cargo benchmark --bench instruction-count --features gungraun
+
+    - uses: actions/upload-artifact@v6
+      if: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+      with:
+        name: gungraun-output
+        path: gungraun-output.json
+
+    - name: Prepare devhub
+      if: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+      run: |
+        cargo install tokei --version 14.0.0
+        cargo install ripgrep --version 15.1.0
+
+        wget https://github.com/jqlang/jq/releases/download/jq-1.8.1/jq-linux-amd64 -O jq
+        echo "020468de7539ce70ef1bceaf7cde2e8c4f2ca6c3afb84642aabc5c97d9fc2a0d  jq" | sha256sum -c
+        mv jq $HOME/.cargo/bin/jq
+        chmod +x $HOME/.cargo/bin/jq
+
+    - name: Show dependencies
+      if: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+      run: |
+        set -x
+        rustc -vV
+        cargo -vV
+        valgrind --version
+        tokei --version
+        rg --version
+        jq --version
+
+    - name: Run devhub
+      if: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+      run: |
+        echo "commit sha ${GITHUB_SHA}"
+        cargo devhub run
+        cat data.json
+
+    - name: Commit new devhub data
+      if: ${{ github.event_name == 'push' && github.ref_name == 'main' }}
+      env:
+        DEVHUBDB_TOKEN: ${{ secrets.DEVHUBDB_TOKEN }}
+      run: |
+        git config --global user.name "Devhub Bot"
+        git config --global user.email devhub-bot@mozilla.com
+        cargo devhub commit data.json

Makefile

@@ -203,12 +203,12 @@ upload-wheels: setup-python
 		$(GLEAN_PYENV)/bin/python3 -m twine upload target/wheels/*
 .PHONY: upload-wheels
 
-clean: ## Clean up the object directories (alias for clobber)
-clobber:  ## Clean up the object directories
+clean: clobber  ## Clean up the object directories and build artifacts (alias for clobber)
+clobber:  ## Clean up the object directories and build artifacts
 	# Rust
 	cargo clean
-	cargo clean --manifest-path tools/devhub/Cargo.toml
 	cargo clean --manifest-path glean-core/benchmark/Cargo.toml
+	cargo clean --manifest-path tools/devhub/Cargo.toml
 	# General build folder
 	rm -rf build
 	# Swift artifacts

bin/build-rust-docs.sh

@@ -90,6 +90,9 @@ cp -a docs/dev/book/. build/docs/dev
 mkdir -p build/docs/shared
 cp -a docs/shared/. build/docs/shared
 
+mkdir -p build/docs/devhub
+cp -a tools/devhub/www/. build/docs/devhub
+
 mkdir -p build/docs/docs
 cp -a target/doc/. build/docs/docs
 printf '<meta http-equiv=refresh content=0;url=%s/index.html>\n' "$CRATE_NAME" > build/docs/docs/index.html

tools/devhub/Cargo.toml

@@ -0,0 +1,13 @@
+[workspace]
+# Its own workspace to keep its dependencies separate
+
+[package]
+name = "devhub"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+serde = { version = "1.0", features = ["derive", "serde_derive"] }
+serde_json = "1.0.145"
+xflags = "0.3.2"
+xshell = "0.2.7"

tools/devhub/README.md

@@ -0,0 +1,38 @@
+# Glean Devhub
+
+Collect different metrics from the Glean SDK build and publish them.
+Results will be displayed on <https://mozilla.github.io/glean/devhub>.
+
+* This script is run by the CI infrastructure on every merge to `main`.
+* It collects a set of "metrics", where a "metric" can be anything that can be measured with a single numeric value.
+* The results of all measurements are serialized as a single JSON object.
+* The key part: this JSON is then stored in a "distributed database" for our visualization
+  front-end to pick up. This "database" is just a newline-delimited JSON file in a Git repository.
+
+To generate a `DEVHUBDB_TOKEN` (used on CI to publish to the database repository):
+
+1. Go to <https://github.com/settings/personal-access-tokens/new>
+2. Fill out token name (e.g. "Glean CI devhubdb token").
+3. Resource owner: "mozilla"
+4. Expiry: "366 days" (maximum available)
+5. Repository access: "Only select repositories"
+6. Select repositories: "mozilla/glean-devhubdb"
+7. Add permissions: "Metadata"
+8. Add permissions: "Contents"; Access: "Read and write".
+9. "Generate token".
+10. (Copy token.)
+
+To update token in Glean CI:
+
+1. <https://github.com/mozilla/glean/settings/environments>
+2. Click "production".
+3. Environment Secrets > Edit `DEVHUBDB_TOKEN`
+4. Paste token; "Update secret"
+
+## References
+
+This code is based on ideas from:
+
+* [TigerBeetle's devhub](https://github.com/tigerbeetle/tigerbeetle/blob/b6d541562290f23c10760ea20b559cf21b9010b0/src/scripts/devhub.zig)
+* [PerfHerder](https://treeherder.mozilla.org/perfherder/),
+  and specifically [`PerfStats`](https://firefox-source-docs.mozilla.org/performance/perfstats.html) and [`MozGTestBench`](https://firefox-source-docs.mozilla.org/gtest/index.html#mozgtestbench)

tools/devhub/src/benchmarks.rs

@@ -0,0 +1,63 @@
+use serde::Deserialize;
+use xshell::{Shell, cmd};
+
+use super::{Metric, MetricRecorder, Result};
+
+/// Extract the listed metrics out from all benchmarks, grouped by the benchmark name.
+const JQ_SCRIPT: &str = r#"
+  .profiles[0].summaries.parts[0].metrics_summary.Cachegrind as $cachegrind
+| .function_name + " " + .details as $name
+| [ "Ir", "EstimatedCycles", "TotalRW", "L1hits", "LLhits", "RamHits" ] as $keys
+| $keys
+| map({
+  name: $name + " -- " + .,
+  value: $cachegrind[.].metrics.Both[0].Int
+})
+"#;
+
+/// Parse Gungraun benchmark results
+///
+/// Expects a `gungraun-output.json` file in the current directory.
+pub struct Benchmark;
+
+impl MetricRecorder for Benchmark {
+    fn name(&self) -> &'static str {
+        "benchmark"
+    }
+
+    fn description(&self) -> &'static str {
+        "Gungraun Benchmark Results"
+    }
+
+    fn record(&self, sh: &Shell) -> Result<Vec<Metric>> {
+        let temp = sh.create_temp_dir()?;
+
+        let jq_script_path = temp.path().join("transform-gungraun.jq");
+        sh.write_file(&jq_script_path, JQ_SCRIPT)?;
+
+        let input = sh.current_dir().join("gungraun-output.json");
+
+        let mut metrics = Vec::new();
+
+        let benchmarks = cmd!(sh, "jq -cf {jq_script_path} {input}").read()?;
+        let benchmarks = benchmarks.lines();
+        for line in benchmarks {
+            let bench: Vec<Bench> = serde_json::from_str(line)?;
+
+            for b in bench {
+                metrics.push(Metric {
+                    name: b.name,
+                    value: b.value,
+                })
+            }
+        }
+
+        Ok(metrics)
+    }
+}
+
+#[derive(Deserialize)]
+struct Bench {
+    name: String,
+    value: u64,
+}