Feat: add the exec subcommand to run an arbitrary rust binary

.config/nextest.toml

@@ -0,0 +1,8 @@
+# Valgrind and memtrack tests must never run concurrently, even across multiple processes, else valgrind crashes.
+# We have a semaphore setup in the test code, but it's not sufficient since nextest runs tests in multiple processes.
+[test-groups]
+bpf-instrumentation = { max-threads = 1 }
+
+[[profile.default.overrides]]
+filter = 'test(~executor::tests::valgrind) | test(~executor::tests::memory)'
+test-group = 'bpf-instrumentation'

.gitattributes

@@ -1,2 +1,2 @@
 testdata/perf_map/* filter=lfs diff=lfs merge=lfs -text
-src/run/runner/wall_time/perf/snapshots/*.snap filter=lfs diff=lfs merge=lfs -text
+*.snap filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,3 +1,2 @@
 /target
 .DS_Store
-samples

Cargo.toml

@@ -80,7 +80,7 @@ rstest_reuse = "0.7.0"
 shell-quote = "0.7.2"
 
 [workspace]
-members = ["crates/runner-shared", "crates/memtrack"]
+members = ["crates/runner-shared", "crates/memtrack", "crates/exec-harness"]
 
 [workspace.dependencies]
 anyhow = "1.0"

crates/exec-harness/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "exec-harness"
+version = "4.4.2-beta.1"
+edition = "2024"
+
+[dependencies]
+anyhow = { workspace = true }
+codspeed = "4.1.0"
+clap = { workspace = true }
+serde_json = { workspace = true }
+serde = { workspace = true }

crates/exec-harness/src/main.rs

@@ -0,0 +1,105 @@
+use crate::walltime::WalltimeResults;
+use anyhow::Context;
+use anyhow::Result;
+use anyhow::bail;
+use clap::Parser;
+use codspeed::instrument_hooks::InstrumentHooks;
+use codspeed::walltime_results::WalltimeBenchmark;
+use std::path::PathBuf;
+use std::process;
+
+mod walltime;
+
+#[derive(Parser, Debug)]
+#[command(name = "exec-harness")]
+#[command(about = "CodSpeed exec harness - wraps commands with performance instrumentation")]
+struct Args {
+    /// Optional benchmark name (defaults to command filename)
+    #[arg(long)]
+    name: Option<String>,
+
+    /// The command and arguments to execute
+    command: Vec<String>,
+}
+
+fn main() -> Result<()> {
+    let args = Args::parse();
+
+    if args.command.is_empty() {
+        bail!("Error: No command provided");
+    }
+
+    // Derive benchmark name from command if not provided
+    let bench_name = args.name.unwrap_or_else(|| {
+        // Extract filename from command path
+        let cmd = &args.command[0];
+        std::path::Path::new(cmd).to_string_lossy().into_owned()
+    });
+
+    // TODO: Better URI generation
+    let bench_uri = format!("standalone_run::{bench_name}");
+
+    let hooks = InstrumentHooks::instance();
+
+    // TODO: Stop impersonating codspeed-rust 🥸
+    hooks
+        .set_integration("codspeed-rust", env!("CARGO_PKG_VERSION"))
+        .unwrap();
+
+    const NUM_ITERATIONS: usize = 1;
+    let mut times_per_round_ns = Vec::with_capacity(NUM_ITERATIONS);
+
+    hooks.start_benchmark().unwrap();
+    for _ in 0..NUM_ITERATIONS {
+        // Spawn the command
+        let mut child = process::Command::new(&args.command[0])
+            .args(&args.command[1..])
+            .spawn()
+            .context("Failed to spawn command")?;
+
+        // Start monotonic timer for this iteration
+        let bench_start = InstrumentHooks::current_timestamp();
+
+        // Wait for the process to complete
+        let status = child.wait().context("Failed to wait for command")?;
+
+        // Measure elapsed time
+        let bench_end = InstrumentHooks::current_timestamp();
+        hooks.add_benchmark_timestamps(bench_start, bench_end);
+
+        // Exit immediately if any iteration fails
+        if !status.success() {
+            bail!("Command failed with exit code: {:?}", status.code());
+        }
+
+        // Calculate and store the elapsed time in nanoseconds
+        let elapsed_ns = (bench_end - bench_start) as u128;
+        times_per_round_ns.push(elapsed_ns);
+    }
+
+    hooks.stop_benchmark().unwrap();
+    hooks.set_executed_benchmark(&bench_uri).unwrap();
+
+    // Collect walltime results
+    let max_time_ns = times_per_round_ns.iter().copied().max();
+    let walltime_benchmark = WalltimeBenchmark::from_runtime_data(
+        bench_name.clone(),
+        bench_uri.clone(),
+        vec![1; NUM_ITERATIONS],
+        times_per_round_ns,
+        max_time_ns,
+    );
+
+    let walltime_results = WalltimeResults::from_benchmarks(vec![walltime_benchmark])
+        .expect("Failed to create walltime results");
+
+    walltime_results
+        .save_to_file(
+            std::env::var("CODSPEED_PROFILE_FOLDER")
+                .map(PathBuf::from)
+                .unwrap_or_else(|_| std::env::current_dir().unwrap().join(".codspeed")),
+        )
+        .expect("Failed to save walltime results");
+
+    Ok(())
+}

crates/exec-harness/src/walltime.rs

@@ -0,0 +1,63 @@
+use anyhow::Context;
+use anyhow::Result;
+use codspeed::walltime_results::WalltimeBenchmark;
+use serde::Deserialize;
+use serde::Serialize;
+use std::path::Path;
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Instrument {
+    #[serde(rename = "type")]
+    type_: String,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Creator {
+    name: String,
+    version: String,
+    pid: u32,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct WalltimeResults {
+    creator: Creator,
+    instrument: Instrument,
+    benchmarks: Vec<WalltimeBenchmark>,
+}
+
+impl WalltimeResults {
+    pub fn from_benchmarks(benchmarks: Vec<WalltimeBenchmark>) -> Result<Self> {
+        Ok(WalltimeResults {
+            instrument: Instrument {
+                type_: "walltime".to_string(),
+            },
+            creator: Creator {
+                // TODO: Stop impersonating codspeed-rust 🥸
+                name: "codspeed-rust".to_string(),
+                version: env!("CARGO_PKG_VERSION").to_string(),
+                pid: std::process::id(),
+            },
+            benchmarks,
+        })
+    }
+
+    pub fn save_to_file<P: AsRef<Path>>(&self, profile_folder: P) -> Result<()> {
+        let results_path = {
+            let results_dir = profile_folder.as_ref().join("results");
+            std::fs::create_dir_all(&results_dir).with_context(|| {
+                format!(
+                    "Failed to create results directory: {}",
+                    results_dir.display()
+                )
+            })?;
+
+            results_dir.join(format!("{}.json", self.creator.pid))
+        };
+
+        let file = std::fs::File::create(&results_path)
+            .with_context(|| format!("Failed to create file: {}", results_path.display()))?;
+        serde_json::to_writer_pretty(file, &self)
+            .with_context(|| format!("Failed to write JSON to file: {}", results_path.display()))?;
+        Ok(())
+    }
+}

crates/memtrack/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "memtrack"
-version = "4.4.1"
+version = "4.4.2-beta.1"
 edition = "2024"
 repository = "https://github.com/CodSpeedHQ/runner"
 publish = false
@@ -16,11 +16,7 @@ required-features = ["ebpf"]
 
 [features]
 default = ["ebpf"]
-ebpf = [
-    "dep:libbpf-rs",
-    "dep:libbpf-cargo",
-    "dep:vmlinux",
-]
+ebpf = ["dep:libbpf-rs", "dep:libbpf-cargo", "dep:vmlinux"]
 
 [dependencies]
 anyhow = { workspace = true }

src/api_client.rs

@@ -83,6 +83,13 @@ pub struct FetchLocalRunReportVars {
     pub name: String,
     pub run_id: String,
 }
+
+#[derive(Serialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct FetchLocalExecReportVars {
+    pub name: String,
+    pub run_id: String,
+}
 #[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
 pub enum ReportConclusion {
     AcknowledgedFailure,
@@ -157,11 +164,25 @@ nest! {
     }
 }
 
+nest! {
+    #[derive(Debug, Deserialize, Serialize)]*
+    #[serde(rename_all = "camelCase")]*
+    struct FetchLocalExecReportData {
+        project: pub struct FetchLocalExecReportProject {
+            run: FetchLocalRunReportRun,
+        }
+    }
+}
+
 pub struct FetchLocalRunReportResponse {
     pub allowed_regression: f64,
     pub run: FetchLocalRunReportRun,
 }
 
+pub struct FetchLocalExecReportResponse {
+    pub run: FetchLocalRunReportRun,
+}
+
 impl CodSpeedAPIClient {
     pub async fn create_login_session(&self) -> Result<CreateLoginSessionPayload> {
         let response = self
@@ -215,4 +236,26 @@ impl CodSpeedAPIClient {
             Err(err) => bail!("Failed to fetch local run report: {err}"),
         }
     }
+
+    pub async fn fetch_local_exec_report(
+        &self,
+        vars: FetchLocalExecReportVars,
+    ) -> Result<FetchLocalExecReportResponse> {
+        let response = self
+            .gql_client
+            .query_with_vars_unwrap::<FetchLocalExecReportData, FetchLocalExecReportVars>(
+                include_str!("queries/FetchLocalExecReport.gql"),
+                vars.clone(),
+            )
+            .await;
+        match response {
+            Ok(response) => Ok(FetchLocalExecReportResponse {
+                run: response.project.run,
+            }),
+            Err(err) if err.contains_error_code("UNAUTHENTICATED") => {
+                bail!("Your session has expired, please login again using `codspeed auth login`")
+            }
+            Err(err) => bail!("Failed to fetch local run report: {err}"),
+        }
+    }
 }