feat(vitest-runner): add vitest-runner

Author: adriencaccia

packages/vitest-runner/README.md

@@ -0,0 +1,85 @@
+<div align="center">
+<h1><code>@codspeed/vitest-runner</code></h1>
+
+Custom [Vitest](https://vitest.dev) runner for [CodSpeed](https://codspeed.io)
+
+[![CI](https://github.com/CodSpeedHQ/codspeed-node/actions/workflows/ci.yml/badge.svg)](https://github.com/CodSpeedHQ/codspeed-node/actions/workflows/ci.yml)
+[![npm (scoped)](https://img.shields.io/npm/v/@codspeed/tinybench-plugin)](https://www.npmjs.com/package/@codspeed/tinybench-plugin)
+[![Discord](https://img.shields.io/badge/chat%20on-discord-7289da.svg)](https://discord.com/invite/MxpaCfKSqF)
+
+</div>
+
+## Documentation
+
+Check out the [documentation](https://docs.codspeed.io/benchmarks/nodejs) for complete integration instructions.
+
+## Installation
+
+First install the plugin [`@codspeed/vitest-runner`](https://www.npmjs.com/package/@codspeed/vitest-runner) and `vitest` (if not already installed):
+
+```sh
+npm install --save-dev @codspeed/vitest-runner vitest
+```
+
+or with `yarn`:
+
+```sh
+yarn add --dev @codspeed/vitest-runner vitest
+```
+
+or with `pnpm`:
+
+```sh
+pnpm add --save-dev @codspeed/vitest-runner vitest
+```
+
+## Usage
+
+Let's create a fibonacci function and benchmark it with `vitest.bench`:
+
+```ts title="benches/fibo.bench.ts"
+import { describe, bench } from "vitest";
+
+function fibonacci(n: number): number {
+  if (n < 2) {
+    return n;
+  }
+  return fibonacci(n - 1) + fibonacci(n - 2);
+}
+
+describe("fibonacci", () => {
+  bench("fibonacci10", () => {
+    fibonacci(10);
+  });
+
+  bench("fibonacci15", () => {
+    fibonacci(15);
+  });
+});
+```
+
+Create or update your `vitest.config.ts` file to use the CodSpeed runner:
+
+```ts title="vitest.config.ts"
+import { defineConfig } from "vitest/config";
+import { withCodSpeed } from "@codspeed/vitest-runner";
+
+export default withCodSpeed(
+  defineConfig({
+    // ...
+  })
+);
+```
+
+Finally, run your benchmarks (here with `pnpm`):
+
+```bash
+$ pnpm vitest bench --run
+[CodSpeed] bench detected but no instrumentation found, falling back to default vitest runner
+
+... Regular `vitest bench` output
+```
+
+And... Congrats🎉, CodSpeed is installed in your benchmarking suite! Locally, CodSpeed will fallback to vitest since the instrumentation is only available in the CI environment for now.
+
+You can now [run those benchmark in your CI](https://docs.codspeed.io/benchmarks/nodejs#running-the-benchmarks-in-your-ci) to continuously get consistent performance measurements.

packages/vitest-runner/benches/flat.bench.ts

@@ -0,0 +1,33 @@
+import { bench, describe } from "vitest";
+import parsePr from "./parsePr";
+
+const LONG_BODY =
+  new Array(1_000)
+    .fill(
+      "Lorem ipsum dolor sit amet consectetur adipisicing elit. Sunt, earum. Atque architecto vero veniam est tempora fugiat sint quo praesentium quia. Autem, veritatis omnis beatae iste delectus recusandae animi non."
+    )
+    .join("\n") + "fixes #123";
+
+describe("parsePr", () => {
+  bench("short body", () => {
+    parsePr({ body: "fixes #123", title: "test-1", number: 1 });
+  });
+
+  bench("long body", () => {
+    parsePr({ body: LONG_BODY, title: "test-2", number: 2 });
+  });
+});
+
+function fibo(n: number): number {
+  if (n < 2) return 1;
+  return fibo(n - 1) + fibo(n - 2);
+}
+
+describe("fibo", () => {
+  bench("fibo 10", () => {
+    fibo(10);
+  });
+  bench("fibo 15", () => {
+    fibo(15);
+  });
+});

packages/vitest-runner/benches/parsePr.bench.ts

@@ -0,0 +1,51 @@
+import { bench, describe } from "vitest";
+import parsePr from "./parsePr";
+
+const LONG_BODY =
+  new Array(1_000)
+    .fill(
+      "Lorem ipsum dolor sit amet consectetur adipisicing elit. Sunt, earum. Atque architecto vero veniam est tempora fugiat sint quo praesentium quia. Autem, veritatis omnis beatae iste delectus recusandae animi non."
+    )
+    .join("\n") + "fixes #123";
+
+describe("parsePr", () => {
+  bench("short body", () => {
+    parsePr({ body: "fixes #123", title: "test", number: 124 });
+  });
+
+  bench("long body", () => {
+    parsePr({ body: LONG_BODY, title: "test", number: 124 });
+  });
+
+  describe("nested suite", () => {
+    bench("short body", () => {
+      parsePr({ body: "fixes #123", title: "test", number: 124 });
+    });
+
+    bench("long body", () => {
+      parsePr({ body: LONG_BODY, title: "test", number: 124 });
+    });
+
+    describe("deeply nested suite", () => {
+      bench("short body", () => {
+        parsePr({ body: "fixes #123", title: "test", number: 124 });
+      });
+    });
+  });
+});
+
+describe("another parsePr", () => {
+  bench("short body", () => {
+    parsePr({ body: "fixes #123", title: "test", number: 124 });
+  });
+
+  bench("long body", () => {
+    parsePr({ body: LONG_BODY, title: "test", number: 124 });
+  });
+
+  describe("nested suite", () => {
+    bench("short body", () => {
+      parsePr({ body: "fixes #123", title: "test", number: 124 });
+    });
+  });
+});

packages/vitest-runner/benches/parsePr.ts

@@ -0,0 +1,75 @@
+interface PullRequest {
+  number: number;
+  title: string;
+  body: string;
+}
+
+function sendEvent(numberOfOperations: number): void {
+  for (let i = 0; i < numberOfOperations; i++) {
+    let a = i;
+    a = a + 1;
+  }
+}
+
+function logMetrics(
+  numberOfOperations: number,
+  numberOfDeepOperations: number
+): void {
+  for (let i = 0; i < numberOfOperations; i++) {
+    for (let i = 0; i < numberOfOperations; i++) {
+      let a = i;
+      a = a + 1;
+      a = a + 1;
+    }
+    sendEvent(numberOfDeepOperations);
+  }
+}
+
+function parseTitle(title: string): void {
+  logMetrics(10, 10);
+  modifyTitle(title);
+}
+
+function modifyTitle(title: string): void {
+  for (let i = 0; i < 100; i++) {
+    let a = i;
+    a = a + 1 + title.length;
+  }
+}
+
+function prepareParsingBody(body: string): void {
+  for (let i = 0; i < 100; i++) {
+    let a = i;
+    a = a + 1;
+  }
+  parseBody(body);
+}
+
+function parseBody(body: string): void {
+  logMetrics(10, 10);
+  for (let i = 0; i < 200; i++) {
+    let a = i;
+    a = a + 1;
+  }
+  parseIssueFixed(body);
+}
+
+function parseIssueFixed(body: string): number | null {
+  const prefix = "fixes #";
+  const index = body.indexOf(prefix);
+  if (index === -1) {
+    return null;
+  }
+
+  const start = index + prefix.length;
+  let end = start;
+  while (end < body.length && /\d/.test(body[end])) {
+    end += 1;
+  }
+  return parseInt(body.slice(start, end));
+}
+
+export default function parsePr(pullRequest: PullRequest): void {
+  parseTitle(pullRequest.title);
+  prepareParsingBody(pullRequest.body);
+}

packages/vitest-runner/moon.yml

@@ -0,0 +1,16 @@
+tasks:
+  bench:
+    command: vitest bench
+    inputs:
+      - "benches/**"
+    local: true
+    options:
+      cache: false
+
+  test:
+    command: vitest --run
+    inputs:
+      - "./vitest.config.ts"
+
+  test/integ:
+    command: noop

packages/vitest-runner/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@codspeed/vitest-runner",
+  "version": "2.2.0",
+  "description": "vitest runner for CodSpeed",
+  "keywords": [
+    "codspeed",
+    "benchmark",
+    "vitest",
+    "performance"
+  ],
+  "main": "dist/index.es5.js",
+  "types": "dist/index.d.ts",
+  "type": "module",
+  "files": [
+    "dist"
+  ],
+  "author": "Adrien Cacciaguerra <adrien@codspeed.io>",
+  "repository": "https://github.com/CodSpeedHQ/codspeed-node",
+  "homepage": "https://codspeed.io",
+  "license": "Apache-2.0",
+  "scripts": {
+    "bench": "vitest bench"
+  },
+  "dependencies": {
+    "@codspeed/core": "workspace:^2.2.0"
+  },
+  "devDependencies": {
+    "@total-typescript/shoehorn": "^0.1.1",
+    "execa": "^8.0.1",
+    "vitest": "1.0.0-beta.4"
+  },
+  "peerDependencies": {
+    "vitest": "^1.0.0-beta.4"
+  }
+}

packages/vitest-runner/rollup.config.ts

@@ -0,0 +1,33 @@
+import { defineConfig } from "rollup";
+import { declarationsPlugin, jsPlugins } from "../../rollup.options";
+import pkg from "./package.json" assert { type: "json" };
+
+export default defineConfig([
+  {
+    input: "src/index.ts",
+    // for some reasons, vitest only wants to require the `main` entrypoint
+    // but fails when its CJS since it cannot require `vitest/*` modules, as
+    // they are ESM only 🤷
+    // we can circumvent this by exposing the `main` entrypoint as ESM
+    output: { file: pkg.main, format: "es" },
+    plugins: jsPlugins(pkg.version),
+    external: ["@codspeed/core", /^vitest/],
+  },
+  {
+    input: "src/index.ts",
+    output: { file: pkg.types, format: "es" },
+    plugins: declarationsPlugin({ compilerOptions: { composite: false } }),
+  },
+  {
+    input: "src/globalSetup.ts",
+    output: { file: "dist/globalSetup.es5.js", format: "es" },
+    plugins: jsPlugins(pkg.version),
+    external: ["@codspeed/core", /^vitest/],
+  },
+  {
+    input: "src/runner.ts",
+    output: { file: "dist/runner.es5.js", format: "es" },
+    plugins: jsPlugins(pkg.version),
+    external: ["@codspeed/core", /^vitest/],
+  },
+]);

packages/vitest-runner/src/__tests__/globalSetup.test.ts

@@ -0,0 +1,24 @@
+import { describe, expect, it, vi } from "vitest";
+import globalSetup from "../globalSetup";
+
+console.log = vi.fn();
+
+vi.stubGlobal("__VERSION__", "1.0.0");
+
+describe("globalSetup", () => {
+  it("should log the correct message on setup and teardown, and fail when teardown is called twice", async () => {
+    const teardown = globalSetup();
+
+    expect(console.log).toHaveBeenCalledWith(
+      "[CodSpeed] @codspeed/vitest-runner v1.0.0 - setup"
+    );
+
+    teardown();
+
+    expect(console.log).toHaveBeenCalledWith(
+      "[CodSpeed] @codspeed/vitest-runner v1.0.0 - teardown"
+    );
+
+    expect(() => teardown()).toThrowError("teardown called twice");
+  });
+});